merge from master at 17.2-30-gf7deaf15

4 files changed, 555 insertions, 0 deletions

diff --git a/tests/cloud_tests/platforms/ec2/image.py b/tests/cloud_tests/platforms/ec2/image.py
new file mode 100644
index 00000000..7bedf59d
--- /dev/null
+++ b/tests/cloud_tests/platforms/ec2/image.py
@@ -0,0 +1,99 @@
+# This file is part of cloud-init. See LICENSE file for license information.
+
+"""EC2 Image Base Class."""
+
+from ..images import Image
+from .snapshot import EC2Snapshot
+from tests.cloud_tests import LOG
+
+
+class EC2Image(Image):
+    """EC2 backed image."""
+
+    platform_name = 'ec2'
+
+    def __init__(self, platform, config, image_ami):
+        """Set up image.
+
+        @param platform: platform object
+        @param config: image configuration
+        @param image_ami: string of image ami ID
+        """
+        super(EC2Image, self).__init__(platform, config)
+        self._img_instance = None
+        self.image_ami = image_ami
+
+    @property
+    def _instance(self):
+        """Internal use only, returns a running instance"""
+        if not self._img_instance:
+            self._img_instance = self.platform.create_instance(
+                self.properties, self.config, self.features,
+                self.image_ami, user_data=None)
+            self._img_instance.start(wait=True, wait_for_cloud_init=True)
+        return self._img_instance
+
+    def destroy(self):
+        """Delete the instance used to create a custom image."""
+        if self._img_instance:
+            LOG.debug('terminating backing instance %s',
+                      self._img_instance.instance.instance_id)
+            self._img_instance.instance.terminate()
+            self._img_instance.instance.wait_until_terminated()
+
+        super(EC2Image, self).destroy()
+
+    def _execute(self, *args, **kwargs):
+        """Execute command in image, modifying image."""
+        self._instance.start(wait=True)
+        return self._instance._execute(*args, **kwargs)
+
+    def push_file(self, local_path, remote_path):
+        """Copy file at 'local_path' to instance at 'remote_path'."""
+        self._instance.start(wait=True)
+        return self._instance.push_file(local_path, remote_path)
+
+    def run_script(self, *args, **kwargs):
+        """Run script in image, modifying image.
+
+        @return_value: script output
+        """
+        self._instance.start(wait=True)
+        return self._instance.run_script(*args, **kwargs)
+
+    def snapshot(self):
+        """Create snapshot of image, block until done.
+
+        Will return base image_ami if no instance has been booted, otherwise
+        will run the clean script, shutdown the instance, create a custom
+        AMI, and use that AMI once available.
+        """
+        if not self._img_instance:
+            return EC2Snapshot(self.platform, self.properties, self.config,
+                               self.features, self.image_ami,
+                               delete_on_destroy=False)
+
+        if self.config.get('boot_clean_script'):
+            self._img_instance.run_script(self.config.get('boot_clean_script'))
+
+        self._img_instance.shutdown(wait=True)
+
+        LOG.debug('creating custom ami from instance %s',
+                  self._img_instance.instance.instance_id)
+        response = self.platform.ec2_client.create_image(
+            Name='%s-%s' % (self.platform.tag, self.image_ami),
+            InstanceId=self._img_instance.instance.instance_id
+        )
+        image_ami_edited = response['ImageId']
+
+        # Create image and wait until it is in the 'available' state
+        image = self.platform.ec2_resource.Image(image_ami_edited)
+        image.wait_until_exists()
+        waiter = self.platform.ec2_client.get_waiter('image_available')
+        waiter.wait(ImageIds=[image.id])
+        image.reload()
+
+        return EC2Snapshot(self.platform, self.properties, self.config,
+                           self.features, image_ami_edited)
+
+# vi: ts=4 expandtab
diff --git a/tests/cloud_tests/platforms/ec2/instance.py b/tests/cloud_tests/platforms/ec2/instance.py
new file mode 100644
index 00000000..ab6037b1
--- /dev/null
+++ b/tests/cloud_tests/platforms/ec2/instance.py
@@ -0,0 +1,132 @@
+# This file is part of cloud-init. See LICENSE file for license information.
+
+"""Base EC2 instance."""
+import os
+
+import botocore
+
+from ..instances import Instance
+from tests.cloud_tests import LOG, util
+
+
+class EC2Instance(Instance):
+    """EC2 backed instance."""
+
+    platform_name = "ec2"
+    _ssh_client = None
+
+    def __init__(self, platform, properties, config, features,
+                 image_ami, user_data=None):
+        """Set up instance.
+
+        @param platform: platform object
+        @param properties: dictionary of properties
+        @param config: dictionary of configuration values
+        @param features: dictionary of supported feature flags
+        @param image_ami: AWS AMI ID for image to use
+        @param user_data: test user-data to pass to instance
+        """
+        super(EC2Instance, self).__init__(
+            platform, image_ami, properties, config, features)
+
+        self.image_ami = image_ami
+        self.instance = None
+        self.user_data = user_data
+        self.ssh_ip = None
+        self.ssh_port = 22
+        self.ssh_key_file = os.path.join(
+            platform.config['data_dir'], platform.config['private_key'])
+        self.ssh_pubkey_file = os.path.join(
+            platform.config['data_dir'], platform.config['public_key'])
+
+    def console_log(self):
+        """Collect console log from instance.
+
+        The console log is buffered and not always present, therefore
+        may return empty string.
+        """
+        try:
+            # OutputBytes comes from platform._decode_console_output_as_bytes
+            response = self.instance.console_output()
+            return response['OutputBytes']
+        except KeyError:
+            if 'Output' in response:
+                msg = ("'OutputBytes' did not exist in console_output() but "
+                       "'Output' did: %s..." % response['Output'][0:128])
+                raise util.PlatformError('console_log', msg)
+            return ('No Console Output [%s]' % self.instance).encode()
+
+    def destroy(self):
+        """Clean up instance."""
+        if self.instance:
+            LOG.debug('destroying instance %s', self.instance.id)
+            self.instance.terminate()
+            self.instance.wait_until_terminated()
+
+        self._ssh_close()
+
+        super(EC2Instance, self).destroy()
+
+    def _execute(self, command, stdin=None, env=None):
+        """Execute command on instance."""
+        env_args = []
+        if env:
+            env_args = ['env'] + ["%s=%s" for k, v in env.items()]
+
+        return self._ssh(['sudo'] + env_args + list(command), stdin=stdin)
+
+    def start(self, wait=True, wait_for_cloud_init=False):
+        """Start instance on EC2 with the platfrom's VPC."""
+        if self.instance:
+            if self.instance.state['Name'] == 'running':
+                return
+
+            LOG.debug('starting instance %s', self.instance.id)
+            self.instance.start()
+        else:
+            LOG.debug('launching instance')
+
+            args = {
+                'ImageId': self.image_ami,
+                'InstanceType': self.platform.instance_type,
+                'KeyName': self.platform.key_name,
+                'MaxCount': 1,
+                'MinCount': 1,
+                'SecurityGroupIds': [self.platform.security_group.id],
+                'SubnetId': self.platform.subnet.id,
+                'TagSpecifications': [{
+                    'ResourceType': 'instance',
+                    'Tags': [{
+                        'Key': 'Name', 'Value': self.platform.tag
+                    }]
+                }],
+            }
+
+            if self.user_data:
+                args['UserData'] = self.user_data
+
+            try:
+                instances = self.platform.ec2_resource.create_instances(**args)
+            except botocore.exceptions.ClientError as error:
+                error_msg = error.response['Error']['Message']
+                raise util.PlatformError('start', error_msg)
+
+            self.instance = instances[0]
+
+        LOG.debug('instance id: %s', self.instance.id)
+        if wait:
+            self.instance.wait_until_running()
+            self.instance.reload()
+            self.ssh_ip = self.instance.public_ip_address
+            self._wait_for_system(wait_for_cloud_init)
+
+    def shutdown(self, wait=True):
+        """Shutdown instance."""
+        LOG.debug('stopping instance %s', self.instance.id)
+        self.instance.stop()
+
+        if wait:
+            self.instance.wait_until_stopped()
+            self.instance.reload()
+
+# vi: ts=4 expandtab
diff --git a/tests/cloud_tests/platforms/ec2/platform.py b/tests/cloud_tests/platforms/ec2/platform.py
new file mode 100644
index 00000000..f188c27b
--- /dev/null
+++ b/tests/cloud_tests/platforms/ec2/platform.py
@@ -0,0 +1,258 @@
+# This file is part of cloud-init. See LICENSE file for license information.
+
+"""Base EC2 platform."""
+from datetime import datetime
+import os
+
+import boto3
+import botocore
+from botocore import session, handlers
+import base64
+
+from ..platforms import Platform
+from .image import EC2Image
+from .instance import EC2Instance
+from tests.cloud_tests import LOG
+
+
+class EC2Platform(Platform):
+    """EC2 test platform."""
+
+    platform_name = 'ec2'
+    ipv4_cidr = '192.168.1.0/20'
+
+    def __init__(self, config):
+        """Set up platform."""
+        super(EC2Platform, self).__init__(config)
+        # Used for unique VPC, SSH key, and custom AMI generation naming
+        self.tag = '%s-%s' % (
+            config['tag'], datetime.now().strftime('%Y%m%d%H%M%S'))
+        self.instance_type = config['instance-type']
+
+        try:
+            b3session = get_session()
+            self.ec2_client = b3session.client('ec2')
+            self.ec2_resource = b3session.resource('ec2')
+            self.ec2_region = b3session.region_name
+            self.key_name = self._upload_public_key(config)
+        except botocore.exceptions.NoRegionError:
+            raise RuntimeError(
+                'Please configure default region in $HOME/.aws/config')
+        except botocore.exceptions.NoCredentialsError:
+            raise RuntimeError(
+                'Please configure ec2 credentials in $HOME/.aws/credentials')
+
+        self.vpc = self._create_vpc()
+        self.internet_gateway = self._create_internet_gateway()
+        self.subnet = self._create_subnet()
+        self.routing_table = self._create_routing_table()
+        self.security_group = self._create_security_group()
+
+    def create_instance(self, properties, config, features,
+                        image_ami, user_data=None):
+        """Create an instance
+
+        @param src_img_path: image path to launch from
+        @param properties: image properties
+        @param config: image configuration
+        @param features: image features
+        @param image_ami: string of image ami ID
+        @param user_data: test user-data to pass to instance
+        @return_value: cloud_tests.instances instance
+        """
+        return EC2Instance(self, properties, config, features,
+                           image_ami, user_data)
+
+    def destroy(self):
+        """Delete SSH keys, terminate all instances, and delete VPC."""
+        for instance in self.vpc.instances.all():
+            LOG.debug('waiting for instance %s termination', instance.id)
+            instance.terminate()
+            instance.wait_until_terminated()
+
+        if self.key_name:
+            LOG.debug('deleting SSH key %s', self.key_name)
+            self.ec2_client.delete_key_pair(KeyName=self.key_name)
+
+        if self.security_group:
+            LOG.debug('deleting security group %s', self.security_group.id)
+            self.security_group.delete()
+
+        if self.subnet:
+            LOG.debug('deleting subnet %s', self.subnet.id)
+            self.subnet.delete()
+
+        if self.routing_table:
+            LOG.debug('deleting routing table %s', self.routing_table.id)
+            self.routing_table.delete()
+
+        if self.internet_gateway:
+            LOG.debug('deleting internet gateway %s', self.internet_gateway.id)
+            self.internet_gateway.detach_from_vpc(VpcId=self.vpc.id)
+            self.internet_gateway.delete()
+
+        if self.vpc:
+            LOG.debug('deleting vpc %s', self.vpc.id)
+            self.vpc.delete()
+
+    def get_image(self, img_conf):
+        """Get image using specified image configuration.
+
+        Hard coded for 'amd64' based images.
+
+        @param img_conf: configuration for image
+        @return_value: cloud_tests.images instance
+        """
+        if img_conf['root-store'] == 'ebs':
+            root_store = 'ssd'
+        elif img_conf['root-store'] == 'instance-store':
+            root_store = 'instance'
+        else:
+            raise RuntimeError('Unknown root-store type: %s' %
+                               (img_conf['root-store']))
+
+        filters = [
+            'arch=%s' % 'amd64',
+            'endpoint=https://ec2.%s.amazonaws.com' % self.ec2_region,
+            'region=%s' % self.ec2_region,
+            'release=%s' % img_conf['release'],
+            'root_store=%s' % root_store,
+            'virt=hvm',
+        ]
+
+        LOG.debug('finding image using streams')
+        image = self._query_streams(img_conf, filters)
+
+        try:
+            image_ami = image['id']
+        except KeyError:
+            raise RuntimeError('No images found for %s!' % img_conf['release'])
+
+        LOG.debug('found image: %s', image_ami)
+        image = EC2Image(self, img_conf, image_ami)
+        return image
+
+    def _create_internet_gateway(self):
+        """Create Internet Gateway and assign to VPC."""
+        LOG.debug('creating internet gateway')
+        internet_gateway = self.ec2_resource.create_internet_gateway()
+        internet_gateway.attach_to_vpc(VpcId=self.vpc.id)
+        self._tag_resource(internet_gateway)
+
+        return internet_gateway
+
+    def _create_routing_table(self):
+        """Update default routing table with internet gateway.
+
+        This sets up internet access between the VPC via the internet gateway
+        by configuring routing tables for IPv4 and IPv6.
+        """
+        LOG.debug('creating routing table')
+        route_table = self.vpc.create_route_table()
+        route_table.create_route(DestinationCidrBlock='0.0.0.0/0',
+                                 GatewayId=self.internet_gateway.id)
+        route_table.create_route(DestinationIpv6CidrBlock='::/0',
+                                 GatewayId=self.internet_gateway.id)
+        route_table.associate_with_subnet(SubnetId=self.subnet.id)
+        self._tag_resource(route_table)
+
+        return route_table
+
+    def _create_security_group(self):
+        """Enables ingress to default VPC security group."""
+        LOG.debug('creating security group')
+        security_group = self.vpc.create_security_group(
+            GroupName=self.tag, Description='integration test security group')
+        security_group.authorize_ingress(
+            IpProtocol='-1', FromPort=-1, ToPort=-1, CidrIp='0.0.0.0/0')
+        self._tag_resource(security_group)
+
+        return security_group
+
+    def _create_subnet(self):
+        """Generate IPv4 and IPv6 subnets for use."""
+        ipv6_cidr = self.vpc.ipv6_cidr_block_association_set[0][
+            'Ipv6CidrBlock'][:-2] + '64'
+
+        LOG.debug('creating subnet with following ranges:')
+        LOG.debug('ipv4: %s', self.ipv4_cidr)
+        LOG.debug('ipv6: %s', ipv6_cidr)
+        subnet = self.vpc.create_subnet(CidrBlock=self.ipv4_cidr,
+                                        Ipv6CidrBlock=ipv6_cidr)
+        modify_subnet = subnet.meta.client.modify_subnet_attribute
+        modify_subnet(SubnetId=subnet.id,
+                      MapPublicIpOnLaunch={'Value': True})
+        self._tag_resource(subnet)
+
+        return subnet
+
+    def _create_vpc(self):
+        """Setup AWS EC2 VPC or return existing VPC."""
+        LOG.debug('creating new vpc')
+        try:
+            vpc = self.ec2_resource.create_vpc(
+                CidrBlock=self.ipv4_cidr,
+                AmazonProvidedIpv6CidrBlock=True)
+        except botocore.exceptions.ClientError as e:
+            raise RuntimeError(e)
+
+        vpc.wait_until_available()
+        self._tag_resource(vpc)
+
+        return vpc
+
+    def _tag_resource(self, resource):
+        """Tag a resource with the specified tag.
+
+        This makes finding and deleting resources specific to this testing
+        much easier to find.
+
+        @param resource: resource to tag
+        """
+        tag = {
+            'Key': 'Name',
+            'Value': self.tag
+        }
+        resource.create_tags(Tags=[tag])
+
+    def _upload_public_key(self, config):
+        """Generate random name and upload SSH key with that name.
+
+        @param config: platform config
+        @return: string of ssh key name
+        """
+        key_file = os.path.join(config['data_dir'], config['public_key'])
+        with open(key_file, 'r') as file:
+            public_key = file.read().strip('\n')
+
+        LOG.debug('uploading SSH key %s', self.tag)
+        self.ec2_client.import_key_pair(KeyName=self.tag,
+                                        PublicKeyMaterial=public_key)
+
+        return self.tag
+
+
+def _decode_console_output_as_bytes(parsed, **kwargs):
+    """Provide console output as bytes in OutputBytes.
+
+       For this to be useful, the session has to have had the
+       decode_console_output handler unregistered already.
+
+       https://github.com/boto/botocore/issues/1351 ."""
+    if 'Output' not in parsed:
+        return
+    orig = parsed['Output']
+    handlers.decode_console_output(parsed, **kwargs)
+    parsed['OutputBytes'] = base64.b64decode(orig)
+
+
+def get_session():
+    mysess = session.get_session()
+    mysess.unregister('after-call.ec2.GetConsoleOutput',
+                      handlers.decode_console_output)
+    mysess.register('after-call.ec2.GetConsoleOutput',
+                    _decode_console_output_as_bytes)
+    return boto3.Session(botocore_session=mysess)
+
+
+# vi: ts=4 expandtab
diff --git a/tests/cloud_tests/platforms/ec2/snapshot.py b/tests/cloud_tests/platforms/ec2/snapshot.py
new file mode 100644
index 00000000..2c48cb54
--- /dev/null
+++ b/tests/cloud_tests/platforms/ec2/snapshot.py
@@ -0,0 +1,66 @@
+# This file is part of cloud-init. See LICENSE file for license information.
+
+"""Base EC2 snapshot."""
+
+from ..snapshots import Snapshot
+from tests.cloud_tests import LOG
+
+
+class EC2Snapshot(Snapshot):
+    """EC2 image copy backed snapshot."""
+
+    platform_name = 'ec2'
+
+    def __init__(self, platform, properties, config, features, image_ami,
+                 delete_on_destroy=True):
+        """Set up snapshot.
+
+        @param platform: platform object
+        @param properties: image properties
+        @param config: image config
+        @param features: supported feature flags
+        @param image_ami: string of image ami ID
+        @param delete_on_destroy: boolean to delete on destroy
+        """
+        super(EC2Snapshot, self).__init__(
+            platform, properties, config, features)
+
+        self.image_ami = image_ami
+        self.delete_on_destroy = delete_on_destroy
+
+    def destroy(self):
+        """Deregister the backing AMI."""
+        if self.delete_on_destroy:
+            image = self.platform.ec2_resource.Image(self.image_ami)
+            snapshot_id = image.block_device_mappings[0]['Ebs']['SnapshotId']
+
+            LOG.debug('removing custom ami %s', self.image_ami)
+            self.platform.ec2_client.deregister_image(ImageId=self.image_ami)
+
+            LOG.debug('removing custom snapshot %s', snapshot_id)
+            self.platform.ec2_client.delete_snapshot(SnapshotId=snapshot_id)
+
+    def launch(self, user_data, meta_data=None, block=True, start=True,
+               use_desc=None):
+        """Launch instance.
+
+        @param user_data: user-data for the instance
+        @param meta_data: meta_data for the instance
+        @param block: wait until instance is created
+        @param start: start instance and wait until fully started
+        @param use_desc: string of test name
+        @return_value: an Instance
+        """
+        if meta_data is not None:
+            raise ValueError("metadata not supported on Ec2")
+
+        instance = self.platform.create_instance(
+            self.properties, self.config, self.features,
+            self.image_ami, user_data)
+
+        if start:
+            instance.start()
+
+        return instance
+
+# vi: ts=4 expandtab