From 34595e9b4abacc10ac599aad97c95861af34ea54 Mon Sep 17 00:00:00 2001 From: Joshua Powers Date: Thu, 7 Dec 2017 12:54:46 -0800 Subject: tests: Enable AWS EC2 Integration Testing This enables integration tests to utilize AWS EC2 as a testing platform by utilizing the boto3 Python library. Usage will create and delete a custom VPC for every run. All resources will be tagged with the ec2 tag, 'cii', and the date (e.g. cii-20171220-102452). The VPC is setup with both IPv4 and IPv6 capabilities, but will only hand out IPv4 addresses by default. Instances will have complete Internet access and have full ingress and egress access (i.e. no firewall). SSH keys are generated with each run of the integration tests with the key getting uploaded to AWS at the start of tests and deleted on exit. To enable creation when the platform is setup the SSH generation code is moved to be completed by the platform setup and not during image setup. The nocloud-kvm platform was updated with this change. Creating a custom image will utilize the same clean script, boot_clean_script, that the LXD platform uses as well. The custom AMI is generated, used, and de-registered after a test run. The default instance type is set to t2.micro. This is one of the smallest instance types and is free tier eligible. The default timeout for ec2 was increased to 300 from 120 as many tests hit up against the 2 minute timeout and depending on region load can go over. Documentation for the AWS platform was added with the expected configuration files for the platform to be used. There are some additional whitespace changes included as well. pylint exception was added for paramiko and simplestreams. In the past these were not already flagged due to no __init__.py in the subdirectories of files that used these. boto3 was added to the list of dependencies in the tox ci-test runner. In order to grab console logs on EC2 the harness will now shut down an instance before terminating and before collecting the console log. This is to address a behavior of EC2 where the console log is refreshed very infrequently, but one point when it is refreshed is after shutdown. --- tests/cloud_tests/platforms/ec2/image.py | 109 +++++++++++++++++++++++++++++++ 1 file changed, 109 insertions(+) create mode 100644 tests/cloud_tests/platforms/ec2/image.py (limited to 'tests/cloud_tests/platforms/ec2/image.py') diff --git a/tests/cloud_tests/platforms/ec2/image.py b/tests/cloud_tests/platforms/ec2/image.py new file mode 100644 index 00000000..53706b1d --- /dev/null +++ b/tests/cloud_tests/platforms/ec2/image.py @@ -0,0 +1,109 @@ +# This file is part of cloud-init. See LICENSE file for license information. + +"""EC2 Image Base Class.""" + +from ..images import Image +from .snapshot import EC2Snapshot +from tests.cloud_tests import LOG + + +class EC2Image(Image): + """EC2 backed image.""" + + platform_name = 'ec2' + + def __init__(self, platform, config, image_ami): + """Set up image. + + @param platform: platform object + @param config: image configuration + @param image_ami: string of image ami ID + """ + super(EC2Image, self).__init__(platform, config) + self._img_instance = None + self.image_ami = image_ami + + @property + def _instance(self): + """Internal use only, returns a running instance""" + if not self._img_instance: + self._img_instance = self.platform.create_instance( + self.properties, self.config, self.features, + self.image_ami, user_data=None) + self._img_instance.start(wait=True, wait_for_cloud_init=True) + return self._img_instance + + @property + def properties(self): + """Dictionary containing: 'arch', 'os', 'version', 'release'.""" + return { + 'arch': self.config['arch'], + 'os': self.config['family'], + 'release': self.config['release'], + 'version': self.config['version'], + } + + def destroy(self): + """Delete the instance used to create a custom image.""" + if self._img_instance: + LOG.debug('terminating backing instance %s', + self._img_instance.instance.instance_id) + self._img_instance.instance.terminate() + self._img_instance.instance.wait_until_terminated() + + super(EC2Image, self).destroy() + + def _execute(self, *args, **kwargs): + """Execute command in image, modifying image.""" + self._instance.start(wait=True) + return self._instance._execute(*args, **kwargs) + + def push_file(self, local_path, remote_path): + """Copy file at 'local_path' to instance at 'remote_path'.""" + self._instance.start(wait=True) + return self._instance.push_file(local_path, remote_path) + + def run_script(self, *args, **kwargs): + """Run script in image, modifying image. + + @return_value: script output + """ + self._instance.start(wait=True) + return self._instance.run_script(*args, **kwargs) + + def snapshot(self): + """Create snapshot of image, block until done. + + Will return base image_ami if no instance has been booted, otherwise + will run the clean script, shutdown the instance, create a custom + AMI, and use that AMI once available. + """ + if not self._img_instance: + return EC2Snapshot(self.platform, self.properties, self.config, + self.features, self.image_ami, + delete_on_destroy=False) + + if self.config.get('boot_clean_script'): + self._img_instance.run_script(self.config.get('boot_clean_script')) + + self._img_instance.shutdown(wait=True) + + LOG.debug('creating custom ami from instance %s', + self._img_instance.instance.instance_id) + response = self.platform.ec2_client.create_image( + Name='%s-%s' % (self.platform.tag, self.image_ami), + InstanceId=self._img_instance.instance.instance_id + ) + image_ami_edited = response['ImageId'] + + # Create image and wait until it is in the 'available' state + image = self.platform.ec2_resource.Image(image_ami_edited) + image.wait_until_exists() + waiter = self.platform.ec2_client.get_waiter('image_available') + waiter.wait(ImageIds=[image.id]) + image.reload() + + return EC2Snapshot(self.platform, self.properties, self.config, + self.features, image_ami_edited) + +# vi: ts=4 expandtab -- cgit v1.2.3 From 72270e8c311efc8b9ba8bb92492d8728d84bd9f2 Mon Sep 17 00:00:00 2001 From: Joshua Powers Date: Mon, 8 Jan 2018 10:00:35 -0800 Subject: tests: clean up image properties This fixes the incorrectly named 'family' value for images as 'os'. Families are already defined in util.py:OS_FAMILY_MAPPING and a family is a collection of OSes. This makes the properties function part of the super class of image as it is only overrided by the lxd backend. --- tests/cloud_tests/platforms/ec2/image.py | 10 ---------- tests/cloud_tests/platforms/images.py | 3 ++- tests/cloud_tests/platforms/nocloudkvm/image.py | 10 ---------- tests/cloud_tests/releases.yaml | 10 +++++----- 4 files changed, 7 insertions(+), 26 deletions(-) (limited to 'tests/cloud_tests/platforms/ec2/image.py') diff --git a/tests/cloud_tests/platforms/ec2/image.py b/tests/cloud_tests/platforms/ec2/image.py index 53706b1d..7bedf59d 100644 --- a/tests/cloud_tests/platforms/ec2/image.py +++ b/tests/cloud_tests/platforms/ec2/image.py @@ -33,16 +33,6 @@ class EC2Image(Image): self._img_instance.start(wait=True, wait_for_cloud_init=True) return self._img_instance - @property - def properties(self): - """Dictionary containing: 'arch', 'os', 'version', 'release'.""" - return { - 'arch': self.config['arch'], - 'os': self.config['family'], - 'release': self.config['release'], - 'version': self.config['version'], - } - def destroy(self): """Delete the instance used to create a custom image.""" if self._img_instance: diff --git a/tests/cloud_tests/platforms/images.py b/tests/cloud_tests/platforms/images.py index d503108a..557a5cf6 100644 --- a/tests/cloud_tests/platforms/images.py +++ b/tests/cloud_tests/platforms/images.py @@ -26,7 +26,8 @@ class Image(TargetBase): @property def properties(self): """{} containing: 'arch', 'os', 'version', 'release'.""" - raise NotImplementedError + return {k: self.config[k] + for k in ('arch', 'os', 'release', 'version')} @property def features(self): diff --git a/tests/cloud_tests/platforms/nocloudkvm/image.py b/tests/cloud_tests/platforms/nocloudkvm/image.py index 09ff2a3b..bc2b6e75 100644 --- a/tests/cloud_tests/platforms/nocloudkvm/image.py +++ b/tests/cloud_tests/platforms/nocloudkvm/image.py @@ -35,16 +35,6 @@ class NoCloudKVMImage(Image): super(NoCloudKVMImage, self).__init__(platform, config) - @property - def properties(self): - """Dictionary containing: 'arch', 'os', 'version', 'release'.""" - return { - 'arch': self.config['arch'], - 'os': self.config['family'], - 'release': self.config['release'], - 'version': self.config['version'], - } - def _execute(self, command, stdin=None, env=None): """Execute command in image, modifying image.""" return self.mount_image_callback(command, stdin=stdin, env=env) diff --git a/tests/cloud_tests/releases.yaml b/tests/cloud_tests/releases.yaml index 48f903b8..0a9fa602 100644 --- a/tests/cloud_tests/releases.yaml +++ b/tests/cloud_tests/releases.yaml @@ -132,7 +132,7 @@ releases: enabled: true release: bionic version: 18.04 - family: ubuntu + os: ubuntu feature_groups: - base - debian_base @@ -148,7 +148,7 @@ releases: enabled: true release: artful version: 17.10 - family: ubuntu + os: ubuntu feature_groups: - base - debian_base @@ -164,7 +164,7 @@ releases: enabled: true release: zesty version: 17.04 - family: ubuntu + os: ubuntu feature_groups: - base - debian_base @@ -180,7 +180,7 @@ releases: enabled: true release: xenial version: 16.04 - family: ubuntu + os: ubuntu feature_groups: - base - debian_base @@ -196,7 +196,7 @@ releases: enabled: true release: trusty version: 14.04 - family: ubuntu + os: ubuntu feature_groups: - base - debian_base -- cgit v1.2.3