From 34595e9b4abacc10ac599aad97c95861af34ea54 Mon Sep 17 00:00:00 2001 From: Joshua Powers Date: Thu, 7 Dec 2017 12:54:46 -0800 Subject: tests: Enable AWS EC2 Integration Testing This enables integration tests to utilize AWS EC2 as a testing platform by utilizing the boto3 Python library. Usage will create and delete a custom VPC for every run. All resources will be tagged with the ec2 tag, 'cii', and the date (e.g. cii-20171220-102452). The VPC is setup with both IPv4 and IPv6 capabilities, but will only hand out IPv4 addresses by default. Instances will have complete Internet access and have full ingress and egress access (i.e. no firewall). SSH keys are generated with each run of the integration tests with the key getting uploaded to AWS at the start of tests and deleted on exit. To enable creation when the platform is setup the SSH generation code is moved to be completed by the platform setup and not during image setup. The nocloud-kvm platform was updated with this change. Creating a custom image will utilize the same clean script, boot_clean_script, that the LXD platform uses as well. The custom AMI is generated, used, and de-registered after a test run. The default instance type is set to t2.micro. This is one of the smallest instance types and is free tier eligible. The default timeout for ec2 was increased to 300 from 120 as many tests hit up against the 2 minute timeout and depending on region load can go over. Documentation for the AWS platform was added with the expected configuration files for the platform to be used. There are some additional whitespace changes included as well. pylint exception was added for paramiko and simplestreams. In the past these were not already flagged due to no __init__.py in the subdirectories of files that used these. boto3 was added to the list of dependencies in the tox ci-test runner. In order to grab console logs on EC2 the harness will now shut down an instance before terminating and before collecting the console log. This is to address a behavior of EC2 where the console log is refreshed very infrequently, but one point when it is refreshed is after shutdown. --- tests/cloud_tests/platforms/ec2/instance.py | 126 ++++++++++++++++++++++++++++ 1 file changed, 126 insertions(+) create mode 100644 tests/cloud_tests/platforms/ec2/instance.py (limited to 'tests/cloud_tests/platforms/ec2/instance.py') diff --git a/tests/cloud_tests/platforms/ec2/instance.py b/tests/cloud_tests/platforms/ec2/instance.py new file mode 100644 index 00000000..4ba737ab --- /dev/null +++ b/tests/cloud_tests/platforms/ec2/instance.py @@ -0,0 +1,126 @@ +# This file is part of cloud-init. See LICENSE file for license information. + +"""Base EC2 instance.""" +import os + +import botocore + +from ..instances import Instance +from tests.cloud_tests import LOG, util + + +class EC2Instance(Instance): + """EC2 backed instance.""" + + platform_name = "ec2" + _ssh_client = None + + def __init__(self, platform, properties, config, features, + image_ami, user_data=None): + """Set up instance. + + @param platform: platform object + @param properties: dictionary of properties + @param config: dictionary of configuration values + @param features: dictionary of supported feature flags + @param image_ami: AWS AMI ID for image to use + @param user_data: test user-data to pass to instance + """ + super(EC2Instance, self).__init__( + platform, image_ami, properties, config, features) + + self.image_ami = image_ami + self.instance = None + self.user_data = user_data + self.ssh_ip = None + self.ssh_port = 22 + self.ssh_key_file = os.path.join( + platform.config['data_dir'], platform.config['private_key']) + self.ssh_pubkey_file = os.path.join( + platform.config['data_dir'], platform.config['public_key']) + + def console_log(self): + """Collect console log from instance. + + The console log is buffered and not always present, therefore + may return empty string. + """ + try: + return self.instance.console_output()['Output'].encode() + except KeyError: + return b'' + + def destroy(self): + """Clean up instance.""" + if self.instance: + LOG.debug('destroying instance %s', self.instance.id) + self.instance.terminate() + self.instance.wait_until_terminated() + + self._ssh_close() + + super(EC2Instance, self).destroy() + + def _execute(self, command, stdin=None, env=None): + """Execute command on instance.""" + env_args = [] + if env: + env_args = ['env'] + ["%s=%s" for k, v in env.items()] + + return self._ssh(['sudo'] + env_args + list(command), stdin=stdin) + + def start(self, wait=True, wait_for_cloud_init=False): + """Start instance on EC2 with the platfrom's VPC.""" + if self.instance: + if self.instance.state['Name'] == 'running': + return + + LOG.debug('starting instance %s', self.instance.id) + self.instance.start() + else: + LOG.debug('launching instance') + + args = { + 'ImageId': self.image_ami, + 'InstanceType': self.platform.instance_type, + 'KeyName': self.platform.key_name, + 'MaxCount': 1, + 'MinCount': 1, + 'SecurityGroupIds': [self.platform.security_group.id], + 'SubnetId': self.platform.subnet.id, + 'TagSpecifications': [{ + 'ResourceType': 'instance', + 'Tags': [{ + 'Key': 'Name', 'Value': self.platform.tag + }] + }], + } + + if self.user_data: + args['UserData'] = self.user_data + + try: + instances = self.platform.ec2_resource.create_instances(**args) + except botocore.exceptions.ClientError as error: + error_msg = error.response['Error']['Message'] + raise util.PlatformError('start', error_msg) + + self.instance = instances[0] + + LOG.debug('instance id: %s', self.instance.id) + if wait: + self.instance.wait_until_running() + self.instance.reload() + self.ssh_ip = self.instance.public_ip_address + self._wait_for_system(wait_for_cloud_init) + + def shutdown(self, wait=True): + """Shutdown instance.""" + LOG.debug('stopping instance %s', self.instance.id) + self.instance.stop() + + if wait: + self.instance.wait_until_stopped() + self.instance.reload() + +# vi: ts=4 expandtab -- cgit v1.2.3 From 32a6a1764e902c31dd3af9b674cea14cd6501187 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Fri, 19 Jan 2018 09:43:55 -0500 Subject: tests: Fix EC2 Platform to return console output as bytes. The EC2 test platform uses boto, and boto decodes console output with decode('utf-8', 'replace). It is known that Ubuntu consoles contain non-utf8 characters, making this call lossy. The change here is to patch the boto session to include a OutputBytes entry in the console_output response, and then to utilize that in console_log. More information on problem and solution at: https://github.com/boto/botocore/issues/1351 --- tests/cloud_tests/platforms/ec2/instance.py | 10 +++++++-- tests/cloud_tests/platforms/ec2/platform.py | 33 ++++++++++++++++++++++++++--- 2 files changed, 38 insertions(+), 5 deletions(-) (limited to 'tests/cloud_tests/platforms/ec2/instance.py') diff --git a/tests/cloud_tests/platforms/ec2/instance.py b/tests/cloud_tests/platforms/ec2/instance.py index 4ba737ab..ab6037b1 100644 --- a/tests/cloud_tests/platforms/ec2/instance.py +++ b/tests/cloud_tests/platforms/ec2/instance.py @@ -46,9 +46,15 @@ class EC2Instance(Instance): may return empty string. """ try: - return self.instance.console_output()['Output'].encode() + # OutputBytes comes from platform._decode_console_output_as_bytes + response = self.instance.console_output() + return response['OutputBytes'] except KeyError: - return b'' + if 'Output' in response: + msg = ("'OutputBytes' did not exist in console_output() but " + "'Output' did: %s..." % response['Output'][0:128]) + raise util.PlatformError('console_log', msg) + return ('No Console Output [%s]' % self.instance).encode() def destroy(self): """Clean up instance.""" diff --git a/tests/cloud_tests/platforms/ec2/platform.py b/tests/cloud_tests/platforms/ec2/platform.py index fdb17ba0..f188c27b 100644 --- a/tests/cloud_tests/platforms/ec2/platform.py +++ b/tests/cloud_tests/platforms/ec2/platform.py @@ -6,6 +6,8 @@ import os import boto3 import botocore +from botocore import session, handlers +import base64 from ..platforms import Platform from .image import EC2Image @@ -28,9 +30,10 @@ class EC2Platform(Platform): self.instance_type = config['instance-type'] try: - self.ec2_client = boto3.client('ec2') - self.ec2_resource = boto3.resource('ec2') - self.ec2_region = boto3.Session().region_name + b3session = get_session() + self.ec2_client = b3session.client('ec2') + self.ec2_resource = b3session.resource('ec2') + self.ec2_region = b3session.region_name self.key_name = self._upload_public_key(config) except botocore.exceptions.NoRegionError: raise RuntimeError( @@ -228,4 +231,28 @@ class EC2Platform(Platform): return self.tag + +def _decode_console_output_as_bytes(parsed, **kwargs): + """Provide console output as bytes in OutputBytes. + + For this to be useful, the session has to have had the + decode_console_output handler unregistered already. + + https://github.com/boto/botocore/issues/1351 .""" + if 'Output' not in parsed: + return + orig = parsed['Output'] + handlers.decode_console_output(parsed, **kwargs) + parsed['OutputBytes'] = base64.b64decode(orig) + + +def get_session(): + mysess = session.get_session() + mysess.unregister('after-call.ec2.GetConsoleOutput', + handlers.decode_console_output) + mysess.register('after-call.ec2.GetConsoleOutput', + _decode_console_output_as_bytes) + return boto3.Session(botocore_session=mysess) + + # vi: ts=4 expandtab -- cgit v1.2.3