tests: Fix EC2 Platform to return console output as bytes.

The EC2 test platform uses boto, and boto decodes console output
with decode('utf-8', 'replace). It is known that Ubuntu consoles
contain non-utf8 characters, making this call lossy.

The change here is to patch the boto session to include a OutputBytes
entry in the console_output response, and then to utilize that in
console_log.

More information on problem and solution at:
   https://github.com/boto/botocore/issues/1351

2 files changed, 38 insertions, 5 deletions

diff --git a/tests/cloud_tests/platforms/ec2/instance.py b/tests/cloud_tests/platforms/ec2/instance.py
index 4ba737ab..ab6037b1 100644
--- a/tests/cloud_tests/platforms/ec2/instance.py
+++ b/tests/cloud_tests/platforms/ec2/instance.py
@@ -46,9 +46,15 @@ class EC2Instance(Instance):
         may return empty string.
         """
         try:
-            return self.instance.console_output()['Output'].encode()
+            # OutputBytes comes from platform._decode_console_output_as_bytes
+            response = self.instance.console_output()
+            return response['OutputBytes']
         except KeyError:
-            return b''
+            if 'Output' in response:
+                msg = ("'OutputBytes' did not exist in console_output() but "
+                       "'Output' did: %s..." % response['Output'][0:128])
+                raise util.PlatformError('console_log', msg)
+            return ('No Console Output [%s]' % self.instance).encode()
 
     def destroy(self):
         """Clean up instance."""
diff --git a/tests/cloud_tests/platforms/ec2/platform.py b/tests/cloud_tests/platforms/ec2/platform.py
index fdb17ba0..f188c27b 100644
--- a/tests/cloud_tests/platforms/ec2/platform.py
+++ b/tests/cloud_tests/platforms/ec2/platform.py
@@ -6,6 +6,8 @@ import os
 
 import boto3
 import botocore
+from botocore import session, handlers
+import base64
 
 from ..platforms import Platform
 from .image import EC2Image
@@ -28,9 +30,10 @@ class EC2Platform(Platform):
         self.instance_type = config['instance-type']
 
         try:
-            self.ec2_client = boto3.client('ec2')
-            self.ec2_resource = boto3.resource('ec2')
-            self.ec2_region = boto3.Session().region_name
+            b3session = get_session()
+            self.ec2_client = b3session.client('ec2')
+            self.ec2_resource = b3session.resource('ec2')
+            self.ec2_region = b3session.region_name
             self.key_name = self._upload_public_key(config)
         except botocore.exceptions.NoRegionError:
             raise RuntimeError(
@@ -228,4 +231,28 @@ class EC2Platform(Platform):
 
         return self.tag
 
+
+def _decode_console_output_as_bytes(parsed, **kwargs):
+    """Provide console output as bytes in OutputBytes.
+
+       For this to be useful, the session has to have had the
+       decode_console_output handler unregistered already.
+
+       https://github.com/boto/botocore/issues/1351 ."""
+    if 'Output' not in parsed:
+        return
+    orig = parsed['Output']
+    handlers.decode_console_output(parsed, **kwargs)
+    parsed['OutputBytes'] = base64.b64decode(orig)
+
+
+def get_session():
+    mysess = session.get_session()
+    mysess.unregister('after-call.ec2.GetConsoleOutput',
+                      handlers.decode_console_output)
+    mysess.register('after-call.ec2.GetConsoleOutput',
+                    _decode_console_output_as_bytes)
+    return boto3.Session(botocore_session=mysess)
+
+
 # vi: ts=4 expandtab