move towards user-data being binary

UrlResponse: biggest change... make readurl return bytes, making user
             know what to do with it.
util: add load_tfile_or_url for loading text file or url
      as  read_file_or_url now returns bytes

ec2_utils: all meta-data is text, remove non-obvious string translations
DigitalOcean: adjust for ec2_utils

DataSourceGCE, DataSourceMAAS: user-data is binary other fields are text.
openstack.py: read paths without decoding to text.  This is ok as paths
              other than user-data are json, and load_json will handle

load_file still returns text, and that is what most things use.

9 files changed, 41 insertions, 33 deletions

diff --git a/tests/unittests/helpers.py b/tests/unittests/helpers.py
index 7516bd02..24e1e881 100644
--- a/tests/unittests/helpers.py
+++ b/tests/unittests/helpers.py
@@ -288,7 +288,10 @@ def populate_dir(path, files):
         os.makedirs(path)
     for (name, content) in files.items():
         with open(os.path.join(path, name), "wb") as fp:
-            fp.write(content.encode('utf-8'))
+            if isinstance(content, six.binary_type):
+                fp.write(content)
+            else:
+                fp.write(content.encode('utf-8'))
             fp.close()
 
 
diff --git a/tests/unittests/test_datasource/test_configdrive.py b/tests/unittests/test_datasource/test_configdrive.py
index e28bdd84..83aca505 100644
--- a/tests/unittests/test_datasource/test_configdrive.py
+++ b/tests/unittests/test_datasource/test_configdrive.py
@@ -2,6 +2,7 @@ from copy import copy
 import json
 import os
 import shutil
+import six
 import tempfile
 
 try:
@@ -45,7 +46,7 @@ EC2_META = {
     'reservation-id': 'r-iru5qm4m',
     'security-groups': ['default']
 }
-USER_DATA = '#!/bin/sh\necho This is user data\n'
+USER_DATA = b'#!/bin/sh\necho This is user data\n'
 OSTACK_META = {
     'availability_zone': 'nova',
     'files': [{'content_path': '/content/0000', 'path': '/etc/foo.cfg'},
@@ -56,8 +57,8 @@ OSTACK_META = {
     'public_keys': {'mykey': PUBKEY},
     'uuid': 'b0fa911b-69d4-4476-bbe2-1c92bff6535c'}
 
-CONTENT_0 = 'This is contents of /etc/foo.cfg\n'
-CONTENT_1 = '# this is /etc/bar/bar.cfg\n'
+CONTENT_0 = b'This is contents of /etc/foo.cfg\n'
+CONTENT_1 = b'# this is /etc/bar/bar.cfg\n'
 
 CFG_DRIVE_FILES_V2 = {
   'ec2/2009-04-04/meta-data.json': json.dumps(EC2_META),
@@ -346,8 +347,12 @@ def populate_dir(seed_dir, files):
         dirname = os.path.dirname(path)
         if not os.path.isdir(dirname):
             os.makedirs(dirname)
-        with open(path, "w") as fp:
+        if isinstance(content, six.text_type):
+            mode = "w"
+        else:
+            mode = "wb"
+
+        with open(path, mode) as fp:
             fp.write(content)
-            fp.close()
 
 # vi: ts=4 expandtab
diff --git a/tests/unittests/test_datasource/test_gce.py b/tests/unittests/test_datasource/test_gce.py
index 6dd4b5ed..d28f3b08 100644
--- a/tests/unittests/test_datasource/test_gce.py
+++ b/tests/unittests/test_datasource/test_gce.py
@@ -32,7 +32,7 @@ GCE_META = {
     'instance/zone': 'foo/bar',
     'project/attributes/sshKeys': 'user:ssh-rsa AA2..+aRD0fyVw== root@server',
     'instance/hostname': 'server.project-foo.local',
-    'instance/attributes/user-data': '/bin/echo foo\n',
+    'instance/attributes/user-data': b'/bin/echo foo\n',
 }
 
 GCE_META_PARTIAL = {
diff --git a/tests/unittests/test_datasource/test_maas.py b/tests/unittests/test_datasource/test_maas.py
index d25e1adc..f109bb04 100644
--- a/tests/unittests/test_datasource/test_maas.py
+++ b/tests/unittests/test_datasource/test_maas.py
@@ -26,7 +26,7 @@ class TestMAASDataSource(TestCase):
 
         data = {'instance-id': 'i-valid01',
             'local-hostname': 'valid01-hostname',
-            'user-data': 'valid01-userdata',
+            'user-data': b'valid01-userdata',
             'public-keys': 'ssh-rsa AAAAB3Nz...aC1yc2E= keyname'}
 
         my_d = os.path.join(self.tmp, "valid")
@@ -46,7 +46,7 @@ class TestMAASDataSource(TestCase):
 
         data = {'instance-id': 'i-valid-extra',
             'local-hostname': 'valid-extra-hostname',
-            'user-data': 'valid-extra-userdata', 'foo': 'bar'}
+            'user-data': b'valid-extra-userdata', 'foo': 'bar'}
 
         my_d = os.path.join(self.tmp, "valid_extra")
         populate_dir(my_d, data)
@@ -103,7 +103,7 @@ class TestMAASDataSource(TestCase):
             'meta-data/instance-id': 'i-instanceid',
             'meta-data/local-hostname': 'test-hostname',
             'meta-data/public-keys': 'test-hostname',
-            'user-data': 'foodata',
+            'user-data': b'foodata',
             }
         valid_order = [
             'meta-data/local-hostname',
@@ -143,7 +143,7 @@ class TestMAASDataSource(TestCase):
             userdata, metadata = DataSourceMAAS.read_maas_seed_url(
                 my_seed, header_cb=my_headers_cb, version=my_ver)
 
-            self.assertEqual("foodata", userdata)
+            self.assertEqual(b"foodata", userdata)
             self.assertEqual(metadata['instance-id'],
                              valid['meta-data/instance-id'])
             self.assertEqual(metadata['local-hostname'],
diff --git a/tests/unittests/test_datasource/test_nocloud.py b/tests/unittests/test_datasource/test_nocloud.py
index 4f967f58..85b4c25a 100644
--- a/tests/unittests/test_datasource/test_nocloud.py
+++ b/tests/unittests/test_datasource/test_nocloud.py
@@ -37,7 +37,7 @@ class TestNoCloudDataSource(TestCase):
 
     def test_nocloud_seed_dir(self):
         md = {'instance-id': 'IID', 'dsmode': 'local'}
-        ud = "USER_DATA_HERE"
+        ud = b"USER_DATA_HERE"
         populate_dir(os.path.join(self.paths.seed_dir, "nocloud"),
                      {'user-data': ud, 'meta-data': yaml.safe_dump(md)})
 
@@ -92,20 +92,20 @@ class TestNoCloudDataSource(TestCase):
         data = {
             'fs_label': None,
             'meta-data': yaml.safe_dump({'instance-id': 'IID'}),
-            'user-data': "USER_DATA_RAW",
+            'user-data': b"USER_DATA_RAW",
         }
 
         sys_cfg = {'datasource': {'NoCloud': data}}
         dsrc = ds(sys_cfg=sys_cfg, distro=None, paths=self.paths)
         ret = dsrc.get_data()
-        self.assertEqual(dsrc.userdata_raw, "USER_DATA_RAW")
+        self.assertEqual(dsrc.userdata_raw, b"USER_DATA_RAW")
         self.assertEqual(dsrc.metadata.get('instance-id'), 'IID')
         self.assertTrue(ret)
 
     def test_nocloud_seed_with_vendordata(self):
         md = {'instance-id': 'IID', 'dsmode': 'local'}
-        ud = "USER_DATA_HERE"
-        vd = "THIS IS MY VENDOR_DATA"
+        ud = b"USER_DATA_HERE"
+        vd = b"THIS IS MY VENDOR_DATA"
 
         populate_dir(os.path.join(self.paths.seed_dir, "nocloud"),
                      {'user-data': ud, 'meta-data': yaml.safe_dump(md),
@@ -126,7 +126,7 @@ class TestNoCloudDataSource(TestCase):
 
     def test_nocloud_no_vendordata(self):
         populate_dir(os.path.join(self.paths.seed_dir, "nocloud"),
-                     {'user-data': "ud", 'meta-data': "instance-id: IID\n"})
+                     {'user-data': b"ud", 'meta-data': "instance-id: IID\n"})
 
         sys_cfg = {'datasource': {'NoCloud': {'fs_label': None}}}
 
@@ -134,7 +134,7 @@ class TestNoCloudDataSource(TestCase):
 
         dsrc = ds(sys_cfg=sys_cfg, distro=None, paths=self.paths)
         ret = dsrc.get_data()
-        self.assertEqual(dsrc.userdata_raw, "ud")
+        self.assertEqual(dsrc.userdata_raw, b"ud")
         self.assertFalse(dsrc.vendordata)
         self.assertTrue(ret)
 
diff --git a/tests/unittests/test_datasource/test_openstack.py b/tests/unittests/test_datasource/test_openstack.py
index 81ef1546..81411ced 100644
--- a/tests/unittests/test_datasource/test_openstack.py
+++ b/tests/unittests/test_datasource/test_openstack.py
@@ -49,7 +49,7 @@ EC2_META = {
     'public-ipv4': '0.0.0.1',
     'reservation-id': 'r-iru5qm4m',
 }
-USER_DATA = '#!/bin/sh\necho This is user data\n'
+USER_DATA = b'#!/bin/sh\necho This is user data\n'
 VENDOR_DATA = {
     'magic': '',
 }
@@ -63,8 +63,8 @@ OSTACK_META = {
     'public_keys': {'mykey': PUBKEY},
     'uuid': 'b0fa911b-69d4-4476-bbe2-1c92bff6535c',
 }
-CONTENT_0 = 'This is contents of /etc/foo.cfg\n'
-CONTENT_1 = '# this is /etc/bar/bar.cfg\n'
+CONTENT_0 = b'This is contents of /etc/foo.cfg\n'
+CONTENT_1 = b'# this is /etc/bar/bar.cfg\n'
 OS_FILES = {
     'openstack/latest/meta_data.json': json.dumps(OSTACK_META),
     'openstack/latest/user_data': USER_DATA,
diff --git a/tests/unittests/test_ec2_util.py b/tests/unittests/test_ec2_util.py
index 84aa002e..bd43accf 100644
--- a/tests/unittests/test_ec2_util.py
+++ b/tests/unittests/test_ec2_util.py
@@ -16,7 +16,7 @@ class TestEc2Util(helpers.HttprettyTestCase):
                         body='stuff',
                         status=200)
         userdata = eu.get_instance_userdata(self.VERSION)
-        self.assertEquals('stuff', userdata)
+        self.assertEquals('stuff', userdata.decode('utf-8'))
 
     @hp.activate
     def test_userdata_fetch_fail_not_found(self):
diff --git a/tests/unittests/test_handler/test_handler_apt_configure.py b/tests/unittests/test_handler/test_handler_apt_configure.py
index d8fe9a4f..02cad8b2 100644
--- a/tests/unittests/test_handler/test_handler_apt_configure.py
+++ b/tests/unittests/test_handler/test_handler_apt_configure.py
@@ -30,7 +30,7 @@ class TestAptProxyConfig(TestCase):
         self.assertTrue(os.path.isfile(self.pfile))
         self.assertFalse(os.path.isfile(self.cfile))
 
-        contents = str(util.read_file_or_url(self.pfile))
+        contents = util.load_tfile_or_url(self.pfile)
         self.assertTrue(self._search_apt_config(contents, "http", "myproxy"))
 
     def test_apt_http_proxy_written(self):
@@ -40,7 +40,7 @@ class TestAptProxyConfig(TestCase):
         self.assertTrue(os.path.isfile(self.pfile))
         self.assertFalse(os.path.isfile(self.cfile))
 
-        contents = str(util.read_file_or_url(self.pfile))
+        contents = util.load_tfile_or_url(self.pfile)
         self.assertTrue(self._search_apt_config(contents, "http", "myproxy"))
 
     def test_apt_all_proxy_written(self):
@@ -58,7 +58,7 @@ class TestAptProxyConfig(TestCase):
         self.assertTrue(os.path.isfile(self.pfile))
         self.assertFalse(os.path.isfile(self.cfile))
 
-        contents = str(util.read_file_or_url(self.pfile))
+        contents = util.load_tfile_or_url(self.pfile)
 
         for ptype, pval in values.items():
             self.assertTrue(self._search_apt_config(contents, ptype, pval))
@@ -74,7 +74,7 @@ class TestAptProxyConfig(TestCase):
         cc_apt_configure.apply_apt_config({'apt_proxy': "foo"},
                                           self.pfile, self.cfile)
         self.assertTrue(os.path.isfile(self.pfile))
-        contents = str(util.read_file_or_url(self.pfile))
+        contents = util.load_tfile_or_url(self.pfile)
         self.assertTrue(self._search_apt_config(contents, "http", "foo"))
 
     def test_config_written(self):
@@ -86,14 +86,14 @@ class TestAptProxyConfig(TestCase):
         self.assertTrue(os.path.isfile(self.cfile))
         self.assertFalse(os.path.isfile(self.pfile))
 
-        self.assertEqual(str(util.read_file_or_url(self.cfile)), payload)
+        self.assertEqual(util.load_tfile_or_url(self.cfile), payload)
 
     def test_config_replaced(self):
         util.write_file(self.pfile, "content doesnt matter")
         cc_apt_configure.apply_apt_config({'apt_config': "foo"},
                                           self.pfile, self.cfile)
         self.assertTrue(os.path.isfile(self.cfile))
-        self.assertEqual(str(util.read_file_or_url(self.cfile)), "foo")
+        self.assertEqual(util.load_tfile_or_url(self.cfile), "foo")
 
     def test_config_deleted(self):
         # if no 'apt_config' is provided, delete any previously written file
diff --git a/tests/unittests/test_pathprefix2dict.py b/tests/unittests/test_pathprefix2dict.py
index 7089bde6..38fd75b6 100644
--- a/tests/unittests/test_pathprefix2dict.py
+++ b/tests/unittests/test_pathprefix2dict.py
@@ -14,28 +14,28 @@ class TestPathPrefix2Dict(TestCase):
         self.addCleanup(shutil.rmtree, self.tmp)
 
     def test_required_only(self):
-        dirdata = {'f1': 'f1content', 'f2': 'f2content'}
+        dirdata = {'f1': b'f1content', 'f2': b'f2content'}
         populate_dir(self.tmp, dirdata)
 
         ret = util.pathprefix2dict(self.tmp, required=['f1', 'f2'])
         self.assertEqual(dirdata, ret)
 
     def test_required_missing(self):
-        dirdata = {'f1': 'f1content'}
+        dirdata = {'f1': b'f1content'}
         populate_dir(self.tmp, dirdata)
         kwargs = {'required': ['f1', 'f2']}
         self.assertRaises(ValueError, util.pathprefix2dict, self.tmp, **kwargs)
 
     def test_no_required_and_optional(self):
-        dirdata = {'f1': 'f1c', 'f2': 'f2c'}
+        dirdata = {'f1': b'f1c', 'f2': b'f2c'}
         populate_dir(self.tmp, dirdata)
 
         ret = util.pathprefix2dict(self.tmp, required=None,
-                                  optional=['f1', 'f2'])
+                                   optional=['f1', 'f2'])
         self.assertEqual(dirdata, ret)
 
     def test_required_and_optional(self):
-        dirdata = {'f1': 'f1c', 'f2': 'f2c'}
+        dirdata = {'f1': b'f1c', 'f2': b'f2c'}
         populate_dir(self.tmp, dirdata)
 
         ret = util.pathprefix2dict(self.tmp, required=['f1'], optional=['f2'])