From f895cb12141281702b34da18f2384deb64c881e7 Mon Sep 17 00:00:00 2001
From: Barry Warsaw <barry@python.org>
Date: Wed, 21 Jan 2015 17:56:53 -0500
Subject: Largely merge lp:~harlowja/cloud-init/py2-3 albeit manually because
 it seemed to be behind trunk.

`tox -e py27` passes full test suite.  Now to work on replacing mocker.
---
 tests/unittests/test_handler/test_handler_seed_random.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'tests/unittests/test_handler/test_handler_seed_random.py')

diff --git a/tests/unittests/test_handler/test_handler_seed_random.py b/tests/unittests/test_handler/test_handler_seed_random.py
index 40481f16..579377fb 100644
--- a/tests/unittests/test_handler/test_handler_seed_random.py
+++ b/tests/unittests/test_handler/test_handler_seed_random.py
@@ -22,7 +22,7 @@ import base64
 import gzip
 import tempfile
 
-from StringIO import StringIO
+from six import StringIO
 
 from cloudinit import cloud
 from cloudinit import distros
-- 
cgit v1.2.3


From 841db73600e3f203243c773109d71ab88d3334bc Mon Sep 17 00:00:00 2001
From: Barry Warsaw <barry@python.org>
Date: Mon, 26 Jan 2015 11:14:06 -0500
Subject: More test repairs.

---
 cloudinit/distros/__init__.py                      |  2 +-
 cloudinit/user_data.py                             |  9 +++++++
 tests/unittests/helpers.py                         | 12 ++++++---
 tests/unittests/test_builtin_handlers.py           |  1 -
 tests/unittests/test_datasource/test_azure.py      | 31 +++++++++++++---------
 tests/unittests/test_datasource/test_gce.py        |  2 +-
 tests/unittests/test_datasource/test_opennebula.py | 10 +++++--
 tests/unittests/test_datasource/test_smartos.py    | 12 ++++++---
 tests/unittests/test_filters/test_launch_index.py  |  8 +++---
 tests/unittests/test_handler/test_handler_chef.py  |  3 ++-
 .../test_handler/test_handler_seed_random.py       | 11 ++++++--
 tests/unittests/test_util.py                       |  6 ++---
 12 files changed, 73 insertions(+), 34 deletions(-)

(limited to 'tests/unittests/test_handler/test_handler_seed_random.py')

diff --git a/cloudinit/distros/__init__.py b/cloudinit/distros/__init__.py
index 00fb95fb..ab874b45 100644
--- a/cloudinit/distros/__init__.py
+++ b/cloudinit/distros/__init__.py
@@ -857,7 +857,7 @@ def extract_default(users, default_name=None, default_config=None):
     if not tmp_users:
         return (default_name, default_config)
     else:
-        name = tmp_users.keys()[0]
+        name = list(tmp_users)[0]
         config = tmp_users[name]
         config.pop('default', None)
         return (name, config)
diff --git a/cloudinit/user_data.py b/cloudinit/user_data.py
index 9111bd39..ff21259c 100644
--- a/cloudinit/user_data.py
+++ b/cloudinit/user_data.py
@@ -109,6 +109,15 @@ class UserDataProcessor(object):
             ctype = None
             ctype_orig = part.get_content_type()
             payload = part.get_payload(decode=True)
+            # In Python 3, decoding the payload will ironically hand us a
+            # bytes object.  'decode' means to decode according to
+            # Content-Transfer-Encoding, not according to any charset in the
+            # Content-Type.  So, if we end up with bytes, first try to decode
+            # to str via CT charset, and failing that, try utf-8 using
+            # surrogate escapes.
+            if six.PY3 and isinstance(payload, bytes):
+                charset = part.get_charset() or 'utf-8'
+                payload = payload.decode(charset, errors='surrogateescape')
             was_compressed = False
 
             # When the message states it is of a gzipped content type ensure
diff --git a/tests/unittests/helpers.py b/tests/unittests/helpers.py
index 70b8116f..4b8dcc5c 100644
--- a/tests/unittests/helpers.py
+++ b/tests/unittests/helpers.py
@@ -1,8 +1,11 @@
 import os
 import sys
+import shutil
 import tempfile
 import unittest
 
+import six
+
 try:
     from unittest import mock
 except ImportError:
@@ -15,8 +18,6 @@ except ImportError:
 from cloudinit import helpers as ch
 from cloudinit import util
 
-import shutil
-
 # Used for detecting different python versions
 PY2 = False
 PY26 = False
@@ -115,7 +116,12 @@ def retarget_many_wrapper(new_base, am, old_func):
             nam = len(n_args)
         for i in range(0, nam):
             path = args[i]
-            n_args[i] = rebase_path(path, new_base)
+            # patchOS() wraps various os and os.path functions, however in
+            # Python 3 some of these now accept file-descriptors (integers).
+            # That breaks rebase_path() so in lieu of a better solution, just
+            # don't rebase if we get a fd.
+            if isinstance(path, six.string_types):
+                n_args[i] = rebase_path(path, new_base)
         return old_func(*n_args, **kwds)
     return wrapper
 
diff --git a/tests/unittests/test_builtin_handlers.py b/tests/unittests/test_builtin_handlers.py
index 47ff6318..ad32d0b2 100644
--- a/tests/unittests/test_builtin_handlers.py
+++ b/tests/unittests/test_builtin_handlers.py
@@ -21,7 +21,6 @@ from cloudinit.settings import (PER_ALWAYS, PER_INSTANCE)
 
 
 class TestBuiltins(test_helpers.FilesystemMockingTestCase):
-
     def test_upstart_frequency_no_out(self):
         c_root = tempfile.mkdtemp()
         self.addCleanup(shutil.rmtree, c_root)
diff --git a/tests/unittests/test_datasource/test_azure.py b/tests/unittests/test_datasource/test_azure.py
index 2dbcd389..1f0330b3 100644
--- a/tests/unittests/test_datasource/test_azure.py
+++ b/tests/unittests/test_datasource/test_azure.py
@@ -22,6 +22,13 @@ import tempfile
 import unittest
 
 
+def b64(source):
+    # In Python 3, b64encode only accepts bytes and returns bytes.
+    if not isinstance(source, bytes):
+        source = source.encode('utf-8')
+    return base64.b64encode(source).decode('us-ascii')
+
+
 def construct_valid_ovf_env(data=None, pubkeys=None, userdata=None):
     if data is None:
         data = {'HostName': 'FOOHOST'}
@@ -51,7 +58,7 @@ def construct_valid_ovf_env(data=None, pubkeys=None, userdata=None):
         content += "<%s%s>%s</%s>\n" % (key, attrs, val, key)
 
     if userdata:
-        content += "<UserData>%s</UserData>\n" % (base64.b64encode(userdata))
+        content += "<UserData>%s</UserData>\n" % (b64(userdata))
 
     if pubkeys:
         content += "<SSH><PublicKeys>\n"
@@ -181,7 +188,7 @@ class TestAzureDataSource(unittest.TestCase):
         # set dscfg in via base64 encoded yaml
         cfg = {'agent_command': "my_command"}
         odata = {'HostName': "myhost", 'UserName': "myuser",
-                'dscfg': {'text': base64.b64encode(yaml.dump(cfg)),
+                'dscfg': {'text': b64(yaml.dump(cfg)),
                           'encoding': 'base64'}}
         data = {'ovfcontent': construct_valid_ovf_env(data=odata)}
 
@@ -233,13 +240,13 @@ class TestAzureDataSource(unittest.TestCase):
 
     def test_userdata_found(self):
         mydata = "FOOBAR"
-        odata = {'UserData': base64.b64encode(mydata)}
+        odata = {'UserData': b64(mydata)}
         data = {'ovfcontent': construct_valid_ovf_env(data=odata)}
 
         dsrc = self._get_ds(data)
         ret = dsrc.get_data()
         self.assertTrue(ret)
-        self.assertEqual(dsrc.userdata_raw, mydata)
+        self.assertEqual(dsrc.userdata_raw, mydata.encode('utf-8'))
 
     def test_no_datasource_expected(self):
         # no source should be found if no seed_dir and no devs
@@ -281,7 +288,7 @@ class TestAzureDataSource(unittest.TestCase):
                                    'command': 'my-bounce-command',
                                    'hostname_command': 'my-hostname-command'}}
         odata = {'HostName': "xhost",
-                'dscfg': {'text': base64.b64encode(yaml.dump(cfg)),
+                'dscfg': {'text': b64(yaml.dump(cfg)),
                           'encoding': 'base64'}}
         data = {'ovfcontent': construct_valid_ovf_env(data=odata)}
         self._get_ds(data).get_data()
@@ -296,7 +303,7 @@ class TestAzureDataSource(unittest.TestCase):
         # config specifying set_hostname off should not bounce
         cfg = {'set_hostname': False}
         odata = {'HostName': "xhost",
-                'dscfg': {'text': base64.b64encode(yaml.dump(cfg)),
+                'dscfg': {'text': b64(yaml.dump(cfg)),
                           'encoding': 'base64'}}
         data = {'ovfcontent': construct_valid_ovf_env(data=odata)}
         self._get_ds(data).get_data()
@@ -325,7 +332,7 @@ class TestAzureDataSource(unittest.TestCase):
         # Make sure that user can affect disk aliases
         dscfg = {'disk_aliases': {'ephemeral0': '/dev/sdc'}}
         odata = {'HostName': "myhost", 'UserName': "myuser",
-                'dscfg': {'text': base64.b64encode(yaml.dump(dscfg)),
+                'dscfg': {'text': b64(yaml.dump(dscfg)),
                           'encoding': 'base64'}}
         usercfg = {'disk_setup': {'/dev/sdc': {'something': '...'},
                                   'ephemeral0': False}}
@@ -347,7 +354,7 @@ class TestAzureDataSource(unittest.TestCase):
         dsrc = self._get_ds(data)
         dsrc.get_data()
 
-        self.assertEqual(userdata, dsrc.userdata_raw)
+        self.assertEqual(userdata.encode('us-ascii'), dsrc.userdata_raw)
 
     def test_ovf_env_arrives_in_waagent_dir(self):
         xml = construct_valid_ovf_env(data={}, userdata="FOODATA")
@@ -362,7 +369,7 @@ class TestAzureDataSource(unittest.TestCase):
 
     def test_existing_ovf_same(self):
         # waagent/SharedConfig left alone if found ovf-env.xml same as cached
-        odata = {'UserData': base64.b64encode("SOMEUSERDATA")}
+        odata = {'UserData': b64("SOMEUSERDATA")}
         data = {'ovfcontent': construct_valid_ovf_env(data=odata)}
 
         populate_dir(self.waagent_d,
@@ -386,9 +393,9 @@ class TestAzureDataSource(unittest.TestCase):
         # 'get_data' should remove SharedConfig.xml in /var/lib/waagent
         # if ovf-env.xml differs.
         cached_ovfenv = construct_valid_ovf_env(
-            {'userdata': base64.b64encode("FOO_USERDATA")})
+            {'userdata': b64("FOO_USERDATA")})
         new_ovfenv = construct_valid_ovf_env(
-            {'userdata': base64.b64encode("NEW_USERDATA")})
+            {'userdata': b64("NEW_USERDATA")})
 
         populate_dir(self.waagent_d,
             {'ovf-env.xml': cached_ovfenv,
@@ -398,7 +405,7 @@ class TestAzureDataSource(unittest.TestCase):
         dsrc = self._get_ds({'ovfcontent': new_ovfenv})
         ret = dsrc.get_data()
         self.assertTrue(ret)
-        self.assertEqual(dsrc.userdata_raw, "NEW_USERDATA")
+        self.assertEqual(dsrc.userdata_raw, b"NEW_USERDATA")
         self.assertTrue(os.path.exists(
             os.path.join(self.waagent_d, 'otherfile')))
         self.assertFalse(
diff --git a/tests/unittests/test_datasource/test_gce.py b/tests/unittests/test_datasource/test_gce.py
index aa60eb33..6dd4b5ed 100644
--- a/tests/unittests/test_datasource/test_gce.py
+++ b/tests/unittests/test_datasource/test_gce.py
@@ -45,7 +45,7 @@ GCE_META_ENCODING = {
     'instance/id': '12345',
     'instance/hostname': 'server.project-baz.local',
     'instance/zone': 'baz/bang',
-    'instance/attributes/user-data': b64encode('/bin/echo baz\n'),
+    'instance/attributes/user-data': b64encode(b'/bin/echo baz\n'),
     'instance/attributes/user-data-encoding': 'base64',
 }
 
diff --git a/tests/unittests/test_datasource/test_opennebula.py b/tests/unittests/test_datasource/test_opennebula.py
index b79237f0..1a8d2122 100644
--- a/tests/unittests/test_datasource/test_opennebula.py
+++ b/tests/unittests/test_datasource/test_opennebula.py
@@ -10,6 +10,12 @@ import shutil
 import tempfile
 import unittest
 
+def b64(source):
+    # In Python 3, b64encode only accepts bytes and returns bytes.
+    if not isinstance(source, bytes):
+        source = source.encode('utf-8')
+    return b64encode(source).decode('us-ascii')
+
 
 TEST_VARS = {
     'VAR1': 'single',
@@ -180,7 +186,7 @@ class TestOpenNebulaDataSource(unittest.TestCase):
             self.assertEqual(USER_DATA, results['userdata'])
 
     def test_user_data_encoding_required_for_decode(self):
-        b64userdata = b64encode(USER_DATA)
+        b64userdata = b64(USER_DATA)
         for k in ('USER_DATA', 'USERDATA'):
             my_d = os.path.join(self.tmp, k)
             populate_context_dir(my_d, {k: b64userdata})
@@ -192,7 +198,7 @@ class TestOpenNebulaDataSource(unittest.TestCase):
     def test_user_data_base64_encoding(self):
         for k in ('USER_DATA', 'USERDATA'):
             my_d = os.path.join(self.tmp, k)
-            populate_context_dir(my_d, {k: b64encode(USER_DATA),
+            populate_context_dir(my_d, {k: b64(USER_DATA),
                                         'USERDATA_ENCODING': 'base64'})
             results = ds.read_context_disk_dir(my_d)
 
diff --git a/tests/unittests/test_datasource/test_smartos.py b/tests/unittests/test_datasource/test_smartos.py
index 01b9b73e..2fb9e1b6 100644
--- a/tests/unittests/test_datasource/test_smartos.py
+++ b/tests/unittests/test_datasource/test_smartos.py
@@ -36,6 +36,12 @@ import tempfile
 import stat
 import uuid
 
+def b64(source):
+    # In Python 3, b64encode only accepts bytes and returns bytes.
+    if not isinstance(source, bytes):
+        source = source.encode('utf-8')
+    return base64.b64encode(source).decode('us-ascii')
+
 
 MOCK_RETURNS = {
     'hostname': 'test-host',
@@ -233,7 +239,7 @@ class TestSmartOSDataSource(helpers.FilesystemMockingTestCase):
         my_returns = MOCK_RETURNS.copy()
         my_returns['base64_all'] = "true"
         for k in ('hostname', 'cloud-init:user-data'):
-            my_returns[k] = base64.b64encode(my_returns[k])
+            my_returns[k] = b64(my_returns[k])
 
         dsrc = self._get_ds(mockdata=my_returns)
         ret = dsrc.get_data()
@@ -254,7 +260,7 @@ class TestSmartOSDataSource(helpers.FilesystemMockingTestCase):
         my_returns['b64-cloud-init:user-data'] = "true"
         my_returns['b64-hostname'] = "true"
         for k in ('hostname', 'cloud-init:user-data'):
-            my_returns[k] = base64.b64encode(my_returns[k])
+            my_returns[k] = b64(my_returns[k])
 
         dsrc = self._get_ds(mockdata=my_returns)
         ret = dsrc.get_data()
@@ -270,7 +276,7 @@ class TestSmartOSDataSource(helpers.FilesystemMockingTestCase):
         my_returns = MOCK_RETURNS.copy()
         my_returns['base64_keys'] = 'hostname,ignored'
         for k in ('hostname',):
-            my_returns[k] = base64.b64encode(my_returns[k])
+            my_returns[k] = b64(my_returns[k])
 
         dsrc = self._get_ds(mockdata=my_returns)
         ret = dsrc.get_data()
diff --git a/tests/unittests/test_filters/test_launch_index.py b/tests/unittests/test_filters/test_launch_index.py
index 2f4c2fda..95d24b9b 100644
--- a/tests/unittests/test_filters/test_launch_index.py
+++ b/tests/unittests/test_filters/test_launch_index.py
@@ -2,7 +2,7 @@ import copy
 
 from .. import helpers
 
-import itertools
+from six.moves import filterfalse
 
 from cloudinit.filters import launch_index
 from cloudinit import user_data as ud
@@ -36,11 +36,9 @@ class TestLaunchFilter(helpers.ResourceUsingTestCase):
             return False
         # Do some basic payload checking
         msg1_msgs = [m for m in msg1.walk()]
-        msg1_msgs = [m for m in
-                     itertools.ifilterfalse(ud.is_skippable, msg1_msgs)]
+        msg1_msgs = [m for m in filterfalse(ud.is_skippable, msg1_msgs)]
         msg2_msgs = [m for m in msg2.walk()]
-        msg2_msgs = [m for m in
-                     itertools.ifilterfalse(ud.is_skippable, msg2_msgs)]
+        msg2_msgs = [m for m in filterfalse(ud.is_skippable, msg2_msgs)]
         for i in range(0, len(msg2_msgs)):
             m1_msg = msg1_msgs[i]
             m2_msg = msg2_msgs[i]
diff --git a/tests/unittests/test_handler/test_handler_chef.py b/tests/unittests/test_handler/test_handler_chef.py
index b06a160c..8ab27911 100644
--- a/tests/unittests/test_handler/test_handler_chef.py
+++ b/tests/unittests/test_handler/test_handler_chef.py
@@ -11,6 +11,7 @@ from cloudinit.sources import DataSourceNone
 
 from .. import helpers as t_help
 
+import six
 import logging
 import shutil
 import tempfile
@@ -77,7 +78,7 @@ class TestChef(t_help.FilesystemMockingTestCase):
         for k, v in cfg['chef'].items():
             self.assertIn(v, c)
         for k, v in cc_chef.CHEF_RB_TPL_DEFAULTS.items():
-            if isinstance(v, basestring):
+            if isinstance(v, six.string_types):
                 self.assertIn(v, c)
         c = util.load_file(cc_chef.CHEF_FB_PATH)
         self.assertEqual({}, json.loads(c))
diff --git a/tests/unittests/test_handler/test_handler_seed_random.py b/tests/unittests/test_handler/test_handler_seed_random.py
index 579377fb..c2da5ced 100644
--- a/tests/unittests/test_handler/test_handler_seed_random.py
+++ b/tests/unittests/test_handler/test_handler_seed_random.py
@@ -38,6 +38,13 @@ import logging
 LOG = logging.getLogger(__name__)
 
 
+def b64(source):
+    # In Python 3, b64encode only accepts bytes and returns bytes.
+    if not isinstance(source, bytes):
+        source = source.encode('utf-8')
+    return base64.b64encode(source).decode('us-ascii')
+
+
 class TestRandomSeed(t_help.TestCase):
     def setUp(self):
         super(TestRandomSeed, self).setUp()
@@ -134,7 +141,7 @@ class TestRandomSeed(t_help.TestCase):
         self.assertEquals("big-toe", contents)
 
     def test_append_random_base64(self):
-        data = base64.b64encode('bubbles')
+        data = b64('bubbles')
         cfg = {
             'random_seed': {
                 'file': self._seed_file,
@@ -147,7 +154,7 @@ class TestRandomSeed(t_help.TestCase):
         self.assertEquals("bubbles", contents)
 
     def test_append_random_b64(self):
-        data = base64.b64encode('kit-kat')
+        data = b64('kit-kat')
         cfg = {
             'random_seed': {
                 'file': self._seed_file,
diff --git a/tests/unittests/test_util.py b/tests/unittests/test_util.py
index b1f5d62c..b0207ace 100644
--- a/tests/unittests/test_util.py
+++ b/tests/unittests/test_util.py
@@ -119,7 +119,7 @@ class TestWriteFile(unittest.TestCase):
 
         # Create file first with basic content
         with open(path, "wb") as f:
-            f.write("LINE1\n")
+            f.write(b"LINE1\n")
         util.write_file(path, contents, omode="a")
 
         self.assertTrue(os.path.exists(path))
@@ -194,7 +194,7 @@ class TestDeleteDirContents(unittest.TestCase):
         os.mkdir(os.path.join(self.tmp, "new_dir"))
         f_name = os.path.join(self.tmp, "new_dir", "new_file.txt")
         with open(f_name, "wb") as f:
-            f.write("DELETE ME")
+            f.write(b"DELETE ME")
 
         util.delete_dir_contents(self.tmp)
 
@@ -205,7 +205,7 @@ class TestDeleteDirContents(unittest.TestCase):
         file_name = os.path.join(self.tmp, "new_file.txt")
         link_name = os.path.join(self.tmp, "new_file_link.txt")
         with open(file_name, "wb") as f:
-            f.write("DELETE ME")
+            f.write(b"DELETE ME")
         os.symlink(file_name, link_name)
 
         util.delete_dir_contents(self.tmp)
-- 
cgit v1.2.3


From f5d6d0e6433f12d05676bea03f78d57966c35b0a Mon Sep 17 00:00:00 2001
From: Barry Warsaw <barry@python.org>
Date: Mon, 26 Jan 2015 15:09:48 -0500
Subject: Down to it.

---
 tests/unittests/test_handler/test_handler_seed_random.py | 10 +++++-----
 tests/unittests/test_util.py                             |  2 +-
 2 files changed, 6 insertions(+), 6 deletions(-)

(limited to 'tests/unittests/test_handler/test_handler_seed_random.py')

diff --git a/tests/unittests/test_handler/test_handler_seed_random.py b/tests/unittests/test_handler/test_handler_seed_random.py
index c2da5ced..d3f18fa0 100644
--- a/tests/unittests/test_handler/test_handler_seed_random.py
+++ b/tests/unittests/test_handler/test_handler_seed_random.py
@@ -22,7 +22,7 @@ import base64
 import gzip
 import tempfile
 
-from six import StringIO
+from six import BytesIO
 
 from cloudinit import cloud
 from cloudinit import distros
@@ -76,7 +76,7 @@ class TestRandomSeed(t_help.TestCase):
         return
 
     def _compress(self, text):
-        contents = StringIO()
+        contents = BytesIO()
         gz_fh = gzip.GzipFile(mode='wb', fileobj=contents)
         gz_fh.write(text)
         gz_fh.close()
@@ -103,7 +103,7 @@ class TestRandomSeed(t_help.TestCase):
         self.assertEquals("tiny-tim-was-here", contents)
 
     def test_append_random_unknown_encoding(self):
-        data = self._compress("tiny-toe")
+        data = self._compress(b"tiny-toe")
         cfg = {
             'random_seed': {
                 'file': self._seed_file,
@@ -115,7 +115,7 @@ class TestRandomSeed(t_help.TestCase):
                           self._get_cloud('ubuntu'), LOG, [])
 
     def test_append_random_gzip(self):
-        data = self._compress("tiny-toe")
+        data = self._compress(b"tiny-toe")
         cfg = {
             'random_seed': {
                 'file': self._seed_file,
@@ -128,7 +128,7 @@ class TestRandomSeed(t_help.TestCase):
         self.assertEquals("tiny-toe", contents)
 
     def test_append_random_gz(self):
-        data = self._compress("big-toe")
+        data = self._compress(b"big-toe")
         cfg = {
             'random_seed': {
                 'file': self._seed_file,
diff --git a/tests/unittests/test_util.py b/tests/unittests/test_util.py
index b0207ace..f537d332 100644
--- a/tests/unittests/test_util.py
+++ b/tests/unittests/test_util.py
@@ -166,7 +166,7 @@ class TestDeleteDirContents(unittest.TestCase):
     def test_deletes_files(self):
         """Single file should be deleted."""
         with open(os.path.join(self.tmp, "new_file.txt"), "wb") as f:
-            f.write("DELETE ME")
+            f.write(b"DELETE ME")
 
         util.delete_dir_contents(self.tmp)
 
-- 
cgit v1.2.3


From 6e742d20e9ed56498925c7c850cd5da65d063b4b Mon Sep 17 00:00:00 2001
From: Barry Warsaw <barry@python.org>
Date: Tue, 27 Jan 2015 15:03:52 -0500
Subject: Respond to review:

- Refactor both the base64 encoding and decoding into utility functions.

Also:

- Mechanically fix some other broken untested code.
---
 cloudinit/config/cc_seed_random.py                 |  8 +------
 cloudinit/config/cc_ssh_authkey_fingerprints.py    |  2 +-
 cloudinit/sources/DataSourceOpenNebula.py          |  7 +-----
 cloudinit/sources/DataSourceSmartOS.py             | 11 +--------
 cloudinit/util.py                                  | 20 ++++++++++++++++
 tests/unittests/test_datasource/test_azure.py      | 28 ++++++++--------------
 tests/unittests/test_datasource/test_opennebula.py | 11 ++-------
 tests/unittests/test_datasource/test_smartos.py    | 14 ++++-------
 .../test_handler/test_handler_seed_random.py       | 12 ++--------
 9 files changed, 42 insertions(+), 71 deletions(-)

(limited to 'tests/unittests/test_handler/test_handler_seed_random.py')

diff --git a/cloudinit/config/cc_seed_random.py b/cloudinit/config/cc_seed_random.py
index 981e1b08..bb64b0f5 100644
--- a/cloudinit/config/cc_seed_random.py
+++ b/cloudinit/config/cc_seed_random.py
@@ -38,13 +38,7 @@ def _decode(data, encoding=None):
     if not encoding or encoding.lower() in ['raw']:
         return data
     elif encoding.lower() in ['base64', 'b64']:
-        # Try to give us a native string in both Python 2 and 3, and remember
-        # that b64decode() returns bytes in Python 3.
-        decoded = base64.b64decode(data)
-        try:
-            return decoded.decode('utf-8')
-        except UnicodeDecodeError:
-            return decoded
+        return util.b64d(data)
     elif encoding.lower() in ['gzip', 'gz']:
         return util.decomp_gzip(data, quiet=False)
     else:
diff --git a/cloudinit/config/cc_ssh_authkey_fingerprints.py b/cloudinit/config/cc_ssh_authkey_fingerprints.py
index 51580633..6ce831bc 100644
--- a/cloudinit/config/cc_ssh_authkey_fingerprints.py
+++ b/cloudinit/config/cc_ssh_authkey_fingerprints.py
@@ -32,7 +32,7 @@ from cloudinit import util
 
 def _split_hash(bin_hash):
     split_up = []
-    for i in xrange(0, len(bin_hash), 2):
+    for i in range(0, len(bin_hash), 2):
         split_up.append(bin_hash[i:i + 2])
     return split_up
 
diff --git a/cloudinit/sources/DataSourceOpenNebula.py b/cloudinit/sources/DataSourceOpenNebula.py
index a0275cda..61709c1b 100644
--- a/cloudinit/sources/DataSourceOpenNebula.py
+++ b/cloudinit/sources/DataSourceOpenNebula.py
@@ -426,12 +426,7 @@ def read_context_disk_dir(source_dir, asuser=None):
                                context.get('USER_DATA_ENCODING'))
         if encoding == "base64":
             try:
-                userdata = base64.b64decode(results['userdata'])
-                # In Python 3 we still expect a str, but b64decode will return
-                # bytes.  Convert to str.
-                if isinstance(userdata, bytes):
-                    userdata = userdata.decode('utf-8')
-                results['userdata'] = userdata
+                results['userdata'] = util.b64d(results['userdata'])
             except TypeError:
                 LOG.warn("Failed base64 decoding of userdata")
 
diff --git a/cloudinit/sources/DataSourceSmartOS.py b/cloudinit/sources/DataSourceSmartOS.py
index f59ad3d6..9d48beab 100644
--- a/cloudinit/sources/DataSourceSmartOS.py
+++ b/cloudinit/sources/DataSourceSmartOS.py
@@ -351,16 +351,7 @@ def query_data(noun, seed_device, seed_timeout, strip=False, default=None,
 
     if b64:
         try:
-            # Generally, we want native strings in the values.  Python 3's
-            # b64decode will return bytes though, so decode them to utf-8 if
-            # possible.  If that fails, return the bytes.
-            decoded = base64.b64decode(resp)
-            try:
-                if isinstance(decoded, bytes):
-                    return decoded.decode('utf-8')
-            except UnicodeDecodeError:
-                pass
-            return decoded
+            return util.b64d(resp)
         # Bogus input produces different errors in Python 2 and 3; catch both.
         except (TypeError, binascii.Error):
             LOG.warn("Failed base64 decoding key '%s'", noun)
diff --git a/cloudinit/util.py b/cloudinit/util.py
index 766f8e32..8916cc11 100644
--- a/cloudinit/util.py
+++ b/cloudinit/util.py
@@ -44,6 +44,7 @@ import sys
 import tempfile
 import time
 
+from base64 import b64decode, b64encode
 from six.moves.urllib import parse as urlparse
 
 import six
@@ -90,6 +91,25 @@ def encode_text(text, encoding='utf-8'):
         return text
     return text.encode(encoding)
 
+
+def b64d(source):
+    # Base64 decode some data, accepting bytes or unicode/str, and returning
+    # str/unicode if the result is utf-8 compatible, otherwise returning bytes.
+    decoded = b64decode(source)
+    if isinstance(decoded, bytes):
+        try:
+            return decoded.decode('utf-8')
+        except UnicodeDecodeError:
+            return decoded
+
+def b64e(source):
+    # Base64 encode some data, accepting bytes or unicode/str, and returning
+    # str/unicode if the result is utf-8 compatible, otherwise returning bytes.
+    if not isinstance(source, bytes):
+        source = source.encode('utf-8')
+    return b64encode(source).decode('utf-8')
+
+
 # Path for DMI Data
 DMI_SYS_PATH = "/sys/class/dmi/id"
 
diff --git a/tests/unittests/test_datasource/test_azure.py b/tests/unittests/test_datasource/test_azure.py
index 97a53bee..965bce4b 100644
--- a/tests/unittests/test_datasource/test_azure.py
+++ b/tests/unittests/test_datasource/test_azure.py
@@ -1,5 +1,5 @@
 from cloudinit import helpers
-from cloudinit.util import load_file
+from cloudinit.util import b64e, load_file
 from cloudinit.sources import DataSourceAzure
 from ..helpers import TestCase, populate_dir
 
@@ -12,7 +12,6 @@ try:
 except ImportError:
     from contextlib2 import ExitStack
 
-import base64
 import crypt
 import os
 import stat
@@ -22,13 +21,6 @@ import tempfile
 import unittest
 
 
-def b64(source):
-    # In Python 3, b64encode only accepts bytes and returns bytes.
-    if not isinstance(source, bytes):
-        source = source.encode('utf-8')
-    return base64.b64encode(source).decode('us-ascii')
-
-
 def construct_valid_ovf_env(data=None, pubkeys=None, userdata=None):
     if data is None:
         data = {'HostName': 'FOOHOST'}
@@ -58,7 +50,7 @@ def construct_valid_ovf_env(data=None, pubkeys=None, userdata=None):
         content += "<%s%s>%s</%s>\n" % (key, attrs, val, key)
 
     if userdata:
-        content += "<UserData>%s</UserData>\n" % (b64(userdata))
+        content += "<UserData>%s</UserData>\n" % (b64e(userdata))
 
     if pubkeys:
         content += "<SSH><PublicKeys>\n"
@@ -189,7 +181,7 @@ class TestAzureDataSource(TestCase):
         # set dscfg in via base64 encoded yaml
         cfg = {'agent_command': "my_command"}
         odata = {'HostName': "myhost", 'UserName': "myuser",
-                'dscfg': {'text': b64(yaml.dump(cfg)),
+                'dscfg': {'text': b64e(yaml.dump(cfg)),
                           'encoding': 'base64'}}
         data = {'ovfcontent': construct_valid_ovf_env(data=odata)}
 
@@ -241,7 +233,7 @@ class TestAzureDataSource(TestCase):
 
     def test_userdata_found(self):
         mydata = "FOOBAR"
-        odata = {'UserData': b64(mydata)}
+        odata = {'UserData': b64e(mydata)}
         data = {'ovfcontent': construct_valid_ovf_env(data=odata)}
 
         dsrc = self._get_ds(data)
@@ -289,7 +281,7 @@ class TestAzureDataSource(TestCase):
                                    'command': 'my-bounce-command',
                                    'hostname_command': 'my-hostname-command'}}
         odata = {'HostName': "xhost",
-                'dscfg': {'text': b64(yaml.dump(cfg)),
+                'dscfg': {'text': b64e(yaml.dump(cfg)),
                           'encoding': 'base64'}}
         data = {'ovfcontent': construct_valid_ovf_env(data=odata)}
         self._get_ds(data).get_data()
@@ -304,7 +296,7 @@ class TestAzureDataSource(TestCase):
         # config specifying set_hostname off should not bounce
         cfg = {'set_hostname': False}
         odata = {'HostName': "xhost",
-                'dscfg': {'text': b64(yaml.dump(cfg)),
+                'dscfg': {'text': b64e(yaml.dump(cfg)),
                           'encoding': 'base64'}}
         data = {'ovfcontent': construct_valid_ovf_env(data=odata)}
         self._get_ds(data).get_data()
@@ -333,7 +325,7 @@ class TestAzureDataSource(TestCase):
         # Make sure that user can affect disk aliases
         dscfg = {'disk_aliases': {'ephemeral0': '/dev/sdc'}}
         odata = {'HostName': "myhost", 'UserName': "myuser",
-                'dscfg': {'text': b64(yaml.dump(dscfg)),
+                'dscfg': {'text': b64e(yaml.dump(dscfg)),
                           'encoding': 'base64'}}
         usercfg = {'disk_setup': {'/dev/sdc': {'something': '...'},
                                   'ephemeral0': False}}
@@ -370,7 +362,7 @@ class TestAzureDataSource(TestCase):
 
     def test_existing_ovf_same(self):
         # waagent/SharedConfig left alone if found ovf-env.xml same as cached
-        odata = {'UserData': b64("SOMEUSERDATA")}
+        odata = {'UserData': b64e("SOMEUSERDATA")}
         data = {'ovfcontent': construct_valid_ovf_env(data=odata)}
 
         populate_dir(self.waagent_d,
@@ -394,9 +386,9 @@ class TestAzureDataSource(TestCase):
         # 'get_data' should remove SharedConfig.xml in /var/lib/waagent
         # if ovf-env.xml differs.
         cached_ovfenv = construct_valid_ovf_env(
-            {'userdata': b64("FOO_USERDATA")})
+            {'userdata': b64e("FOO_USERDATA")})
         new_ovfenv = construct_valid_ovf_env(
-            {'userdata': b64("NEW_USERDATA")})
+            {'userdata': b64e("NEW_USERDATA")})
 
         populate_dir(self.waagent_d,
             {'ovf-env.xml': cached_ovfenv,
diff --git a/tests/unittests/test_datasource/test_opennebula.py b/tests/unittests/test_datasource/test_opennebula.py
index e5a4bd18..27adf21b 100644
--- a/tests/unittests/test_datasource/test_opennebula.py
+++ b/tests/unittests/test_datasource/test_opennebula.py
@@ -3,19 +3,12 @@ from cloudinit.sources import DataSourceOpenNebula as ds
 from cloudinit import util
 from ..helpers import TestCase, populate_dir
 
-from base64 import b64encode
 import os
 import pwd
 import shutil
 import tempfile
 import unittest
 
-def b64(source):
-    # In Python 3, b64encode only accepts bytes and returns bytes.
-    if not isinstance(source, bytes):
-        source = source.encode('utf-8')
-    return b64encode(source).decode('us-ascii')
-
 
 TEST_VARS = {
     'VAR1': 'single',
@@ -186,7 +179,7 @@ class TestOpenNebulaDataSource(TestCase):
             self.assertEqual(USER_DATA, results['userdata'])
 
     def test_user_data_encoding_required_for_decode(self):
-        b64userdata = b64(USER_DATA)
+        b64userdata = util.b64e(USER_DATA)
         for k in ('USER_DATA', 'USERDATA'):
             my_d = os.path.join(self.tmp, k)
             populate_context_dir(my_d, {k: b64userdata})
@@ -198,7 +191,7 @@ class TestOpenNebulaDataSource(TestCase):
     def test_user_data_base64_encoding(self):
         for k in ('USER_DATA', 'USERDATA'):
             my_d = os.path.join(self.tmp, k)
-            populate_context_dir(my_d, {k: b64(USER_DATA),
+            populate_context_dir(my_d, {k: util.b64e(USER_DATA),
                                         'USERDATA_ENCODING': 'base64'})
             results = ds.read_context_disk_dir(my_d)
 
diff --git a/tests/unittests/test_datasource/test_smartos.py b/tests/unittests/test_datasource/test_smartos.py
index b5ebf94d..8b62b1b1 100644
--- a/tests/unittests/test_datasource/test_smartos.py
+++ b/tests/unittests/test_datasource/test_smartos.py
@@ -24,9 +24,9 @@
 
 from __future__ import print_function
 
-import base64
 from cloudinit import helpers as c_helpers
 from cloudinit.sources import DataSourceSmartOS
+from cloudinit.util import b64e
 from .. import helpers
 import os
 import os.path
@@ -36,12 +36,6 @@ import tempfile
 import stat
 import uuid
 
-def b64(source):
-    # In Python 3, b64encode only accepts bytes and returns bytes.
-    if not isinstance(source, bytes):
-        source = source.encode('utf-8')
-    return base64.b64encode(source).decode('us-ascii')
-
 
 MOCK_RETURNS = {
     'hostname': 'test-host',
@@ -239,7 +233,7 @@ class TestSmartOSDataSource(helpers.FilesystemMockingTestCase):
         my_returns = MOCK_RETURNS.copy()
         my_returns['base64_all'] = "true"
         for k in ('hostname', 'cloud-init:user-data'):
-            my_returns[k] = b64(my_returns[k])
+            my_returns[k] = b64e(my_returns[k])
 
         dsrc = self._get_ds(mockdata=my_returns)
         ret = dsrc.get_data()
@@ -260,7 +254,7 @@ class TestSmartOSDataSource(helpers.FilesystemMockingTestCase):
         my_returns['b64-cloud-init:user-data'] = "true"
         my_returns['b64-hostname'] = "true"
         for k in ('hostname', 'cloud-init:user-data'):
-            my_returns[k] = b64(my_returns[k])
+            my_returns[k] = b64e(my_returns[k])
 
         dsrc = self._get_ds(mockdata=my_returns)
         ret = dsrc.get_data()
@@ -276,7 +270,7 @@ class TestSmartOSDataSource(helpers.FilesystemMockingTestCase):
         my_returns = MOCK_RETURNS.copy()
         my_returns['base64_keys'] = 'hostname,ignored'
         for k in ('hostname',):
-            my_returns[k] = b64(my_returns[k])
+            my_returns[k] = b64e(my_returns[k])
 
         dsrc = self._get_ds(mockdata=my_returns)
         ret = dsrc.get_data()
diff --git a/tests/unittests/test_handler/test_handler_seed_random.py b/tests/unittests/test_handler/test_handler_seed_random.py
index d3f18fa0..0bcdcb31 100644
--- a/tests/unittests/test_handler/test_handler_seed_random.py
+++ b/tests/unittests/test_handler/test_handler_seed_random.py
@@ -18,7 +18,6 @@
 
 from cloudinit.config import cc_seed_random
 
-import base64
 import gzip
 import tempfile
 
@@ -38,13 +37,6 @@ import logging
 LOG = logging.getLogger(__name__)
 
 
-def b64(source):
-    # In Python 3, b64encode only accepts bytes and returns bytes.
-    if not isinstance(source, bytes):
-        source = source.encode('utf-8')
-    return base64.b64encode(source).decode('us-ascii')
-
-
 class TestRandomSeed(t_help.TestCase):
     def setUp(self):
         super(TestRandomSeed, self).setUp()
@@ -141,7 +133,7 @@ class TestRandomSeed(t_help.TestCase):
         self.assertEquals("big-toe", contents)
 
     def test_append_random_base64(self):
-        data = b64('bubbles')
+        data = util.b64e('bubbles')
         cfg = {
             'random_seed': {
                 'file': self._seed_file,
@@ -154,7 +146,7 @@ class TestRandomSeed(t_help.TestCase):
         self.assertEquals("bubbles", contents)
 
     def test_append_random_b64(self):
-        data = b64('kit-kat')
+        data = util.b64e('kit-kat')
         cfg = {
             'random_seed': {
                 'file': self._seed_file,
-- 
cgit v1.2.3


From 8cb7c3f7b5339e686bfbf95996b51afafeaf9c9e Mon Sep 17 00:00:00 2001
From: Ryan Harper <ryan.harper@canonical.com>
Date: Thu, 3 Mar 2016 16:20:10 -0600
Subject: Update pep8 runner and fix pep8 issues

---
 Makefile                                           |  9 ++--
 bin/cloud-init                                     | 43 +++++++++---------
 cloudinit/config/cc_apt_configure.py               |  6 ++-
 cloudinit/config/cc_disk_setup.py                  | 31 +++++++------
 cloudinit/config/cc_grub_dpkg.py                   |  8 ++--
 cloudinit/config/cc_keys_to_console.py             |  2 +-
 cloudinit/config/cc_lxd.py                         |  2 +-
 cloudinit/config/cc_mounts.py                      | 12 ++---
 cloudinit/config/cc_power_state_change.py          |  2 +-
 cloudinit/config/cc_puppet.py                      |  6 +--
 cloudinit/config/cc_resizefs.py                    |  2 +-
 cloudinit/config/cc_rh_subscription.py             |  4 +-
 cloudinit/config/cc_set_hostname.py                |  2 +-
 cloudinit/config/cc_ssh.py                         |  7 +--
 cloudinit/config/cc_update_etc_hosts.py            |  6 +--
 cloudinit/config/cc_update_hostname.py             |  2 +-
 cloudinit/config/cc_yum_add_repo.py                |  2 +-
 cloudinit/distros/__init__.py                      | 12 ++---
 cloudinit/distros/arch.py                          |  6 +--
 cloudinit/distros/debian.py                        |  5 ++-
 cloudinit/distros/freebsd.py                       |  4 +-
 cloudinit/distros/gentoo.py                        |  4 +-
 cloudinit/distros/parsers/hostname.py              |  2 +-
 cloudinit/distros/parsers/resolv_conf.py           |  2 +-
 cloudinit/distros/parsers/sys_conf.py              |  7 ++-
 cloudinit/filters/launch_index.py                  |  2 +-
 cloudinit/helpers.py                               |  7 +--
 cloudinit/sources/DataSourceAzure.py               | 21 +++++----
 cloudinit/sources/DataSourceConfigDrive.py         |  2 +-
 cloudinit/sources/DataSourceEc2.py                 | 10 ++---
 cloudinit/sources/DataSourceMAAS.py                | 15 ++++---
 cloudinit/sources/DataSourceOVF.py                 |  4 +-
 cloudinit/sources/DataSourceOpenNebula.py          |  3 +-
 cloudinit/sources/DataSourceSmartOS.py             |  7 ++-
 cloudinit/ssh_util.py                              |  3 +-
 cloudinit/stages.py                                | 18 ++++----
 cloudinit/url_helper.py                            |  6 +--
 cloudinit/util.py                                  | 15 ++++---
 tests/unittests/test_data.py                       |  5 ++-
 tests/unittests/test_datasource/test_altcloud.py   | 23 +++++-----
 tests/unittests/test_datasource/test_azure.py      | 15 ++++---
 .../unittests/test_datasource/test_configdrive.py  | 12 ++---
 tests/unittests/test_datasource/test_maas.py       | 16 +++----
 tests/unittests/test_datasource/test_smartos.py    |  6 +--
 .../test_handler/test_handler_power_state.py       |  3 +-
 .../test_handler/test_handler_seed_random.py       |  3 +-
 .../unittests/test_handler/test_handler_snappy.py  |  3 +-
 tests/unittests/test_sshutil.py                    |  3 +-
 tests/unittests/test_templating.py                 |  3 +-
 tools/hacking.py                                   | 16 +++----
 tools/mock-meta.py                                 | 27 +++++++-----
 tools/run-pep8                                     | 51 ++++++++--------------
 52 files changed, 244 insertions(+), 243 deletions(-)

(limited to 'tests/unittests/test_handler/test_handler_seed_random.py')

diff --git a/Makefile b/Makefile
index 058ac199..fb65b70b 100644
--- a/Makefile
+++ b/Makefile
@@ -20,13 +20,14 @@ all: test check_version
 check: pep8 pyflakes pyflakes3 unittest
 
 pep8:
-	@$(CWD)/tools/run-pep8 $(PY_FILES)
+	@$(CWD)/tools/run-pep8
 
 pyflakes:
-	@$(CWD)/tools/tox-venv py27 pyflakes $(PY_FILES)
+	@$(CWD)/tools/run-pyflakes
 
-pyflakes:
-	@$(CWD)/tools/tox-venv py34 pyflakes $(PY_FILES)
+pyflakes3:
+	@$(CWD)/tools/run-pyflakes3
+	
 
 unittest:
 	nosetests $(noseopts) tests/unittests
diff --git a/bin/cloud-init b/bin/cloud-init
index 9b90c45e..7f665e7e 100755
--- a/bin/cloud-init
+++ b/bin/cloud-init
@@ -194,7 +194,7 @@ def main_init(name, args):
     if args.debug:
         # Reset so that all the debug handlers are closed out
         LOG.debug(("Logging being reset, this logger may no"
-                    " longer be active shortly"))
+                   " longer be active shortly"))
         logging.resetLogging()
     logging.setupLogging(init.cfg)
     apply_reporting_cfg(init.cfg)
@@ -276,9 +276,9 @@ def main_init(name, args):
         # This may run user-data handlers and/or perform
         # url downloads and such as needed.
         (ran, _results) = init.cloudify().run('consume_data',
-                                             init.consume_data,
-                                             args=[PER_INSTANCE],
-                                             freq=PER_INSTANCE)
+                                              init.consume_data,
+                                              args=[PER_INSTANCE],
+                                              freq=PER_INSTANCE)
         if not ran:
             # Just consume anything that is set to run per-always
             # if nothing ran in the per-instance code
@@ -349,7 +349,7 @@ def main_modules(action_name, args):
     if args.debug:
         # Reset so that all the debug handlers are closed out
         LOG.debug(("Logging being reset, this logger may no"
-                    " longer be active shortly"))
+                   " longer be active shortly"))
         logging.resetLogging()
     logging.setupLogging(mods.cfg)
     apply_reporting_cfg(init.cfg)
@@ -534,7 +534,8 @@ def status_wrapper(name, args, data_d=None, link_d=None):
                 errors.extend(v1[m].get('errors', []))
 
         atomic_write_json(result_path,
-            {'v1': {'datasource': v1['datasource'], 'errors': errors}})
+                          {'v1': {'datasource': v1['datasource'],
+                                  'errors': errors}})
         util.sym_link(os.path.relpath(result_path, link_d), result_link,
                       force=True)
 
@@ -578,13 +579,13 @@ def main():
 
     # These settings are used for the 'config' and 'final' stages
     parser_mod = subparsers.add_parser('modules',
-                                      help=('activates modules '
-                                            'using a given configuration key'))
+                                       help=('activates modules using '
+                                             'a given configuration key'))
     parser_mod.add_argument("--mode", '-m', action='store',
-                             help=("module configuration name "
-                                    "to use (default: %(default)s)"),
-                             default='config',
-                             choices=('init', 'config', 'final'))
+                            help=("module configuration name "
+                                  "to use (default: %(default)s)"),
+                            default='config',
+                            choices=('init', 'config', 'final'))
     parser_mod.set_defaults(action=('modules', main_modules))
 
     # These settings are used when you want to query information
@@ -600,22 +601,22 @@ def main():
 
     # This subcommand allows you to run a single module
     parser_single = subparsers.add_parser('single',
-                                         help=('run a single module '))
+                                          help=('run a single module '))
     parser_single.set_defaults(action=('single', main_single))
     parser_single.add_argument("--name", '-n', action="store",
-                              help="module name to run",
-                              required=True)
+                               help="module name to run",
+                               required=True)
     parser_single.add_argument("--frequency", action="store",
-                              help=("frequency of the module"),
-                              required=False,
-                              choices=list(FREQ_SHORT_NAMES.keys()))
+                               help=("frequency of the module"),
+                               required=False,
+                               choices=list(FREQ_SHORT_NAMES.keys()))
     parser_single.add_argument("--report", action="store_true",
                                help="enable reporting",
                                required=False)
     parser_single.add_argument("module_args", nargs="*",
-                              metavar='argument',
-                              help=('any additional arguments to'
-                                    ' pass to this module'))
+                               metavar='argument',
+                               help=('any additional arguments to'
+                                     ' pass to this module'))
     parser_single.set_defaults(action=('single', main_single))
 
     args = parser.parse_args()
diff --git a/cloudinit/config/cc_apt_configure.py b/cloudinit/config/cc_apt_configure.py
index 9e9e9e26..702977cb 100644
--- a/cloudinit/config/cc_apt_configure.py
+++ b/cloudinit/config/cc_apt_configure.py
@@ -91,7 +91,8 @@ def handle(name, cfg, cloud, log, _args):
         if matchcfg:
             matcher = re.compile(matchcfg).search
         else:
-            matcher = lambda f: False
+            def matcher(x):
+                return False
 
         errors = add_sources(cfg['apt_sources'], params,
                              aa_repo_match=matcher)
@@ -173,7 +174,8 @@ def add_sources(srclist, template_params=None, aa_repo_match=None):
         template_params = {}
 
     if aa_repo_match is None:
-        aa_repo_match = lambda f: False
+        def aa_repo_match(x):
+            return False
 
     errorlist = []
     for ent in srclist:
diff --git a/cloudinit/config/cc_disk_setup.py b/cloudinit/config/cc_disk_setup.py
index d5b0d1d7..0ecc2e4c 100644
--- a/cloudinit/config/cc_disk_setup.py
+++ b/cloudinit/config/cc_disk_setup.py
@@ -167,11 +167,12 @@ def enumerate_disk(device, nodeps=False):
     parts = [x for x in (info.strip()).splitlines() if len(x.split()) > 0]
 
     for part in parts:
-        d = {'name': None,
-             'type': None,
-             'fstype': None,
-             'label': None,
-            }
+        d = {
+            'name': None,
+            'type': None,
+            'fstype': None,
+            'label': None,
+        }
 
         for key, value in value_splitter(part):
             d[key.lower()] = value
@@ -701,11 +702,12 @@ def lookup_force_flag(fs):
     """
     A force flag might be -F or -F, this look it up
     """
-    flags = {'ext': '-F',
-             'btrfs': '-f',
-             'xfs': '-f',
-             'reiserfs': '-f',
-            }
+    flags = {
+        'ext': '-F',
+        'btrfs': '-f',
+        'xfs': '-f',
+        'reiserfs': '-f',
+    }
 
     if 'ext' in fs.lower():
         fs = 'ext'
@@ -824,10 +826,11 @@ def mkfs(fs_cfg):
 
     # Create the commands
     if fs_cmd:
-        fs_cmd = fs_cfg['cmd'] % {'label': label,
-                                  'filesystem': fs_type,
-                                  'device': device,
-                                 }
+        fs_cmd = fs_cfg['cmd'] % {
+            'label': label,
+            'filesystem': fs_type,
+            'device': device,
+        }
     else:
         # Find the mkfs command
         mkfs_cmd = util.which("mkfs.%s" % fs_type)
diff --git a/cloudinit/config/cc_grub_dpkg.py b/cloudinit/config/cc_grub_dpkg.py
index 456597af..acd3e60a 100644
--- a/cloudinit/config/cc_grub_dpkg.py
+++ b/cloudinit/config/cc_grub_dpkg.py
@@ -38,11 +38,11 @@ def handle(name, cfg, _cloud, log, _args):
 
     idevs = util.get_cfg_option_str(mycfg, "grub-pc/install_devices", None)
     idevs_empty = util.get_cfg_option_str(mycfg,
-        "grub-pc/install_devices_empty", None)
+                                          "grub-pc/install_devices_empty",
+                                          None)
 
     if ((os.path.exists("/dev/sda1") and not os.path.exists("/dev/sda")) or
-            (os.path.exists("/dev/xvda1")
-            and not os.path.exists("/dev/xvda"))):
+       (os.path.exists("/dev/xvda1") and not os.path.exists("/dev/xvda"))):
         if idevs is None:
             idevs = ""
         if idevs_empty is None:
@@ -66,7 +66,7 @@ def handle(name, cfg, _cloud, log, _args):
                  (idevs, idevs_empty))
 
     log.debug("Setting grub debconf-set-selections with '%s','%s'" %
-        (idevs, idevs_empty))
+              (idevs, idevs_empty))
 
     try:
         util.subp(['debconf-set-selections'], dconf_sel)
diff --git a/cloudinit/config/cc_keys_to_console.py b/cloudinit/config/cc_keys_to_console.py
index f1c1adff..aa844ee9 100644
--- a/cloudinit/config/cc_keys_to_console.py
+++ b/cloudinit/config/cc_keys_to_console.py
@@ -48,7 +48,7 @@ def handle(name, cfg, cloud, log, _args):
                                             "ssh_fp_console_blacklist", [])
     key_blacklist = util.get_cfg_option_list(cfg,
                                              "ssh_key_console_blacklist",
-                                              ["ssh-dss"])
+                                             ["ssh-dss"])
 
     try:
         cmd = [helper_path]
diff --git a/cloudinit/config/cc_lxd.py b/cloudinit/config/cc_lxd.py
index 7d8a0202..e2fdf68e 100644
--- a/cloudinit/config/cc_lxd.py
+++ b/cloudinit/config/cc_lxd.py
@@ -59,7 +59,7 @@ def handle(name, cfg, cloud, log, args):
     if init_cfg:
         if not isinstance(init_cfg, dict):
             log.warn("lxd/init config must be a dictionary. found a '%s'",
-                      type(init_cfg))
+                     type(init_cfg))
             return
         cmd = ['lxd', 'init', '--auto']
         for k in init_keys:
diff --git a/cloudinit/config/cc_mounts.py b/cloudinit/config/cc_mounts.py
index 11089d8d..4fe3ee21 100644
--- a/cloudinit/config/cc_mounts.py
+++ b/cloudinit/config/cc_mounts.py
@@ -204,12 +204,12 @@ def setup_swapfile(fname, size=None, maxsize=None):
     try:
         util.ensure_dir(tdir)
         util.log_time(LOG.debug, msg, func=util.subp,
-            args=[['sh', '-c',
-                   ('rm -f "$1" && umask 0066 && '
-                    '{ fallocate -l "${2}M" "$1" || '
-                    '  dd if=/dev/zero "of=$1" bs=1M "count=$2"; } && '
-                    'mkswap "$1" || { r=$?; rm -f "$1"; exit $r; }'),
-                   'setup_swap', fname, mbsize]])
+                      args=[['sh', '-c',
+                            ('rm -f "$1" && umask 0066 && '
+                             '{ fallocate -l "${2}M" "$1" || '
+                             ' dd if=/dev/zero "of=$1" bs=1M "count=$2"; } && '
+                             'mkswap "$1" || { r=$?; rm -f "$1"; exit $r; }'),
+                             'setup_swap', fname, mbsize]])
 
     except Exception as e:
         raise IOError("Failed %s: %s" % (msg, e))
diff --git a/cloudinit/config/cc_power_state_change.py b/cloudinit/config/cc_power_state_change.py
index 7d9567e3..cc3f7f70 100644
--- a/cloudinit/config/cc_power_state_change.py
+++ b/cloudinit/config/cc_power_state_change.py
@@ -105,7 +105,7 @@ def handle(_name, cfg, _cloud, log, _args):
 
     log.debug("After pid %s ends, will execute: %s" % (mypid, ' '.join(args)))
 
-    util.fork_cb(run_after_pid_gone, mypid, cmdline, timeout, log, 
+    util.fork_cb(run_after_pid_gone, mypid, cmdline, timeout, log,
                  condition, execmd, [args, devnull_fp])
 
 
diff --git a/cloudinit/config/cc_puppet.py b/cloudinit/config/cc_puppet.py
index 4501598e..774d3322 100644
--- a/cloudinit/config/cc_puppet.py
+++ b/cloudinit/config/cc_puppet.py
@@ -36,8 +36,8 @@ def _autostart_puppet(log):
     # Set puppet to automatically start
     if os.path.exists('/etc/default/puppet'):
         util.subp(['sed', '-i',
-                  '-e', 's/^START=.*/START=yes/',
-                  '/etc/default/puppet'], capture=False)
+                   '-e', 's/^START=.*/START=yes/',
+                   '/etc/default/puppet'], capture=False)
     elif os.path.exists('/bin/systemctl'):
         util.subp(['/bin/systemctl', 'enable', 'puppet.service'],
                   capture=False)
@@ -65,7 +65,7 @@ def handle(name, cfg, cloud, log, _args):
                   " doing nothing."))
     elif install:
         log.debug(("Attempting to install puppet %s,"),
-                   version if version else 'latest')
+                  version if version else 'latest')
         cloud.distro.install_packages(('puppet', version))
 
     # ... and then update the puppet configuration
diff --git a/cloudinit/config/cc_resizefs.py b/cloudinit/config/cc_resizefs.py
index cbc07853..2a2a9f59 100644
--- a/cloudinit/config/cc_resizefs.py
+++ b/cloudinit/config/cc_resizefs.py
@@ -166,7 +166,7 @@ def handle(name, cfg, _cloud, log, args):
             func=do_resize, args=(resize_cmd, log))
     else:
         util.log_time(logfunc=log.debug, msg="Resizing",
-            func=do_resize, args=(resize_cmd, log))
+                      func=do_resize, args=(resize_cmd, log))
 
     action = 'Resized'
     if resize_root == NOBLOCK:
diff --git a/cloudinit/config/cc_rh_subscription.py b/cloudinit/config/cc_rh_subscription.py
index 3b30c47e..6f474aed 100644
--- a/cloudinit/config/cc_rh_subscription.py
+++ b/cloudinit/config/cc_rh_subscription.py
@@ -127,8 +127,8 @@ class SubscriptionManager(object):
             return False, not_bool
 
         if (self.servicelevel is not None) and \
-                ((not self.auto_attach)
-                 or (util.is_false(str(self.auto_attach)))):
+           ((not self.auto_attach) or
+           (util.is_false(str(self.auto_attach)))):
 
             no_auto = ("The service-level key must be used in conjunction "
                        "with the auto-attach key.  Please re-run with "
diff --git a/cloudinit/config/cc_set_hostname.py b/cloudinit/config/cc_set_hostname.py
index 5d7f4331..f43d8d5a 100644
--- a/cloudinit/config/cc_set_hostname.py
+++ b/cloudinit/config/cc_set_hostname.py
@@ -24,7 +24,7 @@ from cloudinit import util
 def handle(name, cfg, cloud, log, _args):
     if util.get_cfg_option_bool(cfg, "preserve_hostname", False):
         log.debug(("Configuration option 'preserve_hostname' is set,"
-                    " not setting the hostname in module %s"), name)
+                   " not setting the hostname in module %s"), name)
         return
 
     (hostname, fqdn) = util.get_hostname_fqdn(cfg, cloud)
diff --git a/cloudinit/config/cc_ssh.py b/cloudinit/config/cc_ssh.py
index 5bd2dec6..d24e43c0 100644
--- a/cloudinit/config/cc_ssh.py
+++ b/cloudinit/config/cc_ssh.py
@@ -30,9 +30,10 @@ from cloudinit import distros as ds
 from cloudinit import ssh_util
 from cloudinit import util
 
-DISABLE_ROOT_OPTS = ("no-port-forwarding,no-agent-forwarding,"
-"no-X11-forwarding,command=\"echo \'Please login as the user \\\"$USER\\\" "
-"rather than the user \\\"root\\\".\';echo;sleep 10\"")
+DISABLE_ROOT_OPTS = (
+    "no-port-forwarding,no-agent-forwarding,"
+    "no-X11-forwarding,command=\"echo \'Please login as the user \\\"$USER\\\""
+    " rather than the user \\\"root\\\".\';echo;sleep 10\"")
 
 GENERATE_KEY_NAMES = ['rsa', 'dsa', 'ecdsa', 'ed25519']
 KEY_FILE_TPL = '/etc/ssh/ssh_host_%s_key'
diff --git a/cloudinit/config/cc_update_etc_hosts.py b/cloudinit/config/cc_update_etc_hosts.py
index d3dd1f32..15703efe 100644
--- a/cloudinit/config/cc_update_etc_hosts.py
+++ b/cloudinit/config/cc_update_etc_hosts.py
@@ -41,10 +41,10 @@ def handle(name, cfg, cloud, log, _args):
         if not tpl_fn_name:
             raise RuntimeError(("No hosts template could be"
                                 " found for distro %s") %
-                                (cloud.distro.osfamily))
+                               (cloud.distro.osfamily))
 
         templater.render_to_file(tpl_fn_name, '/etc/hosts',
-                                {'hostname': hostname, 'fqdn': fqdn})
+                                 {'hostname': hostname, 'fqdn': fqdn})
 
     elif manage_hosts == "localhost":
         (hostname, fqdn) = util.get_hostname_fqdn(cfg, cloud)
@@ -57,4 +57,4 @@ def handle(name, cfg, cloud, log, _args):
         cloud.distro.update_etc_hosts(hostname, fqdn)
     else:
         log.debug(("Configuration option 'manage_etc_hosts' is not set,"
-                    " not managing /etc/hosts in module %s"), name)
+                   " not managing /etc/hosts in module %s"), name)
diff --git a/cloudinit/config/cc_update_hostname.py b/cloudinit/config/cc_update_hostname.py
index e396ba13..5b78afe1 100644
--- a/cloudinit/config/cc_update_hostname.py
+++ b/cloudinit/config/cc_update_hostname.py
@@ -29,7 +29,7 @@ frequency = PER_ALWAYS
 def handle(name, cfg, cloud, log, _args):
     if util.get_cfg_option_bool(cfg, "preserve_hostname", False):
         log.debug(("Configuration option 'preserve_hostname' is set,"
-                    " not updating the hostname in module %s"), name)
+                   " not updating the hostname in module %s"), name)
         return
 
     (hostname, fqdn) = util.get_hostname_fqdn(cfg, cloud)
diff --git a/cloudinit/config/cc_yum_add_repo.py b/cloudinit/config/cc_yum_add_repo.py
index 3b821af9..64fba869 100644
--- a/cloudinit/config/cc_yum_add_repo.py
+++ b/cloudinit/config/cc_yum_add_repo.py
@@ -92,7 +92,7 @@ def handle(name, cfg, _cloud, log, _args):
         for req_field in ['baseurl']:
             if req_field not in repo_config:
                 log.warn(("Repository %s does not contain a %s"
-                           " configuration 'required' entry"),
+                          " configuration 'required' entry"),
                          repo_id, req_field)
                 missing_required += 1
         if not missing_required:
diff --git a/cloudinit/distros/__init__.py b/cloudinit/distros/__init__.py
index 71884b32..661a9fd2 100644
--- a/cloudinit/distros/__init__.py
+++ b/cloudinit/distros/__init__.py
@@ -211,8 +211,8 @@ class Distro(object):
 
         # If the system hostname is different than the previous
         # one or the desired one lets update it as well
-        if (not sys_hostname) or (sys_hostname == prev_hostname
-                                  and sys_hostname != hostname):
+        if ((not sys_hostname) or (sys_hostname == prev_hostname and
+           sys_hostname != hostname)):
             update_files.append(sys_fn)
 
         # If something else has changed the hostname after we set it
@@ -221,7 +221,7 @@ class Distro(object):
         if (sys_hostname and prev_hostname and
                 sys_hostname != prev_hostname):
             LOG.info("%s differs from %s, assuming user maintained hostname.",
-                       prev_hostname_fn, sys_fn)
+                     prev_hostname_fn, sys_fn)
             return
 
         # Remove duplicates (incase the previous config filename)
@@ -289,7 +289,7 @@ class Distro(object):
     def _bring_up_interface(self, device_name):
         cmd = ['ifup', device_name]
         LOG.debug("Attempting to run bring up interface %s using command %s",
-                   device_name, cmd)
+                  device_name, cmd)
         try:
             (_out, err) = util.subp(cmd)
             if len(err):
@@ -548,7 +548,7 @@ class Distro(object):
             for member in members:
                 if not util.is_user(member):
                     LOG.warn("Unable to add group member '%s' to group '%s'"
-                            "; user does not exist.", member, name)
+                             "; user does not exist.", member, name)
                     continue
 
                 util.subp(['usermod', '-a', '-G', name, member])
@@ -886,7 +886,7 @@ def fetch(name):
     locs, looked_locs = importer.find_module(name, ['', __name__], ['Distro'])
     if not locs:
         raise ImportError("No distribution found for distro %s (searched %s)"
-                           % (name, looked_locs))
+                          % (name, looked_locs))
     mod = importer.import_module(locs[0])
     cls = getattr(mod, 'Distro')
     return cls
diff --git a/cloudinit/distros/arch.py b/cloudinit/distros/arch.py
index 45fcf26f..93a2e008 100644
--- a/cloudinit/distros/arch.py
+++ b/cloudinit/distros/arch.py
@@ -74,7 +74,7 @@ class Distro(distros.Distro):
                 'Interface': dev,
                 'IP': info.get('bootproto'),
                 'Address': "('%s/%s')" % (info.get('address'),
-                        info.get('netmask')),
+                                          info.get('netmask')),
                 'Gateway': info.get('gateway'),
                 'DNS': str(tuple(info.get('dns-nameservers'))).replace(',', '')
             }
@@ -86,7 +86,7 @@ class Distro(distros.Distro):
 
         if nameservers:
             util.write_file(self.resolve_conf_fn,
-                    convert_resolv_conf(nameservers))
+                            convert_resolv_conf(nameservers))
 
         return dev_names
 
@@ -102,7 +102,7 @@ class Distro(distros.Distro):
     def _bring_up_interface(self, device_name):
         cmd = ['netctl', 'restart', device_name]
         LOG.debug("Attempting to run bring up interface %s using command %s",
-                   device_name, cmd)
+                  device_name, cmd)
         try:
             (_out, err) = util.subp(cmd)
             if len(err):
diff --git a/cloudinit/distros/debian.py b/cloudinit/distros/debian.py
index 6d3a82bf..db5890b1 100644
--- a/cloudinit/distros/debian.py
+++ b/cloudinit/distros/debian.py
@@ -159,8 +159,9 @@ class Distro(distros.Distro):
 
         # Allow the output of this to flow outwards (ie not be captured)
         util.log_time(logfunc=LOG.debug,
-            msg="apt-%s [%s]" % (command, ' '.join(cmd)), func=util.subp,
-            args=(cmd,), kwargs={'env': e, 'capture': False})
+                      msg="apt-%s [%s]" % (command, ' '.join(cmd)),
+                      func=util.subp,
+                      args=(cmd,), kwargs={'env': e, 'capture': False})
 
     def update_package_sources(self):
         self._runner.run("update-sources", self.package_command,
diff --git a/cloudinit/distros/freebsd.py b/cloudinit/distros/freebsd.py
index 4c484639..72012056 100644
--- a/cloudinit/distros/freebsd.py
+++ b/cloudinit/distros/freebsd.py
@@ -205,8 +205,8 @@ class Distro(distros.Distro):
         redact_opts = ['passwd']
 
         for key, val in kwargs.items():
-            if (key in adduser_opts and val
-                    and isinstance(val, six.string_types)):
+            if (key in adduser_opts and val and
+               isinstance(val, six.string_types)):
                 adduser_cmd.extend([adduser_opts[key], val])
 
                 # Redact certain fields from the logs
diff --git a/cloudinit/distros/gentoo.py b/cloudinit/distros/gentoo.py
index 9e80583c..6267dd6e 100644
--- a/cloudinit/distros/gentoo.py
+++ b/cloudinit/distros/gentoo.py
@@ -66,7 +66,7 @@ class Distro(distros.Distro):
     def _bring_up_interface(self, device_name):
         cmd = ['/etc/init.d/net.%s' % device_name, 'restart']
         LOG.debug("Attempting to run bring up interface %s using command %s",
-                   device_name, cmd)
+                  device_name, cmd)
         try:
             (_out, err) = util.subp(cmd)
             if len(err):
@@ -88,7 +88,7 @@ class Distro(distros.Distro):
                 (_out, err) = util.subp(cmd)
                 if len(err):
                     LOG.warn("Running %s resulted in stderr output: %s", cmd,
-                            err)
+                             err)
             except util.ProcessExecutionError:
                 util.logexc(LOG, "Running interface command %s failed", cmd)
                 return False
diff --git a/cloudinit/distros/parsers/hostname.py b/cloudinit/distros/parsers/hostname.py
index 84a1de42..efb185d4 100644
--- a/cloudinit/distros/parsers/hostname.py
+++ b/cloudinit/distros/parsers/hostname.py
@@ -84,5 +84,5 @@ class HostnameConf(object):
             hostnames_found.add(head)
         if len(hostnames_found) > 1:
             raise IOError("Multiple hostnames (%s) found!"
-                           % (hostnames_found))
+                          % (hostnames_found))
         return entries
diff --git a/cloudinit/distros/parsers/resolv_conf.py b/cloudinit/distros/parsers/resolv_conf.py
index 8aee03a4..2ed13d9c 100644
--- a/cloudinit/distros/parsers/resolv_conf.py
+++ b/cloudinit/distros/parsers/resolv_conf.py
@@ -132,7 +132,7 @@ class ResolvConf(object):
             # Some hard limit on 256 chars total
             raise ValueError(("Adding %r would go beyond the "
                               "256 maximum search list character limit")
-                              % (search_domain))
+                             % (search_domain))
         self._remove_option('search')
         self._contents.append(('option', ['search', s_list, '']))
         return flat_sds
diff --git a/cloudinit/distros/parsers/sys_conf.py b/cloudinit/distros/parsers/sys_conf.py
index d795e12f..6157cf32 100644
--- a/cloudinit/distros/parsers/sys_conf.py
+++ b/cloudinit/distros/parsers/sys_conf.py
@@ -77,8 +77,7 @@ class SysConf(configobj.ConfigObj):
         quot_func = None
         if value[0] in ['"', "'"] and value[-1] in ['"', "'"]:
             if len(value) == 1:
-                quot_func = (lambda x:
-                                self._get_single_quote(x) % x)
+                quot_func = (lambda x: self._get_single_quote(x) % x)
         else:
             # Quote whitespace if it isn't the start + end of a shell command
             if value.strip().startswith("$(") and value.strip().endswith(")"):
@@ -91,10 +90,10 @@ class SysConf(configobj.ConfigObj):
                         # to use single quotes which won't get expanded...
                         if re.search(r"[\n\"']", value):
                             quot_func = (lambda x:
-                                            self._get_triple_quote(x) % x)
+                                         self._get_triple_quote(x) % x)
                         else:
                             quot_func = (lambda x:
-                                            self._get_single_quote(x) % x)
+                                         self._get_single_quote(x) % x)
                     else:
                         quot_func = pipes.quote
         if not quot_func:
diff --git a/cloudinit/filters/launch_index.py b/cloudinit/filters/launch_index.py
index 5bebd318..baecdac9 100644
--- a/cloudinit/filters/launch_index.py
+++ b/cloudinit/filters/launch_index.py
@@ -61,7 +61,7 @@ class Filter(object):
                     discarded += 1
             LOG.debug(("Discarding %s multipart messages "
                        "which do not match launch index %s"),
-                       discarded, self.wanted_idx)
+                      discarded, self.wanted_idx)
             new_message = copy.copy(message)
             new_message.set_payload(new_msgs)
             new_message[ud.ATTACHMENT_FIELD] = str(len(new_msgs))
diff --git a/cloudinit/helpers.py b/cloudinit/helpers.py
index 5e99d185..a6eb20fe 100644
--- a/cloudinit/helpers.py
+++ b/cloudinit/helpers.py
@@ -139,9 +139,10 @@ class FileSemaphores(object):
         # but the item had run before we did canon_sem_name.
         if cname != name and os.path.exists(self._get_path(name, freq)):
             LOG.warn("%s has run without canonicalized name [%s].\n"
-                "likely the migrator has not yet run. It will run next boot.\n"
-                "run manually with: cloud-init single --name=migrator"
-                % (name, cname))
+                     "likely the migrator has not yet run. "
+                     "It will run next boot.\n"
+                     "run manually with: cloud-init single --name=migrator"
+                     % (name, cname))
             return True
 
         return False
diff --git a/cloudinit/sources/DataSourceAzure.py b/cloudinit/sources/DataSourceAzure.py
index bd80a8a6..b03ab895 100644
--- a/cloudinit/sources/DataSourceAzure.py
+++ b/cloudinit/sources/DataSourceAzure.py
@@ -38,7 +38,8 @@ LOG = logging.getLogger(__name__)
 DS_NAME = 'Azure'
 DEFAULT_METADATA = {"instance-id": "iid-AZURE-NODE"}
 AGENT_START = ['service', 'walinuxagent', 'start']
-BOUNCE_COMMAND = ['sh', '-xc',
+BOUNCE_COMMAND = [
+    'sh', '-xc',
     "i=$interface; x=0; ifdown $i || x=$?; ifup $i || x=$?; exit $x"]
 
 BUILTIN_DS_CONFIG = {
@@ -91,9 +92,9 @@ def temporary_hostname(temp_hostname, cfg, hostname_command='hostname'):
     """
     policy = cfg['hostname_bounce']['policy']
     previous_hostname = get_hostname(hostname_command)
-    if (not util.is_true(cfg.get('set_hostname'))
-            or util.is_false(policy)
-            or (previous_hostname == temp_hostname and policy != 'force')):
+    if (not util.is_true(cfg.get('set_hostname')) or
+       util.is_false(policy) or
+       (previous_hostname == temp_hostname and policy != 'force')):
         yield None
         return
     set_hostname(temp_hostname, hostname_command)
@@ -123,8 +124,8 @@ class DataSourceAzureNet(sources.DataSource):
         with temporary_hostname(temp_hostname, self.ds_cfg,
                                 hostname_command=hostname_command) \
                 as previous_hostname:
-            if (previous_hostname is not None
-                    and util.is_true(self.ds_cfg.get('set_hostname'))):
+            if (previous_hostname is not None and
+               util.is_true(self.ds_cfg.get('set_hostname'))):
                 cfg = self.ds_cfg['hostname_bounce']
                 try:
                     perform_hostname_bounce(hostname=temp_hostname,
@@ -152,7 +153,8 @@ class DataSourceAzureNet(sources.DataSource):
                 else:
                     bname = str(pk['fingerprint'] + ".crt")
                     fp_files += [os.path.join(ddir, bname)]
-                    LOG.debug("ssh authentication: using fingerprint from fabirc")
+                    LOG.debug("ssh authentication: "
+                              "using fingerprint from fabirc")
 
             missing = util.log_time(logfunc=LOG.debug, msg="waiting for files",
                                     func=wait_for_files,
@@ -506,7 +508,7 @@ def read_azure_ovf(contents):
         raise BrokenAzureDataSource("invalid xml: %s" % e)
 
     results = find_child(dom.documentElement,
-        lambda n: n.localName == "ProvisioningSection")
+                         lambda n: n.localName == "ProvisioningSection")
 
     if len(results) == 0:
         raise NonAzureDataSource("No ProvisioningSection")
@@ -516,7 +518,8 @@ def read_azure_ovf(contents):
     provSection = results[0]
 
     lpcs_nodes = find_child(provSection,
-        lambda n: n.localName == "LinuxProvisioningConfigurationSet")
+                            lambda n:
+                            n.localName == "LinuxProvisioningConfigurationSet")
 
     if len(results) == 0:
         raise NonAzureDataSource("No LinuxProvisioningConfigurationSet")
diff --git a/cloudinit/sources/DataSourceConfigDrive.py b/cloudinit/sources/DataSourceConfigDrive.py
index eb474079..e3916208 100644
--- a/cloudinit/sources/DataSourceConfigDrive.py
+++ b/cloudinit/sources/DataSourceConfigDrive.py
@@ -39,7 +39,7 @@ FS_TYPES = ('vfat', 'iso9660')
 LABEL_TYPES = ('config-2',)
 POSSIBLE_MOUNTS = ('sr', 'cd')
 OPTICAL_DEVICES = tuple(('/dev/%s%s' % (z, i) for z in POSSIBLE_MOUNTS
-                  for i in range(0, 2)))
+                        for i in range(0, 2)))
 
 
 class DataSourceConfigDrive(openstack.SourceMixin, sources.DataSource):
diff --git a/cloudinit/sources/DataSourceEc2.py b/cloudinit/sources/DataSourceEc2.py
index 0032d06c..6a897f7d 100644
--- a/cloudinit/sources/DataSourceEc2.py
+++ b/cloudinit/sources/DataSourceEc2.py
@@ -61,12 +61,12 @@ class DataSourceEc2(sources.DataSource):
             if not self.wait_for_metadata_service():
                 return False
             start_time = time.time()
-            self.userdata_raw = ec2.get_instance_userdata(self.api_ver,
-                self.metadata_address)
+            self.userdata_raw = \
+                ec2.get_instance_userdata(self.api_ver, self.metadata_address)
             self.metadata = ec2.get_instance_metadata(self.api_ver,
                                                       self.metadata_address)
             LOG.debug("Crawl of metadata service took %s seconds",
-                       int(time.time() - start_time))
+                      int(time.time() - start_time))
             return True
         except Exception:
             util.logexc(LOG, "Failed reading from metadata address %s",
@@ -132,13 +132,13 @@ class DataSourceEc2(sources.DataSource):
 
         start_time = time.time()
         url = uhelp.wait_for_url(urls=urls, max_wait=max_wait,
-                                timeout=timeout, status_cb=LOG.warn)
+                                 timeout=timeout, status_cb=LOG.warn)
 
         if url:
             LOG.debug("Using metadata source: '%s'", url2base[url])
         else:
             LOG.critical("Giving up on md from %s after %s seconds",
-                            urls, int(time.time() - start_time))
+                         urls, int(time.time() - start_time))
 
         self.metadata_address = url2base.get(url)
         return bool(url)
diff --git a/cloudinit/sources/DataSourceMAAS.py b/cloudinit/sources/DataSourceMAAS.py
index cfc59ca5..f18c4cee 100644
--- a/cloudinit/sources/DataSourceMAAS.py
+++ b/cloudinit/sources/DataSourceMAAS.py
@@ -275,17 +275,18 @@ if __name__ == "__main__":
 
         parser = argparse.ArgumentParser(description='Interact with MAAS DS')
         parser.add_argument("--config", metavar="file",
-            help="specify DS config file", default=None)
+                            help="specify DS config file", default=None)
         parser.add_argument("--ckey", metavar="key",
-            help="the consumer key to auth with", default=None)
+                            help="the consumer key to auth with", default=None)
         parser.add_argument("--tkey", metavar="key",
-            help="the token key to auth with", default=None)
+                            help="the token key to auth with", default=None)
         parser.add_argument("--csec", metavar="secret",
-            help="the consumer secret (likely '')", default="")
+                            help="the consumer secret (likely '')", default="")
         parser.add_argument("--tsec", metavar="secret",
-            help="the token secret to auth with", default=None)
+                            help="the token secret to auth with", default=None)
         parser.add_argument("--apiver", metavar="version",
-            help="the apiver to use ("" can be used)", default=MD_VERSION)
+                            help="the apiver to use ("" can be used)",
+                            default=MD_VERSION)
 
         subcmds = parser.add_subparsers(title="subcommands", dest="subcmd")
         subcmds.add_parser('crawl', help="crawl the datasource")
@@ -297,7 +298,7 @@ if __name__ == "__main__":
         args = parser.parse_args()
 
         creds = {'consumer_key': args.ckey, 'token_key': args.tkey,
-            'token_secret': args.tsec, 'consumer_secret': args.csec}
+                 'token_secret': args.tsec, 'consumer_secret': args.csec}
 
         if args.config:
             cfg = util.read_conf(args.config)
diff --git a/cloudinit/sources/DataSourceOVF.py b/cloudinit/sources/DataSourceOVF.py
index 58a4b2a2..adf9b12e 100644
--- a/cloudinit/sources/DataSourceOVF.py
+++ b/cloudinit/sources/DataSourceOVF.py
@@ -264,14 +264,14 @@ def get_properties(contents):
     # could also check here that elem.namespaceURI ==
     #   "http://schemas.dmtf.org/ovf/environment/1"
     propSections = find_child(dom.documentElement,
-        lambda n: n.localName == "PropertySection")
+                              lambda n: n.localName == "PropertySection")
 
     if len(propSections) == 0:
         raise XmlError("No 'PropertySection's")
 
     props = {}
     propElems = find_child(propSections[0],
-                            (lambda n: n.localName == "Property"))
+                           (lambda n: n.localName == "Property"))
 
     for elem in propElems:
         key = elem.attributes.getNamedItemNS(envNsURI, "key").value
diff --git a/cloudinit/sources/DataSourceOpenNebula.py b/cloudinit/sources/DataSourceOpenNebula.py
index ac2c3b45..b26940d1 100644
--- a/cloudinit/sources/DataSourceOpenNebula.py
+++ b/cloudinit/sources/DataSourceOpenNebula.py
@@ -404,7 +404,8 @@ def read_context_disk_dir(source_dir, asuser=None):
     if ssh_key_var:
         lines = context.get(ssh_key_var).splitlines()
         results['metadata']['public-keys'] = [l for l in lines
-            if len(l) and not l.startswith("#")]
+                                              if len(l) and not
+                                              l.startswith("#")]
 
     # custom hostname -- try hostname or leave cloud-init
     # itself create hostname from IP address later
diff --git a/cloudinit/sources/DataSourceSmartOS.py b/cloudinit/sources/DataSourceSmartOS.py
index 7453379a..139ee52c 100644
--- a/cloudinit/sources/DataSourceSmartOS.py
+++ b/cloudinit/sources/DataSourceSmartOS.py
@@ -90,8 +90,7 @@ BUILTIN_DS_CONFIG = {
                          'user-data',
                          'user-script',
                          'sdc:datacenter_name',
-                         'sdc:uuid',
-                        ],
+                         'sdc:uuid'],
     'base64_keys': [],
     'base64_all': False,
     'disk_aliases': {'ephemeral0': '/dev/vdb'},
@@ -450,7 +449,7 @@ class JoyentMetadataClient(object):
 
         response = bytearray()
         response.extend(self.metasource.read(1))
-        while response[-1:] !=  b'\n':
+        while response[-1:] != b'\n':
             response.extend(self.metasource.read(1))
         response = response.rstrip().decode('ascii')
         LOG.debug('Read "%s" from metadata transport.', response)
@@ -513,7 +512,7 @@ def write_boot_content(content, content_f, link=None, shebang=False,
 
         except Exception as e:
             util.logexc(LOG, ("Failed to identify script type for %s" %
-                             content_f, e))
+                              content_f, e))
 
     if link:
         try:
diff --git a/cloudinit/ssh_util.py b/cloudinit/ssh_util.py
index 9b2f5ed5..c74a7ae2 100644
--- a/cloudinit/ssh_util.py
+++ b/cloudinit/ssh_util.py
@@ -31,7 +31,8 @@ LOG = logging.getLogger(__name__)
 DEF_SSHD_CFG = "/etc/ssh/sshd_config"
 
 # taken from openssh source key.c/key_type_from_name
-VALID_KEY_TYPES = ("rsa", "dsa", "ssh-rsa", "ssh-dss", "ecdsa",
+VALID_KEY_TYPES = (
+    "rsa", "dsa", "ssh-rsa", "ssh-dss", "ecdsa",
     "ssh-rsa-cert-v00@openssh.com", "ssh-dss-cert-v00@openssh.com",
     "ssh-rsa-cert-v00@openssh.com", "ssh-dss-cert-v00@openssh.com",
     "ssh-rsa-cert-v01@openssh.com", "ssh-dss-cert-v01@openssh.com",
diff --git a/cloudinit/stages.py b/cloudinit/stages.py
index 9f192c8d..dbcf3d55 100644
--- a/cloudinit/stages.py
+++ b/cloudinit/stages.py
@@ -509,13 +509,13 @@ class Init(object):
     def consume_data(self, frequency=PER_INSTANCE):
         # Consume the userdata first, because we need want to let the part
         # handlers run first (for merging stuff)
-        with events.ReportEventStack(
-            "consume-user-data", "reading and applying user-data",
-            parent=self.reporter):
+        with events.ReportEventStack("consume-user-data",
+                                     "reading and applying user-data",
+                                     parent=self.reporter):
                 self._consume_userdata(frequency)
-        with events.ReportEventStack(
-            "consume-vendor-data", "reading and applying vendor-data",
-            parent=self.reporter):
+        with events.ReportEventStack("consume-vendor-data",
+                                     "reading and applying vendor-data",
+                                     parent=self.reporter):
                 self._consume_vendordata(frequency)
 
         # Perform post-consumption adjustments so that
@@ -655,7 +655,7 @@ class Modules(object):
             else:
                 raise TypeError(("Failed to read '%s' item in config,"
                                  " unknown type %s") %
-                                 (item, type_utils.obj_name(item)))
+                                (item, type_utils.obj_name(item)))
         return module_list
 
     def _fixup_modules(self, raw_mods):
@@ -762,8 +762,8 @@ class Modules(object):
 
         if skipped:
             LOG.info("Skipping modules %s because they are not verified "
-                      "on distro '%s'.  To run anyway, add them to "
-                      "'unverified_modules' in config.", skipped, d_name)
+                     "on distro '%s'.  To run anyway, add them to "
+                     "'unverified_modules' in config.", skipped, d_name)
         if forced:
             LOG.info("running unverified_modules: %s", forced)
 
diff --git a/cloudinit/url_helper.py b/cloudinit/url_helper.py
index f2e1390e..936f7da5 100644
--- a/cloudinit/url_helper.py
+++ b/cloudinit/url_helper.py
@@ -252,9 +252,9 @@ def readurl(url, data=None, timeout=None, retries=0, sec_between=1,
             # attrs
             return UrlResponse(r)
         except exceptions.RequestException as e:
-            if (isinstance(e, (exceptions.HTTPError))
-                    and hasattr(e, 'response')  # This appeared in v 0.10.8
-                    and hasattr(e.response, 'status_code')):
+            if (isinstance(e, (exceptions.HTTPError)) and
+               hasattr(e, 'response') and  # This appeared in v 0.10.8
+               hasattr(e.response, 'status_code')):
                 excps.append(UrlError(e, code=e.response.status_code,
                                       headers=e.response.headers,
                                       url=url))
diff --git a/cloudinit/util.py b/cloudinit/util.py
index 45d49e66..de37b0f5 100644
--- a/cloudinit/util.py
+++ b/cloudinit/util.py
@@ -612,7 +612,7 @@ def redirect_output(outfmt, errfmt, o_out=None, o_err=None):
 
 
 def make_url(scheme, host, port=None,
-                path='', params='', query='', fragment=''):
+             path='', params='', query='', fragment=''):
 
     pieces = []
     pieces.append(scheme or '')
@@ -804,8 +804,8 @@ def load_yaml(blob, default=None, allowed=(dict,)):
     blob = decode_binary(blob)
     try:
         LOG.debug("Attempting to load yaml from string "
-                 "of length %s with allowed root types %s",
-                 len(blob), allowed)
+                  "of length %s with allowed root types %s",
+                  len(blob), allowed)
         converted = safeyaml.load(blob)
         if not isinstance(converted, allowed):
             # Yes this will just be caught, but thats ok for now...
@@ -878,7 +878,7 @@ def read_conf_with_confd(cfgfile):
             if not isinstance(confd, six.string_types):
                 raise TypeError(("Config file %s contains 'conf_d' "
                                  "with non-string type %s") %
-                                 (cfgfile, type_utils.obj_name(confd)))
+                                (cfgfile, type_utils.obj_name(confd)))
             else:
                 confd = str(confd).strip()
     elif os.path.isdir("%s.d" % cfgfile):
@@ -1041,7 +1041,8 @@ def is_resolvable(name):
         for iname in badnames:
             try:
                 result = socket.getaddrinfo(iname, None, 0, 0,
-                    socket.SOCK_STREAM, socket.AI_CANONNAME)
+                                            socket.SOCK_STREAM,
+                                            socket.AI_CANONNAME)
                 badresults[iname] = []
                 for (_fam, _stype, _proto, cname, sockaddr) in result:
                     badresults[iname].append("%s: %s" % (cname, sockaddr[0]))
@@ -1109,7 +1110,7 @@ def close_stdin():
 
 
 def find_devs_with(criteria=None, oformat='device',
-                    tag=None, no_cache=False, path=None):
+                   tag=None, no_cache=False, path=None):
     """
     find devices matching given criteria (via blkid)
     criteria can be *one* of:
@@ -1628,7 +1629,7 @@ def write_file(filename, content, mode=0o644, omode="wb"):
         content = decode_binary(content)
         write_type = 'characters'
     LOG.debug("Writing to %s - %s: [%s] %s %s",
-               filename, omode, mode, len(content), write_type)
+              filename, omode, mode, len(content), write_type)
     with SeLinuxGuard(path=filename):
         with open(filename, omode) as fh:
             fh.write(content)
diff --git a/tests/unittests/test_data.py b/tests/unittests/test_data.py
index c603bfdb..9c1ec1d4 100644
--- a/tests/unittests/test_data.py
+++ b/tests/unittests/test_data.py
@@ -27,11 +27,12 @@ from cloudinit import stages
 from cloudinit import user_data as ud
 from cloudinit import util
 
-INSTANCE_ID = "i-testing"
-
 from . import helpers
 
 
+INSTANCE_ID = "i-testing"
+
+
 class FakeDataSource(sources.DataSource):
 
     def __init__(self, userdata=None, vendordata=None):
diff --git a/tests/unittests/test_datasource/test_altcloud.py b/tests/unittests/test_datasource/test_altcloud.py
index e9cd2fa5..85759c68 100644
--- a/tests/unittests/test_datasource/test_altcloud.py
+++ b/tests/unittests/test_datasource/test_altcloud.py
@@ -134,8 +134,7 @@ class TestGetCloudType(TestCase):
         '''
         util.read_dmi_data = _dmi_data('RHEV')
         dsrc = DataSourceAltCloud({}, None, self.paths)
-        self.assertEquals('RHEV', \
-            dsrc.get_cloud_type())
+        self.assertEquals('RHEV', dsrc.get_cloud_type())
 
     def test_vsphere(self):
         '''
@@ -144,8 +143,7 @@ class TestGetCloudType(TestCase):
         '''
         util.read_dmi_data = _dmi_data('VMware Virtual Platform')
         dsrc = DataSourceAltCloud({}, None, self.paths)
-        self.assertEquals('VSPHERE', \
-            dsrc.get_cloud_type())
+        self.assertEquals('VSPHERE', dsrc.get_cloud_type())
 
     def test_unknown(self):
         '''
@@ -154,8 +152,7 @@ class TestGetCloudType(TestCase):
         '''
         util.read_dmi_data = _dmi_data('Unrecognized Platform')
         dsrc = DataSourceAltCloud({}, None, self.paths)
-        self.assertEquals('UNKNOWN', \
-            dsrc.get_cloud_type())
+        self.assertEquals('UNKNOWN', dsrc.get_cloud_type())
 
 
 class TestGetDataCloudInfoFile(TestCase):
@@ -412,27 +409,27 @@ class TestReadUserDataCallback(TestCase):
         '''Test read_user_data_callback() with both files.'''
 
         self.assertEquals('test user data',
-            read_user_data_callback(self.mount_dir))
+                          read_user_data_callback(self.mount_dir))
 
     def test_callback_dc(self):
         '''Test read_user_data_callback() with only DC file.'''
 
         _remove_user_data_files(self.mount_dir,
-            dc_file=False,
-            non_dc_file=True)
+                                dc_file=False,
+                                non_dc_file=True)
 
         self.assertEquals('test user data',
-            read_user_data_callback(self.mount_dir))
+                          read_user_data_callback(self.mount_dir))
 
     def test_callback_non_dc(self):
         '''Test read_user_data_callback() with only non-DC file.'''
 
         _remove_user_data_files(self.mount_dir,
-            dc_file=True,
-            non_dc_file=False)
+                                dc_file=True,
+                                non_dc_file=False)
 
         self.assertEquals('test user data',
-            read_user_data_callback(self.mount_dir))
+                          read_user_data_callback(self.mount_dir))
 
     def test_callback_none(self):
         '''Test read_user_data_callback() no files are found.'''
diff --git a/tests/unittests/test_datasource/test_azure.py b/tests/unittests/test_datasource/test_azure.py
index 3933794f..4c9c7d8b 100644
--- a/tests/unittests/test_datasource/test_azure.py
+++ b/tests/unittests/test_datasource/test_azure.py
@@ -207,7 +207,7 @@ class TestAzureDataSource(TestCase):
         yaml_cfg = "{agent_command: my_command}\n"
         cfg = yaml.safe_load(yaml_cfg)
         odata = {'HostName': "myhost", 'UserName': "myuser",
-                'dscfg': {'text': yaml_cfg, 'encoding': 'plain'}}
+                 'dscfg': {'text': yaml_cfg, 'encoding': 'plain'}}
         data = {'ovfcontent': construct_valid_ovf_env(data=odata)}
 
         dsrc = self._get_ds(data)
@@ -219,8 +219,8 @@ class TestAzureDataSource(TestCase):
         # set dscfg in via base64 encoded yaml
         cfg = {'agent_command': "my_command"}
         odata = {'HostName': "myhost", 'UserName': "myuser",
-                'dscfg': {'text': b64e(yaml.dump(cfg)),
-                          'encoding': 'base64'}}
+                 'dscfg': {'text': b64e(yaml.dump(cfg)),
+                           'encoding': 'base64'}}
         data = {'ovfcontent': construct_valid_ovf_env(data=odata)}
 
         dsrc = self._get_ds(data)
@@ -267,7 +267,8 @@ class TestAzureDataSource(TestCase):
         # should equal that after the '$'
         pos = defuser['passwd'].rfind("$") + 1
         self.assertEqual(defuser['passwd'],
-            crypt.crypt(odata['UserPassword'], defuser['passwd'][0:pos]))
+                         crypt.crypt(odata['UserPassword'],
+                         defuser['passwd'][0:pos]))
 
     def test_userdata_plain(self):
         mydata = "FOOBAR"
@@ -364,8 +365,8 @@ class TestAzureDataSource(TestCase):
         # Make sure that user can affect disk aliases
         dscfg = {'disk_aliases': {'ephemeral0': '/dev/sdc'}}
         odata = {'HostName': "myhost", 'UserName': "myuser",
-                'dscfg': {'text': b64e(yaml.dump(dscfg)),
-                          'encoding': 'base64'}}
+                 'dscfg': {'text': b64e(yaml.dump(dscfg)),
+                           'encoding': 'base64'}}
         usercfg = {'disk_setup': {'/dev/sdc': {'something': '...'},
                                   'ephemeral0': False}}
         userdata = '#cloud-config' + yaml.dump(usercfg) + "\n"
@@ -634,7 +635,7 @@ class TestReadAzureOvf(TestCase):
     def test_invalid_xml_raises_non_azure_ds(self):
         invalid_xml = "<foo>" + construct_valid_ovf_env(data={})
         self.assertRaises(DataSourceAzure.BrokenAzureDataSource,
-            DataSourceAzure.read_azure_ovf, invalid_xml)
+                          DataSourceAzure.read_azure_ovf, invalid_xml)
 
     def test_load_with_pubkeys(self):
         mypklist = [{'fingerprint': 'fp1', 'path': 'path1', 'value': ''}]
diff --git a/tests/unittests/test_datasource/test_configdrive.py b/tests/unittests/test_datasource/test_configdrive.py
index 83aca505..3954ceb3 100644
--- a/tests/unittests/test_datasource/test_configdrive.py
+++ b/tests/unittests/test_datasource/test_configdrive.py
@@ -293,9 +293,8 @@ class TestConfigDriveDataSource(TestCase):
             util.is_partition = my_is_partition
 
             devs_with_answers = {"TYPE=vfat": [],
-                "TYPE=iso9660": ["/dev/vdb"],
-                "LABEL=config-2": ["/dev/vdb"],
-            }
+                                 "TYPE=iso9660": ["/dev/vdb"],
+                                 "LABEL=config-2": ["/dev/vdb"]}
             self.assertEqual(["/dev/vdb"], ds.find_candidate_devs())
 
             # add a vfat item
@@ -306,9 +305,10 @@ class TestConfigDriveDataSource(TestCase):
 
             # verify that partitions are considered, that have correct label.
             devs_with_answers = {"TYPE=vfat": ["/dev/sda1"],
-                "TYPE=iso9660": [], "LABEL=config-2": ["/dev/vdb3"]}
+                                 "TYPE=iso9660": [],
+                                 "LABEL=config-2": ["/dev/vdb3"]}
             self.assertEqual(["/dev/vdb3"],
-                              ds.find_candidate_devs())
+                             ds.find_candidate_devs())
 
         finally:
             util.find_devs_with = orig_find_devs_with
@@ -319,7 +319,7 @@ class TestConfigDriveDataSource(TestCase):
         populate_dir(self.tmp, CFG_DRIVE_FILES_V2)
         myds = cfg_ds_from_dir(self.tmp)
         self.assertEqual(myds.get_public_ssh_keys(),
-           [OSTACK_META['public_keys']['mykey']])
+                         [OSTACK_META['public_keys']['mykey']])
 
 
 def cfg_ds_from_dir(seed_d):
diff --git a/tests/unittests/test_datasource/test_maas.py b/tests/unittests/test_datasource/test_maas.py
index eb97b692..77d15cac 100644
--- a/tests/unittests/test_datasource/test_maas.py
+++ b/tests/unittests/test_datasource/test_maas.py
@@ -25,9 +25,9 @@ class TestMAASDataSource(TestCase):
         """Verify a valid seeddir is read as such."""
 
         data = {'instance-id': 'i-valid01',
-            'local-hostname': 'valid01-hostname',
-            'user-data': b'valid01-userdata',
-            'public-keys': 'ssh-rsa AAAAB3Nz...aC1yc2E= keyname'}
+                'local-hostname': 'valid01-hostname',
+                'user-data': b'valid01-userdata',
+                'public-keys': 'ssh-rsa AAAAB3Nz...aC1yc2E= keyname'}
 
         my_d = os.path.join(self.tmp, "valid")
         populate_dir(my_d, data)
@@ -45,8 +45,8 @@ class TestMAASDataSource(TestCase):
         """Verify extra files do not affect seed_dir validity."""
 
         data = {'instance-id': 'i-valid-extra',
-            'local-hostname': 'valid-extra-hostname',
-            'user-data': b'valid-extra-userdata', 'foo': 'bar'}
+                'local-hostname': 'valid-extra-hostname',
+                'user-data': b'valid-extra-userdata', 'foo': 'bar'}
 
         my_d = os.path.join(self.tmp, "valid_extra")
         populate_dir(my_d, data)
@@ -64,7 +64,7 @@ class TestMAASDataSource(TestCase):
         """Verify that invalid seed_dir raises MAASSeedDirMalformed."""
 
         valid = {'instance-id': 'i-instanceid',
-            'local-hostname': 'test-hostname', 'user-data': ''}
+                 'local-hostname': 'test-hostname', 'user-data': ''}
 
         my_based = os.path.join(self.tmp, "valid_extra")
 
@@ -94,8 +94,8 @@ class TestMAASDataSource(TestCase):
     def test_seed_dir_missing(self):
         """Verify that missing seed_dir raises MAASSeedDirNone."""
         self.assertRaises(DataSourceMAAS.MAASSeedDirNone,
-            DataSourceMAAS.read_maas_seed_dir,
-            os.path.join(self.tmp, "nonexistantdirectory"))
+                          DataSourceMAAS.read_maas_seed_dir,
+                          os.path.join(self.tmp, "nonexistantdirectory"))
 
     def test_seed_url_valid(self):
         """Verify that valid seed_url is read as such."""
diff --git a/tests/unittests/test_datasource/test_smartos.py b/tests/unittests/test_datasource/test_smartos.py
index 1235436d..5e617b83 100644
--- a/tests/unittests/test_datasource/test_smartos.py
+++ b/tests/unittests/test_datasource/test_smartos.py
@@ -463,8 +463,8 @@ class TestJoyentMetadataClient(helpers.FilesystemMockingTestCase):
                 payloadstr = ' {0}'.format(self.response_parts['payload'])
             return ('V2 {length} {crc} {request_id} '
                     '{command}{payloadstr}\n'.format(
-                    payloadstr=payloadstr,
-                    **self.response_parts).encode('ascii'))
+                     payloadstr=payloadstr,
+                     **self.response_parts).encode('ascii'))
 
         self.metasource_data = None
 
@@ -501,7 +501,7 @@ class TestJoyentMetadataClient(helpers.FilesystemMockingTestCase):
         written_line = self.serial.write.call_args[0][0]
         print(type(written_line))
         self.assertEndsWith(written_line.decode('ascii'),
-            b'\n'.decode('ascii'))
+                            b'\n'.decode('ascii'))
         self.assertEqual(1, written_line.count(b'\n'))
 
     def _get_written_line(self, key='some_key'):
diff --git a/tests/unittests/test_handler/test_handler_power_state.py b/tests/unittests/test_handler/test_handler_power_state.py
index 5687b10d..f9660ff6 100644
--- a/tests/unittests/test_handler/test_handler_power_state.py
+++ b/tests/unittests/test_handler/test_handler_power_state.py
@@ -74,7 +74,7 @@ class TestLoadPowerState(t_help.TestCase):
 class TestCheckCondition(t_help.TestCase):
     def cmd_with_exit(self, rc):
         return([sys.executable, '-c', 'import sys; sys.exit(%s)' % rc])
-        
+
     def test_true_is_true(self):
         self.assertEqual(psc.check_condition(True), True)
 
@@ -94,7 +94,6 @@ class TestCheckCondition(t_help.TestCase):
         self.assertEqual(mocklog.warn.call_count, 1)
 
 
-
 def check_lps_ret(psc_return, mode=None):
     if len(psc_return) != 3:
         raise TypeError("length returned = %d" % len(psc_return))
diff --git a/tests/unittests/test_handler/test_handler_seed_random.py b/tests/unittests/test_handler/test_handler_seed_random.py
index 0bcdcb31..34d11f21 100644
--- a/tests/unittests/test_handler/test_handler_seed_random.py
+++ b/tests/unittests/test_handler/test_handler_seed_random.py
@@ -190,7 +190,8 @@ class TestRandomSeed(t_help.TestCase):
         c = self._get_cloud('ubuntu', {})
         self.whichdata = {}
         self.assertRaises(ValueError, cc_seed_random.handle,
-            'test', {'random_seed': {'command_required': True}}, c, LOG, [])
+                          'test', {'random_seed': {'command_required': True}},
+                          c, LOG, [])
 
     def test_seed_command_and_required(self):
         c = self._get_cloud('ubuntu', {})
diff --git a/tests/unittests/test_handler/test_handler_snappy.py b/tests/unittests/test_handler/test_handler_snappy.py
index eceb14d9..8aeff53c 100644
--- a/tests/unittests/test_handler/test_handler_snappy.py
+++ b/tests/unittests/test_handler/test_handler_snappy.py
@@ -125,8 +125,7 @@ class TestInstallPackages(t_help.TestCase):
              "pkg1.smoser.config": "pkg1.smoser.config-data",
              "pkg1.config": "pkg1.config-data",
              "pkg2.smoser_0.0_amd64.snap": "pkg2-snapdata",
-             "pkg2.smoser_0.0_amd64.config": "pkg2.config",
-            })
+             "pkg2.smoser_0.0_amd64.config": "pkg2.config"})
 
         ret = get_package_ops(
             packages=[], configs={}, installed=[], fspath=self.tmp)
diff --git a/tests/unittests/test_sshutil.py b/tests/unittests/test_sshutil.py
index 3b317121..9aeb1cde 100644
--- a/tests/unittests/test_sshutil.py
+++ b/tests/unittests/test_sshutil.py
@@ -32,7 +32,8 @@ VALID_CONTENT = {
     ),
 }
 
-TEST_OPTIONS = ("no-port-forwarding,no-agent-forwarding,no-X11-forwarding,"
+TEST_OPTIONS = (
+    "no-port-forwarding,no-agent-forwarding,no-X11-forwarding,"
     'command="echo \'Please login as the user \"ubuntu\" rather than the'
     'user \"root\".\';echo;sleep 10"')
 
diff --git a/tests/unittests/test_templating.py b/tests/unittests/test_templating.py
index 0c19a2c2..b9863650 100644
--- a/tests/unittests/test_templating.py
+++ b/tests/unittests/test_templating.py
@@ -114,5 +114,6 @@ $a,$b'''
                                                                codename)
 
         out_data = templater.basic_render(in_data,
-            {'mirror': mirror, 'codename': codename})
+                                          {'mirror': mirror,
+                                           'codename': codename})
         self.assertEqual(ex_data, out_data)
diff --git a/tools/hacking.py b/tools/hacking.py
index 3175df38..1a0631c2 100755
--- a/tools/hacking.py
+++ b/tools/hacking.py
@@ -47,10 +47,10 @@ def import_normalize(line):
     # handle "from x import y as z" to "import x.y as z"
     split_line = line.split()
     if (line.startswith("from ") and "," not in line and
-           split_line[2] == "import" and split_line[3] != "*" and
-           split_line[1] != "__future__" and
-           (len(split_line) == 4 or
-           (len(split_line) == 6 and split_line[4] == "as"))):
+       split_line[2] == "import" and split_line[3] != "*" and
+       split_line[1] != "__future__" and
+       (len(split_line) == 4 or
+       (len(split_line) == 6 and split_line[4] == "as"))):
         return "import %s.%s" % (split_line[1], split_line[3])
     else:
         return line
@@ -74,7 +74,7 @@ def cloud_import_alphabetical(physical_line, line_number, lines):
             split_line[0] == "import" and split_previous[0] == "import"):
         if split_line[1] < split_previous[1]:
             return (0, "N306: imports not in alphabetical order (%s, %s)"
-                % (split_previous[1], split_line[1]))
+                    % (split_previous[1], split_line[1]))
 
 
 def cloud_docstring_start_space(physical_line):
@@ -87,8 +87,8 @@ def cloud_docstring_start_space(physical_line):
     pos = max([physical_line.find(i) for i in DOCSTRING_TRIPLE])  # start
     if (pos != -1 and len(physical_line) > pos + 1):
         if (physical_line[pos + 3] == ' '):
-            return (pos, "N401: one line docstring should not start with"
-                " a space")
+            return (pos,
+                    "N401: one line docstring should not start with a space")
 
 
 def cloud_todo_format(physical_line):
@@ -167,4 +167,4 @@ if __name__ == "__main__":
     finally:
         if len(_missingImport) > 0:
             print >> sys.stderr, ("%i imports missing in this test environment"
-                    % len(_missingImport))
+                                  % len(_missingImport))
diff --git a/tools/mock-meta.py b/tools/mock-meta.py
index dfbc2a71..1c746f17 100755
--- a/tools/mock-meta.py
+++ b/tools/mock-meta.py
@@ -126,11 +126,11 @@ class WebException(Exception):
 
 def yamlify(data):
     formatted = yaml.dump(data,
-        line_break="\n",
-        indent=4,
-        explicit_start=True,
-        explicit_end=True,
-        default_flow_style=False)
+                          line_break="\n",
+                          indent=4,
+                          explicit_start=True,
+                          explicit_end=True,
+                          default_flow_style=False)
     return formatted
 
 
@@ -282,7 +282,7 @@ class MetaDataHandler(object):
         else:
             log.warn(("Did not implement action %s, "
                       "returning empty response: %r"),
-                      action, NOT_IMPL_RESPONSE)
+                     action, NOT_IMPL_RESPONSE)
             return NOT_IMPL_RESPONSE
 
 
@@ -404,14 +404,17 @@ def setup_logging(log_level, fmt='%(levelname)s: @%(name)s : %(message)s'):
 def extract_opts():
     parser = OptionParser()
     parser.add_option("-p", "--port", dest="port", action="store", type=int,
-        default=80, metavar="PORT",
-        help="port from which to serve traffic (default: %default)")
+                      default=80, metavar="PORT",
+                      help=("port from which to serve traffic"
+                            " (default: %default)"))
     parser.add_option("-a", "--addr", dest="address", action="store", type=str,
-        default='0.0.0.0', metavar="ADDRESS",
-        help="address from which to serve traffic (default: %default)")
+                      default='0.0.0.0', metavar="ADDRESS",
+                      help=("address from which to serve traffic"
+                            " (default: %default)"))
     parser.add_option("-f", '--user-data-file', dest='user_data_file',
-        action='store', metavar='FILE',
-        help="user data filename to serve back to incoming requests")
+                      action='store', metavar='FILE',
+                      help=("user data filename to serve back to"
+                            "incoming requests"))
     (options, args) = parser.parse_args()
     out = dict()
     out['extra'] = args
diff --git a/tools/run-pep8 b/tools/run-pep8
index ccd6be5a..086400fc 100755
--- a/tools/run-pep8
+++ b/tools/run-pep8
@@ -1,39 +1,22 @@
 #!/bin/bash
 
-if [ $# -eq 0 ]; then
-   files=( bin/cloud-init $(find * -name "*.py" -type f) )
+pycheck_dirs=( "cloudinit/" "bin/" "tests/" "tools/" )
+# FIXME: cloud-init modifies sys module path, pep8 does not like
+# bin_files=( "bin/cloud-init" )
+CR="
+"
+[ "$1" = "-v" ] && { verbose="$1"; shift; } || verbose=""
+
+set -f
+if [ $# -eq 0 ]; then unset IFS
+   IFS="$CR"
+   files=( "${bin_files[@]}" "${pycheck_dirs[@]}" )
+   unset IFS
 else
-   files=( "$@" );
+   files=( "$@" )
 fi
 
-if [ -f 'hacking.py' ]
-then
-    base=`pwd`
-else
-    base=`pwd`/tools/
-fi
-
-IGNORE=""
-
-# King Arthur: Be quiet! ... Be Quiet! I Order You to Be Quiet.
-IGNORE="$IGNORE,E121" # Continuation line indentation is not a multiple of four
-IGNORE="$IGNORE,E123" # Closing bracket does not match indentation of opening bracket's line
-IGNORE="$IGNORE,E124" # Closing bracket missing visual indentation
-IGNORE="$IGNORE,E125" # Continuation line does not distinguish itself from next logical line
-IGNORE="$IGNORE,E126" # Continuation line over-indented for hanging indent
-IGNORE="$IGNORE,E127" # Continuation line over-indented for visual indent
-IGNORE="$IGNORE,E128" # Continuation line under-indented for visual indent
-IGNORE="$IGNORE,E502" # The backslash is redundant between brackets
-IGNORE="${IGNORE#,}"  # remove the leading ',' added above
-
-cmd=(
-    ${base}/hacking.py
-
-    --ignore="$IGNORE"
-
-    "${files[@]}"
-)
-
-echo -e "\nRunning 'cloudinit' pep8:"
-echo "${cmd[@]}"
-"${cmd[@]}"
+myname=${0##*/}
+cmd=( "${myname#run-}" $verbose "${files[@]}" )
+echo "Running: " "${cmd[@]}" 1>&2
+exec "${cmd[@]}"
-- 
cgit v1.2.3


From 6e31038b9cccbcb4a33693060b96fc4f71d86789 Mon Sep 17 00:00:00 2001
From: Scott Moser <smoser@ubuntu.com>
Date: Mon, 7 Mar 2016 21:31:25 -0500
Subject: No longer run pollinate by default in seed_random

The user can still choose to run pollinate here to seed their
random data.  And in an environment with network datasource, that
would be expected to work.  However, we do not want to run it any
more from cloud-init because
a.) pollinate's own init system jobs should get it ran before ssh,
    which is the primary purpose of wanting cloud-init to run it.
b.) with a local datasource, there is no network guarantee when
    init_modules run, so pollinate -q would often cause issues then.
c.) cloud-init would run pollinate and log the failure causing
    many cloud-init specific failures that it could do nothing about.

LP: #1554152
---
 ChangeLog                                                |  1 +
 cloudinit/config/cc_seed_random.py                       |  2 +-
 tests/unittests/test_handler/test_handler_seed_random.py | 14 ++++++++------
 3 files changed, 10 insertions(+), 7 deletions(-)

(limited to 'tests/unittests/test_handler/test_handler_seed_random.py')

diff --git a/ChangeLog b/ChangeLog
index a80a5d5f..6da276b5 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -85,6 +85,7 @@
    unless it is already a file (LP: #1543025).
  - Enable password changing via a hashed string [Alex Sirbu]
  - Added BigStep datasource [Alex Sirbu]
+ - No longer run pollinate in seed_random (LP: #1554152)
 
 0.7.6:
  - open 0.7.6
diff --git a/cloudinit/config/cc_seed_random.py b/cloudinit/config/cc_seed_random.py
index 3288a853..1b011216 100644
--- a/cloudinit/config/cc_seed_random.py
+++ b/cloudinit/config/cc_seed_random.py
@@ -83,7 +83,7 @@ def handle(name, cfg, cloud, log, _args):
                   len(seed_data), seed_path)
         util.append_file(seed_path, seed_data)
 
-    command = mycfg.get('command', ['pollinate', '-q'])
+    command = mycfg.get('command', None)
     req = mycfg.get('command_required', False)
     try:
         env = os.environ.copy()
diff --git a/tests/unittests/test_handler/test_handler_seed_random.py b/tests/unittests/test_handler/test_handler_seed_random.py
index 34d11f21..98bc9b81 100644
--- a/tests/unittests/test_handler/test_handler_seed_random.py
+++ b/tests/unittests/test_handler/test_handler_seed_random.py
@@ -170,28 +170,30 @@ class TestRandomSeed(t_help.TestCase):
         contents = util.load_file(self._seed_file)
         self.assertEquals('tiny-tim-was-here-so-was-josh', contents)
 
-    def test_seed_command_not_provided_pollinate_available(self):
+    def test_seed_command_provided_and_available(self):
         c = self._get_cloud('ubuntu', {})
         self.whichdata = {'pollinate': '/usr/bin/pollinate'}
-        cc_seed_random.handle('test', {}, c, LOG, [])
+        cfg = {'random_seed': {'command': ['pollinate', '-q']}}
+        cc_seed_random.handle('test', cfg, c, LOG, [])
 
         subp_args = [f['args'] for f in self.subp_called]
         self.assertIn(['pollinate', '-q'], subp_args)
 
-    def test_seed_command_not_provided_pollinate_not_available(self):
+    def test_seed_command_not_provided(self):
         c = self._get_cloud('ubuntu', {})
         self.whichdata = {}
         cc_seed_random.handle('test', {}, c, LOG, [])
 
         # subp should not have been called as which would say not available
-        self.assertEquals(self.subp_called, list())
+        self.assertFalse(self.subp_called)
 
     def test_unavailable_seed_command_and_required_raises_error(self):
         c = self._get_cloud('ubuntu', {})
         self.whichdata = {}
+        cfg = {'random_seed': {'command': ['THIS_NO_COMMAND'],
+                               'command_required': True}}
         self.assertRaises(ValueError, cc_seed_random.handle,
-                          'test', {'random_seed': {'command_required': True}},
-                          c, LOG, [])
+                          'test', cfg, c, LOG, [])
 
     def test_seed_command_and_required(self):
         c = self._get_cloud('ubuntu', {})
-- 
cgit v1.2.3