merge with last upstream to avoid merging conflicts on MP

8 files changed, 40 insertions, 30 deletions

diff --git a/ChangeLog b/ChangeLog
index e06578fc..8db29e2e 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -109,6 +109,9 @@
    [Wido den Hollander]
  - Paths: fix instance path if datasource's id has a '/'. (LP: #1575938)
    [Robert Jennings]
+ - Ec2: do not retry requests for user-data path on 404.
+ - settings on the kernel command line (cc:) override all local settings
+   rather than only those in /etc/cloud/cloud.cfg (LP: #1582323)
 
 0.7.6:
  - open 0.7.6
diff --git a/cloudinit/ec2_utils.py b/cloudinit/ec2_utils.py
index 37b92a83..76dda042 100644
--- a/cloudinit/ec2_utils.py
+++ b/cloudinit/ec2_utils.py
@@ -144,9 +144,7 @@ def _skip_retry_on_codes(status_codes, _request_args, cause):
     """Returns if a request should retry based on a given set of codes that
     case retrying to be stopped/skipped.
     """
-    if cause.code in status_codes:
-        return False
-    return True
+    return cause.code in status_codes
 
 
 def get_instance_userdata(api_version='latest',
diff --git a/cloudinit/sources/DataSourceOpenStack.py b/cloudinit/sources/DataSourceOpenStack.py
index 3af17b10..dfd96035 100644
--- a/cloudinit/sources/DataSourceOpenStack.py
+++ b/cloudinit/sources/DataSourceOpenStack.py
@@ -103,7 +103,7 @@ class DataSourceOpenStack(openstack.SourceMixin, sources.DataSource):
         self.metadata_address = url2base.get(avail_url)
         return bool(avail_url)
 
-    def get_data(self):
+    def get_data(self, retries=5, timeout=5):
         try:
             if not self.wait_for_metadata_service():
                 return False
@@ -115,7 +115,9 @@ class DataSourceOpenStack(openstack.SourceMixin, sources.DataSource):
                                     'Crawl of openstack metadata service',
                                     read_metadata_service,
                                     args=[self.metadata_address],
-                                    kwargs={'ssl_details': self.ssl_details})
+                                    kwargs={'ssl_details': self.ssl_details,
+                                            'retries': retries,
+                                            'timeout': timeout})
         except openstack.NonReadable:
             return False
         except (openstack.BrokenMetadata, IOError):
@@ -153,8 +155,10 @@ class DataSourceOpenStack(openstack.SourceMixin, sources.DataSource):
         return sources.instance_id_matches_system_uuid(self.get_instance_id())
 
 
-def read_metadata_service(base_url, ssl_details=None):
-    reader = openstack.MetadataReader(base_url, ssl_details=ssl_details)
+def read_metadata_service(base_url, ssl_details=None,
+                          timeout=5, retries=5):
+    reader = openstack.MetadataReader(base_url, ssl_details=ssl_details,
+                                      timeout=timeout, retries=retries)
     return reader.read_v2()
 
 
diff --git a/cloudinit/stages.py b/cloudinit/stages.py
index 62d066de..002e5832 100644
--- a/cloudinit/stages.py
+++ b/cloudinit/stages.py
@@ -794,16 +794,16 @@ class Modules(object):
 def fetch_base_config():
     base_cfgs = []
     default_cfg = util.get_builtin_cfg()
-    kern_contents = util.read_cc_from_cmdline()
-
-    # Kernel/cmdline parameters override system config
-    if kern_contents:
-        base_cfgs.append(util.load_yaml(kern_contents, default={}))
 
     # Anything in your conf.d location??
     # or the 'default' cloud.cfg location???
     base_cfgs.append(util.read_conf_with_confd(CLOUD_CONFIG))
 
+    # Kernel/cmdline parameters override system config
+    kern_contents = util.read_cc_from_cmdline()
+    if kern_contents:
+        base_cfgs.append(util.load_yaml(kern_contents, default={}))
+
     # And finally the default gets to play
     if default_cfg:
         base_cfgs.append(default_cfg)
diff --git a/packages/bddeb b/packages/bddeb
index c141b1ab..1b0f642c 100755
--- a/packages/bddeb
+++ b/packages/bddeb
@@ -40,6 +40,8 @@ STD_NAMED_PACKAGES = [
     'mock',
     'nose',
     'setuptools',
+    'flake8',
+    'hacking',
 ]
 NONSTD_NAMED_PACKAGES = {
     'argparse': ('python-argparse', None),
diff --git a/test-requirements.txt b/test-requirements.txt
index 2ab53707..651af11b 100644
--- a/test-requirements.txt
+++ b/test-requirements.txt
@@ -3,6 +3,9 @@ httpretty>=0.7.1
 mock
 nose
 
+# Only needed if you want to know the test times
+# nose-timer
+
 # Only really needed on older versions of python
 contextlib2
 setuptools
diff --git a/tests/unittests/test_datasource/test_openstack.py b/tests/unittests/test_datasource/test_openstack.py
index 4140d054..5c8592c5 100644
--- a/tests/unittests/test_datasource/test_openstack.py
+++ b/tests/unittests/test_datasource/test_openstack.py
@@ -135,13 +135,17 @@ def _register_uris(version, ec2_files, ec2_meta, os_files):
                     body=get_request_callback)
 
 
+def _read_metadata_service():
+    return ds.read_metadata_service(BASE_URL, retries=0, timeout=0.1)
+
+
 class TestOpenStackDataSource(test_helpers.HttprettyTestCase):
     VERSION = 'latest'
 
     @hp.activate
     def test_successful(self):
         _register_uris(self.VERSION, EC2_FILES, EC2_META, OS_FILES)
-        f = ds.read_metadata_service(BASE_URL)
+        f = _read_metadata_service()
         self.assertEqual(VENDOR_DATA, f.get('vendordata'))
         self.assertEqual(CONTENT_0, f['files']['/etc/foo.cfg'])
         self.assertEqual(CONTENT_1, f['files']['/etc/bar/bar.cfg'])
@@ -163,7 +167,7 @@ class TestOpenStackDataSource(test_helpers.HttprettyTestCase):
     @hp.activate
     def test_no_ec2(self):
         _register_uris(self.VERSION, {}, {}, OS_FILES)
-        f = ds.read_metadata_service(BASE_URL)
+        f = _read_metadata_service()
         self.assertEqual(VENDOR_DATA, f.get('vendordata'))
         self.assertEqual(CONTENT_0, f['files']['/etc/foo.cfg'])
         self.assertEqual(CONTENT_1, f['files']['/etc/bar/bar.cfg'])
@@ -178,8 +182,7 @@ class TestOpenStackDataSource(test_helpers.HttprettyTestCase):
             if k.endswith('meta_data.json'):
                 os_files.pop(k, None)
         _register_uris(self.VERSION, {}, {}, os_files)
-        self.assertRaises(openstack.NonReadable, ds.read_metadata_service,
-                          BASE_URL)
+        self.assertRaises(openstack.NonReadable, _read_metadata_service)
 
     @hp.activate
     def test_bad_uuid(self):
@@ -190,8 +193,7 @@ class TestOpenStackDataSource(test_helpers.HttprettyTestCase):
             if k.endswith('meta_data.json'):
                 os_files[k] = json.dumps(os_meta)
         _register_uris(self.VERSION, {}, {}, os_files)
-        self.assertRaises(openstack.BrokenMetadata, ds.read_metadata_service,
-                          BASE_URL)
+        self.assertRaises(openstack.BrokenMetadata, _read_metadata_service)
 
     @hp.activate
     def test_userdata_empty(self):
@@ -200,7 +202,7 @@ class TestOpenStackDataSource(test_helpers.HttprettyTestCase):
             if k.endswith('user_data'):
                 os_files.pop(k, None)
         _register_uris(self.VERSION, {}, {}, os_files)
-        f = ds.read_metadata_service(BASE_URL)
+        f = _read_metadata_service()
         self.assertEqual(VENDOR_DATA, f.get('vendordata'))
         self.assertEqual(CONTENT_0, f['files']['/etc/foo.cfg'])
         self.assertEqual(CONTENT_1, f['files']['/etc/bar/bar.cfg'])
@@ -213,7 +215,7 @@ class TestOpenStackDataSource(test_helpers.HttprettyTestCase):
             if k.endswith('vendor_data.json'):
                 os_files.pop(k, None)
         _register_uris(self.VERSION, {}, {}, os_files)
-        f = ds.read_metadata_service(BASE_URL)
+        f = _read_metadata_service()
         self.assertEqual(CONTENT_0, f['files']['/etc/foo.cfg'])
         self.assertEqual(CONTENT_1, f['files']['/etc/bar/bar.cfg'])
         self.assertFalse(f.get('vendordata'))
@@ -225,8 +227,7 @@ class TestOpenStackDataSource(test_helpers.HttprettyTestCase):
             if k.endswith('vendor_data.json'):
                 os_files[k] = '{'  # some invalid json
         _register_uris(self.VERSION, {}, {}, os_files)
-        self.assertRaises(openstack.BrokenMetadata, ds.read_metadata_service,
-                          BASE_URL)
+        self.assertRaises(openstack.BrokenMetadata, _read_metadata_service)
 
     @hp.activate
     def test_metadata_invalid(self):
@@ -235,8 +236,7 @@ class TestOpenStackDataSource(test_helpers.HttprettyTestCase):
             if k.endswith('meta_data.json'):
                 os_files[k] = '{'  # some invalid json
         _register_uris(self.VERSION, {}, {}, os_files)
-        self.assertRaises(openstack.BrokenMetadata, ds.read_metadata_service,
-                          BASE_URL)
+        self.assertRaises(openstack.BrokenMetadata, _read_metadata_service)
 
     @hp.activate
     def test_datasource(self):
@@ -245,7 +245,7 @@ class TestOpenStackDataSource(test_helpers.HttprettyTestCase):
                                        None,
                                        helpers.Paths({}))
         self.assertIsNone(ds_os.version)
-        found = ds_os.get_data()
+        found = ds_os.get_data(timeout=0.1, retries=0)
         self.assertTrue(found)
         self.assertEqual(2, ds_os.version)
         md = dict(ds_os.metadata)
@@ -269,7 +269,7 @@ class TestOpenStackDataSource(test_helpers.HttprettyTestCase):
                                        None,
                                        helpers.Paths({}))
         self.assertIsNone(ds_os.version)
-        found = ds_os.get_data()
+        found = ds_os.get_data(timeout=0.1, retries=0)
         self.assertFalse(found)
         self.assertIsNone(ds_os.version)
 
@@ -288,7 +288,7 @@ class TestOpenStackDataSource(test_helpers.HttprettyTestCase):
             'timeout': 0,
         }
         self.assertIsNone(ds_os.version)
-        found = ds_os.get_data()
+        found = ds_os.get_data(timeout=0.1, retries=0)
         self.assertFalse(found)
         self.assertIsNone(ds_os.version)
 
@@ -311,7 +311,7 @@ class TestOpenStackDataSource(test_helpers.HttprettyTestCase):
             'timeout': 0,
         }
         self.assertIsNone(ds_os.version)
-        found = ds_os.get_data()
+        found = ds_os.get_data(timeout=0.1, retries=0)
         self.assertFalse(found)
         self.assertIsNone(ds_os.version)
 
diff --git a/tests/unittests/test_merging.py b/tests/unittests/test_merging.py
index 681f3780..a33ec184 100644
--- a/tests/unittests/test_merging.py
+++ b/tests/unittests/test_merging.py
@@ -125,9 +125,9 @@ class TestSimpleRun(helpers.ResourceUsingTestCase):
 
     def test_seed_runs(self):
         test_dicts = []
-        for i in range(1, 50):
+        for i in range(1, 10):
             base_dicts = []
-            for j in range(1, 50):
+            for j in range(1, 10):
                 base_dicts.append(make_dict(5, i * j))
             test_dicts.append(base_dicts)
         for test in test_dicts: