From 75211bdfd58b3209455c88ab00c96e9e7c7acecc Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Thu, 12 May 2016 14:29:50 -0700 Subject: Enable nose-timer and always show top 10 slow tests --- test-requirements.txt | 1 + tox.ini | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/test-requirements.txt b/test-requirements.txt index 2ab53707..88a67a26 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -2,6 +2,7 @@ httpretty>=0.7.1 mock nose +nose-timer # Only really needed on older versions of python contextlib2 diff --git a/tox.ini b/tox.ini index dafaaf6d..d051f2ac 100644 --- a/tox.ini +++ b/tox.ini @@ -3,7 +3,7 @@ envlist = py27,py3,flake8 recreate = True [testenv] -commands = python -m nose {posargs:tests} +commands = python -m nose --with-timer --timer-top-n 10 {posargs:tests} deps = -r{toxinidir}/test-requirements.txt -r{toxinidir}/requirements.txt @@ -19,7 +19,7 @@ setenv = LC_ALL = en_US.utf-8 [testenv:py26] -commands = nosetests {posargs:tests} +commands = nosetests --with-timer --timer-top-n 10 {posargs:tests} deps = contextlib2 httpretty>=0.7.1 -- cgit v1.2.3 From f2665c246a3e6dec55064eced09919d912ae0e52 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Mon, 16 May 2016 16:08:19 -0700 Subject: Fix slow tests Timeouts and retries were triggering so make it so that tests do not use the typical timesouts and retries so that the tests finish faster. --- cloudinit/sources/DataSourceOpenStack.py | 12 ++++++--- tests/unittests/test_datasource/test_openstack.py | 32 +++++++++++------------ 2 files changed, 24 insertions(+), 20 deletions(-) diff --git a/cloudinit/sources/DataSourceOpenStack.py b/cloudinit/sources/DataSourceOpenStack.py index 3af17b10..dfd96035 100644 --- a/cloudinit/sources/DataSourceOpenStack.py +++ b/cloudinit/sources/DataSourceOpenStack.py @@ -103,7 +103,7 @@ class DataSourceOpenStack(openstack.SourceMixin, sources.DataSource): self.metadata_address = url2base.get(avail_url) return bool(avail_url) - def get_data(self): + def get_data(self, retries=5, timeout=5): try: if not self.wait_for_metadata_service(): return False @@ -115,7 +115,9 @@ class DataSourceOpenStack(openstack.SourceMixin, sources.DataSource): 'Crawl of openstack metadata service', read_metadata_service, args=[self.metadata_address], - kwargs={'ssl_details': self.ssl_details}) + kwargs={'ssl_details': self.ssl_details, + 'retries': retries, + 'timeout': timeout}) except openstack.NonReadable: return False except (openstack.BrokenMetadata, IOError): @@ -153,8 +155,10 @@ class DataSourceOpenStack(openstack.SourceMixin, sources.DataSource): return sources.instance_id_matches_system_uuid(self.get_instance_id()) -def read_metadata_service(base_url, ssl_details=None): - reader = openstack.MetadataReader(base_url, ssl_details=ssl_details) +def read_metadata_service(base_url, ssl_details=None, + timeout=5, retries=5): + reader = openstack.MetadataReader(base_url, ssl_details=ssl_details, + timeout=timeout, retries=retries) return reader.read_v2() diff --git a/tests/unittests/test_datasource/test_openstack.py b/tests/unittests/test_datasource/test_openstack.py index 4140d054..5c8592c5 100644 --- a/tests/unittests/test_datasource/test_openstack.py +++ b/tests/unittests/test_datasource/test_openstack.py @@ -135,13 +135,17 @@ def _register_uris(version, ec2_files, ec2_meta, os_files): body=get_request_callback) +def _read_metadata_service(): + return ds.read_metadata_service(BASE_URL, retries=0, timeout=0.1) + + class TestOpenStackDataSource(test_helpers.HttprettyTestCase): VERSION = 'latest' @hp.activate def test_successful(self): _register_uris(self.VERSION, EC2_FILES, EC2_META, OS_FILES) - f = ds.read_metadata_service(BASE_URL) + f = _read_metadata_service() self.assertEqual(VENDOR_DATA, f.get('vendordata')) self.assertEqual(CONTENT_0, f['files']['/etc/foo.cfg']) self.assertEqual(CONTENT_1, f['files']['/etc/bar/bar.cfg']) @@ -163,7 +167,7 @@ class TestOpenStackDataSource(test_helpers.HttprettyTestCase): @hp.activate def test_no_ec2(self): _register_uris(self.VERSION, {}, {}, OS_FILES) - f = ds.read_metadata_service(BASE_URL) + f = _read_metadata_service() self.assertEqual(VENDOR_DATA, f.get('vendordata')) self.assertEqual(CONTENT_0, f['files']['/etc/foo.cfg']) self.assertEqual(CONTENT_1, f['files']['/etc/bar/bar.cfg']) @@ -178,8 +182,7 @@ class TestOpenStackDataSource(test_helpers.HttprettyTestCase): if k.endswith('meta_data.json'): os_files.pop(k, None) _register_uris(self.VERSION, {}, {}, os_files) - self.assertRaises(openstack.NonReadable, ds.read_metadata_service, - BASE_URL) + self.assertRaises(openstack.NonReadable, _read_metadata_service) @hp.activate def test_bad_uuid(self): @@ -190,8 +193,7 @@ class TestOpenStackDataSource(test_helpers.HttprettyTestCase): if k.endswith('meta_data.json'): os_files[k] = json.dumps(os_meta) _register_uris(self.VERSION, {}, {}, os_files) - self.assertRaises(openstack.BrokenMetadata, ds.read_metadata_service, - BASE_URL) + self.assertRaises(openstack.BrokenMetadata, _read_metadata_service) @hp.activate def test_userdata_empty(self): @@ -200,7 +202,7 @@ class TestOpenStackDataSource(test_helpers.HttprettyTestCase): if k.endswith('user_data'): os_files.pop(k, None) _register_uris(self.VERSION, {}, {}, os_files) - f = ds.read_metadata_service(BASE_URL) + f = _read_metadata_service() self.assertEqual(VENDOR_DATA, f.get('vendordata')) self.assertEqual(CONTENT_0, f['files']['/etc/foo.cfg']) self.assertEqual(CONTENT_1, f['files']['/etc/bar/bar.cfg']) @@ -213,7 +215,7 @@ class TestOpenStackDataSource(test_helpers.HttprettyTestCase): if k.endswith('vendor_data.json'): os_files.pop(k, None) _register_uris(self.VERSION, {}, {}, os_files) - f = ds.read_metadata_service(BASE_URL) + f = _read_metadata_service() self.assertEqual(CONTENT_0, f['files']['/etc/foo.cfg']) self.assertEqual(CONTENT_1, f['files']['/etc/bar/bar.cfg']) self.assertFalse(f.get('vendordata')) @@ -225,8 +227,7 @@ class TestOpenStackDataSource(test_helpers.HttprettyTestCase): if k.endswith('vendor_data.json'): os_files[k] = '{' # some invalid json _register_uris(self.VERSION, {}, {}, os_files) - self.assertRaises(openstack.BrokenMetadata, ds.read_metadata_service, - BASE_URL) + self.assertRaises(openstack.BrokenMetadata, _read_metadata_service) @hp.activate def test_metadata_invalid(self): @@ -235,8 +236,7 @@ class TestOpenStackDataSource(test_helpers.HttprettyTestCase): if k.endswith('meta_data.json'): os_files[k] = '{' # some invalid json _register_uris(self.VERSION, {}, {}, os_files) - self.assertRaises(openstack.BrokenMetadata, ds.read_metadata_service, - BASE_URL) + self.assertRaises(openstack.BrokenMetadata, _read_metadata_service) @hp.activate def test_datasource(self): @@ -245,7 +245,7 @@ class TestOpenStackDataSource(test_helpers.HttprettyTestCase): None, helpers.Paths({})) self.assertIsNone(ds_os.version) - found = ds_os.get_data() + found = ds_os.get_data(timeout=0.1, retries=0) self.assertTrue(found) self.assertEqual(2, ds_os.version) md = dict(ds_os.metadata) @@ -269,7 +269,7 @@ class TestOpenStackDataSource(test_helpers.HttprettyTestCase): None, helpers.Paths({})) self.assertIsNone(ds_os.version) - found = ds_os.get_data() + found = ds_os.get_data(timeout=0.1, retries=0) self.assertFalse(found) self.assertIsNone(ds_os.version) @@ -288,7 +288,7 @@ class TestOpenStackDataSource(test_helpers.HttprettyTestCase): 'timeout': 0, } self.assertIsNone(ds_os.version) - found = ds_os.get_data() + found = ds_os.get_data(timeout=0.1, retries=0) self.assertFalse(found) self.assertIsNone(ds_os.version) @@ -311,7 +311,7 @@ class TestOpenStackDataSource(test_helpers.HttprettyTestCase): 'timeout': 0, } self.assertIsNone(ds_os.version) - found = ds_os.get_data() + found = ds_os.get_data(timeout=0.1, retries=0) self.assertFalse(found) self.assertIsNone(ds_os.version) -- cgit v1.2.3 From 70ee3c4e6806996401d747e4f8b25855e85d86eb Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Tue, 24 May 2016 14:05:20 -0700 Subject: Make the usage of 'nose-timer' optional --- test-requirements.txt | 4 +++- tox.ini | 4 ++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/test-requirements.txt b/test-requirements.txt index 88a67a26..aa01fa88 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -2,7 +2,9 @@ httpretty>=0.7.1 mock nose -nose-timer + +# Only needed if u want to know the test times +# nose-timer # Only really needed on older versions of python contextlib2 diff --git a/tox.ini b/tox.ini index d051f2ac..dafaaf6d 100644 --- a/tox.ini +++ b/tox.ini @@ -3,7 +3,7 @@ envlist = py27,py3,flake8 recreate = True [testenv] -commands = python -m nose --with-timer --timer-top-n 10 {posargs:tests} +commands = python -m nose {posargs:tests} deps = -r{toxinidir}/test-requirements.txt -r{toxinidir}/requirements.txt @@ -19,7 +19,7 @@ setenv = LC_ALL = en_US.utf-8 [testenv:py26] -commands = nosetests --with-timer --timer-top-n 10 {posargs:tests} +commands = nosetests {posargs:tests} deps = contextlib2 httpretty>=0.7.1 -- cgit v1.2.3 From 2ec72c91b7ba60b260ba0a50097df16f82960dd8 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Tue, 24 May 2016 19:27:08 -0400 Subject: fix logic error in ec2 get_instance_userdata and slow tests The change to get_instance_userdata is to fix an issue that was causing retry in the test when it was not desired. if user_data returned 404 it means "there was no user-data", so dont bother retrying. However, _skip_retry_on_codes was returning False indicating that readurl should retry. test_merging was creating 2500 random tests, shrink that down to 100. test_seed_runs is still on my system the slowest test, but taking < .5 seconds where it was taking > 3. --- cloudinit/ec2_utils.py | 7 ++++--- tests/unittests/test_merging.py | 4 ++-- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/cloudinit/ec2_utils.py b/cloudinit/ec2_utils.py index 37b92a83..90b34eff 100644 --- a/cloudinit/ec2_utils.py +++ b/cloudinit/ec2_utils.py @@ -144,9 +144,10 @@ def _skip_retry_on_codes(status_codes, _request_args, cause): """Returns if a request should retry based on a given set of codes that case retrying to be stopped/skipped. """ - if cause.code in status_codes: - return False - return True + print("status_codes=%s" % status_codes) + print("_request_args=%s" % _request_args) + print("cause=%s" % cause) + return cause.code in status_codes def get_instance_userdata(api_version='latest', diff --git a/tests/unittests/test_merging.py b/tests/unittests/test_merging.py index 681f3780..a33ec184 100644 --- a/tests/unittests/test_merging.py +++ b/tests/unittests/test_merging.py @@ -125,9 +125,9 @@ class TestSimpleRun(helpers.ResourceUsingTestCase): def test_seed_runs(self): test_dicts = [] - for i in range(1, 50): + for i in range(1, 10): base_dicts = [] - for j in range(1, 50): + for j in range(1, 10): base_dicts.append(make_dict(5, i * j)) test_dicts.append(base_dicts) for test in test_dicts: -- cgit v1.2.3 From 8bd380c5b17b012bcbf00013ab095ea7c6d8b533 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Tue, 24 May 2016 19:34:28 -0400 Subject: update changelog --- ChangeLog | 1 + 1 file changed, 1 insertion(+) diff --git a/ChangeLog b/ChangeLog index e06578fc..6748e8fa 100644 --- a/ChangeLog +++ b/ChangeLog @@ -109,6 +109,7 @@ [Wido den Hollander] - Paths: fix instance path if datasource's id has a '/'. (LP: #1575938) [Robert Jennings] + - Ec2: do not retry requests for user-data path on 404. 0.7.6: - open 0.7.6 -- cgit v1.2.3 From db54fca319954d72f3fd48eabd27aad8be31e7b3 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Wed, 25 May 2016 09:08:29 -0400 Subject: remove debug print statements --- cloudinit/ec2_utils.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/cloudinit/ec2_utils.py b/cloudinit/ec2_utils.py index 90b34eff..76dda042 100644 --- a/cloudinit/ec2_utils.py +++ b/cloudinit/ec2_utils.py @@ -144,9 +144,6 @@ def _skip_retry_on_codes(status_codes, _request_args, cause): """Returns if a request should retry based on a given set of codes that case retrying to be stopped/skipped. """ - print("status_codes=%s" % status_codes) - print("_request_args=%s" % _request_args) - print("cause=%s" % cause) return cause.code in status_codes -- cgit v1.2.3 From 4728b0de649bf231fa667373d5d8ebf451e19d0b Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Wed, 25 May 2016 20:21:04 -0400 Subject: packages/bddeb: fix to know about packages flake8 and hacking --- packages/bddeb | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages/bddeb b/packages/bddeb index c141b1ab..1b0f642c 100755 --- a/packages/bddeb +++ b/packages/bddeb @@ -40,6 +40,8 @@ STD_NAMED_PACKAGES = [ 'mock', 'nose', 'setuptools', + 'flake8', + 'hacking', ] NONSTD_NAMED_PACKAGES = { 'argparse': ('python-argparse', None), -- cgit v1.2.3 From 63501f44eff7ef2d6083900c47180faf444662fc Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Thu, 26 May 2016 09:02:17 -0400 Subject: kernel command line: override all local settings settings on the kernel command line (cc:) were documented to override all local settings, but a bug in implementation meant they would only override those that are in /etc/cloud/cloud.cfg, not any found in /etc/cloud/cloud.cfg.d. LP: #1582323 --- ChangeLog | 2 ++ cloudinit/stages.py | 10 +++++----- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/ChangeLog b/ChangeLog index 6748e8fa..8db29e2e 100644 --- a/ChangeLog +++ b/ChangeLog @@ -110,6 +110,8 @@ - Paths: fix instance path if datasource's id has a '/'. (LP: #1575938) [Robert Jennings] - Ec2: do not retry requests for user-data path on 404. + - settings on the kernel command line (cc:) override all local settings + rather than only those in /etc/cloud/cloud.cfg (LP: #1582323) 0.7.6: - open 0.7.6 diff --git a/cloudinit/stages.py b/cloudinit/stages.py index 62d066de..002e5832 100644 --- a/cloudinit/stages.py +++ b/cloudinit/stages.py @@ -794,16 +794,16 @@ class Modules(object): def fetch_base_config(): base_cfgs = [] default_cfg = util.get_builtin_cfg() - kern_contents = util.read_cc_from_cmdline() - - # Kernel/cmdline parameters override system config - if kern_contents: - base_cfgs.append(util.load_yaml(kern_contents, default={})) # Anything in your conf.d location?? # or the 'default' cloud.cfg location??? base_cfgs.append(util.read_conf_with_confd(CLOUD_CONFIG)) + # Kernel/cmdline parameters override system config + kern_contents = util.read_cc_from_cmdline() + if kern_contents: + base_cfgs.append(util.load_yaml(kern_contents, default={})) + # And finally the default gets to play if default_cfg: base_cfgs.append(default_cfg) -- cgit v1.2.3