From ba84f51f0143a8ca1ca5113ae932505ce1bfe5e5 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Thu, 23 Jan 2014 14:41:09 -0800 Subject: Skip retry and continued fetch of userdata when NOT_FOUND When a 404 http code comes back from the fetching of ec2 data, instead of retrying immediatly stop the fetching process and in the userdata fetching function handle this case as a special case of no userdata being fetched (an empty string in this case). --- cloudinit/ec2_utils.py | 27 ++++++++++++++++++++++++--- cloudinit/url_helper.py | 17 +++++++++-------- cloudinit/util.py | 5 +++-- tests/unittests/test_datasource/test_maas.py | 3 ++- tests/unittests/test_ec2_util.py | 8 ++++++++ 5 files changed, 46 insertions(+), 14 deletions(-) diff --git a/cloudinit/ec2_utils.py b/cloudinit/ec2_utils.py index 92a22747..cd94ad4c 100644 --- a/cloudinit/ec2_utils.py +++ b/cloudinit/ec2_utils.py @@ -16,6 +16,7 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . +import httplib from urlparse import (urlparse, urlunparse) import functools @@ -23,9 +24,11 @@ import json import urllib from cloudinit import log as logging +from cloudinit import url_helper from cloudinit import util LOG = logging.getLogger(__name__) +SKIP_USERDATA_CODES = frozenset([httplib.NOT_FOUND]) def maybe_json_object(text): @@ -138,20 +141,38 @@ class MetadataMaterializer(object): return joined +def _skip_retry_on_codes(status_codes, request_args, cause): + """Returns if a request should retry based on a given set of codes that + case retrying to be stopped/skipped. + """ + if cause.code in status_codes: + return False + return True + + def get_instance_userdata(api_version='latest', metadata_address='http://169.254.169.254', ssl_details=None, timeout=5, retries=5): ud_url = combine_url(metadata_address, api_version) ud_url = combine_url(ud_url, 'user-data') + user_data = '' try: + # It is ok for userdata to not exist (thats why we are stopping if + # NOT_FOUND occurs) and just in that case returning an empty string. + exception_cb = functools.partial(_skip_retry_on_codes, + SKIP_USERDATA_CODES) response = util.read_file_or_url(ud_url, ssl_details=ssl_details, timeout=timeout, - retries=retries) - return str(response) + retries=retries, + exception_cb=exception_cb) + user_data = str(response) + except url_helper.UrlError as e: + if e.code not in SKIP_USERDATA_CODES: + util.logexc(LOG, "Failed fetching userdata from url %s", ud_url) except Exception: util.logexc(LOG, "Failed fetching userdata from url %s", ud_url) - return '' + return user_data def get_instance_metadata(api_version='latest', diff --git a/cloudinit/url_helper.py b/cloudinit/url_helper.py index 19a30409..42edf9cf 100644 --- a/cloudinit/url_helper.py +++ b/cloudinit/url_helper.py @@ -103,7 +103,7 @@ class UrlError(IOError): def readurl(url, data=None, timeout=None, retries=0, sec_between=1, headers=None, headers_cb=None, ssl_details=None, - check_status=True, allow_redirects=True): + check_status=True, allow_redirects=True, exception_cb=None): url = _cleanurl(url) req_args = { 'url': url, @@ -163,14 +163,13 @@ def readurl(url, data=None, timeout=None, retries=0, sec_between=1, # Handle retrying ourselves since the built-in support # doesn't handle sleeping between tries... for i in range(0, manual_tries): + req_args['headers'] = headers_cb(url) + filtered_req_args = {} + for (k, v) in req_args.items(): + if k == 'data': + continue + filtered_req_args[k] = v try: - req_args['headers'] = headers_cb(url) - filtered_req_args = {} - for (k, v) in req_args.items(): - if k == 'data': - continue - filtered_req_args[k] = v - LOG.debug("[%s/%s] open '%s' with %s configuration", i, manual_tries, url, filtered_req_args) @@ -196,6 +195,8 @@ def readurl(url, data=None, timeout=None, retries=0, sec_between=1, # ssl exceptions are not going to get fixed by waiting a # few seconds break + if exception_cb and not exception_cb(filtered_req_args, e): + break if i + 1 < manual_tries and sec_between > 0: LOG.debug("Please wait %s seconds while we wait to try again", sec_between) diff --git a/cloudinit/util.py b/cloudinit/util.py index 77f9ab36..e1263f47 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -691,7 +691,7 @@ def fetch_ssl_details(paths=None): def read_file_or_url(url, timeout=5, retries=10, headers=None, data=None, sec_between=1, ssl_details=None, - headers_cb=None): + headers_cb=None, exception_cb=None): url = url.lstrip() if url.startswith("/"): url = "file://%s" % url @@ -708,7 +708,8 @@ def read_file_or_url(url, timeout=5, retries=10, headers_cb=headers_cb, data=data, sec_between=sec_between, - ssl_details=ssl_details) + ssl_details=ssl_details, + exception_cb=exception_cb) def load_yaml(blob, default=None, allowed=(dict,)): diff --git a/tests/unittests/test_datasource/test_maas.py b/tests/unittests/test_datasource/test_maas.py index 2007a6df..ebfb24da 100644 --- a/tests/unittests/test_datasource/test_maas.py +++ b/tests/unittests/test_datasource/test_maas.py @@ -119,7 +119,8 @@ class TestMAASDataSource(mocker.MockerTestCase): mock_request(url, headers=None, timeout=mocker.ANY, data=mocker.ANY, sec_between=mocker.ANY, ssl_details=mocker.ANY, retries=mocker.ANY, - headers_cb=my_headers_cb) + headers_cb=my_headers_cb, + exception_cb=mocker.ANY) resp = valid.get(key) self.mocker.result(util.StringResponse(resp)) self.mocker.replay() diff --git a/tests/unittests/test_ec2_util.py b/tests/unittests/test_ec2_util.py index dd588aca..957dc3f2 100644 --- a/tests/unittests/test_ec2_util.py +++ b/tests/unittests/test_ec2_util.py @@ -33,6 +33,14 @@ class TestEc2Util(helpers.TestCase): userdata = eu.get_instance_userdata(self.VERSION, retries=0) self.assertEquals('', userdata) + @hp.activate + def test_userdata_fetch_fail_server_not_found(self): + hp.register_uri(hp.GET, + 'http://169.254.169.254/%s/user-data' % (self.VERSION), + status=404) + userdata = eu.get_instance_userdata(self.VERSION) + self.assertEquals('', userdata) + @hp.activate def test_metadata_fetch_no_keys(self): base_url = 'http://169.254.169.254/%s/meta-data' % (self.VERSION) -- cgit v1.2.3