diff options
Diffstat (limited to 'azurelinuxagent/common/utils/restutil.py')
| -rw-r--r-- | azurelinuxagent/common/utils/restutil.py | 105 |
1 files changed, 83 insertions, 22 deletions
diff --git a/azurelinuxagent/common/utils/restutil.py b/azurelinuxagent/common/utils/restutil.py index ddd930b..807be29 100644 --- a/azurelinuxagent/common/utils/restutil.py +++ b/azurelinuxagent/common/utils/restutil.py @@ -18,6 +18,7 @@ # import os +import threading import time import traceback @@ -32,10 +33,11 @@ from azurelinuxagent.common.version import PY_VERSION_MAJOR SECURE_WARNING_EMITTED = False -DEFAULT_RETRIES = 3 +DEFAULT_RETRIES = 6 +DELAY_IN_SECONDS = 1 -SHORT_DELAY_IN_SECONDS = 5 -LONG_DELAY_IN_SECONDS = 15 +THROTTLE_RETRIES = 25 +THROTTLE_DELAY_IN_SECONDS = 1 RETRY_CODES = [ httpclient.RESET_CONTENT, @@ -63,7 +65,8 @@ OK_CODES = [ THROTTLE_CODES = [ httpclient.FORBIDDEN, - httpclient.SERVICE_UNAVAILABLE + httpclient.SERVICE_UNAVAILABLE, + 429, # Request Rate Limit Exceeded ] RETRY_EXCEPTIONS = [ @@ -76,6 +79,48 @@ RETRY_EXCEPTIONS = [ HTTP_PROXY_ENV = "http_proxy" HTTPS_PROXY_ENV = "https_proxy" +DEFAULT_PROTOCOL_ENDPOINT='168.63.129.16' +HOST_PLUGIN_PORT = 32526 + + +class IOErrorCounter(object): + _lock = threading.RLock() + _protocol_endpoint = DEFAULT_PROTOCOL_ENDPOINT + _counts = {"hostplugin":0, "protocol":0, "other":0} + + @staticmethod + def increment(host=None, port=None): + with IOErrorCounter._lock: + if host == IOErrorCounter._protocol_endpoint: + if port == HOST_PLUGIN_PORT: + IOErrorCounter._counts["hostplugin"] += 1 + else: + IOErrorCounter._counts["protocol"] += 1 + else: + IOErrorCounter._counts["other"] += 1 + + @staticmethod + def get_and_reset(): + with IOErrorCounter._lock: + counts = IOErrorCounter._counts.copy() + IOErrorCounter.reset() + return counts + + @staticmethod + def reset(): + with IOErrorCounter._lock: + IOErrorCounter._counts = {"hostplugin":0, "protocol":0, "other":0} + + @staticmethod + def set_protocol_endpoint(endpoint=DEFAULT_PROTOCOL_ENDPOINT): + IOErrorCounter._protocol_endpoint = endpoint + + +def _compute_delay(retry_attempt=1, delay=DELAY_IN_SECONDS): + fib = (1, 1) + for n in range(retry_attempt): + fib = (fib[1], fib[0]+fib[1]) + return delay*fib[1] def _is_retry_status(status, retry_codes=RETRY_CODES): return status in retry_codes @@ -166,7 +211,7 @@ def http_request(method, use_proxy=False, max_retry=DEFAULT_RETRIES, retry_codes=RETRY_CODES, - retry_delay=SHORT_DELAY_IN_SECONDS): + retry_delay=DELAY_IN_SECONDS): global SECURE_WARNING_EMITTED @@ -208,18 +253,31 @@ def http_request(method, msg = '' attempt = 0 - delay = retry_delay + delay = 0 + was_throttled = False while attempt < max_retry: if attempt > 0: - logger.info("[HTTP Retry] Attempt {0} of {1}: {2}", + # Compute the request delay + # -- Use a fixed delay if the server ever rate-throttles the request + # (with a safe, minimum number of retry attempts) + # -- Otherwise, compute a delay that is the product of the next + # item in the Fibonacci series and the initial delay value + delay = THROTTLE_DELAY_IN_SECONDS \ + if was_throttled \ + else _compute_delay(retry_attempt=attempt, + delay=retry_delay) + + logger.verbose("[HTTP Retry] " + "Attempt {0} of {1} will delay {2} seconds: {3}", attempt+1, max_retry, + delay, msg) + time.sleep(delay) attempt += 1 - delay = retry_delay try: resp = _http_request(method, @@ -235,13 +293,13 @@ def http_request(method, if request_failed(resp): if _is_retry_status(resp.status, retry_codes=retry_codes): - msg = '[HTTP Retry] HTTP {0} Status Code {1}'.format( - method, resp.status) + msg = '[HTTP Retry] {0} {1} -- Status Code {2}'.format( + method, url, resp.status) + # Note if throttled and ensure a safe, minimum number of + # retry attempts if _is_throttle_status(resp.status): - delay = LONG_DELAY_IN_SECONDS - logger.info("[HTTP Delay] Delay {0} seconds for " \ - "Status Code {1}".format( - delay, resp.status)) + was_throttled = True + max_retry = max(max_retry, THROTTLE_RETRIES) continue if resp.status in RESOURCE_GONE_CODES: @@ -250,22 +308,25 @@ def http_request(method, return resp except httpclient.HTTPException as e: - msg = '[HTTP Failed] HTTP {0} HttpException {1}'.format(method, e) + msg = '[HTTP Failed] {0} {1} -- HttpException {2}'.format( + method, url, e) if _is_retry_exception(e): continue break except IOError as e: - msg = '[HTTP Failed] HTTP {0} IOError {1}'.format(method, e) + IOErrorCounter.increment(host=host, port=port) + msg = '[HTTP Failed] {0} {1} -- IOError {2}'.format( + method, url, e) continue - raise HttpError(msg) + raise HttpError("{0} -- {1} attempts made".format(msg,attempt)) def http_get(url, headers=None, use_proxy=False, max_retry=DEFAULT_RETRIES, retry_codes=RETRY_CODES, - retry_delay=SHORT_DELAY_IN_SECONDS): + retry_delay=DELAY_IN_SECONDS): return http_request("GET", url, None, headers=headers, use_proxy=use_proxy, @@ -277,7 +338,7 @@ def http_get(url, headers=None, use_proxy=False, def http_head(url, headers=None, use_proxy=False, max_retry=DEFAULT_RETRIES, retry_codes=RETRY_CODES, - retry_delay=SHORT_DELAY_IN_SECONDS): + retry_delay=DELAY_IN_SECONDS): return http_request("HEAD", url, None, headers=headers, use_proxy=use_proxy, @@ -289,7 +350,7 @@ def http_head(url, headers=None, use_proxy=False, def http_post(url, data, headers=None, use_proxy=False, max_retry=DEFAULT_RETRIES, retry_codes=RETRY_CODES, - retry_delay=SHORT_DELAY_IN_SECONDS): + retry_delay=DELAY_IN_SECONDS): return http_request("POST", url, data, headers=headers, use_proxy=use_proxy, @@ -301,7 +362,7 @@ def http_post(url, data, headers=None, use_proxy=False, def http_put(url, data, headers=None, use_proxy=False, max_retry=DEFAULT_RETRIES, retry_codes=RETRY_CODES, - retry_delay=SHORT_DELAY_IN_SECONDS): + retry_delay=DELAY_IN_SECONDS): return http_request("PUT", url, data, headers=headers, use_proxy=use_proxy, @@ -313,7 +374,7 @@ def http_put(url, data, headers=None, use_proxy=False, def http_delete(url, headers=None, use_proxy=False, max_retry=DEFAULT_RETRIES, retry_codes=RETRY_CODES, - retry_delay=SHORT_DELAY_IN_SECONDS): + retry_delay=DELAY_IN_SECONDS): return http_request("DELETE", url, None, headers=headers, use_proxy=use_proxy, |