Import patches-applied version 2.2.21-0ubuntu1 to applied/ubuntu/bionic-proposed

Imported using git-ubuntu import.

Changelog parent: 110d301b04a64d680fc7d102424e303a8e3ca1a6
Unapplied parent: d5298bbf0f5696fc948877304e86f43d477d6b71

New changelog entries:
  * New upstream release (LP: #1746628).
  * debian/patches/disable_import_test.patch: refreshed patch.

1 files changed, 83 insertions, 22 deletions

diff --git a/azurelinuxagent/common/utils/restutil.py b/azurelinuxagent/common/utils/restutil.py
index ddd930b..807be29 100644
--- a/azurelinuxagent/common/utils/restutil.py
+++ b/azurelinuxagent/common/utils/restutil.py
@@ -18,6 +18,7 @@
 #
 
 import os
+import threading
 import time
 import traceback
 
@@ -32,10 +33,11 @@ from azurelinuxagent.common.version import PY_VERSION_MAJOR
 
 SECURE_WARNING_EMITTED = False
 
-DEFAULT_RETRIES = 3
+DEFAULT_RETRIES = 6
+DELAY_IN_SECONDS = 1
 
-SHORT_DELAY_IN_SECONDS = 5
-LONG_DELAY_IN_SECONDS = 15
+THROTTLE_RETRIES = 25
+THROTTLE_DELAY_IN_SECONDS = 1
 
 RETRY_CODES = [
     httpclient.RESET_CONTENT,
@@ -63,7 +65,8 @@ OK_CODES = [
 
 THROTTLE_CODES = [
     httpclient.FORBIDDEN,
-    httpclient.SERVICE_UNAVAILABLE
+    httpclient.SERVICE_UNAVAILABLE,
+    429, # Request Rate Limit Exceeded
 ]
 
 RETRY_EXCEPTIONS = [
@@ -76,6 +79,48 @@ RETRY_EXCEPTIONS = [
 HTTP_PROXY_ENV = "http_proxy"
 HTTPS_PROXY_ENV = "https_proxy"
 
+DEFAULT_PROTOCOL_ENDPOINT='168.63.129.16'
+HOST_PLUGIN_PORT = 32526
+
+
+class IOErrorCounter(object):
+    _lock = threading.RLock()
+    _protocol_endpoint = DEFAULT_PROTOCOL_ENDPOINT
+    _counts = {"hostplugin":0, "protocol":0, "other":0}
+
+    @staticmethod
+    def increment(host=None, port=None):
+        with IOErrorCounter._lock:
+            if host == IOErrorCounter._protocol_endpoint:
+                if port == HOST_PLUGIN_PORT:
+                    IOErrorCounter._counts["hostplugin"] += 1
+                else:
+                    IOErrorCounter._counts["protocol"] += 1
+            else:
+                IOErrorCounter._counts["other"] += 1
+
+    @staticmethod
+    def get_and_reset():
+        with IOErrorCounter._lock:
+            counts = IOErrorCounter._counts.copy()
+            IOErrorCounter.reset()
+            return counts
+
+    @staticmethod
+    def reset():
+        with IOErrorCounter._lock:
+            IOErrorCounter._counts = {"hostplugin":0, "protocol":0, "other":0}
+
+    @staticmethod
+    def set_protocol_endpoint(endpoint=DEFAULT_PROTOCOL_ENDPOINT):
+        IOErrorCounter._protocol_endpoint = endpoint
+
+
+def _compute_delay(retry_attempt=1, delay=DELAY_IN_SECONDS):
+    fib = (1, 1)
+    for n in range(retry_attempt):
+        fib = (fib[1], fib[0]+fib[1])
+    return delay*fib[1]
 
 def _is_retry_status(status, retry_codes=RETRY_CODES):
     return status in retry_codes
@@ -166,7 +211,7 @@ def http_request(method,
                 use_proxy=False,
                 max_retry=DEFAULT_RETRIES,
                 retry_codes=RETRY_CODES,
-                retry_delay=SHORT_DELAY_IN_SECONDS):
+                retry_delay=DELAY_IN_SECONDS):
 
     global SECURE_WARNING_EMITTED
 
@@ -208,18 +253,31 @@ def http_request(method,
 
     msg = ''
     attempt = 0
-    delay = retry_delay
+    delay = 0
+    was_throttled = False
 
     while attempt < max_retry:
         if attempt > 0:
-            logger.info("[HTTP Retry] Attempt {0} of {1}: {2}",
+            # Compute the request delay
+            # -- Use a fixed delay if the server ever rate-throttles the request
+            #    (with a safe, minimum number of retry attempts)
+            # -- Otherwise, compute a delay that is the product of the next
+            #    item in the Fibonacci series and the initial delay value
+            delay = THROTTLE_DELAY_IN_SECONDS \
+                        if was_throttled \
+                        else _compute_delay(retry_attempt=attempt,
+                                            delay=retry_delay)
+
+            logger.verbose("[HTTP Retry] "
+                        "Attempt {0} of {1} will delay {2} seconds: {3}",
                         attempt+1,
                         max_retry,
+                        delay,
                         msg)
+
             time.sleep(delay)
 
         attempt += 1
-        delay = retry_delay
 
         try:
             resp = _http_request(method,
@@ -235,13 +293,13 @@ def http_request(method,
 
             if request_failed(resp):
                 if _is_retry_status(resp.status, retry_codes=retry_codes):
-                    msg = '[HTTP Retry] HTTP {0} Status Code {1}'.format(
-                        method, resp.status)
+                    msg = '[HTTP Retry] {0} {1} -- Status Code {2}'.format(
+                        method, url, resp.status)
+                    # Note if throttled and ensure a safe, minimum number of
+                    # retry attempts
                     if _is_throttle_status(resp.status):
-                        delay = LONG_DELAY_IN_SECONDS
-                        logger.info("[HTTP Delay] Delay {0} seconds for " \
-                                    "Status Code {1}".format(
-                                        delay, resp.status))
+                        was_throttled = True
+                        max_retry = max(max_retry, THROTTLE_RETRIES)
                     continue
 
             if resp.status in RESOURCE_GONE_CODES:
@@ -250,22 +308,25 @@ def http_request(method,
             return resp
 
         except httpclient.HTTPException as e:
-            msg = '[HTTP Failed] HTTP {0} HttpException {1}'.format(method, e)
+            msg = '[HTTP Failed] {0} {1} -- HttpException {2}'.format(
+                method, url, e)
             if _is_retry_exception(e):
                 continue
             break
 
         except IOError as e:
-            msg = '[HTTP Failed] HTTP {0} IOError {1}'.format(method, e)
+            IOErrorCounter.increment(host=host, port=port)
+            msg = '[HTTP Failed] {0} {1} -- IOError {2}'.format(
+                method, url, e)
             continue
 
-    raise HttpError(msg)
+    raise HttpError("{0} -- {1} attempts made".format(msg,attempt))
 
 
 def http_get(url, headers=None, use_proxy=False,
                 max_retry=DEFAULT_RETRIES,
                 retry_codes=RETRY_CODES,
-                retry_delay=SHORT_DELAY_IN_SECONDS):
+                retry_delay=DELAY_IN_SECONDS):
     return http_request("GET",
                         url, None, headers=headers,
                         use_proxy=use_proxy,
@@ -277,7 +338,7 @@ def http_get(url, headers=None, use_proxy=False,
 def http_head(url, headers=None, use_proxy=False,
                 max_retry=DEFAULT_RETRIES,
                 retry_codes=RETRY_CODES,
-                retry_delay=SHORT_DELAY_IN_SECONDS):
+                retry_delay=DELAY_IN_SECONDS):
     return http_request("HEAD",
                         url, None, headers=headers,
                         use_proxy=use_proxy,
@@ -289,7 +350,7 @@ def http_head(url, headers=None, use_proxy=False,
 def http_post(url, data, headers=None, use_proxy=False,
                 max_retry=DEFAULT_RETRIES,
                 retry_codes=RETRY_CODES,
-                retry_delay=SHORT_DELAY_IN_SECONDS):
+                retry_delay=DELAY_IN_SECONDS):
     return http_request("POST",
                         url, data, headers=headers,
                         use_proxy=use_proxy,
@@ -301,7 +362,7 @@ def http_post(url, data, headers=None, use_proxy=False,
 def http_put(url, data, headers=None, use_proxy=False,
                 max_retry=DEFAULT_RETRIES,
                 retry_codes=RETRY_CODES,
-                retry_delay=SHORT_DELAY_IN_SECONDS):
+                retry_delay=DELAY_IN_SECONDS):
     return http_request("PUT",
                         url, data, headers=headers,
                         use_proxy=use_proxy,
@@ -313,7 +374,7 @@ def http_put(url, data, headers=None, use_proxy=False,
 def http_delete(url, headers=None, use_proxy=False,
                 max_retry=DEFAULT_RETRIES,
                 retry_codes=RETRY_CODES,
-                retry_delay=SHORT_DELAY_IN_SECONDS):
+                retry_delay=DELAY_IN_SECONDS):
     return http_request("DELETE",
                         url, None, headers=headers,
                         use_proxy=use_proxy,