Reduce AzurePreprovisioning HTTP timeouts.

Reducing timeout to 1 second as IMDS responds within a handful
of milliseconds. Also get rid of max_retries to prevent exiting
out of polling loop early due to IMDS outage / upgrade.

Reduce Azure PreProvisioning HTTP timeouts during polling to
avoid waiting an extra minute.

LP: #1752977

2 files changed, 17 insertions, 27 deletions

diff --git a/cloudinit/sources/DataSourceAzure.py b/cloudinit/sources/DataSourceAzure.py
index 0bb7fad9..0ee622e2 100644
--- a/cloudinit/sources/DataSourceAzure.py
+++ b/cloudinit/sources/DataSourceAzure.py
@@ -20,7 +20,7 @@ from cloudinit import net
 from cloudinit.net.dhcp import EphemeralDHCPv4
 from cloudinit import sources
 from cloudinit.sources.helpers.azure import get_metadata_from_fabric
-from cloudinit.url_helper import readurl, wait_for_url, UrlError
+from cloudinit.url_helper import readurl, UrlError
 from cloudinit import util
 
 LOG = logging.getLogger(__name__)
@@ -49,7 +49,6 @@ DEFAULT_FS = 'ext4'
 AZURE_CHASSIS_ASSET_TAG = '7783-7084-3265-9085-8269-3286-77'
 REPROVISION_MARKER_FILE = "/var/lib/cloud/data/poll_imds"
 IMDS_URL = "http://169.254.169.254/metadata/reprovisiondata"
-IMDS_RETRIES = 5
 
 
 def find_storvscid_from_sysctl_pnpinfo(sysctl_out, deviceid):
@@ -451,36 +450,24 @@ class DataSourceAzure(sources.DataSource):
         headers = {"Metadata": "true"}
         LOG.debug("Start polling IMDS")
 
-        def sleep_cb(response, loop_n):
-            return 1
-
-        def exception_cb(msg, exception):
+        def exc_cb(msg, exception):
             if isinstance(exception, UrlError) and exception.code == 404:
-                return
-            LOG.warning("Exception during polling. Will try DHCP.",
-                        exc_info=True)
-
+                return True
             # If we get an exception while trying to call IMDS, we
             # call DHCP and setup the ephemeral network to acquire the new IP.
-            raise exception
+            return False
 
         need_report = report_ready
-        for i in range(IMDS_RETRIES):
+        while True:
             try:
                 with EphemeralDHCPv4() as lease:
                     if need_report:
                         self._report_ready(lease=lease)
                         need_report = False
-                    wait_for_url([url], max_wait=None, timeout=60,
-                                 status_cb=LOG.info,
-                                 headers_cb=lambda url: headers, sleep_time=1,
-                                 exception_cb=exception_cb,
-                                 sleep_time_cb=sleep_cb)
-                    return str(readurl(url, headers=headers))
-            except Exception:
-                LOG.debug("Exception during polling-retrying dhcp" +
-                          " %d more time(s).", (IMDS_RETRIES - i),
-                          exc_info=True)
+                    return readurl(url, timeout=1, headers=headers,
+                                   exception_cb=exc_cb, infinite=True).contents
+            except UrlError:
+                pass
 
     def _report_ready(self, lease):
         """Tells the fabric provisioning has completed
diff --git a/cloudinit/url_helper.py b/cloudinit/url_helper.py
index 36289af5..03a573af 100644
--- a/cloudinit/url_helper.py
+++ b/cloudinit/url_helper.py
@@ -16,7 +16,7 @@ import time
 
 from email.utils import parsedate
 from functools import partial
-
+from itertools import count
 from requests import exceptions
 
 from six.moves.urllib.parse import (
@@ -172,7 +172,7 @@ def _get_ssl_args(url, ssl_details):
 def readurl(url, data=None, timeout=None, retries=0, sec_between=1,
             headers=None, headers_cb=None, ssl_details=None,
             check_status=True, allow_redirects=True, exception_cb=None,
-            session=None):
+            session=None, infinite=False):
     url = _cleanurl(url)
     req_args = {
         'url': url,
@@ -220,7 +220,8 @@ def readurl(url, data=None, timeout=None, retries=0, sec_between=1,
     excps = []
     # Handle retrying ourselves since the built-in support
     # doesn't handle sleeping between tries...
-    for i in range(0, manual_tries):
+    # Infinitely retry if infinite is True
+    for i in count() if infinite else range(0, manual_tries):
         req_args['headers'] = headers_cb(url)
         filtered_req_args = {}
         for (k, v) in req_args.items():
@@ -229,7 +230,8 @@ def readurl(url, data=None, timeout=None, retries=0, sec_between=1,
             filtered_req_args[k] = v
         try:
             LOG.debug("[%s/%s] open '%s' with %s configuration", i,
-                      manual_tries, url, filtered_req_args)
+                      "infinite" if infinite else manual_tries, url,
+                      filtered_req_args)
 
             if session is None:
                 session = requests.Session()
@@ -263,7 +265,8 @@ def readurl(url, data=None, timeout=None, retries=0, sec_between=1,
                 # to continue retrying and False to break and re-raise the
                 # exception
                 break
-            if i + 1 < manual_tries and sec_between > 0:
+            if (infinite and sec_between > 0) or \
+               (i + 1 < manual_tries and sec_between > 0):
                 LOG.debug("Please wait %s seconds while we wait to try again",
                           sec_between)
                 time.sleep(sec_between)