diff options
Diffstat (limited to 'cloudinit/url_helper.py')
| -rw-r--r-- | cloudinit/url_helper.py | 55 |
1 files changed, 40 insertions, 15 deletions
diff --git a/cloudinit/url_helper.py b/cloudinit/url_helper.py index 97ed75ad..4a83169a 100644 --- a/cloudinit/url_helper.py +++ b/cloudinit/url_helper.py @@ -22,6 +22,7 @@ import httplib import time +import urllib import requests from requests import exceptions @@ -38,6 +39,7 @@ NOT_FOUND = httplib.NOT_FOUND # Check if requests has ssl support (added in requests >= 0.8.8) SSL_ENABLED = False CONFIG_ENABLED = False # This was added in 0.7 (but taken out in >=1.0) +_REQ_VER = None try: from distutils.version import LooseVersion import pkg_resources @@ -61,6 +63,23 @@ def _cleanurl(url): return urlunparse(parsed_url) +def combine_url(base, *add_ons): + + def combine_single(url, add_on): + url_parsed = list(urlparse(url)) + path = url_parsed[2] + if path and not path.endswith("/"): + path += "/" + path += urllib.quote(str(add_on), safe="/:") + url_parsed[2] = path + return urlunparse(url_parsed) + + url = base + for add_on in add_ons: + url = combine_single(url, add_on) + return url + + # Made to have same accessors as UrlResponse so that the # read_file_or_url can return this or that object and the # 'user' of those objects will not need to know the difference. @@ -129,28 +148,34 @@ class UrlError(IOError): self.headers = {} -def readurl(url, data=None, timeout=None, retries=0, sec_between=1, - headers=None, headers_cb=None, ssl_details=None, - check_status=True, allow_redirects=True, exception_cb=None): - url = _cleanurl(url) - req_args = { - 'url': url, - } +def _get_ssl_args(url, ssl_details): + ssl_args = {} scheme = urlparse(url).scheme # pylint: disable=E1101 if scheme == 'https' and ssl_details: if not SSL_ENABLED: - LOG.warn("SSL is not enabled, cert. verification can not occur!") + LOG.warn("SSL is not supported in requests v%s, " + "cert. verification can not occur!", _REQ_VER) else: if 'ca_certs' in ssl_details and ssl_details['ca_certs']: - req_args['verify'] = ssl_details['ca_certs'] + ssl_args['verify'] = ssl_details['ca_certs'] else: - req_args['verify'] = True + ssl_args['verify'] = True if 'cert_file' in ssl_details and 'key_file' in ssl_details: - req_args['cert'] = [ssl_details['cert_file'], + ssl_args['cert'] = [ssl_details['cert_file'], ssl_details['key_file']] elif 'cert_file' in ssl_details: - req_args['cert'] = str(ssl_details['cert_file']) + ssl_args['cert'] = str(ssl_details['cert_file']) + return ssl_args + +def readurl(url, data=None, timeout=None, retries=0, sec_between=1, + headers=None, headers_cb=None, ssl_details=None, + check_status=True, allow_redirects=True, exception_cb=None): + url = _cleanurl(url) + req_args = { + 'url': url, + } + req_args.update(_get_ssl_args(url, ssl_details)) req_args['allow_redirects'] = allow_redirects req_args['method'] = 'GET' if timeout is not None: @@ -181,12 +206,11 @@ def readurl(url, data=None, timeout=None, retries=0, sec_between=1, def _cb(url): return headers headers_cb = _cb - if data: - # Do this after the log (it might be large) req_args['data'] = data if sec_between is None: sec_between = -1 + excps = [] # Handle retrying ourselves since the built-in support # doesn't handle sleeping between tries... @@ -223,7 +247,7 @@ def readurl(url, data=None, timeout=None, retries=0, sec_between=1, # ssl exceptions are not going to get fixed by waiting a # few seconds break - if exception_cb and not exception_cb(filtered_req_args, excps[-1]): + if exception_cb and not exception_cb(req_args.copy(), excps[-1]): break if i + 1 < manual_tries and sec_between > 0: LOG.debug("Please wait %s seconds while we wait to try again", @@ -242,6 +266,7 @@ def wait_for_url(urls, max_wait=None, timeout=None, max_wait: roughly the maximum time to wait before giving up The max time is *actually* len(urls)*timeout as each url will be tried once and given the timeout provided. + a number <= 0 will always result in only one try timeout: the timeout provided to urlopen status_cb: call method with string message when a url is not available headers_cb: call method with single argument of url to get headers |