summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorScott Moser <smoser@ubuntu.com>2013-04-25 14:03:16 -0400
committerScott Moser <smoser@ubuntu.com>2013-04-25 14:03:16 -0400
commit4697c1afcc7d05951f4717a83dad01d2360301c6 (patch)
tree6d66840f102c71fb7dbd3eafc044cc5e75531d58
parent8e97ca8cfaf75edc0b05b0dc2a2ef35c5c82221b (diff)
parenta69c9c8a35b61f69f6e959448af8d0619989589f (diff)
downloadvyos-cloud-init-4697c1afcc7d05951f4717a83dad01d2360301c6.tar.gz
vyos-cloud-init-4697c1afcc7d05951f4717a83dad01d2360301c6.zip
Fix issue in readurl with requests, support passing through headers_cb
This adds the 'headers_cb' option to pass all the way through to readurl. It also makes requests for user-data not retry (as a 404 indicates "no user-data"). LP: #1172742
-rw-r--r--cloudinit/sources/DataSourceMAAS.py16
-rw-r--r--cloudinit/url_helper.py18
-rw-r--r--cloudinit/util.py4
-rw-r--r--tests/unittests/test_datasource/test_maas.py5
4 files changed, 30 insertions, 13 deletions
diff --git a/cloudinit/sources/DataSourceMAAS.py b/cloudinit/sources/DataSourceMAAS.py
index de822924..dfe90bc6 100644
--- a/cloudinit/sources/DataSourceMAAS.py
+++ b/cloudinit/sources/DataSourceMAAS.py
@@ -218,14 +218,20 @@ def read_maas_seed_url(seed_url, header_cb=None, timeout=None,
md = {}
for name in file_order:
url = files.get(name)
- if header_cb:
- headers = header_cb(url)
+ if not header_cb:
+ def _cb(url):
+ return {}
+ header_cb = _cb
+
+ if name == 'user-data':
+ retries = 0
else:
- headers = {}
+ retries = None
+
try:
ssl_details = util.fetch_ssl_details(paths)
- resp = util.read_file_or_url(url,
- headers=headers,
+ resp = util.read_file_or_url(url, retries=retries,
+ headers_cb=header_cb,
timeout=timeout,
ssl_details=ssl_details)
if resp.ok():
diff --git a/cloudinit/url_helper.py b/cloudinit/url_helper.py
index 6b4516e0..24ce6871 100644
--- a/cloudinit/url_helper.py
+++ b/cloudinit/url_helper.py
@@ -102,8 +102,8 @@ class UrlError(IOError):
def readurl(url, data=None, timeout=None, retries=0, sec_between=1,
- headers=None, ssl_details=None, check_status=True,
- allow_redirects=True):
+ headers=None, headers_cb=None, ssl_details=None,
+ check_status=True, allow_redirects=True):
url = _cleanurl(url)
req_args = {
'url': url,
@@ -149,8 +149,11 @@ def readurl(url, data=None, timeout=None, retries=0, sec_between=1,
headers = {
'User-Agent': 'Cloud-Init/%s' % (version.version_string()),
}
- req_args['headers'] = headers
- LOG.debug("Attempting to open '%s' with %s configuration", url, req_args)
+ if not headers_cb:
+ def _cb(url):
+ return headers
+ headers_cb = _cb
+
if data:
# Do this after the log (it might be large)
req_args['data'] = data
@@ -161,6 +164,11 @@ def readurl(url, data=None, timeout=None, retries=0, sec_between=1,
# doesn't handle sleeping between tries...
for i in range(0, manual_tries):
try:
+ req_args['headers'] = headers_cb(url)
+ LOG.debug("[%s/%s] open '%s' with %s configuration", i,
+ manual_tries, url,
+ {k: req_args[k] for k in req_args if k != 'data'})
+
r = requests.request(**req_args)
if check_status:
r.raise_for_status() # pylint: disable=E1103
@@ -174,7 +182,7 @@ def readurl(url, data=None, timeout=None, retries=0, sec_between=1,
except exceptions.RequestException as e:
if (isinstance(e, (exceptions.HTTPError))
and hasattr(e, 'response') # This appeared in v 0.10.8
- and e.response):
+ and hasattr(e.response, 'status_code')):
excps.append(UrlError(e, code=e.response.status_code,
headers=e.response.headers))
else:
diff --git a/cloudinit/util.py b/cloudinit/util.py
index 50de55fe..053fa95d 100644
--- a/cloudinit/util.py
+++ b/cloudinit/util.py
@@ -671,7 +671,8 @@ def fetch_ssl_details(paths=None):
def read_file_or_url(url, timeout=5, retries=10,
- headers=None, data=None, sec_between=1, ssl_details=None):
+ headers=None, data=None, sec_between=1, ssl_details=None,
+ headers_cb=None):
url = url.lstrip()
if url.startswith("/"):
url = "file://%s" % url
@@ -685,6 +686,7 @@ def read_file_or_url(url, timeout=5, retries=10,
timeout=timeout,
retries=retries,
headers=headers,
+ headers_cb=headers_cb,
data=data,
sec_between=sec_between,
ssl_details=ssl_details)
diff --git a/tests/unittests/test_datasource/test_maas.py b/tests/unittests/test_datasource/test_maas.py
index 47f8caa4..2007a6df 100644
--- a/tests/unittests/test_datasource/test_maas.py
+++ b/tests/unittests/test_datasource/test_maas.py
@@ -116,9 +116,10 @@ class TestMAASDataSource(mocker.MockerTestCase):
for key in valid_order:
url = "%s/%s/%s" % (my_seed, my_ver, key)
- mock_request(url, headers=my_headers, timeout=mocker.ANY,
+ mock_request(url, headers=None, timeout=mocker.ANY,
data=mocker.ANY, sec_between=mocker.ANY,
- ssl_details=mocker.ANY, retries=mocker.ANY)
+ ssl_details=mocker.ANY, retries=mocker.ANY,
+ headers_cb=my_headers_cb)
resp = valid.get(key)
self.mocker.result(util.StringResponse(resp))
self.mocker.replay()