ec2: only redact token request headers in logs, avoid altering request (#230)

Our header redact logic was redacting both logged request headers and
the actual source request. This results in DataSourceEc2 sending the
invalid header "X-aws-ec2-metadata-token-ttl-seconds: REDACTED" which
gets an HTTP status response of 400.

Cloud-init retries this failed token request for 2 minutes before
falling back to IMDSv1.

LP: #1865882

1 files changed, 33 insertions, 1 deletions

diff --git a/cloudinit/tests/test_url_helper.py b/cloudinit/tests/test_url_helper.py
index 1674120f..29b39374 100644
--- a/cloudinit/tests/test_url_helper.py
+++ b/cloudinit/tests/test_url_helper.py
@@ -1,7 +1,8 @@
 # This file is part of cloud-init. See LICENSE file for license information.
 
 from cloudinit.url_helper import (
-    NOT_FOUND, UrlError, oauth_headers, read_file_or_url, retry_on_url_exc)
+    NOT_FOUND, UrlError, REDACTED, oauth_headers, read_file_or_url,
+    retry_on_url_exc)
 from cloudinit.tests.helpers import CiTestCase, mock, skipIf
 from cloudinit import util
 from cloudinit import version
@@ -50,6 +51,9 @@ class TestOAuthHeaders(CiTestCase):
 
 
 class TestReadFileOrUrl(CiTestCase):
+
+    with_logs = True
+
     def test_read_file_or_url_str_from_file(self):
         """Test that str(result.contents) on file is text version of contents.
         It should not be "b'data'", but just "'data'" """
@@ -71,6 +75,34 @@ class TestReadFileOrUrl(CiTestCase):
         self.assertEqual(result.contents, data)
         self.assertEqual(str(result), data.decode('utf-8'))
 
+    @httpretty.activate
+    def test_read_file_or_url_str_from_url_redacting_headers_from_logs(self):
+        """Headers are redacted from logs but unredacted in requests."""
+        url = 'http://hostname/path'
+        headers = {'sensitive': 'sekret', 'server': 'blah'}
+        httpretty.register_uri(httpretty.GET, url)
+
+        read_file_or_url(url, headers=headers, headers_redact=['sensitive'])
+        logs = self.logs.getvalue()
+        for k in headers.keys():
+            self.assertEqual(headers[k],  httpretty.last_request().headers[k])
+        self.assertIn(REDACTED, logs)
+        self.assertNotIn('sekret', logs)
+
+    @httpretty.activate
+    def test_read_file_or_url_str_from_url_redacts_noheaders(self):
+        """When no headers_redact, header values are in logs and requests."""
+        url = 'http://hostname/path'
+        headers = {'sensitive': 'sekret', 'server': 'blah'}
+        httpretty.register_uri(httpretty.GET, url)
+
+        read_file_or_url(url, headers=headers)
+        for k in headers.keys():
+            self.assertEqual(headers[k], httpretty.last_request().headers[k])
+        logs = self.logs.getvalue()
+        self.assertNotIn(REDACTED, logs)
+        self.assertIn('sekret', logs)
+
     @mock.patch(M_PATH + 'readurl')
     def test_read_file_or_url_passes_params_to_readurl(self, m_readurl):
         """read_file_or_url passes all params through to readurl."""