Catch UrlError when #include'ing URLs

Without this the entire stage can fail, which will leave an instance
unaccessible.

Reviewed-by: Tom Kirchner <tjk@amazon.com>
Reviewed-by: Matt Nierzwicki <nierzwic@amazon.com>
Reviewed-by: Ben Cressey <bcressey@amazon.com>

2 files changed, 68 insertions, 10 deletions

diff --git a/cloudinit/user_data.py b/cloudinit/user_data.py
index 88cb7f84..e163c722 100644
--- a/cloudinit/user_data.py
+++ b/cloudinit/user_data.py
@@ -19,6 +19,7 @@ import six
 
 from cloudinit import handlers
 from cloudinit import log as logging
+from cloudinit.url_helper import UrlError
 from cloudinit import util
 
 LOG = logging.getLogger(__name__)
@@ -222,16 +223,23 @@ class UserDataProcessor(object):
             if include_once_on and os.path.isfile(include_once_fn):
                 content = util.load_file(include_once_fn)
             else:
-                resp = util.read_file_or_url(include_url,
-                                             ssl_details=self.ssl_details)
-                if include_once_on and resp.ok():
-                    util.write_file(include_once_fn, resp.contents, mode=0o600)
-                if resp.ok():
-                    content = resp.contents
-                else:
-                    LOG.warning(("Fetching from %s resulted in"
-                                 " a invalid http code of %s"),
-                                include_url, resp.code)
+                try:
+                    resp = util.read_file_or_url(include_url,
+                                                 ssl_details=self.ssl_details)
+                    if include_once_on and resp.ok():
+                        util.write_file(include_once_fn, resp.contents,
+                                        mode=0o600)
+                    if resp.ok():
+                        content = resp.contents
+                    else:
+                        LOG.warning(("Fetching from %s resulted in"
+                                     " a invalid http code of %s"),
+                                    include_url, resp.code)
+                except UrlError as urle:
+                    LOG.warning(urle)
+                except IOError as ioe:
+                    LOG.warning("Fetching from %s resulted in %s",
+                                include_url, ioe)
 
             if content is not None:
                 new_msg = convert_string(content)
diff --git a/tests/unittests/test_data.py b/tests/unittests/test_data.py
index 6d621d26..275b16d2 100644
--- a/tests/unittests/test_data.py
+++ b/tests/unittests/test_data.py
@@ -18,6 +18,8 @@ from email.mime.application import MIMEApplication
 from email.mime.base import MIMEBase
 from email.mime.multipart import MIMEMultipart
 
+import httpretty
+
 from cloudinit import handlers
 from cloudinit import helpers as c_helpers
 from cloudinit import log
@@ -522,6 +524,54 @@ c: 4
         self.assertEqual(cfg.get('password'), 'gocubs')
         self.assertEqual(cfg.get('locale'), 'chicago')
 
+    @httpretty.activate
+    @mock.patch('cloudinit.url_helper.time.sleep')
+    def test_include(self, mock_sleep):
+        """Test #include."""
+        included_url = 'http://hostname/path'
+        included_data = '#cloud-config\nincluded: true\n'
+        httpretty.register_uri(httpretty.GET, included_url, included_data)
+
+        blob = '#include\n%s\n' % included_url
+
+        self.reRoot()
+        ci = stages.Init()
+        ci.datasource = FakeDataSource(blob)
+        ci.fetch()
+        ci.consume_data()
+        cc_contents = util.load_file(ci.paths.get_ipath("cloud_config"))
+        cc = util.load_yaml(cc_contents)
+        self.assertTrue(cc.get('included'))
+
+    @httpretty.activate
+    @mock.patch('cloudinit.url_helper.time.sleep')
+    def test_include_bad_url(self, mock_sleep):
+        """Test #include with a bad URL."""
+        bad_url = 'http://bad/forbidden'
+        bad_data = '#cloud-config\nbad: true\n'
+        httpretty.register_uri(httpretty.GET, bad_url, bad_data, status=403)
+
+        included_url = 'http://hostname/path'
+        included_data = '#cloud-config\nincluded: true\n'
+        httpretty.register_uri(httpretty.GET, included_url, included_data)
+
+        blob = '#include\n%s\n%s' % (bad_url, included_url)
+
+        self.reRoot()
+        ci = stages.Init()
+        ci.datasource = FakeDataSource(blob)
+        log_file = self.capture_log(logging.WARNING)
+        ci.fetch()
+        ci.consume_data()
+
+        self.assertIn("403 Client Error: Forbidden for url: %s" % bad_url,
+                      log_file.getvalue())
+
+        cc_contents = util.load_file(ci.paths.get_ipath("cloud_config"))
+        cc = util.load_yaml(cc_contents)
+        self.assertIsNone(cc.get('bad'))
+        self.assertTrue(cc.get('included'))
+
 
 class TestUDProcess(helpers.ResourceUsingTestCase):