Import patches-applied version 2.2.16-0ubuntu1 to applied/ubuntu/artful-proposed

Imported using git-ubuntu import.

Changelog parent: 70c0ea1ac879b2e1cba0a8edb1f3fbe82652413b
Unapplied parent: 3a1d96a77ccaf023256d16183428e3d895f8a051

New changelog entries:
  * New upstream release (LP: #1714299).

4 files changed, 129 insertions, 110 deletions

diff --git a/tests/protocol/mockwiredata.py b/tests/protocol/mockwiredata.py
index 4e45623..5924719 100644
--- a/tests/protocol/mockwiredata.py
+++ b/tests/protocol/mockwiredata.py
@@ -16,6 +16,7 @@
 #
 
 from tests.tools import *
+from azurelinuxagent.common.exception import HttpError, ResourceGoneError
 from azurelinuxagent.common.future import httpclient
 from azurelinuxagent.common.utils.cryptutil import CryptUtil
 
@@ -53,6 +54,20 @@ DATA_FILE_EXT_AUTOUPGRADE_INTERNALVERSION["ext_conf"] = "wire/ext_conf_autoupgra
 
 class WireProtocolData(object):
     def __init__(self, data_files=DATA_FILE):
+        self.emulate_stale_goal_state = False
+        self.call_counts = {
+            "comp=versions" : 0,
+            "/versions" : 0,
+            "goalstate" : 0,
+            "hostingenvuri" : 0,
+            "sharedconfiguri" : 0,
+            "certificatesuri" : 0,
+            "extensionsconfiguri" : 0,
+            "extensionArtifact" : 0,
+            "manifest.xml" : 0,
+            "manifest_of_ga.xml" : 0,
+            "ExampleHandlerLinux" : 0
+        }
         self.version_info = load_data(data_files.get("version_info"))
         self.goal_state = load_data(data_files.get("goal_state"))
         self.hosting_env = load_data(data_files.get("hosting_env"))
@@ -67,32 +82,70 @@ class WireProtocolData(object):
 
     def mock_http_get(self, url, *args, **kwargs):
         content = None
-        if "versions" in url:
+
+        resp = MagicMock()
+        resp.status = httpclient.OK
+
+        # wire server versions
+        if "comp=versions" in url:
             content = self.version_info
+            self.call_counts["comp=versions"] += 1
+
+        # HostPlugin versions
+        elif "/versions" in url:
+            content = '["2015-09-01"]'
+            self.call_counts["/versions"] += 1
         elif "goalstate" in url:
             content = self.goal_state
+            self.call_counts["goalstate"] += 1
         elif "hostingenvuri" in url:
             content = self.hosting_env
+            self.call_counts["hostingenvuri"] += 1
         elif "sharedconfiguri" in url:
             content = self.shared_config
+            self.call_counts["sharedconfiguri"] += 1
         elif "certificatesuri" in url:
             content = self.certs
+            self.call_counts["certificatesuri"] += 1
         elif "extensionsconfiguri" in url:
             content = self.ext_conf
-        elif "manifest.xml" in url:
-            content = self.manifest
-        elif "manifest_of_ga.xml" in url:
-            content = self.ga_manifest
-        elif "ExampleHandlerLinux" in url:
-            content = self.ext
-            resp = MagicMock()
-            resp.status = httpclient.OK
-            resp.read = Mock(return_value=content)
-            return resp
+            self.call_counts["extensionsconfiguri"] += 1
+
         else:
-            raise Exception("Bad url {0}".format(url))
-        resp = MagicMock()
-        resp.status = httpclient.OK
+            # A stale GoalState results in a 400 from the HostPlugin
+            # for which the HTTP handler in restutil raises ResourceGoneError
+            if self.emulate_stale_goal_state:
+                if "extensionArtifact" in url:
+                    self.emulate_stale_goal_state = False
+                    self.call_counts["extensionArtifact"] += 1
+                    raise ResourceGoneError()
+                else:
+                    raise HttpError()
+
+            # For HostPlugin requests, replace the URL with that passed
+            # via the x-ms-artifact-location header
+            if "extensionArtifact" in url:
+                self.call_counts["extensionArtifact"] += 1
+                if "headers" not in kwargs or \
+                    "x-ms-artifact-location" not in kwargs["headers"]:
+                    raise Exception("Bad HEADERS passed to HostPlugin: {0}",
+                            kwargs)
+                url = kwargs["headers"]["x-ms-artifact-location"]
+
+            if "manifest.xml" in url:
+                content = self.manifest
+                self.call_counts["manifest.xml"] += 1
+            elif "manifest_of_ga.xml" in url:
+                content = self.ga_manifest
+                self.call_counts["manifest_of_ga.xml"] += 1
+            elif "ExampleHandlerLinux" in url:
+                content = self.ext
+                self.call_counts["ExampleHandlerLinux"] += 1
+                resp.read = Mock(return_value=content)
+                return resp
+            else:
+                raise Exception("Bad url {0}".format(url))
+
         resp.read = Mock(return_value=content.encode("utf-8"))
         return resp
 
diff --git a/tests/protocol/test_hostplugin.py b/tests/protocol/test_hostplugin.py
index b18b691..74f7f24 100644
--- a/tests/protocol/test_hostplugin.py
+++ b/tests/protocol/test_hostplugin.py
@@ -146,6 +146,7 @@ class TestHostPlugin(AgentTestCase):
         test_goal_state = wire.GoalState(WireProtocolData(DATA_FILE).goal_state)
         status = restapi.VMStatus(status="Ready",
                                   message="Guest Agent is running")
+        wire.HostPluginProtocol.set_default_channel(False)
         with patch.object(wire.HostPluginProtocol,
                           "ensure_initialized",
                           return_value=True):
@@ -173,6 +174,7 @@ class TestHostPlugin(AgentTestCase):
         test_goal_state = wire.GoalState(WireProtocolData(DATA_FILE).goal_state)
         status = restapi.VMStatus(status="Ready",
                                   message="Guest Agent is running")
+        wire.HostPluginProtocol.set_default_channel(False)
         with patch.object(wire.StatusBlob,
                           "upload",
                           return_value=False):
@@ -211,6 +213,8 @@ class TestHostPlugin(AgentTestCase):
                         bytearray(faux_status, encoding='utf-8'))
 
         with patch.object(restutil, "http_request") as patch_http:
+            patch_http.return_value = Mock(status=httpclient.OK)
+
             wire_protocol_client.get_goal_state = Mock(return_value=test_goal_state)
             plugin = wire_protocol_client.get_host_plugin()
 
@@ -224,61 +228,6 @@ class TestHostPlugin(AgentTestCase):
                     test_goal_state,
                     exp_method, exp_url, exp_data)
 
-    def test_read_response_error(self):
-        """
-        Validate the read_response_error method handles encoding correctly
-        """
-        responses = ['message', b'message', '\x80message\x80']
-        response = MagicMock()
-        response.status = 'status'
-        response.reason = 'reason'
-        with patch.object(response, 'read') as patch_response:
-            for s in responses:
-                patch_response.return_value = s
-                result = hostplugin.HostPluginProtocol.read_response_error(response)
-                self.assertTrue('[status: reason]' in result)
-                self.assertTrue('message' in result)
-
-    def test_read_response_bytes(self):
-        response_bytes = '7b:0a:20:20:20:20:22:65:72:72:6f:72:43:6f:64:65:22:' \
-                         '3a:20:22:54:68:65:20:62:6c:6f:62:20:74:79:70:65:20:' \
-                         '69:73:20:69:6e:76:61:6c:69:64:20:66:6f:72:20:74:68:' \
-                         '69:73:20:6f:70:65:72:61:74:69:6f:6e:2e:22:2c:0a:20:' \
-                         '20:20:20:22:6d:65:73:73:61:67:65:22:3a:20:22:c3:af:' \
-                         'c2:bb:c2:bf:3c:3f:78:6d:6c:20:76:65:72:73:69:6f:6e:' \
-                         '3d:22:31:2e:30:22:20:65:6e:63:6f:64:69:6e:67:3d:22:' \
-                         '75:74:66:2d:38:22:3f:3e:3c:45:72:72:6f:72:3e:3c:43:' \
-                         '6f:64:65:3e:49:6e:76:61:6c:69:64:42:6c:6f:62:54:79:' \
-                         '70:65:3c:2f:43:6f:64:65:3e:3c:4d:65:73:73:61:67:65:' \
-                         '3e:54:68:65:20:62:6c:6f:62:20:74:79:70:65:20:69:73:' \
-                         '20:69:6e:76:61:6c:69:64:20:66:6f:72:20:74:68:69:73:' \
-                         '20:6f:70:65:72:61:74:69:6f:6e:2e:0a:52:65:71:75:65:' \
-                         '73:74:49:64:3a:63:37:34:32:39:30:63:62:2d:30:30:30:' \
-                         '31:2d:30:30:62:35:2d:30:36:64:61:2d:64:64:36:36:36:' \
-                         '61:30:30:30:22:2c:0a:20:20:20:20:22:64:65:74:61:69:' \
-                         '6c:73:22:3a:20:22:22:0a:7d'.split(':')
-        expected_response = '[status: reason] {\n    "errorCode": "The blob ' \
-                            'type is invalid for this operation.",\n    ' \
-                            '"message": "<?xml version="1.0" ' \
-                            'encoding="utf-8"?>' \
-                            '<Error><Code>InvalidBlobType</Code><Message>The ' \
-                            'blob type is invalid for this operation.\n' \
-                            'RequestId:c74290cb-0001-00b5-06da-dd666a000",' \
-                            '\n    "details": ""\n}'
-
-        response_string = ''.join(chr(int(b, 16)) for b in response_bytes)
-        response = MagicMock()
-        response.status = 'status'
-        response.reason = 'reason'
-        with patch.object(response, 'read') as patch_response:
-            patch_response.return_value = response_string
-            result = hostplugin.HostPluginProtocol.read_response_error(response)
-            self.assertEqual(result, expected_response)
-            try:
-                raise HttpError("{0}".format(result))
-            except HttpError as e:
-                self.assertTrue(result in ustr(e))
-
     def test_no_fallback(self):
         """
         Validate fallback to upload status using HostGAPlugin is not happening
@@ -318,6 +267,8 @@ class TestHostPlugin(AgentTestCase):
                         bytearray(faux_status, encoding='utf-8'))
 
         with patch.object(restutil, "http_request") as patch_http:
+            patch_http.return_value = Mock(status=httpclient.OK)
+
             with patch.object(wire.HostPluginProtocol,
                           "get_api_versions") as patch_get:
                 patch_get.return_value = api_versions
diff --git a/tests/protocol/test_metadata.py b/tests/protocol/test_metadata.py
index ee4ba3e..5047b86 100644
--- a/tests/protocol/test_metadata.py
+++ b/tests/protocol/test_metadata.py
@@ -31,17 +31,15 @@ class TestMetadataProtocolGetters(AgentTestCase):
         return json.loads(ustr(load_data(path)), encoding="utf-8")
 
     @patch("time.sleep")
-    @patch("azurelinuxagent.common.protocol.metadata.restutil")
-    def _test_getters(self, test_data, mock_restutil ,_):
-        mock_restutil.http_get.side_effect = test_data.mock_http_get
-
-        protocol = MetadataProtocol()
-        protocol.detect()
-        protocol.get_vminfo()
-        protocol.get_certs()
-        ext_handlers, etag = protocol.get_ext_handlers()
-        for ext_handler in ext_handlers.extHandlers:
-            protocol.get_ext_handler_pkgs(ext_handler)
+    def _test_getters(self, test_data ,_):
+        with patch.object(restutil, 'http_get', test_data.mock_http_get):
+            protocol = MetadataProtocol()
+            protocol.detect()
+            protocol.get_vminfo()
+            protocol.get_certs()
+            ext_handlers, etag = protocol.get_ext_handlers()
+            for ext_handler in ext_handlers.extHandlers:
+                protocol.get_ext_handler_pkgs(ext_handler)
 
     def test_getters(self, *args):
         test_data = MetadataProtocolData(DATA_FILE)
diff --git a/tests/protocol/test_wire.py b/tests/protocol/test_wire.py
index 02976ca..d19bab1 100644
--- a/tests/protocol/test_wire.py
+++ b/tests/protocol/test_wire.py
@@ -25,30 +25,34 @@ wireserver_url = '168.63.129.16'
 
 @patch("time.sleep")
 @patch("azurelinuxagent.common.protocol.wire.CryptUtil")
-@patch("azurelinuxagent.common.protocol.wire.restutil")
 class TestWireProtocolGetters(AgentTestCase):
-    def _test_getters(self, test_data, mock_restutil, MockCryptUtil, _):
-        mock_restutil.http_get.side_effect = test_data.mock_http_get
+
+    def setUp(self):
+        super(TestWireProtocolGetters, self).setUp()
+        HostPluginProtocol.set_default_channel(False)
+    
+    def _test_getters(self, test_data, MockCryptUtil, _):
         MockCryptUtil.side_effect = test_data.mock_crypt_util
 
-        protocol = WireProtocol(wireserver_url)
-        protocol.detect()
-        protocol.get_vminfo()
-        protocol.get_certs()
-        ext_handlers, etag = protocol.get_ext_handlers()
-        for ext_handler in ext_handlers.extHandlers:
-            protocol.get_ext_handler_pkgs(ext_handler)
-
-        crt1 = os.path.join(self.tmp_dir,
-                            '33B0ABCE4673538650971C10F7D7397E71561F35.crt')
-        crt2 = os.path.join(self.tmp_dir,
-                            '4037FBF5F1F3014F99B5D6C7799E9B20E6871CB3.crt')
-        prv2 = os.path.join(self.tmp_dir,
-                            '4037FBF5F1F3014F99B5D6C7799E9B20E6871CB3.prv')
-
-        self.assertTrue(os.path.isfile(crt1))
-        self.assertTrue(os.path.isfile(crt2))
-        self.assertTrue(os.path.isfile(prv2))
+        with patch.object(restutil, 'http_get', test_data.mock_http_get):
+            protocol = WireProtocol(wireserver_url)
+            protocol.detect()
+            protocol.get_vminfo()
+            protocol.get_certs()
+            ext_handlers, etag = protocol.get_ext_handlers()
+            for ext_handler in ext_handlers.extHandlers:
+                protocol.get_ext_handler_pkgs(ext_handler)
+
+            crt1 = os.path.join(self.tmp_dir,
+                                '33B0ABCE4673538650971C10F7D7397E71561F35.crt')
+            crt2 = os.path.join(self.tmp_dir,
+                                '4037FBF5F1F3014F99B5D6C7799E9B20E6871CB3.crt')
+            prv2 = os.path.join(self.tmp_dir,
+                                '4037FBF5F1F3014F99B5D6C7799E9B20E6871CB3.prv')
+
+            self.assertTrue(os.path.isfile(crt1))
+            self.assertTrue(os.path.isfile(crt2))
+            self.assertTrue(os.path.isfile(prv2))
 
     def test_getters(self, *args):
         """Normal case"""
@@ -70,8 +74,21 @@ class TestWireProtocolGetters(AgentTestCase):
         test_data = WireProtocolData(DATA_FILE_EXT_NO_PUBLIC)
         self._test_getters(test_data, *args)
 
+    def test_getters_with_stale_goal_state(self, *args):
+        test_data = WireProtocolData(DATA_FILE)
+        test_data.emulate_stale_goal_state = True
+
+        self._test_getters(test_data, *args)
+        # Ensure HostPlugin was invoked
+        self.assertEqual(1, test_data.call_counts["/versions"])
+        self.assertEqual(2, test_data.call_counts["extensionArtifact"])
+        # Ensure the expected number of HTTP calls were made
+        # -- Tracking calls to retrieve GoalState is problematic since it is
+        #    fetched often; however, the dependent documents, such as the
+        #    HostingEnvironmentConfig, will be retrieved the expected number
+        self.assertEqual(2, test_data.call_counts["hostingenvuri"])
+
     def test_call_storage_kwargs(self,
-                                 mock_restutil,
                                  mock_cryptutil,
                                  mock_sleep):
         from azurelinuxagent.common.utils import restutil
@@ -83,32 +100,32 @@ class TestWireProtocolGetters(AgentTestCase):
             # no kwargs -- Default to True
             WireClient.call_storage_service(http_req)
 
-            # kwargs, no chk_proxy -- Default to True
+            # kwargs, no use_proxy -- Default to True
             WireClient.call_storage_service(http_req,
                                             url,
                                             headers)
 
-            # kwargs, chk_proxy None -- Default to True
+            # kwargs, use_proxy None -- Default to True
             WireClient.call_storage_service(http_req,
                                             url,
                                             headers,
-                                            chk_proxy=None)
+                                            use_proxy=None)
 
-            # kwargs, chk_proxy False -- Keep False
+            # kwargs, use_proxy False -- Keep False
             WireClient.call_storage_service(http_req,
                                             url,
                                             headers,
-                                            chk_proxy=False)
+                                            use_proxy=False)
 
-            # kwargs, chk_proxy True -- Keep True
+            # kwargs, use_proxy True -- Keep True
             WireClient.call_storage_service(http_req,
                                             url,
                                             headers,
-                                            chk_proxy=True)
+                                            use_proxy=True)
             # assert
             self.assertTrue(http_patch.call_count == 5)
             for i in range(0,5):
-                c = http_patch.call_args_list[i][-1]['chk_proxy']
+                c = http_patch.call_args_list[i][-1]['use_proxy']
                 self.assertTrue(c == (True if i != 3 else False))
 
     def test_status_blob_parsing(self, *args):