Import patches-unapplied version 2.2.12-0ubuntu1 to ubuntu/artful-proposed

Imported using git-ubuntu import.

Changelog parent: 473ad6fbfe0b9c3b362b530492928303f2b4c7f3

New changelog entries:
  * New upstream release (LP: #1690854).
    - Refreshed debian/patches/disable_import_test.patch.

5 files changed, 142 insertions, 126 deletions

diff --git a/azurelinuxagent/common/protocol/hostplugin.py b/azurelinuxagent/common/protocol/hostplugin.py
index 70bf8b4..464fd35 100644
--- a/azurelinuxagent/common/protocol/hostplugin.py
+++ b/azurelinuxagent/common/protocol/hostplugin.py
@@ -19,6 +19,7 @@
 
 import base64
 import json
+import traceback
 
 from azurelinuxagent.common import logger
 from azurelinuxagent.common.exception import ProtocolError, HttpError
@@ -177,8 +178,7 @@ class HostPluginProtocol(object):
                 logger.info("HostGAPlugin: Setting host plugin as default channel")
                 HostPluginProtocol.set_default_channel(True)
         except Exception as e:
-            message = "HostGAPlugin: Exception Put VM status: {0}".format(e)
-            logger.error(message)
+            message = "HostGAPlugin: Exception Put VM status: {0}, {1}".format(e, traceback.format_exc())
             from azurelinuxagent.common.event import WALAEventOperation, report_event
             report_event(op=WALAEventOperation.ReportStatus,
                          is_success=False,
diff --git a/azurelinuxagent/common/protocol/metadata.py b/azurelinuxagent/common/protocol/metadata.py
index c61e373..c50b3dd 100644
--- a/azurelinuxagent/common/protocol/metadata.py
+++ b/azurelinuxagent/common/protocol/metadata.py
@@ -21,17 +21,19 @@ import json
 import os
 import shutil
 import re
+
 import azurelinuxagent.common.conf as conf
 import azurelinuxagent.common.utils.fileutil as fileutil
 import azurelinuxagent.common.utils.shellutil as shellutil
 import azurelinuxagent.common.utils.textutil as textutil
+
 from azurelinuxagent.common.future import httpclient
 from azurelinuxagent.common.protocol.restapi import *
 from azurelinuxagent.common.utils.cryptutil import CryptUtil
 
 METADATA_ENDPOINT = '169.254.169.254'
 APIVERSION = '2015-05-01-preview'
-BASE_URI = "http://{0}/Microsoft.Compute/{1}?api-version={2}{3}"
+BASE_URI = "http://{0}/Microsoft.Compute/{1}?api-version={2}"
 
 TRANSPORT_PRV_FILE_NAME = "V2TransportPrivate.pem"
 TRANSPORT_CERT_FILE_NAME = "V2TransportCert.pem"
@@ -39,6 +41,9 @@ P7M_FILE_NAME = "Certificates.p7m"
 P7B_FILE_NAME = "Certificates.p7b"
 PEM_FILE_NAME = "Certificates.pem"
 
+KEY_AGENT_VERSION_URIS = "versionsManifestUris"
+KEY_URI = "uri"
+
 # TODO remote workaround for azure stack
 MAX_PING = 30
 RETRY_PING_INTERVAL = 10
@@ -56,13 +61,13 @@ class MetadataProtocol(Protocol):
         self.apiversion = apiversion
         self.endpoint = endpoint
         self.identity_uri = BASE_URI.format(self.endpoint, "identity",
-                                            self.apiversion, "&$expand=*")
+                                            self.apiversion)
         self.cert_uri = BASE_URI.format(self.endpoint, "certificates",
-                                        self.apiversion, "&$expand=*")
+                                        self.apiversion)
         self.ext_uri = BASE_URI.format(self.endpoint, "extensionHandlers",
-                                       self.apiversion, "&$expand=*")
+                                       self.apiversion)
         self.vmagent_uri = BASE_URI.format(self.endpoint, "vmAgentVersions",
-                                           self.apiversion, "&$expand=*")
+                                           self.apiversion)
         self.provision_status_uri = BASE_URI.format(self.endpoint,
                                                     "provisioningStatus",
                                                     self.apiversion, "")
@@ -74,6 +79,8 @@ class MetadataProtocol(Protocol):
         self.event_uri = BASE_URI.format(self.endpoint, "status/telemetry",
                                          self.apiversion, "")
         self.certs = None
+        self.agent_manifests = None
+        self.agent_etag = None
 
     def _get_data(self, url, headers=None):
         try:
@@ -166,32 +173,55 @@ class MetadataProtocol(Protocol):
             return None
         return self.certs
 
-    def get_vmagent_manifests(self, last_etag=None):
-        manifests = VMAgentManifestList()
+    def get_vmagent_manifests(self):
         self.update_goal_state()
+
         data, etag = self._get_data(self.vmagent_uri)
-        if last_etag is None or last_etag < etag:
-            set_properties("vmAgentManifests",
-                           manifests.vmAgentManifests,
-                           data)
-        return manifests, etag
+        if self.agent_etag is None or self.agent_etag < etag:
+            self.agent_etag = etag
+
+            # Create a list with a single manifest
+            # -- The protocol lacks "family," use the configured family
+            self.agent_manifests = VMAgentManifestList()
+
+            manifest = VMAgentManifest()
+            manifest.family = family=conf.get_autoupdate_gafamily()
+            
+            if not KEY_AGENT_VERSION_URIS in data:
+                raise ProtocolError(
+                    "Agent versions missing '{0}': {1}".format(
+                        KEY_AGENT_VERSION_URIS, data))
+
+            for version in data[KEY_AGENT_VERSION_URIS]:
+                if not KEY_URI in version:
+                    raise ProtocolError(
+                        "Agent versions missing '{0': {1}".format(
+                            KEY_URI, data))
+                manifest_uri = VMAgentManifestUri(uri=version[KEY_URI])
+                manifest.versionsManifestUris.append(manifest_uri)
+        
+            self.agent_manifests.vmAgentManifests.append(manifest)
+        
+        return self.agent_manifests, self.agent_etag
 
     def get_vmagent_pkgs(self, vmagent_manifest):
-        # Agent package is the same with extension handler
-        vmagent_pkgs = ExtHandlerPackageList()
         data = None
+        etag = None
         for manifest_uri in vmagent_manifest.versionsManifestUris:
             try:
-                data = self._get_data(manifest_uri.uri)
+                data, etag = self._get_data(manifest_uri.uri)
                 break
             except ProtocolError as e:
-                logger.warn("Failed to get vmagent versions: {0}", e)
-                logger.info("Retry getting vmagent versions")
+                logger.verbose(
+                    "Error retrieving agent package from {0}: {1}".format(
+                        manifest_uri, e))
+
         if data is None:
-            raise ProtocolError(("Failed to get versions for vm agent: {0}"
-                                 "").format(vmagent_manifest.family))
+            raise ProtocolError(
+                "Failed retrieving agent package from all URIs")
+
+        vmagent_pkgs = ExtHandlerPackageList()
         set_properties("vmAgentVersions", vmagent_pkgs, data)
-        # TODO: What etag should this return?
         return vmagent_pkgs
 
     def get_ext_handlers(self, last_etag=None):
@@ -251,11 +281,9 @@ class MetadataProtocol(Protocol):
         self._put_data(uri, data)
 
     def report_event(self, events):
-        # TODO disable telemetry for azure stack test
-        # validate_param('events', events, TelemetryEventList)
-        # data = get_properties(events)
-        # self._post_data(self.event_uri, data)
-        pass
+        validate_param('events', events, TelemetryEventList)
+        data = get_properties(events)
+        self._post_data(self.event_uri, data)
 
     def update_certs(self):
         certificates = self.get_certs()
diff --git a/azurelinuxagent/common/protocol/ovfenv.py b/azurelinuxagent/common/protocol/ovfenv.py
index 4901871..3122e3b 100644
--- a/azurelinuxagent/common/protocol/ovfenv.py
+++ b/azurelinuxagent/common/protocol/ovfenv.py
@@ -35,7 +35,7 @@ WA_NAME_SPACE = "http://schemas.microsoft.com/windowsazure"
 
 def _validate_ovf(val, msg):
     if val is None:
-        raise ProtocolError("Failed to parse OVF XML: {0}".format(msg))
+        raise ProtocolError("Failed to validate OVF: {0}".format(msg))
 
 class OvfEnv(object):
     """
diff --git a/azurelinuxagent/common/protocol/util.py b/azurelinuxagent/common/protocol/util.py
index 7e7a74f..0ba03ec 100644
--- a/azurelinuxagent/common/protocol/util.py
+++ b/azurelinuxagent/common/protocol/util.py
@@ -33,25 +33,21 @@ from azurelinuxagent.common.protocol.ovfenv import OvfEnv
 from azurelinuxagent.common.protocol.wire import WireProtocol
 from azurelinuxagent.common.protocol.metadata import MetadataProtocol, \
                                                      METADATA_ENDPOINT
-import azurelinuxagent.common.utils.shellutil as shellutil
 
 OVF_FILE_NAME = "ovf-env.xml"
-
-#Tag file to indicate usage of metadata protocol
-TAG_FILE_NAME = "useMetadataEndpoint.tag" 
-
+TAG_FILE_NAME = "useMetadataEndpoint.tag"
 PROTOCOL_FILE_NAME = "Protocol"
-
-#MAX retry times for protocol probing
 MAX_RETRY = 360
-
 PROBE_INTERVAL = 10
-
 ENDPOINT_FILE_NAME = "WireServerEndpoint"
+PASSWORD_PATTERN = "<UserPassword>.*?<"
+PASSWORD_REPLACEMENT = "<UserPassword>*<"
+
 
 def get_protocol_util():
     return ProtocolUtil()
 
+
 class ProtocolUtil(object):
     """
     ProtocolUtil handles initialization for protocol instance. 2 protocol types 
@@ -71,21 +67,42 @@ class ProtocolUtil(object):
         dvd_mount_point = conf.get_dvd_mount_point()
         ovf_file_path_on_dvd = os.path.join(dvd_mount_point, OVF_FILE_NAME)
         tag_file_path_on_dvd = os.path.join(dvd_mount_point, TAG_FILE_NAME)
+        ovf_file_path = os.path.join(conf.get_lib_dir(), OVF_FILE_NAME)
+        tag_file_path = os.path.join(conf.get_lib_dir(), TAG_FILE_NAME)
+
         try:
             self.osutil.mount_dvd()
+        except OSUtilError as e:
+            raise ProtocolError("[CopyOvfEnv] Error mounting dvd: "
+                                "{0}".format(ustr(e)))
+
+        try:
             ovfxml = fileutil.read_file(ovf_file_path_on_dvd, remove_bom=True)
             ovfenv = OvfEnv(ovfxml)
-            ovfxml = re.sub("<UserPassword>.*?<", "<UserPassword>*<", ovfxml)
-            ovf_file_path = os.path.join(conf.get_lib_dir(), OVF_FILE_NAME)
+        except IOError as e:
+            raise ProtocolError("[CopyOvfEnv] Error reading file "
+                                "{0}: {1}".format(ovf_file_path_on_dvd,
+                                                  ustr(e)))
+
+        try:
+            ovfxml = re.sub(PASSWORD_PATTERN,
+                            PASSWORD_REPLACEMENT,
+                            ovfxml)
             fileutil.write_file(ovf_file_path, ovfxml)
-            
+        except IOError as e:
+            raise ProtocolError("[CopyOvfEnv] Error writing file "
+                                "{0}: {1}".format(ovf_file_path,
+                                                  ustr(e)))
+
+        try:
             if os.path.isfile(tag_file_path_on_dvd):
                 logger.info("Found {0} in provisioning ISO", TAG_FILE_NAME)
-                tag_file_path = os.path.join(conf.get_lib_dir(), TAG_FILE_NAME)
-                shutil.copyfile(tag_file_path_on_dvd, tag_file_path) 
-
-        except (OSUtilError, IOError) as e:
-            raise ProtocolError(ustr(e))
+                shutil.copyfile(tag_file_path_on_dvd, tag_file_path)
+        except IOError as e:
+            raise ProtocolError("[CopyOvfEnv] Error copying file "
+                                "{0} to {1}: {2}".format(tag_file_path,
+                                                         tag_file_path,
+                                                         ustr(e)))
 
         try:
             self.osutil.umount_dvd()
@@ -104,7 +121,7 @@ class ProtocolUtil(object):
             xml_text = fileutil.read_file(ovf_file_path)
             return OvfEnv(xml_text)
         else:
-            raise ProtocolError("ovf-env.xml is missing.")
+            raise ProtocolError("ovf-env.xml is missing from {0}".format(ovf_file_path))
 
     def _get_wireserver_endpoint(self):
         try:
@@ -146,7 +163,7 @@ class ProtocolUtil(object):
         protocol = MetadataProtocol()
         protocol.detect()
         
-        #Only allow root access METADATA_ENDPOINT
+        # only allow root access METADATA_ENDPOINT
         self.osutil.set_admin_access_to_ip(METADATA_ENDPOINT)
         
         self.save_protocol("MetadataProtocol")
@@ -206,7 +223,6 @@ class ProtocolUtil(object):
         except IOError as e:
             logger.error("Failed to save protocol endpoint: {0}", e)
 
-
     def clear_protocol(self):
         """
         Cleanup previous saved endpoint.
@@ -249,7 +265,6 @@ class ProtocolUtil(object):
         finally:
             self.lock.release()
 
-
     def get_protocol_by_file(self):
         """
         Detect protocol by tag file. 
diff --git a/azurelinuxagent/common/protocol/wire.py b/azurelinuxagent/common/protocol/wire.py
index 265e2dd..936be8c 100644
--- a/azurelinuxagent/common/protocol/wire.py
+++ b/azurelinuxagent/common/protocol/wire.py
@@ -21,16 +21,18 @@ import os
 import re
 import time
 import xml.sax.saxutils as saxutils
+
 import azurelinuxagent.common.conf as conf
+import azurelinuxagent.common.utils.fileutil as fileutil
+import azurelinuxagent.common.utils.textutil as textutil
+
 from azurelinuxagent.common.exception import ProtocolNotFoundError
 from azurelinuxagent.common.future import httpclient, bytebuffer
+from azurelinuxagent.common.protocol.hostplugin import HostPluginProtocol
+from azurelinuxagent.common.protocol.restapi import *
+from azurelinuxagent.common.utils.cryptutil import CryptUtil
 from azurelinuxagent.common.utils.textutil import parse_doc, findall, find, \
     findtext, getattrib, gettext, remove_bom, get_bytes_from_pem, parse_json
-import azurelinuxagent.common.utils.fileutil as fileutil
-import azurelinuxagent.common.utils.textutil as textutil
-from azurelinuxagent.common.utils.cryptutil import CryptUtil
-from azurelinuxagent.common.protocol.restapi import *
-from azurelinuxagent.common.protocol.hostplugin import HostPluginProtocol
 
 VERSION_INFO_URI = "http://{0}/?comp=versions"
 GOAL_STATE_URI = "http://{0}/machine/?comp=goalstate"
@@ -376,48 +378,20 @@ class StatusBlob(object):
         self.type = blob_type
 
     def upload(self, url):
-        # TODO upload extension only if content has changed
-        upload_successful = False
-        self.type = self.get_blob_type(url)
         try:
+            if not self.type in ["BlockBlob", "PageBlob"]:
+                raise ProtocolError("Illegal blob type: {0}".format(self.type))
+
             if self.type == "BlockBlob":
                 self.put_block_blob(url, self.data)
-            elif self.type == "PageBlob":
-                self.put_page_blob(url, self.data)
             else:
-                raise ProtocolError("Unknown blob type: {0}".format(self.type))
-        except HttpError as e:
-            message = "Initial upload failed [{0}]".format(e)
-            logger.warn(message)
-            from azurelinuxagent.common.event import WALAEventOperation, report_event
-            report_event(op=WALAEventOperation.ReportStatus,
-                         is_success=False,
-                         message=message)
-        else:
-            logger.verbose("Uploading status blob succeeded")
-            upload_successful = True
-        return upload_successful
-
-    def get_blob_type(self, url):
-        logger.verbose("Get blob type")
-        timestamp = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())
-        try:
-            resp = self.client.call_storage_service(
-                restutil.http_head,
-                url,
-                {
-                    "x-ms-date": timestamp,
-                    "x-ms-version": self.__class__.__storage_version__
-                })
-        except HttpError as e:
-            raise ProtocolError("Failed to get status blob type: {0}", e)
+                self.put_page_blob(url, self.data)
+            return True
 
-        if resp is None or resp.status != httpclient.OK:
-            raise ProtocolError("Failed to get status blob type")
+        except Exception as e:
+            logger.verbose("Initial status upload failed: {0}", e)
 
-        blob_type = resp.getheader("x-ms-blob-type")
-        logger.verbose("Blob type: [{0}]", blob_type)
-        return blob_type
+        return False
 
     def get_block_blob_headers(self, blob_size):
         return {
@@ -538,7 +512,6 @@ class WireClient(object):
         self.req_count = 0
         self.host_plugin = None
         self.status_blob = StatusBlob(self)
-        self.status_blob_type_reported = False
 
     def prevent_throttling(self):
         """
@@ -725,7 +698,6 @@ class WireClient(object):
         xml_text = self.fetch_config(goal_state.ext_uri, self.get_header())
         self.save_cache(local_file, xml_text)
         self.ext_conf = ExtensionsConfig(xml_text)
-        self.status_blob_type_reported = False
 
     def update_goal_state(self, forced=False, max_retry=3):
         uri = GOAL_STATE_URI.format(self.endpoint)
@@ -815,7 +787,6 @@ class WireClient(object):
                 local_file = os.path.join(conf.get_lib_dir(), local_file)
                 xml_text = self.fetch_cache(local_file)
                 self.ext_conf = ExtensionsConfig(xml_text)
-                self.status_blob_type_reported = False
         return self.ext_conf
 
     def get_ext_manifest(self, ext_handler, goal_state):
@@ -852,46 +823,38 @@ class WireClient(object):
 
     def upload_status_blob(self):
         ext_conf = self.get_ext_conf()
-        if ext_conf.status_upload_blob is not None:
-            uploaded = False
+
+        blob_uri = ext_conf.status_upload_blob
+        blob_type = ext_conf.status_upload_blob_type
+
+        if blob_uri is not None:
+
+            if not blob_type in ["BlockBlob", "PageBlob"]:
+                blob_type = "BlockBlob"
+                logger.info("Status Blob type is unspecified "
+                    "-- assuming it is a BlockBlob")
+
             try:
-                self.status_blob.prepare(ext_conf.status_upload_blob_type)
-                if not HostPluginProtocol.is_default_channel():
-                    uploaded = self.status_blob.upload(ext_conf.status_upload_blob)
-                    self.report_blob_type(self.status_blob.type,
-                                          ext_conf.status_upload_blob_type)
-            except (HttpError, ProtocolError):
-                # errors have already been logged
-                pass
+                self.status_blob.prepare(blob_type)
+            except Exception as e:
+                self.report_status_event(
+                    "Exception creating status blob: {0}",
+                    e)
+                return
+
+            uploaded = False
+            if not HostPluginProtocol.is_default_channel():
+                try:
+                    uploaded = self.status_blob.upload(blob_uri)
+                except HttpError as e:
+                    pass
+
             if not uploaded:
                 host = self.get_host_plugin()
                 host.put_vm_status(self.status_blob,
                                    ext_conf.status_upload_blob,
                                    ext_conf.status_upload_blob_type)
 
-    """
-    Emit an event to determine if the type in the extension config
-    matches the actual type from the HTTP HEAD request.
-    """
-    def report_blob_type(self, head_type, config_type):
-        if head_type and config_type:
-            is_match = head_type == config_type
-            if self.status_blob_type_reported is False:
-                message = \
-                    'Blob type match [{0}]'.format(head_type) if is_match else \
-                    'Blob type mismatch [HEAD {0}], [CONFIG {1}]'.format(
-                        head_type,
-                        config_type)
-
-                from azurelinuxagent.common.event import add_event, WALAEventOperation
-                from azurelinuxagent.common.version import AGENT_NAME, CURRENT_VERSION
-                add_event(AGENT_NAME,
-                          version=CURRENT_VERSION,
-                          is_success=is_match,
-                          message=message,
-                          op=WALAEventOperation.HealthCheck)
-                self.status_blob_type_reported = True
-
     def report_role_prop(self, thumbprint):
         goal_state = self.get_goal_state()
         role_prop = _build_role_properties(goal_state.container_id,
@@ -980,6 +943,16 @@ class WireClient(object):
             if len(buf[provider_id]) > 0:
                 self.send_event(provider_id, buf[provider_id])
 
+    def report_status_event(self, message, *args):
+        from azurelinuxagent.common.event import report_event, \
+                WALAEventOperation
+
+        message = message.format(*args)
+        logger.warn(message)
+        report_event(op=WALAEventOperation.ReportStatus,
+                    is_success=False,
+                    message=message)
+
     def get_header(self):
         return {
             "x-ms-agent-name": "WALinuxAgent",