Import patches-unapplied version 2.2.12-0ubuntu1 to ubuntu/artful-proposed

Imported using git-ubuntu import.

Changelog parent: 473ad6fbfe0b9c3b362b530492928303f2b4c7f3

New changelog entries:
  * New upstream release (LP: #1690854).
    - Refreshed debian/patches/disable_import_test.patch.

4 files changed, 276 insertions, 156 deletions

diff --git a/azurelinuxagent/pa/provision/cloudinit.py b/azurelinuxagent/pa/provision/cloudinit.py
new file mode 100644
index 0000000..5789e9a
--- /dev/null
+++ b/azurelinuxagent/pa/provision/cloudinit.py
@@ -0,0 +1,132 @@
+# Microsoft Azure Linux Agent
+#
+# Copyright 2014 Microsoft Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Requires Python 2.4+ and Openssl 1.0+
+#
+
+import os
+import os.path
+import time
+
+from datetime import datetime
+
+import azurelinuxagent.common.conf as conf
+import azurelinuxagent.common.logger as logger
+import azurelinuxagent.common.utils.fileutil as fileutil
+import azurelinuxagent.common.utils.shellutil as shellutil
+
+from azurelinuxagent.common.event import elapsed_milliseconds
+from azurelinuxagent.common.exception import ProvisionError, ProtocolError
+from azurelinuxagent.common.future import ustr
+from azurelinuxagent.common.protocol import OVF_FILE_NAME
+from azurelinuxagent.common.protocol.ovfenv import OvfEnv
+from azurelinuxagent.pa.provision.default import ProvisionHandler
+
+
+class CloudInitProvisionHandler(ProvisionHandler):
+    def __init__(self):
+        super(CloudInitProvisionHandler, self).__init__()
+
+    def run(self):
+        # If provision is enabled, run default provision handler
+        if conf.get_provision_enabled():
+            logger.warn("Provisioning flag is enabled, which overrides using "
+                        "cloud-init; running the default provisioning code")
+            super(CloudInitProvisionHandler, self).run()
+            return
+
+        try:
+            if super(CloudInitProvisionHandler, self).is_provisioned():
+                logger.info("Provisioning already completed, skipping.")
+                return
+
+            utc_start = datetime.utcnow()
+            logger.info("Running CloudInit provisioning handler")
+            self.wait_for_ovfenv()
+            self.protocol_util.get_protocol()
+            self.report_not_ready("Provisioning", "Starting")
+
+            thumbprint = self.wait_for_ssh_host_key()
+            self.write_provisioned()
+            logger.info("Finished provisioning")
+
+            self.report_ready(thumbprint)
+            self.report_event("Provision succeed",
+                is_success=True,
+                duration=elapsed_milliseconds(utc_start))
+
+        except ProvisionError as e:
+            logger.error("Provisioning failed: {0}", ustr(e))
+            self.report_not_ready("ProvisioningFailed", ustr(e))
+            self.report_event(ustr(e))
+            return
+
+    def wait_for_ovfenv(self, max_retry=360, sleep_time=5):
+        """
+        Wait for cloud-init to copy ovf-env.xml file from provision ISO
+        """
+        ovf_file_path = os.path.join(conf.get_lib_dir(), OVF_FILE_NAME)
+        for retry in range(0, max_retry):
+            if os.path.isfile(ovf_file_path):
+                try:
+                    OvfEnv(fileutil.read_file(ovf_file_path))
+                    return
+                except ProtocolError as pe:
+                    raise ProvisionError("OVF xml could not be parsed "
+                                         "[{0}]: {1}".format(ovf_file_path,
+                                                             ustr(pe)))
+            else:
+                if retry < max_retry - 1:
+                    logger.info(
+                        "Waiting for cloud-init to copy ovf-env.xml to {0} "
+                        "[{1} retries remaining, "
+                        "sleeping {2}s]".format(ovf_file_path,
+                                                max_retry - retry,
+                                                sleep_time))
+                    if not self.validate_cloud_init():
+                        logger.warn("cloud-init does not appear to be running")
+                    time.sleep(sleep_time)
+        raise ProvisionError("Giving up, ovf-env.xml was not copied to {0} "
+                             "after {1}s".format(ovf_file_path,
+                                                 max_retry * sleep_time))
+
+    def wait_for_ssh_host_key(self, max_retry=360, sleep_time=5):
+        """
+        Wait for cloud-init to generate ssh host key
+        """
+        keypair_type = conf.get_ssh_host_keypair_type()
+        path = conf.get_ssh_key_public_path()
+        for retry in range(0, max_retry):
+            if os.path.isfile(path):
+                logger.info("ssh host key found at: {0}".format(path))
+                try:
+                    thumbprint = self.get_ssh_host_key_thumbprint(chk_err=False)
+                    logger.info("Thumbprint obtained from : {0}".format(path))
+                    return thumbprint
+                except ProvisionError:
+                    logger.warn("Could not get thumbprint from {0}".format(path))
+            if retry < max_retry - 1:
+                logger.info("Waiting for ssh host key be generated at {0} "
+                            "[{1} attempts remaining, "
+                            "sleeping {2}s]".format(path,
+                                                    max_retry - retry,
+                                                    sleep_time))
+                if not self.validate_cloud_init():
+                    logger.warn("cloud-init does not appear to be running")
+                time.sleep(sleep_time)
+        raise ProvisionError("Giving up, ssh host key was not found at {0} "
+                             "after {1}s".format(path,
+                                                 max_retry * sleep_time))
diff --git a/azurelinuxagent/pa/provision/default.py b/azurelinuxagent/pa/provision/default.py
index 3a3f36f..d4870f1 100644
--- a/azurelinuxagent/pa/provision/default.py
+++ b/azurelinuxagent/pa/provision/default.py
@@ -20,19 +20,31 @@ Provision handler
 """
 
 import os
+import os.path
+import re
+
+from datetime import datetime
+
+import azurelinuxagent.common.conf as conf
 import azurelinuxagent.common.logger as logger
+import azurelinuxagent.common.utils.shellutil as shellutil
+import azurelinuxagent.common.utils.fileutil as fileutil
+
 from azurelinuxagent.common.future import ustr
-import azurelinuxagent.common.conf as conf
-from azurelinuxagent.common.event import add_event, WALAEventOperation
+from azurelinuxagent.common.event import add_event, WALAEventOperation, \
+    elapsed_milliseconds
 from azurelinuxagent.common.exception import ProvisionError, ProtocolError, \
     OSUtilError
-from azurelinuxagent.common.protocol.restapi import ProvisionStatus
-import azurelinuxagent.common.utils.shellutil as shellutil
-import azurelinuxagent.common.utils.fileutil as fileutil
 from azurelinuxagent.common.osutil import get_osutil
+from azurelinuxagent.common.protocol.restapi import ProvisionStatus
 from azurelinuxagent.common.protocol import get_protocol_util
+from azurelinuxagent.common.version import AGENT_NAME
 
 CUSTOM_DATA_FILE = "CustomData"
+CLOUD_INIT_PATTERN = b".*/bin/cloud-init.*"
+CLOUD_INIT_REGEX = re.compile(CLOUD_INIT_PATTERN)
+
+PROVISIONED_FILE = 'provisioned'
 
 
 class ProvisionHandler(object):
@@ -41,54 +53,85 @@ class ProvisionHandler(object):
         self.protocol_util = get_protocol_util()
 
     def run(self):
-        # if provisioning is already done, return
-        provisioned = os.path.join(conf.get_lib_dir(), "provisioned")
-        if os.path.isfile(provisioned):
-            logger.info("Provisioning already completed, skipping.")
-            return
-
-        thumbprint = None
         # If provision is not enabled, report ready and then return
         if not conf.get_provision_enabled():
             logger.info("Provisioning is disabled, skipping.")
-        else:
-            logger.info("Running provisioning handler")
-            try:
-                logger.info("Copying ovf-env.xml")
-                ovf_env = self.protocol_util.copy_ovf_env()
-                self.protocol_util.get_protocol_by_file()
-                self.report_not_ready("Provisioning", "Starting")
-                logger.info("Starting provisioning")
-                self.provision(ovf_env)
-                thumbprint = self.reg_ssh_host_key()
-                self.osutil.restart_ssh_service()
-                self.report_event("Provision succeed", is_success=True)
-            except ProtocolError as e:
-                logger.error("[ProtocolError] Provisioning failed: {0}", e)
-                self.report_not_ready("ProvisioningFailed", ustr(e))
-                self.report_event("Failed to copy ovf-env.xml: {0}".format(e))
-                return
-            except ProvisionError as e:
-                logger.error("[ProvisionError] Provisioning failed: {0}", e)
-                self.report_not_ready("ProvisioningFailed", ustr(e))
-                self.report_event(ustr(e))
+            return
+
+        try:
+            utc_start = datetime.utcnow()
+            thumbprint = None
+
+            # if provisioning is already done, return
+            if self.is_provisioned():
+                logger.info("Provisioning already completed, skipping.")
                 return
-        # write out provisioned file and report Ready
-        fileutil.write_file(provisioned, "")
-        self.report_ready(thumbprint)
-        logger.info("Provisioning complete")
+
+            logger.info("Running default provisioning handler")
+
+            if not self.validate_cloud_init(is_expected=False):
+                raise ProvisionError("cloud-init appears to be running, "
+                                        "this is not expected, cannot continue")
+
+            logger.info("Copying ovf-env.xml")
+            ovf_env = self.protocol_util.copy_ovf_env()
+
+            self.protocol_util.get_protocol_by_file()
+            self.report_not_ready("Provisioning", "Starting")
+            logger.info("Starting provisioning")
+
+            self.provision(ovf_env)
+
+            thumbprint = self.reg_ssh_host_key()
+            self.osutil.restart_ssh_service()
+
+            # write out provisioned file and report Ready
+            self.write_provisioned()
+
+            self.report_event("Provision succeed",
+                is_success=True,
+                duration=elapsed_milliseconds(utc_start))
+
+            self.report_ready(thumbprint)
+            logger.info("Provisioning complete")
+
+        except (ProtocolError, ProvisionError) as e:
+            self.report_not_ready("ProvisioningFailed", ustr(e))
+            self.report_event(ustr(e))
+            logger.error("Provisioning failed: {0}", ustr(e))
+            return
+
+    @staticmethod
+    def validate_cloud_init(is_expected=True):
+        pids = [pid for pid in os.listdir('/proc') if pid.isdigit()]
+        is_running = False
+        for pid in pids:
+            try:
+                pname = open(os.path.join('/proc', pid, 'cmdline'), 'rb').read()
+                if CLOUD_INIT_REGEX.match(pname):
+                    is_running = True
+                    msg = "cloud-init is running [PID {0}, {1}]".format(pid,
+                                                                        pname)
+                    if is_expected:
+                        logger.verbose(msg)
+                    else:
+                        logger.error(msg)
+                    break
+            except IOError:
+                continue
+        return is_running == is_expected
 
     def reg_ssh_host_key(self):
         keypair_type = conf.get_ssh_host_keypair_type()
         if conf.get_regenerate_ssh_host_key():
-            fileutil.rm_files("/etc/ssh/ssh_host_*key*")
-            keygen_cmd = "ssh-keygen -N '' -t {0} -f /etc/ssh/ssh_host_{1}_key"
-            shellutil.run(keygen_cmd.format(keypair_type, keypair_type))
-        thumbprint = self.get_ssh_host_key_thumbprint(keypair_type)
-        return thumbprint
-
-    def get_ssh_host_key_thumbprint(self, keypair_type, chk_err=True):
-        cmd = "ssh-keygen -lf /etc/ssh/ssh_host_{0}_key.pub".format(keypair_type)
+            fileutil.rm_files(conf.get_ssh_key_glob())
+            keygen_cmd = "ssh-keygen -N '' -t {0} -f {1}"
+            shellutil.run(keygen_cmd.format(keypair_type,
+                        conf.get_ssh_key_private_path()))
+        return self.get_ssh_host_key_thumbprint()
+
+    def get_ssh_host_key_thumbprint(self, chk_err=True):
+        cmd = "ssh-keygen -lf {0}".format(conf.get_ssh_key_public_path())
         ret = shellutil.run_get_output(cmd, chk_err=chk_err)
         if ret[0] == 0:
             return ret[1].rstrip().split()[1].replace(':', '')
@@ -96,6 +139,45 @@ class ProvisionHandler(object):
             raise ProvisionError(("Failed to generate ssh host key: "
                                   "ret={0}, out= {1}").format(ret[0], ret[1]))
 
+    def provisioned_file_path(self):
+        return os.path.join(conf.get_lib_dir(), PROVISIONED_FILE)
+
+    def is_provisioned(self):
+        '''
+        A VM is considered provisionend *anytime* the provisioning
+        sentinel file exists and not provisioned *anytime* the file
+        is absent.
+
+        If the VM was provisioned using an agent that did not record
+        the VM unique identifier, the provisioning file will be re-written
+        to include the identifier.
+
+        A warning is logged *if* the VM unique identifier has changed
+        since VM was provisioned.
+        '''
+        if not os.path.isfile(self.provisioned_file_path()):
+            return False
+
+        s = fileutil.read_file(self.provisioned_file_path()).strip()
+        if s != self.osutil.get_instance_id():
+            if len(s) > 0:
+                logger.warn("VM is provisioned, "
+                            "but the VM unique identifier has changed -- "
+                            "clearing cached state")
+                from azurelinuxagent.pa.deprovision \
+                    import get_deprovision_handler
+                deprovision_handler = get_deprovision_handler()
+                deprovision_handler.run_changed_unique_id()
+
+            self.write_provisioned()
+
+        return True
+
+    def write_provisioned(self):
+        fileutil.write_file(
+            self.provisioned_file_path(),
+            get_osutil().get_instance_id())
+
     def provision(self, ovfenv):
         logger.info("Handle ovf-env.xml.")
         try:
@@ -113,7 +195,7 @@ class ProvisionHandler(object):
                 self.osutil.del_root_password()
 
         except OSUtilError as e:
-            raise ProvisionError("Failed to handle ovf-env.xml: {0}".format(e))
+            raise ProvisionError("Failed to provision: {0}".format(ustr(e)))
 
     def config_user_account(self, ovfenv):
         logger.info("Create user account if not exists")
@@ -141,11 +223,12 @@ class ProvisionHandler(object):
         if customdata is None:
             return
 
-        logger.info("Save custom data")
         lib_dir = conf.get_lib_dir()
-        if conf.get_decode_customdata():
+        if conf.get_decode_customdata() or conf.get_execute_customdata():
+            logger.info("Decode custom data")
             customdata = self.osutil.decode_customdata(customdata)
 
+        logger.info("Save custom data")
         customdata_file = os.path.join(lib_dir, CUSTOM_DATA_FILE)
         fileutil.write_file(customdata_file, customdata)
 
@@ -164,9 +247,12 @@ class ProvisionHandler(object):
             logger.info("Deploy ssh key pairs.")
             self.osutil.deploy_ssh_keypair(ovfenv.username, keypair)
 
-    def report_event(self, message, is_success=False):
-        add_event(name="WALA", message=message, is_success=is_success,
-                  op=WALAEventOperation.Provision)
+    def report_event(self, message, is_success=False, duration=0):
+        add_event(name=AGENT_NAME,
+                    message=message,
+                    duration=duration,
+                    is_success=is_success,
+                    op=WALAEventOperation.Provision)
 
     def report_not_ready(self, sub_status, description):
         status = ProvisionStatus(status="NotReady", subStatus=sub_status,
diff --git a/azurelinuxagent/pa/provision/factory.py b/azurelinuxagent/pa/provision/factory.py
index 9bbe35c..d87765f 100644
--- a/azurelinuxagent/pa/provision/factory.py
+++ b/azurelinuxagent/pa/provision/factory.py
@@ -15,18 +15,22 @@
 # Requires Python 2.4+ and Openssl 1.0+
 #
 
+import azurelinuxagent.common.conf as conf
 import azurelinuxagent.common.logger as logger
+
 from azurelinuxagent.common.utils.textutil import Version
 from azurelinuxagent.common.version import DISTRO_NAME, DISTRO_VERSION, \
                                      DISTRO_FULL_NAME
+
 from .default import ProvisionHandler
-from .ubuntu import UbuntuProvisionHandler
+from .cloudinit import CloudInitProvisionHandler
 
 def get_provision_handler(distro_name=DISTRO_NAME, 
                             distro_version=DISTRO_VERSION,
                             distro_full_name=DISTRO_FULL_NAME):
-    if distro_name == "ubuntu":
-        return UbuntuProvisionHandler()
+
+    if conf.get_provision_cloudinit():
+        return CloudInitProvisionHandler()
 
     return ProvisionHandler()
 
diff --git a/azurelinuxagent/pa/provision/ubuntu.py b/azurelinuxagent/pa/provision/ubuntu.py
deleted file mode 100644
index 66866b2..0000000
--- a/azurelinuxagent/pa/provision/ubuntu.py
+++ /dev/null
@@ -1,102 +0,0 @@
-# Microsoft Azure Linux Agent
-#
-# Copyright 2014 Microsoft Corporation
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-# Requires Python 2.4+ and Openssl 1.0+
-#
-
-import os
-import time
-
-import azurelinuxagent.common.conf as conf
-import azurelinuxagent.common.logger as logger
-import azurelinuxagent.common.utils.fileutil as fileutil
-from azurelinuxagent.common.exception import ProvisionError, ProtocolError
-from azurelinuxagent.common.future import ustr
-from azurelinuxagent.pa.provision.default import ProvisionHandler
-
-"""
-On ubuntu image, provision could be disabled.
-"""
-
-
-class UbuntuProvisionHandler(ProvisionHandler):
-    def __init__(self):
-        super(UbuntuProvisionHandler, self).__init__()
-
-    def run(self):
-        # If provision is enabled, run default provision handler
-        if conf.get_provision_enabled():
-            super(UbuntuProvisionHandler, self).run()
-            return
-
-        logger.info("run Ubuntu provision handler")
-        provisioned = os.path.join(conf.get_lib_dir(), "provisioned")
-        if os.path.isfile(provisioned):
-            return
-
-        logger.info("Waiting cloud-init to copy ovf-env.xml.")
-        self.wait_for_ovfenv()
-        self.protocol_util.get_protocol()
-        self.report_not_ready("Provisioning", "Starting")
-        logger.info("Sleeping 1 second to avoid throttling.")
-        time.sleep(1)
-        try:
-            logger.info("Wait for ssh host key to be generated.")
-            thumbprint = self.wait_for_ssh_host_key()
-            fileutil.write_file(provisioned, "")
-            logger.info("Finished provisioning")
-        except ProvisionError as e:
-            logger.error("Provision failed: {0}", e)
-            self.report_not_ready("ProvisioningFailed", ustr(e))
-            self.report_event(ustr(e))
-            return
-
-        self.report_ready(thumbprint)
-        self.report_event("Provision succeed", is_success=True)
-
-    def wait_for_ovfenv(self, max_retry=60):
-        """
-        Wait for cloud-init to copy ovf-env.xml file from provision ISO
-        """
-        for retry in range(0, max_retry):
-            try:
-                self.protocol_util.get_ovf_env()
-                return
-            except ProtocolError:
-                if retry < max_retry - 1:
-                    logger.info("Wait for cloud-init to copy ovf-env.xml")
-                    time.sleep(5)
-        raise ProvisionError("ovf-env.xml is not copied")
-
-    def wait_for_ssh_host_key(self, max_retry=60):
-        """
-        Wait for cloud-init to generate ssh host key
-        """
-        keypair_type = conf.get_ssh_host_keypair_type()
-        path = '/etc/ssh/ssh_host_{0}_key.pub'.format(keypair_type)
-        for retry in range(0, max_retry):
-            if os.path.isfile(path):
-                logger.info("ssh host key found at: {0}".format(path))
-                try:
-                    thumbprint = self.get_ssh_host_key_thumbprint(keypair_type, chk_err=False)
-                    logger.info("Thumbprint obtained from : {0}".format(path))
-                    return thumbprint
-                except ProvisionError:
-                    logger.warn("Could not get thumbprint from {0}".format(path))
-            if retry < max_retry - 1:
-                logger.info("Wait for ssh host key be generated: {0}".format(path))
-                time.sleep(5)
-        raise ProvisionError("ssh host key is not generated.")