Import patches-unapplied version 2.2.12-0ubuntu1 to ubuntu/artful-proposed

Imported using git-ubuntu import.

Changelog parent: 473ad6fbfe0b9c3b362b530492928303f2b4c7f3

New changelog entries:
  * New upstream release (LP: #1690854).
    - Refreshed debian/patches/disable_import_test.patch.

3 files changed, 238 insertions, 31 deletions

diff --git a/azurelinuxagent/ga/exthandlers.py b/azurelinuxagent/ga/exthandlers.py
index e0125aa..b44ed6d 100644
--- a/azurelinuxagent/ga/exthandlers.py
+++ b/azurelinuxagent/ga/exthandlers.py
@@ -20,6 +20,8 @@
 import glob
 import json
 import os
+import os.path
+import re
 import shutil
 import stat
 import subprocess
@@ -31,6 +33,7 @@ import azurelinuxagent.common.logger as logger
 import azurelinuxagent.common.utils.fileutil as fileutil
 import azurelinuxagent.common.utils.restutil as restutil
 import azurelinuxagent.common.utils.shellutil as shellutil
+import azurelinuxagent.common.version as version
 
 from azurelinuxagent.common.event import add_event, WALAEventOperation
 from azurelinuxagent.common.exception import ExtensionError, ProtocolError, HttpError
@@ -55,6 +58,13 @@ VALID_EXTENSION_STATUS = ['transitioning', 'error', 'success', 'warning']
 
 VALID_HANDLER_STATUS = ['Ready', 'NotReady', "Installing", "Unresponsive"]
 
+HANDLER_PATTERN = "^([^-]+)-(\d+(?:\.\d+)*)"
+HANDLER_NAME_PATTERN = re.compile(HANDLER_PATTERN+"$", re.IGNORECASE)
+HANDLER_PKG_EXT = ".zip"
+HANDLER_PKG_PATTERN = re.compile(HANDLER_PATTERN+"\\"+HANDLER_PKG_EXT+"$",
+                                re.IGNORECASE)
+
+
 def validate_has_key(obj, key, fullname):
     if key not in obj:
         raise ExtensionError("Missing: {0}".format(fullname))
@@ -163,6 +173,7 @@ def get_exthandlers_handler():
 class ExtHandlersHandler(object):
     def __init__(self):
         self.protocol_util = get_protocol_util()
+        self.protocol = None
         self.ext_handlers = None
         self.last_etag = None
         self.log_report = False
@@ -188,10 +199,74 @@ class ExtHandlersHandler(object):
         self.last_etag = etag
 
         self.report_ext_handlers_status()
+        self.cleanup_outdated_handlers()
 
     def run_status(self):
         self.report_ext_handlers_status()
         return
+
+    def cleanup_outdated_handlers(self):
+        handlers = []
+        pkgs = []
+
+        # Build a collection of uninstalled handlers and orphaned packages
+        # Note:
+        # -- An orphaned package is one without a corresponding handler
+        #    directory
+        for item in os.listdir(conf.get_lib_dir()):
+            path = os.path.join(conf.get_lib_dir(), item)
+
+            if version.is_agent_package(path) or version.is_agent_path(path):
+                continue
+
+            if os.path.isdir(path):
+                if re.match(HANDLER_NAME_PATTERN, item) is None:
+                    continue
+                try:
+                    eh = ExtHandler()
+
+                    separator = item.rfind('-')
+
+                    eh.name = item[0:separator]
+                    eh.properties.version = str(FlexibleVersion(item[separator+1:]))
+
+                    handler = ExtHandlerInstance(eh, self.protocol)
+                except Exception as e:
+                    continue
+                if handler.get_handler_state() != ExtHandlerState.NotInstalled:
+                    continue
+                handlers.append(handler)
+
+            elif os.path.isfile(path) and \
+                    not os.path.isdir(path[0:-len(HANDLER_PKG_EXT)]):
+                if not re.match(HANDLER_PKG_PATTERN, item):
+                    continue
+                pkgs.append(path)
+
+        # Then, remove the orphaned packages
+        for pkg in pkgs:
+            try:
+                os.remove(pkg)
+                logger.verbose("Removed orphaned extension package "
+                            "{0}".format(pkg))
+            except Exception as e:
+                logger.warn("Failed to remove orphaned package: {0}".format(
+                                pkg))
+
+        # Finally, remove the directories and packages of the
+        # uninstalled handlers
+        for handler in handlers:
+            handler.rm_ext_handler_dir()
+            pkg = os.path.join(conf.get_lib_dir(),
+                    handler.get_full_name() + HANDLER_PKG_EXT)
+            if os.path.isfile(pkg):
+                try:
+                    os.remove(pkg)
+                    logger.verbose("Removed extension package "
+                                "{0}".format(pkg))
+                except Exception as e:
+                    logger.warn("Failed to remove extension package: "
+                                "{0}".format(pkg))
    
     def handle_ext_handlers(self, etag=None):
         if self.ext_handlers.extHandlers is None or \
@@ -478,6 +553,14 @@ class ExtHandlerInstance(object):
 
             separator = path.rfind('-')
             version = FlexibleVersion(path[separator+1:])
+            state_path = os.path.join(path, 'config', 'HandlerState')
+
+            if not os.path.exists(state_path) or \
+                fileutil.read_file(state_path) == \
+                    ExtHandlerState.NotInstalled:
+                logger.verbose("Ignoring version of uninstalled extension: "
+                    "{0}".format(path))
+                continue
 
             if lastest_version is None or lastest_version < version:
                 lastest_version = version
@@ -615,6 +698,7 @@ class ExtHandlerInstance(object):
         except IOError as e:
             message = "Failed to remove extension handler directory: {0}".format(e)
             self.report_event(message=message, is_success=False)
+            self.logger.warn(message)
 
     def update(self):
         self.set_operation(WALAEventOperation.Update)
diff --git a/azurelinuxagent/ga/monitor.py b/azurelinuxagent/ga/monitor.py
index 7ef7f04..dcfd6a4 100644
--- a/azurelinuxagent/ga/monitor.py
+++ b/azurelinuxagent/ga/monitor.py
@@ -178,11 +178,11 @@ class MonitorHandler(object):
             logger.error("{0}", e)
 
     def daemon(self):
-        last_heartbeat = datetime.datetime.min
         period = datetime.timedelta(minutes=30)
+        last_heartbeat = datetime.datetime.utcnow() - period
         while True:
-            if (datetime.datetime.now() - last_heartbeat) > period:
-                last_heartbeat = datetime.datetime.now()
+            if datetime.datetime.utcnow() >= (last_heartbeat + period):
+                last_heartbeat = datetime.datetime.utcnow()
                 add_event(
                     op=WALAEventOperation.HeartBeat,
                     name=CURRENT_AGENT,
diff --git a/azurelinuxagent/ga/update.py b/azurelinuxagent/ga/update.py
index 203bb41..67eb785 100644
--- a/azurelinuxagent/ga/update.py
+++ b/azurelinuxagent/ga/update.py
@@ -30,13 +30,17 @@ import time
 import traceback
 import zipfile
 
+from datetime import datetime, timedelta
+
 import azurelinuxagent.common.conf as conf
 import azurelinuxagent.common.logger as logger
 import azurelinuxagent.common.utils.fileutil as fileutil
 import azurelinuxagent.common.utils.restutil as restutil
 import azurelinuxagent.common.utils.textutil as textutil
 
-from azurelinuxagent.common.event import add_event, WALAEventOperation
+from azurelinuxagent.common.event import add_event, \
+                                    elapsed_milliseconds, \
+                                    WALAEventOperation
 from azurelinuxagent.common.exception import UpdateError, ProtocolError
 from azurelinuxagent.common.future import ustr
 from azurelinuxagent.common.osutil import get_osutil
@@ -53,6 +57,7 @@ from azurelinuxagent.ga.exthandlers import HandlerManifest
 
 AGENT_ERROR_FILE = "error.json" # File name for agent error record
 AGENT_MANIFEST_FILE = "HandlerManifest.json"
+AGENT_SUPPORTED_FILE = "supported.json"
 
 CHILD_HEALTH_INTERVAL = 15 * 60
 CHILD_LAUNCH_INTERVAL = 5 * 60
@@ -60,16 +65,14 @@ CHILD_LAUNCH_RESTART_MAX = 3
 CHILD_POLL_INTERVAL = 60
 
 MAX_FAILURE = 3 # Max failure allowed for agent before blacklisted
-RETAIN_INTERVAL = 24 * 60 * 60 # Retain interval for black list
 
-GOAL_STATE_INTERVAL = 25
+GOAL_STATE_INTERVAL = 3
 REPORT_STATUS_INTERVAL = 15
 
 ORPHAN_WAIT_INTERVAL = 15 * 60 * 60
 
 AGENT_SENTINAL_FILE = "current_version"
 
-
 def get_update_handler():
     return UpdateHandler()
 
@@ -98,7 +101,7 @@ class UpdateHandler(object):
         self.signal_handler = None
         return
 
-    def run_latest(self):
+    def run_latest(self, child_args=None):
         """
         This method is called from the daemon to find and launch the most
         current, downloaded agent.
@@ -127,6 +130,9 @@ class UpdateHandler(object):
             agent_name = latest_agent.name
             agent_version = latest_agent.version
 
+        if child_args is not None:
+            agent_cmd = "{0} {1}".format(agent_cmd, child_args)
+
         try:
 
             # Launch the correct Python version for python-based agents
@@ -198,7 +204,7 @@ class UpdateHandler(object):
                     ret)
                 logger.warn(msg)
                 if latest_agent is not None:
-                    latest_agent.mark_failure()
+                    latest_agent.mark_failure(is_fatal=True)
 
         except Exception as e:
             msg = u"Agent {0} launched with command '{1}' failed with exception: {2}".format(
@@ -237,10 +243,11 @@ class UpdateHandler(object):
         migrate_handler_state()
 
         try:
+            send_event_time = datetime.utcnow()
+
             self._ensure_no_orphans()
             self._emit_restart_event()
 
-            # TODO: Add means to stop running
             while self.running:
                 if self._is_orphaned:
                     logger.info("Goal state agent {0} was orphaned -- exiting", CURRENT_AGENT)
@@ -254,8 +261,29 @@ class UpdateHandler(object):
                             self.agents[0].name)
                     break
 
+                utc_start = datetime.utcnow()
+
+                last_etag = exthandlers_handler.last_etag
                 exthandlers_handler.run()
-                
+
+                log_event = last_etag != exthandlers_handler.last_etag or \
+                            (datetime.utcnow() >= send_event_time)
+                add_event(
+                    AGENT_NAME,
+                    version=CURRENT_VERSION,
+                    op=WALAEventOperation.ProcessGoalState,
+                    is_success=True,
+                    duration=elapsed_milliseconds(utc_start),
+                    log_event=log_event)
+                if log_event:
+                    send_event_time += timedelta(minutes=REPORT_STATUS_INTERVAL)
+
+                test_agent = self.get_test_agent()
+                if test_agent is not None and test_agent.in_slice:
+                    test_agent.enable()
+                    logger.info(u"Enabled Agent {0} as test agent", test_agent.name)
+                    break
+
                 time.sleep(GOAL_STATE_INTERVAL)
 
         except Exception as e:
@@ -305,13 +333,21 @@ class UpdateHandler(object):
         if not conf.get_autoupdate_enabled():
             return None
         
-        self._load_agents()
+        self._find_agents()
         available_agents = [agent for agent in self.agents
                             if agent.is_available
                             and agent.version > FlexibleVersion(AGENT_VERSION)]
 
         return available_agents[0] if len(available_agents) >= 1 else None
 
+    def get_test_agent(self):
+        agent = None
+        agents = [agent for agent in self._load_agents() if agent.is_test]
+        if len(agents) > 0:
+            agents.sort(key=lambda agent: agent.version, reverse=True)
+            agent = agents[0]
+        return agent
+
     def _emit_restart_event(self):
         if not self._is_clean_start:
             msg = u"{0} did not terminate cleanly".format(CURRENT_AGENT)
@@ -390,6 +426,7 @@ class UpdateHandler(object):
         host = None
         if protocol and protocol.client:
             host = protocol.client.get_host_plugin()
+
         self._set_agents([GuestAgent(pkg=pkg, host=host) for pkg in pkg_list.versions])
         self._purge_agents()
         self._filter_blacklisted_agents()
@@ -457,6 +494,17 @@ class UpdateHandler(object):
         self.agents = [agent for agent in self.agents if not agent.is_blacklisted]
         return
 
+    def _find_agents(self):
+        """
+        Load all non-blacklisted agents currently on disk.
+        """
+        try:
+            self._set_agents(self._load_agents())
+            self._filter_blacklisted_agents()
+        except Exception as e:
+            logger.warn(u"Exception occurred loading available agents: {0}", ustr(e))
+        return
+
     def _get_pid_files(self):
         pid_file = conf.get_agent_pid_file_path()
         
@@ -502,17 +550,9 @@ class UpdateHandler(object):
         return fileutil.read_file(conf.get_agent_pid_file_path()) != ustr(parent_pid)
 
     def _load_agents(self):
-        """
-        Load all non-blacklisted agents currently on disk.
-        """
-        try:
-            path = os.path.join(conf.get_lib_dir(), "{0}-*".format(AGENT_NAME))
-            self._set_agents([GuestAgent(path=agent_dir)
-                                for agent_dir in glob.iglob(path) if os.path.isdir(agent_dir)])
-            self._filter_blacklisted_agents()
-        except Exception as e:
-            logger.warn(u"Exception occurred loading available agents: {0}", ustr(e))
-        return
+        path = os.path.join(conf.get_lib_dir(), "{0}-*".format(AGENT_NAME))
+        return [GuestAgent(path=agent_dir)
+                        for agent_dir in glob.iglob(path) if os.path.isdir(agent_dir)]
 
     def _purge_agents(self):
         """
@@ -610,7 +650,11 @@ class GuestAgent(object):
         logger.verbose(u"Instantiating Agent {0} from {1}", self.name, location)
 
         self.error = None
+        self.supported = None
+
         self._load_error()
+        self._load_supported()
+
         self._ensure_downloaded()
         return
 
@@ -633,10 +677,19 @@ class GuestAgent(object):
     def get_agent_pkg_path(self):
         return ".".join((os.path.join(conf.get_lib_dir(), self.name), "zip"))
 
+    def get_agent_supported_file(self):
+        return os.path.join(conf.get_lib_dir(), self.name, AGENT_SUPPORTED_FILE)
+
     def clear_error(self):
         self.error.clear()
         return
 
+    def enable(self):
+        if self.error.is_sentinel:
+            self.error.clear()
+            self.error.save()
+        return
+
     @property
     def is_available(self):
         return self.is_downloaded and not self.is_blacklisted
@@ -649,6 +702,14 @@ class GuestAgent(object):
     def is_downloaded(self):
         return self.is_blacklisted or os.path.isfile(self.get_agent_manifest_path())
 
+    @property
+    def is_test(self):
+        return self.error.is_sentinel and self.supported.is_supported
+
+    @property
+    def in_slice(self):
+        return self.is_test and self.supported.in_slice
+
     def mark_failure(self, is_fatal=False):
         try:
             if not os.path.isdir(self.get_agent_dir()):
@@ -666,7 +727,7 @@ class GuestAgent(object):
             logger.verbose(u"Ensuring Agent {0} is downloaded", self.name)
 
             if self.is_blacklisted:
-                logger.warn(u"Agent {0} is blacklisted - skipping download", self.name)
+                logger.info(u"Agent {0} is blacklisted - skipping download", self.name)
                 return
 
             if self.is_downloaded:
@@ -682,6 +743,7 @@ class GuestAgent(object):
             self._unpack()
             self._load_manifest()
             self._load_error()
+            self._load_supported()
 
             msg = u"Agent {0} downloaded successfully".format(self.name)
             logger.verbose(msg)
@@ -770,6 +832,12 @@ class GuestAgent(object):
             logger.warn(u"Agent {0} failed loading error state: {1}", self.name, ustr(e))
         return
 
+    def _load_supported(self):
+        try:
+            self.supported = Supported(self.get_agent_supported_file())
+        except Exception as e:
+            self.supported = Supported()
+
     def _load_manifest(self):
         path = self.get_agent_manifest_path()
         if not os.path.isfile(path):
@@ -859,18 +927,15 @@ class GuestAgentError(object):
         self.failure_count = 0
         self.was_fatal = False
         return
-    
-    def clear_old_failure(self):
-        if self.last_failure <= 0.0:
-            return
-        if self.last_failure < (time.time() - RETAIN_INTERVAL):
-            self.clear()
-        return
 
     @property
     def is_blacklisted(self):
         return self.was_fatal or self.failure_count >= MAX_FAILURE
 
+    @property
+    def is_sentinel(self):
+        return self.was_fatal and self.last_failure == 0.0
+
     def load(self):
         if self.path is not None and os.path.isfile(self.path):
             with open(self.path, 'r') as f:
@@ -906,3 +971,61 @@ class GuestAgentError(object):
             self.last_failure,
             self.failure_count,
             self.was_fatal)
+
+class Supported(object):
+    def __init__(self, path):
+        if path is None:
+            raise UpdateError(u"Supported requires a path")
+        self.path = path
+
+        self._load()
+        return
+
+    @property
+    def is_supported(self):
+        return self._supported_distribution is not None
+
+    @property
+    def in_slice(self):
+        d = self._supported_distribution
+        return d is not None and d.in_slice
+
+    @property
+    def _supported_distribution(self):
+        for d in self.distributions:
+            dd = self.distributions[d]
+            if dd.is_supported:
+                return dd
+        return None
+
+    def _load(self):
+        self.distributions = {}
+        try:
+            if self.path is not None and os.path.isfile(self.path):
+                j = json.loads(fileutil.read_file(self.path))
+                for d in j:
+                    self.distributions[d] = SupportedDistribution(j[d])
+        except Exception as e:
+            logger.warn("Failed JSON parse of {0}: {1}".format(self.path, e))
+        return
+
+class SupportedDistribution(object):
+    def __init__(self, s):
+        if s is None or not isinstance(s, dict):
+            raise UpdateError(u"SupportedDisribution requires a dictionary")
+
+        self.slice = s['slice']
+        self.versions = s['versions']
+
+    @property
+    def is_supported(self):
+        d = ','.join(platform.linux_distribution())
+        for v in self.versions:
+            if re.match(v, d):
+                return True
+        return False
+
+    @property
+    def in_slice(self):
+        n = int((60 * self.slice) / 100)
+        return (n - datetime.utcnow().second) > 0