diff options
| author | Łukasz 'sil2100' Zemczak <lukasz.zemczak@ubuntu.com> | 2017-05-18 19:58:02 +0200 |
|---|---|---|
| committer | usd-importer <ubuntu-server@lists.ubuntu.com> | 2017-05-31 09:53:12 +0000 |
| commit | 4fb0b5a09b26135ade285844da5d7dfe582a8d4c (patch) | |
| tree | 09b1e5867d6e7501118cdd0af0012b51fc216530 | |
| parent | 473ad6fbfe0b9c3b362b530492928303f2b4c7f3 (diff) | |
| download | vyos-walinuxagent-4fb0b5a09b26135ade285844da5d7dfe582a8d4c.tar.gz vyos-walinuxagent-4fb0b5a09b26135ade285844da5d7dfe582a8d4c.zip | |
Import patches-unapplied version 2.2.12-0ubuntu1 to ubuntu/artful-proposed
Imported using git-ubuntu import.
Changelog parent: 473ad6fbfe0b9c3b362b530492928303f2b4c7f3
New changelog entries:
* New upstream release (LP: #1690854).
- Refreshed debian/patches/disable_import_test.patch.
76 files changed, 2770 insertions, 490 deletions
@@ -170,6 +170,7 @@ waagent. A sample configuration file is shown below: ``` Provisioning.Enabled=y +Provisioning.UseCloudInit=n Provisioning.DeleteRootPassword=n Provisioning.RegenerateSshHostKeyPair=y Provisioning.SshHostKeyPairType=rsa @@ -187,7 +188,9 @@ ResourceDisk.SwapSizeMB=0 LBProbeResponder=y Logs.Verbose=n OS.RootDeviceScsiTimeout=300 +OS.EnableFIPS=n OS.OpensslPath=None +OS.SshDir=/etc/ssh HttpProxy.Host=None HttpProxy.Port=None ``` @@ -208,6 +211,16 @@ agent. Valid values are "y" or "n". If provisioning is disabled, SSH host and user keys in the image are preserved and any configuration specified in the Azure provisioning API is ignored. +# __Provisioning.UseCloudInit__ +_Type: Boolean_ +_Default: n_ + +This options enables / disables support for provisioning by means of cloud-init. +When true ("y"), the agent will wait for cloud-init to complete before installing +extensions and processing the latest goal state. _Provisioning.Enabled_ must be +disabled ("n") for this option to have an effect. Setting _Provisioning.Enabled_ to +true ("y") overrides this option and runs the built-in agent provisioning code. + * __Provisioning.DeleteRootPassword__ _Type: Boolean_ _Default: n_ @@ -340,6 +353,15 @@ _Default: n_ If set, the agent will attempt to install and then load an RDMA kernel driver that matches the version of the firmware on the underlying hardware. +* __OS.EnableFIPS__ +_Type: Boolean_ +_Default: n_ + +If set, the agent will emit into the environment "OPENSSL_FIPS=1" when executing +OpenSSL commands. This signals OpenSSL to use any installed FIPS-compliant libraries. +Note that the agent itself has no FIPS-specific code. _If no FIPS-compliant are +installed, then enabling this option will cause all OpenSSL commands to fail._ + * __OS.RootDeviceScsiTimeout__ _Type: Integer_ _Default: 300_ @@ -354,6 +376,13 @@ _Default: None_ This can be used to specify an alternate path for the openssl binary to use for cryptographic operations. +* __OS.SshDir__ +_Type: String_ +_Default: "/etc/ssh"_ + +This option can be used to override the normal location of the SSH configuration +directory. + * __HttpProxy.Host, HttpProxy.Port__ _Type: String_ _Default: None_ diff --git a/azurelinuxagent/agent.py b/azurelinuxagent/agent.py index bd6dd20..90b4253 100644 --- a/azurelinuxagent/agent.py +++ b/azurelinuxagent/agent.py @@ -21,6 +21,8 @@ Module agent """ +from __future__ import print_function + import os import sys import re @@ -37,17 +39,21 @@ from azurelinuxagent.common.version import AGENT_NAME, AGENT_LONG_VERSION, \ from azurelinuxagent.common.osutil import get_osutil class Agent(object): - def __init__(self, verbose): + def __init__(self, verbose, conf_file_path=None): """ Initialize agent running environment. """ + self.conf_file_path = conf_file_path self.osutil = get_osutil() + #Init stdout log level = logger.LogLevel.VERBOSE if verbose else logger.LogLevel.INFO logger.add_logger_appender(logger.AppenderType.STDOUT, level) #Init config - conf_file_path = self.osutil.get_agent_conf_file_path() + conf_file_path = self.conf_file_path \ + if self.conf_file_path is not None \ + else self.osutil.get_agent_conf_file_path() conf.load_conf_from_file(conf_file_path) #Init log @@ -67,9 +73,13 @@ class Agent(object): """ Run agent daemon """ + child_args = None \ + if self.conf_file_path is None \ + else "-configuration-path:{0}".format(self.conf_file_path) + from azurelinuxagent.daemon import get_daemon_handler daemon_handler = get_daemon_handler() - daemon_handler.run() + daemon_handler.run(child_args=child_args) def provision(self): """ @@ -113,7 +123,7 @@ def main(args=[]): """ if len(args) <= 0: args = sys.argv[1:] - command, force, verbose = parse_args(args) + command, force, verbose, conf_file_path = parse_args(args) if command == "version": version() elif command == "help": @@ -122,7 +132,7 @@ def main(args=[]): start() else: try: - agent = Agent(verbose) + agent = Agent(verbose, conf_file_path=conf_file_path) if command == "deprovision+user": agent.deprovision(force, deluser=True) elif command == "deprovision": @@ -147,8 +157,18 @@ def parse_args(sys_args): cmd = "help" force = False verbose = False + conf_file_path = None for a in sys_args: - if re.match("^([-/]*)deprovision\\+user", a): + m = re.match("^(?:[-/]*)configuration-path:([\w/\.\-_]+)", a) + if not m is None: + conf_file_path = m.group(1) + if not os.path.exists(conf_file_path): + print("Error: Configuration file {0} does not exist".format( + conf_file_path), file=sys.stderr) + usage() + sys.exit(1) + + elif re.match("^([-/]*)deprovision\\+user", a): cmd = "deprovision+user" elif re.match("^([-/]*)deprovision", a): cmd = "deprovision" @@ -171,7 +191,7 @@ def parse_args(sys_args): else: cmd = "help" break - return cmd, force, verbose + return cmd, force, verbose, conf_file_path def version(): """ @@ -191,6 +211,7 @@ def usage(): """ print("") print((("usage: {0} [-verbose] [-force] [-help] " + "-configuration-path:<path to configuration file>" "-deprovision[+user]|-register-service|-version|-daemon|-start|" "-run-exthandlers]" "").format(sys.argv[0]))) diff --git a/azurelinuxagent/common/conf.py b/azurelinuxagent/common/conf.py index 7911699..5422784 100644 --- a/azurelinuxagent/common/conf.py +++ b/azurelinuxagent/common/conf.py @@ -21,13 +21,15 @@ Module conf loads and parses configuration file """ import os +import os.path + import azurelinuxagent.common.utils.fileutil as fileutil from azurelinuxagent.common.exception import AgentConfigError class ConfigurationProvider(object): """ - Parse amd store key:values in /etc/waagent.conf. + Parse and store key:values in /etc/waagent.conf. """ def __init__(self): @@ -38,12 +40,12 @@ class ConfigurationProvider(object): raise AgentConfigError("Can't not parse empty configuration") for line in content.split('\n'): if not line.startswith("#") and "=" in line: - parts = line.split()[0].split('=') + parts = line.split('=') + if len(parts) < 2: + continue + key = parts[0].strip() value = parts[1].strip("\" ") - if value != "None": - self.values[parts[0]] = value - else: - self.values[parts[0]] = None + self.values[key] = value if value != "None" else None def get(self, key, default_val): val = self.values.get(key) @@ -113,38 +115,49 @@ def get_agent_pid_file_path(conf=__conf__): def get_ext_log_dir(conf=__conf__): return conf.get("Extension.LogDir", "/var/log/azure") +def get_fips_enabled(conf=__conf__): + return conf.get_switch("OS.EnableFIPS", False) def get_openssl_cmd(conf=__conf__): return conf.get("OS.OpensslPath", "/usr/bin/openssl") +def get_ssh_dir(conf=__conf__): + return conf.get("OS.SshDir", "/etc/ssh") def get_home_dir(conf=__conf__): return conf.get("OS.HomeDir", "/home") - def get_passwd_file_path(conf=__conf__): return conf.get("OS.PasswordPath", "/etc/shadow") - def get_sudoers_dir(conf=__conf__): return conf.get("OS.SudoersDir", "/etc/sudoers.d") - def get_sshd_conf_file_path(conf=__conf__): - return conf.get("OS.SshdConfigPath", "/etc/ssh/sshd_config") + return os.path.join(get_ssh_dir(conf), "sshd_config") +def get_ssh_key_glob(conf=__conf__): + return os.path.join(get_ssh_dir(conf), 'ssh_host_*key*') + +def get_ssh_key_private_path(conf=__conf__): + return os.path.join(get_ssh_dir(conf), + 'ssh_host_{0}_key'.format(get_ssh_host_keypair_type(conf))) + +def get_ssh_key_public_path(conf=__conf__): + return os.path.join(get_ssh_dir(conf), + 'ssh_host_{0}_key.pub'.format(get_ssh_host_keypair_type(conf))) def get_root_device_scsi_timeout(conf=__conf__): return conf.get("OS.RootDeviceScsiTimeout", None) - def get_ssh_host_keypair_type(conf=__conf__): return conf.get("Provisioning.SshHostKeyPairType", "rsa") - def get_provision_enabled(conf=__conf__): return conf.get_switch("Provisioning.Enabled", True) +def get_provision_cloudinit(conf=__conf__): + return conf.get_switch("Provisioning.UseCloudInit", False) def get_allow_reset_sys_user(conf=__conf__): return conf.get_switch("Provisioning.AllowResetSysUser", False) diff --git a/azurelinuxagent/common/dhcp.py b/azurelinuxagent/common/dhcp.py index 66346b5..6087643 100644 --- a/azurelinuxagent/common/dhcp.py +++ b/azurelinuxagent/common/dhcp.py @@ -130,7 +130,7 @@ class DhcpHandler(object): logger.info("Gateway:{0}", self.gateway) logger.info("Routes:{0}", self.routes) # Add default gateway - if self.gateway is not None: + if self.gateway is not None and self.osutil.is_missing_default_route(): self.osutil.route_add(0, 0, self.gateway) if self.routes is not None: for route in self.routes: diff --git a/azurelinuxagent/common/event.py b/azurelinuxagent/common/event.py index ce79adf..116478b 100644 --- a/azurelinuxagent/common/event.py +++ b/azurelinuxagent/common/event.py @@ -24,7 +24,11 @@ import time import datetime import threading import platform + +from datetime import datetime + import azurelinuxagent.common.logger as logger + from azurelinuxagent.common.exception import EventError, ProtocolError from azurelinuxagent.common.future import ustr from azurelinuxagent.common.protocol.restapi import TelemetryEventParam, \ @@ -45,6 +49,7 @@ class WALAEventOperation: HeartBeat = "HeartBeat" Install = "Install" InitializeHostPlugin = "InitializeHostPlugin" + ProcessGoalState = "ProcessGoalState" Provision = "Provision" ReportStatus = "ReportStatus" Restart = "Restart" @@ -111,6 +116,11 @@ class EventLogger(object): __event_logger__ = EventLogger() +def elapsed_milliseconds(utc_start): + d = datetime.utcnow() - utc_start + return int(((d.days * 24 * 60 * 60 + d.seconds) * 1000) + \ + (d.microseconds / 1000.0)) + def report_event(op, is_success=True, message=''): from azurelinuxagent.common.version import AGENT_NAME, CURRENT_VERSION add_event(AGENT_NAME, @@ -121,10 +131,11 @@ def report_event(op, is_success=True, message=''): def add_event(name, op="", is_success=True, duration=0, version=CURRENT_VERSION, - message="", evt_type="", is_internal=False, + message="", evt_type="", is_internal=False, log_event=True, reporter=__event_logger__): - log = logger.info if is_success else logger.error - log("Event: name={0}, op={1}, message={2}", name, op, message) + if log_event or not is_success: + log = logger.info if is_success else logger.error + log("Event: name={0}, op={1}, message={2}", name, op, message) if reporter.event_dir is None: logger.warn("Event reporter is not initialized.") diff --git a/azurelinuxagent/common/exception.py b/azurelinuxagent/common/exception.py index 457490c..7a0c75e 100644 --- a/azurelinuxagent/common/exception.py +++ b/azurelinuxagent/common/exception.py @@ -20,104 +20,131 @@ Defines all exceptions """ + class AgentError(Exception): """ Base class of agent error. """ + def __init__(self, errno, msg, inner=None): - msg = u"({0}){1}".format(errno, msg) + msg = u"[{0}] {1}".format(errno, msg) if inner is not None: - msg = u"{0} \n inner error: {1}".format(msg, inner) + msg = u"{0}\nInner error: {1}".format(msg, inner) super(AgentError, self).__init__(msg) + class AgentConfigError(AgentError): """ When configure file is not found or malformed. """ + def __init__(self, msg=None, inner=None): super(AgentConfigError, self).__init__('000001', msg, inner) + class AgentNetworkError(AgentError): """ When network is not avaiable. """ + def __init__(self, msg=None, inner=None): super(AgentNetworkError, self).__init__('000002', msg, inner) + class ExtensionError(AgentError): """ When failed to execute an extension """ + def __init__(self, msg=None, inner=None): super(ExtensionError, self).__init__('000003', msg, inner) + class ProvisionError(AgentError): """ When provision failed """ + def __init__(self, msg=None, inner=None): super(ProvisionError, self).__init__('000004', msg, inner) + class ResourceDiskError(AgentError): """ Mount resource disk failed """ + def __init__(self, msg=None, inner=None): super(ResourceDiskError, self).__init__('000005', msg, inner) + class DhcpError(AgentError): """ Failed to handle dhcp response """ + def __init__(self, msg=None, inner=None): super(DhcpError, self).__init__('000006', msg, inner) + class OSUtilError(AgentError): """ Failed to perform operation to OS configuration """ + def __init__(self, msg=None, inner=None): super(OSUtilError, self).__init__('000007', msg, inner) + class ProtocolError(AgentError): """ Azure protocol error """ + def __init__(self, msg=None, inner=None): super(ProtocolError, self).__init__('000008', msg, inner) + class ProtocolNotFoundError(ProtocolError): """ Azure protocol endpoint not found """ + def __init__(self, msg=None, inner=None): super(ProtocolNotFoundError, self).__init__(msg, inner) + class HttpError(AgentError): """ Http request failure """ + def __init__(self, msg=None, inner=None): super(HttpError, self).__init__('000009', msg, inner) + class EventError(AgentError): """ Event reporting error """ + def __init__(self, msg=None, inner=None): super(EventError, self).__init__('000010', msg, inner) + class CryptError(AgentError): """ Encrypt/Decrypt error """ + def __init__(self, msg=None, inner=None): super(CryptError, self).__init__('000011', msg, inner) + class UpdateError(AgentError): """ Update Guest Agent error """ + def __init__(self, msg=None, inner=None): super(UpdateError, self).__init__('000012', msg, inner) - diff --git a/azurelinuxagent/common/osutil/arch.py b/azurelinuxagent/common/osutil/arch.py new file mode 100644 index 0000000..83d3b47 --- /dev/null +++ b/azurelinuxagent/common/osutil/arch.py @@ -0,0 +1,55 @@ +# +# Copyright 2014 Microsoft Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Requires Python 2.4+ and Openssl 1.0+ +# + +import os +import azurelinuxagent.common.utils.shellutil as shellutil +from azurelinuxagent.common.osutil.default import DefaultOSUtil + +class ArchUtil(DefaultOSUtil): + def is_dhcp_enabled(self): + return True + + def start_network(self): + return shellutil.run("systemctl start systemd-networkd", chk_err=False) + + def restart_if(self, iface): + shellutil.run("systemctl restart systemd-networkd") + + def restart_ssh_service(self): + # SSH is socket activated on CoreOS. No need to restart it. + pass + + def stop_dhcp_service(self): + return shellutil.run("systemctl stop systemd-networkd", chk_err=False) + + def start_dhcp_service(self): + return shellutil.run("systemctl start systemd-networkd", chk_err=False) + + def start_agent_service(self): + return shellutil.run("systemctl start waagent", chk_err=False) + + def stop_agent_service(self): + return shellutil.run("systemctl stop waagent", chk_err=False) + + def get_dhcp_pid(self): + ret= shellutil.run_get_output("pidof systemd-networkd") + return ret[1] if ret[0] == 0 else None + + def conf_sshd(self, disable_password): + # Don't whack the system default sshd conf + pass
\ No newline at end of file diff --git a/azurelinuxagent/common/osutil/default.py b/azurelinuxagent/common/osutil/default.py index 59d5985..20dc1f3 100644 --- a/azurelinuxagent/common/osutil/default.py +++ b/azurelinuxagent/common/osutil/default.py @@ -30,13 +30,15 @@ import fcntl import base64 import glob import datetime + import azurelinuxagent.common.logger as logger import azurelinuxagent.common.conf as conf -from azurelinuxagent.common.exception import OSUtilError -from azurelinuxagent.common.future import ustr import azurelinuxagent.common.utils.fileutil as fileutil import azurelinuxagent.common.utils.shellutil as shellutil import azurelinuxagent.common.utils.textutil as textutil + +from azurelinuxagent.common.exception import OSUtilError +from azurelinuxagent.common.future import ustr from azurelinuxagent.common.utils.cryptutil import CryptUtil __RULES_FILES__ = [ "/lib/udev/rules.d/75-persistent-net-generator.rules", @@ -48,6 +50,12 @@ for all distros. Each concrete distro classes could overwrite default behavior if needed. """ +DMIDECODE_CMD = 'dmidecode --string system-uuid' +PRODUCT_ID_FILE = '/sys/class/dmi/id/product_uuid' +UUID_PATTERN = re.compile( + '^\s*[A-F0-9]{8}(?:\-[A-F0-9]{4}){3}\-[A-F0-9]{12}\s*$', + re.IGNORECASE) + class DefaultOSUtil(object): def __init__(self): @@ -58,6 +66,22 @@ class DefaultOSUtil(object): def get_agent_conf_file_path(self): return self.agent_conf_file_path + def get_instance_id(self): + ''' + Azure records a UUID as the instance ID + First check /sys/class/dmi/id/product_uuid. + If that is missing, then extracts from dmidecode + If nothing works (for old VMs), return the empty string + ''' + if os.path.isfile(PRODUCT_ID_FILE): + return fileutil.read_file(PRODUCT_ID_FILE).strip() + + rc, s = shellutil.run_get_output(DMIDECODE_CMD) + if rc != 0 or UUID_PATTERN.match(s) is None: + return "" + + return s.strip() + def get_userentry(self, username): try: return pwd.getpwnam(username) @@ -110,8 +134,8 @@ class DefaultOSUtil(object): def chpasswd(self, username, password, crypt_id=6, salt_len=10): if self.is_sys_user(username): - raise OSUtilError(("User {0} is a system user. " - "Will not set passwd.").format(username)) + raise OSUtilError(("User {0} is a system user, " + "will not set password.").format(username)) passwd_hash = textutil.gen_password_hash(password, crypt_id, salt_len) cmd = "usermod -p '{0}' {1}".format(passwd_hash, username) ret, output = shellutil.run_get_output(cmd, log_cmd=False) @@ -273,46 +297,65 @@ class DefaultOSUtil(object): .format("Disabled" if disable_password else "Enabled")) logger.info("Configured SSH client probing to keep connections alive.") - def get_dvd_device(self, dev_dir='/dev'): - pattern=r'(sr[0-9]|hd[c-z]|cdrom[0-9]|cd[0-9])' - for dvd in [re.match(pattern, dev) for dev in os.listdir(dev_dir)]: + pattern = r'(sr[0-9]|hd[c-z]|cdrom[0-9]|cd[0-9])' + device_list = os.listdir(dev_dir) + for dvd in [re.match(pattern, dev) for dev in device_list]: if dvd is not None: return "/dev/{0}".format(dvd.group(0)) - raise OSUtilError("Failed to get dvd device") - - def mount_dvd(self, max_retry=6, chk_err=True, dvd_device=None, mount_point=None): + inner_detail = "The following devices were found, but none matched " \ + "the pattern [{0}]: {1}\n".format(pattern, device_list) + raise OSUtilError(msg="Failed to get dvd device from {0}".format(dev_dir), + inner=inner_detail) + + def mount_dvd(self, + max_retry=6, + chk_err=True, + dvd_device=None, + mount_point=None, + sleep_time=5): if dvd_device is None: dvd_device = self.get_dvd_device() if mount_point is None: mount_point = conf.get_dvd_mount_point() - mountlist = shellutil.run_get_output("mount")[1] - existing = self.get_mount_point(mountlist, dvd_device) - if existing is not None: #Already mounted + mount_list = shellutil.run_get_output("mount")[1] + existing = self.get_mount_point(mount_list, dvd_device) + + if existing is not None: + # already mounted logger.info("{0} is already mounted at {1}", dvd_device, existing) return + if not os.path.isdir(mount_point): os.makedirs(mount_point) - for retry in range(0, max_retry): - retcode = self.mount(dvd_device, mount_point, option="-o ro -t udf,iso9660", - chk_err=chk_err) - if retcode == 0: + err = '' + for retry in range(1, max_retry): + return_code, err = self.mount(dvd_device, + mount_point, + option="-o ro -t udf,iso9660", + chk_err=chk_err) + if return_code == 0: logger.info("Successfully mounted dvd") return - if retry < max_retry - 1: - logger.warn("Mount dvd failed: retry={0}, ret={1}", retry, - retcode) - time.sleep(5) + else: + logger.warn( + "Mounting dvd failed [retry {0}/{1}, sleeping {2} sec]", + retry, + max_retry - 1, + sleep_time) + if retry < max_retry: + time.sleep(sleep_time) if chk_err: - raise OSUtilError("Failed to mount dvd.") + raise OSUtilError("Failed to mount dvd device", inner=err) def umount_dvd(self, chk_err=True, mount_point=None): if mount_point is None: mount_point = conf.get_dvd_mount_point() - retcode = self.umount(mount_point, chk_err=chk_err) - if chk_err and retcode != 0: - raise OSUtilError("Failed to umount dvd.") + return_code = self.umount(mount_point, chk_err=chk_err) + if chk_err and return_code != 0: + raise OSUtilError("Failed to unmount dvd device at {0}", + mount_point) def eject_dvd(self, chk_err=True): dvd = self.get_dvd_device() @@ -356,7 +399,11 @@ class DefaultOSUtil(object): def mount(self, dvd, mount_point, option="", chk_err=True): cmd = "mount {0} {1} {2}".format(option, dvd, mount_point) - return shellutil.run_get_output(cmd, chk_err)[0] + retcode, err = shellutil.run_get_output(cmd, chk_err) + if retcode != 0: + detail = "[{0}] returned {1}: {2}".format(cmd, retcode, err) + err = detail + return retcode, err def umount(self, mount_point, chk_err=True): return shellutil.run("umount {0}".format(mount_point), chk_err=chk_err) diff --git a/azurelinuxagent/common/osutil/factory.py b/azurelinuxagent/common/osutil/factory.py index eee9f97..3447651 100644 --- a/azurelinuxagent/common/osutil/factory.py +++ b/azurelinuxagent/common/osutil/factory.py @@ -19,6 +19,7 @@ import azurelinuxagent.common.logger as logger from azurelinuxagent.common.utils.textutil import Version from azurelinuxagent.common.version import * from .default import DefaultOSUtil +from .arch import ArchUtil from .clearlinux import ClearLinuxUtil from .coreos import CoreOSUtil from .debian import DebianOSUtil @@ -28,6 +29,7 @@ from .suse import SUSEOSUtil, SUSE11OSUtil from .ubuntu import UbuntuOSUtil, Ubuntu12OSUtil, Ubuntu14OSUtil, UbuntuSnappyOSUtil from .alpine import AlpineOSUtil from .bigip import BigIpOSUtil +from .gaia import GaiaOSUtil def get_osutil(distro_name=DISTRO_NAME, @@ -35,6 +37,9 @@ def get_osutil(distro_name=DISTRO_NAME, distro_version=DISTRO_VERSION, distro_full_name=DISTRO_FULL_NAME): + if distro_name == "arch": + return ArchUtil() + if distro_name == "clear linux software for intel architecture": return ClearLinuxUtil() @@ -85,6 +90,9 @@ def get_osutil(distro_name=DISTRO_NAME, elif distro_name == "bigip": return BigIpOSUtil() + elif distro_name == "gaia": + return GaiaOSUtil() + else: logger.warn("Unable to load distro implementation for {0}. Using " "default distro implementation instead.", diff --git a/azurelinuxagent/common/osutil/freebsd.py b/azurelinuxagent/common/osutil/freebsd.py index d0c40b9..0f465a9 100644 --- a/azurelinuxagent/common/osutil/freebsd.py +++ b/azurelinuxagent/common/osutil/freebsd.py @@ -67,8 +67,8 @@ class FreeBSDOSUtil(DefaultOSUtil): def chpasswd(self, username, password, crypt_id=6, salt_len=10): if self.is_sys_user(username): - raise OSUtilError(("User {0} is a system user. " - "Will not set passwd.").format(username)) + raise OSUtilError(("User {0} is a system user, " + "will not set password.").format(username)) passwd_hash = textutil.gen_password_hash(password, crypt_id, salt_len) cmd = "echo '{0}'|pw usermod {1} -H 0 ".format(passwd_hash, username) ret, output = shellutil.run_get_output(cmd, log_cmd=False) diff --git a/azurelinuxagent/common/osutil/gaia.py b/azurelinuxagent/common/osutil/gaia.py new file mode 100644 index 0000000..a1069d3 --- /dev/null +++ b/azurelinuxagent/common/osutil/gaia.py @@ -0,0 +1,151 @@ +# +# Copyright 2017 Check Point Software Technologies +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Requires Python 2.4+ and Openssl 1.0+ +# + +import socket +import struct +import time + +import azurelinuxagent.common.logger as logger +from azurelinuxagent.common.exception import OSUtilError +import azurelinuxagent.common.utils.shellutil as shellutil +import azurelinuxagent.common.utils.textutil as textutil +from azurelinuxagent.common.osutil.default import DefaultOSUtil + + +class GaiaOSUtil(DefaultOSUtil): + def __init__(self): + super(GaiaOSUtil, self).__init__() + + def _run_clish(self, cmd, log_cmd=True): + for i in xrange(10): + ret, out = shellutil.run_get_output( + "/bin/clish -s -c '" + cmd + "'", log_cmd=log_cmd) + if not ret: + break + if 'NMSHST0025' in out: # Entry for [hostname] already present + ret = 0 + break + time.sleep(2) + return ret, out + + def useradd(self, username, expiration=None): + logger.warn('useradd is not supported on GAiA') + + def chpasswd(self, username, password, crypt_id=6, salt_len=10): + logger.info('chpasswd') + passwd_hash = textutil.gen_password_hash(password, crypt_id, salt_len) + ret, out = self._run_clish( + 'set user admin password-hash ' + passwd_hash, log_cmd=False) + if ret != 0: + raise OSUtilError(("Failed to set password for {0}: {1}" + "").format('admin', out)) + + def conf_sudoer(self, username, nopasswd=False, remove=False): + logger.info('conf_sudoer is not supported on GAiA') + + def del_root_password(self): + logger.info('del_root_password') + ret, out = self._run_clish('set user admin password-hash *LOCK*') + if ret != 0: + raise OSUtilError("Failed to delete root password") + + def _replace_user(path, username): + parts = path.split('/') + for i in xrange(len(parts)): + if parts[i] == '$HOME': + parts[i + 1] = username + break + return '/'.join(parts) + + def deploy_ssh_keypair(self, username, keypair): + logger.info('deploy_ssh_keypair') + username = 'admin' + path, thumbprint = keypair + path = self._replace_user(path, username) + super(GaiaOSUtil, self).deploy_ssh_keypair( + username, (path, thumbprint)) + + def deploy_ssh_pubkey(self, username, pubkey): + logger.info('deploy_ssh_pubkey') + username = 'admin' + path, thumbprint, value = pubkey + path = self._replace_user(path, username) + super(GaiaOSUtil, self).deploy_ssh_pubkey( + 'admin', (path, thumbprint, value)) + + def eject_dvd(self, chk_err=True): + logger.warn('eject is not supported on GAiA') + + def mount(self, dvd, mount_point, option="", chk_err=True): + logger.info('mount {0} {1} {2}', dvd, mount_point, option) + if 'udf,iso9660' in option: + ret, out = super(GaiaOSUtil, self).mount( + dvd, mount_point, option=option.replace('udf,iso9660', 'udf'), + chk_err=chk_err) + if not ret: + return ret, out + return super(GaiaOSUtil, self).mount( + dvd, mount_point, option=option, chk_err=chk_err) + + def allow_dhcp_broadcast(self): + logger.info('allow_dhcp_broadcast is ignored on GAiA') + + def remove_rules_files(self, rules_files=''): + pass + + def restore_rules_files(self, rules_files=''): + logger.info('restore_rules_files is ignored on GAiA') + + def restart_ssh_service(self): + return shellutil.run('/sbin/service sshd condrestart', chk_err=False) + + def _address_to_string(addr): + return socket.inet_ntoa(struct.pack("!I", addr)) + + def _get_prefix(self, mask): + return str(sum([bin(int(x)).count('1') for x in mask.split('.')])) + + def route_add(self, net, mask, gateway): + logger.info('route_add {0} {1} {2}', net, mask, gateway) + + if net == 0 and mask == 0: + cidr = 'default' + else: + cidr = self._address_to_string(net) + '/' + self._get_prefix( + self._address_to_string(mask)) + + ret, out = self._run_clish( + 'set static-route ' + cidr + + ' nexthop gateway address ' + + self._address_to_string(gateway) + ' on') + return ret + + def set_hostname(self, hostname): + logger.warn('set_hostname is ignored on GAiA') + + def set_dhcp_hostname(self, hostname): + logger.warn('set_dhcp_hostname is ignored on GAiA') + + def publish_hostname(self, hostname): + logger.warn('publish_hostname is ignored on GAiA') + + def del_account(self, username): + logger.warn('del_account is ignored on GAiA') + + def set_admin_access_to_ip(self, dest_ip): + logger.warn('set_admin_access_to_ip is ignored on GAiA') diff --git a/azurelinuxagent/common/osutil/redhat.py b/azurelinuxagent/common/osutil/redhat.py index 5254ea5..b94610c 100644 --- a/azurelinuxagent/common/osutil/redhat.py +++ b/azurelinuxagent/common/osutil/redhat.py @@ -103,7 +103,7 @@ class RedhatOSUtil(Redhat6xOSUtil): Due to a bug in systemd in Centos-7.0, if this call fails, fallback to hostname. """ - hostnamectl_cmd = "hostnamectl set-hostname {0}".format(hostname) + hostnamectl_cmd = "hostnamectl set-hostname {0} --static".format(hostname) if shellutil.run(hostnamectl_cmd, chk_err=False) != 0: logger.warn("[{0}] failed, attempting fallback".format(hostnamectl_cmd)) DefaultOSUtil.set_hostname(self, hostname) diff --git a/azurelinuxagent/common/protocol/hostplugin.py b/azurelinuxagent/common/protocol/hostplugin.py index 70bf8b4..464fd35 100644 --- a/azurelinuxagent/common/protocol/hostplugin.py +++ b/azurelinuxagent/common/protocol/hostplugin.py @@ -19,6 +19,7 @@ import base64 import json +import traceback from azurelinuxagent.common import logger from azurelinuxagent.common.exception import ProtocolError, HttpError @@ -177,8 +178,7 @@ class HostPluginProtocol(object): logger.info("HostGAPlugin: Setting host plugin as default channel") HostPluginProtocol.set_default_channel(True) except Exception as e: - message = "HostGAPlugin: Exception Put VM status: {0}".format(e) - logger.error(message) + message = "HostGAPlugin: Exception Put VM status: {0}, {1}".format(e, traceback.format_exc()) from azurelinuxagent.common.event import WALAEventOperation, report_event report_event(op=WALAEventOperation.ReportStatus, is_success=False, diff --git a/azurelinuxagent/common/protocol/metadata.py b/azurelinuxagent/common/protocol/metadata.py index c61e373..c50b3dd 100644 --- a/azurelinuxagent/common/protocol/metadata.py +++ b/azurelinuxagent/common/protocol/metadata.py @@ -21,17 +21,19 @@ import json import os import shutil import re + import azurelinuxagent.common.conf as conf import azurelinuxagent.common.utils.fileutil as fileutil import azurelinuxagent.common.utils.shellutil as shellutil import azurelinuxagent.common.utils.textutil as textutil + from azurelinuxagent.common.future import httpclient from azurelinuxagent.common.protocol.restapi import * from azurelinuxagent.common.utils.cryptutil import CryptUtil METADATA_ENDPOINT = '169.254.169.254' APIVERSION = '2015-05-01-preview' -BASE_URI = "http://{0}/Microsoft.Compute/{1}?api-version={2}{3}" +BASE_URI = "http://{0}/Microsoft.Compute/{1}?api-version={2}" TRANSPORT_PRV_FILE_NAME = "V2TransportPrivate.pem" TRANSPORT_CERT_FILE_NAME = "V2TransportCert.pem" @@ -39,6 +41,9 @@ P7M_FILE_NAME = "Certificates.p7m" P7B_FILE_NAME = "Certificates.p7b" PEM_FILE_NAME = "Certificates.pem" +KEY_AGENT_VERSION_URIS = "versionsManifestUris" +KEY_URI = "uri" + # TODO remote workaround for azure stack MAX_PING = 30 RETRY_PING_INTERVAL = 10 @@ -56,13 +61,13 @@ class MetadataProtocol(Protocol): self.apiversion = apiversion self.endpoint = endpoint self.identity_uri = BASE_URI.format(self.endpoint, "identity", - self.apiversion, "&$expand=*") + self.apiversion) self.cert_uri = BASE_URI.format(self.endpoint, "certificates", - self.apiversion, "&$expand=*") + self.apiversion) self.ext_uri = BASE_URI.format(self.endpoint, "extensionHandlers", - self.apiversion, "&$expand=*") + self.apiversion) self.vmagent_uri = BASE_URI.format(self.endpoint, "vmAgentVersions", - self.apiversion, "&$expand=*") + self.apiversion) self.provision_status_uri = BASE_URI.format(self.endpoint, "provisioningStatus", self.apiversion, "") @@ -74,6 +79,8 @@ class MetadataProtocol(Protocol): self.event_uri = BASE_URI.format(self.endpoint, "status/telemetry", self.apiversion, "") self.certs = None + self.agent_manifests = None + self.agent_etag = None def _get_data(self, url, headers=None): try: @@ -166,32 +173,55 @@ class MetadataProtocol(Protocol): return None return self.certs - def get_vmagent_manifests(self, last_etag=None): - manifests = VMAgentManifestList() + def get_vmagent_manifests(self): self.update_goal_state() + data, etag = self._get_data(self.vmagent_uri) - if last_etag is None or last_etag < etag: - set_properties("vmAgentManifests", - manifests.vmAgentManifests, - data) - return manifests, etag + if self.agent_etag is None or self.agent_etag < etag: + self.agent_etag = etag + + # Create a list with a single manifest + # -- The protocol lacks "family," use the configured family + self.agent_manifests = VMAgentManifestList() + + manifest = VMAgentManifest() + manifest.family = family=conf.get_autoupdate_gafamily() + + if not KEY_AGENT_VERSION_URIS in data: + raise ProtocolError( + "Agent versions missing '{0}': {1}".format( + KEY_AGENT_VERSION_URIS, data)) + + for version in data[KEY_AGENT_VERSION_URIS]: + if not KEY_URI in version: + raise ProtocolError( + "Agent versions missing '{0': {1}".format( + KEY_URI, data)) + manifest_uri = VMAgentManifestUri(uri=version[KEY_URI]) + manifest.versionsManifestUris.append(manifest_uri) + + self.agent_manifests.vmAgentManifests.append(manifest) + + return self.agent_manifests, self.agent_etag def get_vmagent_pkgs(self, vmagent_manifest): - # Agent package is the same with extension handler - vmagent_pkgs = ExtHandlerPackageList() data = None + etag = None for manifest_uri in vmagent_manifest.versionsManifestUris: try: - data = self._get_data(manifest_uri.uri) + data, etag = self._get_data(manifest_uri.uri) break except ProtocolError as e: - logger.warn("Failed to get vmagent versions: {0}", e) - logger.info("Retry getting vmagent versions") + logger.verbose( + "Error retrieving agent package from {0}: {1}".format( + manifest_uri, e)) + if data is None: - raise ProtocolError(("Failed to get versions for vm agent: {0}" - "").format(vmagent_manifest.family)) + raise ProtocolError( + "Failed retrieving agent package from all URIs") + + vmagent_pkgs = ExtHandlerPackageList() set_properties("vmAgentVersions", vmagent_pkgs, data) - # TODO: What etag should this return? return vmagent_pkgs def get_ext_handlers(self, last_etag=None): @@ -251,11 +281,9 @@ class MetadataProtocol(Protocol): self._put_data(uri, data) def report_event(self, events): - # TODO disable telemetry for azure stack test - # validate_param('events', events, TelemetryEventList) - # data = get_properties(events) - # self._post_data(self.event_uri, data) - pass + validate_param('events', events, TelemetryEventList) + data = get_properties(events) + self._post_data(self.event_uri, data) def update_certs(self): certificates = self.get_certs() diff --git a/azurelinuxagent/common/protocol/ovfenv.py b/azurelinuxagent/common/protocol/ovfenv.py index 4901871..3122e3b 100644 --- a/azurelinuxagent/common/protocol/ovfenv.py +++ b/azurelinuxagent/common/protocol/ovfenv.py @@ -35,7 +35,7 @@ WA_NAME_SPACE = "http://schemas.microsoft.com/windowsazure" def _validate_ovf(val, msg): if val is None: - raise ProtocolError("Failed to parse OVF XML: {0}".format(msg)) + raise ProtocolError("Failed to validate OVF: {0}".format(msg)) class OvfEnv(object): """ diff --git a/azurelinuxagent/common/protocol/util.py b/azurelinuxagent/common/protocol/util.py index 7e7a74f..0ba03ec 100644 --- a/azurelinuxagent/common/protocol/util.py +++ b/azurelinuxagent/common/protocol/util.py @@ -33,25 +33,21 @@ from azurelinuxagent.common.protocol.ovfenv import OvfEnv from azurelinuxagent.common.protocol.wire import WireProtocol from azurelinuxagent.common.protocol.metadata import MetadataProtocol, \ METADATA_ENDPOINT -import azurelinuxagent.common.utils.shellutil as shellutil OVF_FILE_NAME = "ovf-env.xml" - -#Tag file to indicate usage of metadata protocol -TAG_FILE_NAME = "useMetadataEndpoint.tag" - +TAG_FILE_NAME = "useMetadataEndpoint.tag" PROTOCOL_FILE_NAME = "Protocol" - -#MAX retry times for protocol probing MAX_RETRY = 360 - PROBE_INTERVAL = 10 - ENDPOINT_FILE_NAME = "WireServerEndpoint" +PASSWORD_PATTERN = "<UserPassword>.*?<" +PASSWORD_REPLACEMENT = "<UserPassword>*<" + def get_protocol_util(): return ProtocolUtil() + class ProtocolUtil(object): """ ProtocolUtil handles initialization for protocol instance. 2 protocol types @@ -71,21 +67,42 @@ class ProtocolUtil(object): dvd_mount_point = conf.get_dvd_mount_point() ovf_file_path_on_dvd = os.path.join(dvd_mount_point, OVF_FILE_NAME) tag_file_path_on_dvd = os.path.join(dvd_mount_point, TAG_FILE_NAME) + ovf_file_path = os.path.join(conf.get_lib_dir(), OVF_FILE_NAME) + tag_file_path = os.path.join(conf.get_lib_dir(), TAG_FILE_NAME) + try: self.osutil.mount_dvd() + except OSUtilError as e: + raise ProtocolError("[CopyOvfEnv] Error mounting dvd: " + "{0}".format(ustr(e))) + + try: ovfxml = fileutil.read_file(ovf_file_path_on_dvd, remove_bom=True) ovfenv = OvfEnv(ovfxml) - ovfxml = re.sub("<UserPassword>.*?<", "<UserPassword>*<", ovfxml) - ovf_file_path = os.path.join(conf.get_lib_dir(), OVF_FILE_NAME) + except IOError as e: + raise ProtocolError("[CopyOvfEnv] Error reading file " + "{0}: {1}".format(ovf_file_path_on_dvd, + ustr(e))) + + try: + ovfxml = re.sub(PASSWORD_PATTERN, + PASSWORD_REPLACEMENT, + ovfxml) fileutil.write_file(ovf_file_path, ovfxml) - + except IOError as e: + raise ProtocolError("[CopyOvfEnv] Error writing file " + "{0}: {1}".format(ovf_file_path, + ustr(e))) + + try: if os.path.isfile(tag_file_path_on_dvd): logger.info("Found {0} in provisioning ISO", TAG_FILE_NAME) - tag_file_path = os.path.join(conf.get_lib_dir(), TAG_FILE_NAME) - shutil.copyfile(tag_file_path_on_dvd, tag_file_path) - - except (OSUtilError, IOError) as e: - raise ProtocolError(ustr(e)) + shutil.copyfile(tag_file_path_on_dvd, tag_file_path) + except IOError as e: + raise ProtocolError("[CopyOvfEnv] Error copying file " + "{0} to {1}: {2}".format(tag_file_path, + tag_file_path, + ustr(e))) try: self.osutil.umount_dvd() @@ -104,7 +121,7 @@ class ProtocolUtil(object): xml_text = fileutil.read_file(ovf_file_path) return OvfEnv(xml_text) else: - raise ProtocolError("ovf-env.xml is missing.") + raise ProtocolError("ovf-env.xml is missing from {0}".format(ovf_file_path)) def _get_wireserver_endpoint(self): try: @@ -146,7 +163,7 @@ class ProtocolUtil(object): protocol = MetadataProtocol() protocol.detect() - #Only allow root access METADATA_ENDPOINT + # only allow root access METADATA_ENDPOINT self.osutil.set_admin_access_to_ip(METADATA_ENDPOINT) self.save_protocol("MetadataProtocol") @@ -206,7 +223,6 @@ class ProtocolUtil(object): except IOError as e: logger.error("Failed to save protocol endpoint: {0}", e) - def clear_protocol(self): """ Cleanup previous saved endpoint. @@ -249,7 +265,6 @@ class ProtocolUtil(object): finally: self.lock.release() - def get_protocol_by_file(self): """ Detect protocol by tag file. diff --git a/azurelinuxagent/common/protocol/wire.py b/azurelinuxagent/common/protocol/wire.py index 265e2dd..936be8c 100644 --- a/azurelinuxagent/common/protocol/wire.py +++ b/azurelinuxagent/common/protocol/wire.py @@ -21,16 +21,18 @@ import os import re import time import xml.sax.saxutils as saxutils + import azurelinuxagent.common.conf as conf +import azurelinuxagent.common.utils.fileutil as fileutil +import azurelinuxagent.common.utils.textutil as textutil + from azurelinuxagent.common.exception import ProtocolNotFoundError from azurelinuxagent.common.future import httpclient, bytebuffer +from azurelinuxagent.common.protocol.hostplugin import HostPluginProtocol +from azurelinuxagent.common.protocol.restapi import * +from azurelinuxagent.common.utils.cryptutil import CryptUtil from azurelinuxagent.common.utils.textutil import parse_doc, findall, find, \ findtext, getattrib, gettext, remove_bom, get_bytes_from_pem, parse_json -import azurelinuxagent.common.utils.fileutil as fileutil -import azurelinuxagent.common.utils.textutil as textutil -from azurelinuxagent.common.utils.cryptutil import CryptUtil -from azurelinuxagent.common.protocol.restapi import * -from azurelinuxagent.common.protocol.hostplugin import HostPluginProtocol VERSION_INFO_URI = "http://{0}/?comp=versions" GOAL_STATE_URI = "http://{0}/machine/?comp=goalstate" @@ -376,48 +378,20 @@ class StatusBlob(object): self.type = blob_type def upload(self, url): - # TODO upload extension only if content has changed - upload_successful = False - self.type = self.get_blob_type(url) try: + if not self.type in ["BlockBlob", "PageBlob"]: + raise ProtocolError("Illegal blob type: {0}".format(self.type)) + if self.type == "BlockBlob": self.put_block_blob(url, self.data) - elif self.type == "PageBlob": - self.put_page_blob(url, self.data) else: - raise ProtocolError("Unknown blob type: {0}".format(self.type)) - except HttpError as e: - message = "Initial upload failed [{0}]".format(e) - logger.warn(message) - from azurelinuxagent.common.event import WALAEventOperation, report_event - report_event(op=WALAEventOperation.ReportStatus, - is_success=False, - message=message) - else: - logger.verbose("Uploading status blob succeeded") - upload_successful = True - return upload_successful - - def get_blob_type(self, url): - logger.verbose("Get blob type") - timestamp = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()) - try: - resp = self.client.call_storage_service( - restutil.http_head, - url, - { - "x-ms-date": timestamp, - "x-ms-version": self.__class__.__storage_version__ - }) - except HttpError as e: - raise ProtocolError("Failed to get status blob type: {0}", e) + self.put_page_blob(url, self.data) + return True - if resp is None or resp.status != httpclient.OK: - raise ProtocolError("Failed to get status blob type") + except Exception as e: + logger.verbose("Initial status upload failed: {0}", e) - blob_type = resp.getheader("x-ms-blob-type") - logger.verbose("Blob type: [{0}]", blob_type) - return blob_type + return False def get_block_blob_headers(self, blob_size): return { @@ -538,7 +512,6 @@ class WireClient(object): self.req_count = 0 self.host_plugin = None self.status_blob = StatusBlob(self) - self.status_blob_type_reported = False def prevent_throttling(self): """ @@ -725,7 +698,6 @@ class WireClient(object): xml_text = self.fetch_config(goal_state.ext_uri, self.get_header()) self.save_cache(local_file, xml_text) self.ext_conf = ExtensionsConfig(xml_text) - self.status_blob_type_reported = False def update_goal_state(self, forced=False, max_retry=3): uri = GOAL_STATE_URI.format(self.endpoint) @@ -815,7 +787,6 @@ class WireClient(object): local_file = os.path.join(conf.get_lib_dir(), local_file) xml_text = self.fetch_cache(local_file) self.ext_conf = ExtensionsConfig(xml_text) - self.status_blob_type_reported = False return self.ext_conf def get_ext_manifest(self, ext_handler, goal_state): @@ -852,46 +823,38 @@ class WireClient(object): def upload_status_blob(self): ext_conf = self.get_ext_conf() - if ext_conf.status_upload_blob is not None: - uploaded = False + + blob_uri = ext_conf.status_upload_blob + blob_type = ext_conf.status_upload_blob_type + + if blob_uri is not None: + + if not blob_type in ["BlockBlob", "PageBlob"]: + blob_type = "BlockBlob" + logger.info("Status Blob type is unspecified " + "-- assuming it is a BlockBlob") + try: - self.status_blob.prepare(ext_conf.status_upload_blob_type) - if not HostPluginProtocol.is_default_channel(): - uploaded = self.status_blob.upload(ext_conf.status_upload_blob) - self.report_blob_type(self.status_blob.type, - ext_conf.status_upload_blob_type) - except (HttpError, ProtocolError): - # errors have already been logged - pass + self.status_blob.prepare(blob_type) + except Exception as e: + self.report_status_event( + "Exception creating status blob: {0}", + e) + return + + uploaded = False + if not HostPluginProtocol.is_default_channel(): + try: + uploaded = self.status_blob.upload(blob_uri) + except HttpError as e: + pass + if not uploaded: host = self.get_host_plugin() host.put_vm_status(self.status_blob, ext_conf.status_upload_blob, ext_conf.status_upload_blob_type) - """ - Emit an event to determine if the type in the extension config - matches the actual type from the HTTP HEAD request. - """ - def report_blob_type(self, head_type, config_type): - if head_type and config_type: - is_match = head_type == config_type - if self.status_blob_type_reported is False: - message = \ - 'Blob type match [{0}]'.format(head_type) if is_match else \ - 'Blob type mismatch [HEAD {0}], [CONFIG {1}]'.format( - head_type, - config_type) - - from azurelinuxagent.common.event import add_event, WALAEventOperation - from azurelinuxagent.common.version import AGENT_NAME, CURRENT_VERSION - add_event(AGENT_NAME, - version=CURRENT_VERSION, - is_success=is_match, - message=message, - op=WALAEventOperation.HealthCheck) - self.status_blob_type_reported = True - def report_role_prop(self, thumbprint): goal_state = self.get_goal_state() role_prop = _build_role_properties(goal_state.container_id, @@ -980,6 +943,16 @@ class WireClient(object): if len(buf[provider_id]) > 0: self.send_event(provider_id, buf[provider_id]) + def report_status_event(self, message, *args): + from azurelinuxagent.common.event import report_event, \ + WALAEventOperation + + message = message.format(*args) + logger.warn(message) + report_event(op=WALAEventOperation.ReportStatus, + is_success=False, + message=message) + def get_header(self): return { "x-ms-agent-name": "WALinuxAgent", diff --git a/azurelinuxagent/common/utils/cryptutil.py b/azurelinuxagent/common/utils/cryptutil.py index b35bda0..6339eb3 100644 --- a/azurelinuxagent/common/utils/cryptutil.py +++ b/azurelinuxagent/common/utils/cryptutil.py @@ -31,7 +31,7 @@ class CryptUtil(object): """ Create ssl certificate for https communication with endpoint server. """ - cmd = ("{0} req -x509 -nodes -subj /CN=LinuxTransport -days 32768 " + cmd = ("{0} req -x509 -nodes -subj /CN=LinuxTransport -days 730 " "-newkey rsa:2048 -keyout {1} " "-out {2}").format(self.openssl_cmd, prv_file, crt_file) shellutil.run(cmd) diff --git a/azurelinuxagent/common/utils/fileutil.py b/azurelinuxagent/common/utils/fileutil.py index 8713d0c..bae1957 100644 --- a/azurelinuxagent/common/utils/fileutil.py +++ b/azurelinuxagent/common/utils/fileutil.py @@ -119,16 +119,20 @@ def rm_files(*args): def rm_dirs(*args): """ - Remove all the contents under the directry + Remove the contents of each directry """ - for dir_name in args: - if os.path.isdir(dir_name): - for item in os.listdir(dir_name): - path = os.path.join(dir_name, item) - if os.path.isfile(path): - os.remove(path) - elif os.path.isdir(path): - shutil.rmtree(path) + for p in args: + if not os.path.isdir(p): + continue + + for pp in os.listdir(p): + path = os.path.join(p, pp) + if os.path.isfile(path): + os.remove(path) + elif os.path.islink(path): + os.unlink(path) + elif os.path.isdir(path): + shutil.rmtree(path) def trim_ext(path, ext): if not ext.startswith("."): diff --git a/azurelinuxagent/common/utils/shellutil.py b/azurelinuxagent/common/utils/shellutil.py index 4efcbc4..fff6aa8 100644 --- a/azurelinuxagent/common/utils/shellutil.py +++ b/azurelinuxagent/common/utils/shellutil.py @@ -76,18 +76,23 @@ def run_get_output(cmd, chk_err=True, log_cmd=True): Reports exceptions to Error if chk_err parameter is True """ if log_cmd: - logger.verbose(u"run cmd '{0}'", cmd) + logger.verbose(u"Run '{0}'", cmd) try: - output = subprocess.check_output(cmd, stderr=subprocess.STDOUT, + output = subprocess.check_output(cmd, + stderr=subprocess.STDOUT, shell=True) - output = ustr(output, encoding='utf-8', errors="backslashreplace") + output = ustr(output, + encoding='utf-8', + errors="backslashreplace") except subprocess.CalledProcessError as e: - output = ustr(e.output, encoding='utf-8', errors="backslashreplace") + output = ustr(e.output, + encoding='utf-8', + errors="backslashreplace") if chk_err: if log_cmd: - logger.error(u"run cmd '{0}' failed", e.cmd) - logger.error(u"Error Code:{0}", e.returncode) - logger.error(u"Result:{0}", output) + logger.error(u"Command: '{0}'", e.cmd) + logger.error(u"Return code: {0}", e.returncode) + logger.error(u"Result: {0}", output) return e.returncode, output return 0, output diff --git a/azurelinuxagent/common/version.py b/azurelinuxagent/common/version.py index 8a81974..dc3592b 100644 --- a/azurelinuxagent/common/version.py +++ b/azurelinuxagent/common/version.py @@ -57,6 +57,22 @@ def get_f5_platform(): return result +def get_checkpoint_platform(): + take = build = release = "" + full_name = open("/etc/cp-release").read().strip() + with open("/etc/cloud-version") as f: + for line in f: + k, _, v = line.partition(": ") + v = v.strip() + if k == "release": + release = v + elif k == "take": + take = v + elif k == "build": + build = v + return ["gaia", take + "." + build, release, full_name] + + def get_distro(): if 'FreeBSD' in platform.system(): release = re.sub('\-.*\Z', '', ustr(platform.release())) @@ -84,6 +100,9 @@ def get_distro(): if os.path.exists("/shared/vadc"): osinfo = get_f5_platform() + if os.path.exists("/etc/cp-release"): + osinfo = get_checkpoint_platform() + # Remove trailing whitespace and quote in distro name osinfo[0] = osinfo[0].strip('"').strip(' ').lower() return osinfo @@ -91,9 +110,9 @@ def get_distro(): AGENT_NAME = "WALinuxAgent" AGENT_LONG_NAME = "Azure Linux Agent" -AGENT_VERSION = '2.2.9' +AGENT_VERSION = '2.2.12' AGENT_LONG_VERSION = "{0}-{1}".format(AGENT_NAME, AGENT_VERSION) -AGENT_DESCRIPTION = """\ +AGENT_DESCRIPTION = """ The Azure Linux Agent supports the provisioning and running of Linux VMs in the Azure cloud. This package should be installed on Linux disk images that are built to run in the Azure environment. @@ -104,6 +123,7 @@ AGENT_PKG_GLOB = "{0}-*.zip".format(AGENT_NAME) AGENT_PATTERN = "{0}-(.*)".format(AGENT_NAME) AGENT_NAME_PATTERN = re.compile(AGENT_PATTERN) +AGENT_PKG_PATTERN = re.compile(AGENT_PATTERN+"\.zip") AGENT_DIR_PATTERN = re.compile(".*/{0}".format(AGENT_PATTERN)) EXT_HANDLER_PATTERN = b".*/WALinuxAgent-(\w.\w.\w[.\w]*)-.*-run-exthandlers" @@ -127,6 +147,13 @@ def set_current_agent(): version = AGENT_VERSION return agent, FlexibleVersion(version) +def is_agent_package(path): + path = os.path.basename(path) + return not re.match(AGENT_PKG_PATTERN, path) is None + +def is_agent_path(path): + path = os.path.basename(path) + return not re.match(AGENT_NAME_PATTERN, path) is None CURRENT_AGENT, CURRENT_VERSION = set_current_agent() diff --git a/azurelinuxagent/daemon/main.py b/azurelinuxagent/daemon/main.py index b0da02a..5b8db2f 100644 --- a/azurelinuxagent/daemon/main.py +++ b/azurelinuxagent/daemon/main.py @@ -25,12 +25,15 @@ import traceback import azurelinuxagent.common.conf as conf import azurelinuxagent.common.logger as logger import azurelinuxagent.common.utils.fileutil as fileutil + from azurelinuxagent.common.event import add_event, WALAEventOperation from azurelinuxagent.common.future import ustr from azurelinuxagent.common.osutil import get_osutil from azurelinuxagent.common.protocol import get_protocol_util +from azurelinuxagent.common.protocol.wire import WireClient from azurelinuxagent.common.rdma import setup_rdma_device -from azurelinuxagent.common.version import AGENT_LONG_NAME, AGENT_VERSION, \ +from azurelinuxagent.common.version import AGENT_NAME, AGENT_LONG_NAME, \ + AGENT_VERSION, \ DISTRO_NAME, DISTRO_VERSION, PY_VERSION_MAJOR, PY_VERSION_MINOR, \ PY_VERSION_MICRO from azurelinuxagent.daemon.resourcedisk import get_resourcedisk_handler @@ -39,6 +42,8 @@ from azurelinuxagent.ga.update import get_update_handler from azurelinuxagent.pa.provision import get_provision_handler from azurelinuxagent.pa.rdma import get_rdma_handler +OPENSSL_FIPS_ENVIRONMENT = "OPENSSL_FIPS" + def get_daemon_handler(): return DaemonHandler() @@ -53,7 +58,7 @@ class DaemonHandler(object): self.running = True self.osutil = get_osutil() - def run(self): + def run(self, child_args=None): logger.info("{0} Version:{1}", AGENT_LONG_NAME, AGENT_VERSION) logger.info("OS: {0} {1}", DISTRO_NAME, DISTRO_VERSION) logger.info("Python: {0}.{1}.{2}", PY_VERSION_MAJOR, PY_VERSION_MINOR, @@ -61,12 +66,18 @@ class DaemonHandler(object): self.check_pid() + # If FIPS is enabled, set the OpenSSL environment variable + # Note: + # -- Subprocesses inherit the current environment + if conf.get_fips_enabled(): + os.environ[OPENSSL_FIPS_ENVIRONMENT] = '1' + while self.running: try: - self.daemon() + self.daemon(child_args) except Exception as e: err_msg = traceback.format_exc() - add_event("WALA", is_success=False, message=ustr(err_msg), + add_event(name=AGENT_NAME, is_success=False, message=ustr(err_msg), op=WALAEventOperation.UnhandledError) logger.info("Sleep 15 seconds and restart daemon") time.sleep(15) @@ -84,7 +95,7 @@ class DaemonHandler(object): fileutil.write_file(pid_file, ustr(os.getpid())) - def daemon(self): + def daemon(self, child_args=None): logger.info("Run daemon") self.protocol_util = get_protocol_util() @@ -117,6 +128,16 @@ class DaemonHandler(object): logger.info("RDMA capabilities are enabled in configuration") try: + # Ensure the most recent SharedConfig is available + # - Changes to RDMA state may not increment the goal state + # incarnation number. A forced update ensures the most + # current values. + protocol = self.protocol_util.get_protocol() + client = protocol.client + if client is None or type(client) is not WireClient: + raise Exception("Attempt to setup RDMA without Wireserver") + client.update_goal_state(forced=True) + setup_rdma_device() except Exception as e: logger.error("Error setting up rdma device: %s" % e) @@ -124,4 +145,4 @@ class DaemonHandler(object): logger.info("RDMA capabilities are not enabled, skipping") while self.running: - self.update_handler.run_latest() + self.update_handler.run_latest(child_args=child_args) diff --git a/azurelinuxagent/daemon/resourcedisk/default.py b/azurelinuxagent/daemon/resourcedisk/default.py index 2b116fb..dadd49c 100644 --- a/azurelinuxagent/daemon/resourcedisk/default.py +++ b/azurelinuxagent/daemon/resourcedisk/default.py @@ -29,6 +29,7 @@ import azurelinuxagent.common.utils.fileutil as fileutil import azurelinuxagent.common.utils.shellutil as shellutil from azurelinuxagent.common.exception import ResourceDiskError from azurelinuxagent.common.osutil import get_osutil +from azurelinuxagent.common.version import AGENT_NAME DATALOSS_WARNING_FILE_NAME = "DATALOSS_WARNING_README.txt" DATA_LOSS_WARNING = """\ @@ -74,7 +75,7 @@ class ResourceDiskHandler(object): return mount_point except ResourceDiskError as e: logger.error("Failed to mount resource disk {0}", e) - add_event(name="WALA", is_success=False, message=ustr(e), + add_event(name=AGENT_NAME, is_success=False, message=ustr(e), op=WALAEventOperation.ActivateResourceDisk) def enable_swap(self, mount_point): @@ -123,7 +124,7 @@ class ResourceDiskHandler(object): force_option = 'F' if self.fs == 'xfs': force_option = 'f' - mkfs_string = "mkfs.{0} {1} -{2}".format(self.fs, partition, force_option) + mkfs_string = "mkfs.{0} -{2} {1}".format(self.fs, partition, force_option) if "gpt" in ret[1]: logger.info("GPT detected, finding partitions") diff --git a/azurelinuxagent/ga/exthandlers.py b/azurelinuxagent/ga/exthandlers.py index e0125aa..b44ed6d 100644 --- a/azurelinuxagent/ga/exthandlers.py +++ b/azurelinuxagent/ga/exthandlers.py @@ -20,6 +20,8 @@ import glob import json import os +import os.path +import re import shutil import stat import subprocess @@ -31,6 +33,7 @@ import azurelinuxagent.common.logger as logger import azurelinuxagent.common.utils.fileutil as fileutil import azurelinuxagent.common.utils.restutil as restutil import azurelinuxagent.common.utils.shellutil as shellutil +import azurelinuxagent.common.version as version from azurelinuxagent.common.event import add_event, WALAEventOperation from azurelinuxagent.common.exception import ExtensionError, ProtocolError, HttpError @@ -55,6 +58,13 @@ VALID_EXTENSION_STATUS = ['transitioning', 'error', 'success', 'warning'] VALID_HANDLER_STATUS = ['Ready', 'NotReady', "Installing", "Unresponsive"] +HANDLER_PATTERN = "^([^-]+)-(\d+(?:\.\d+)*)" +HANDLER_NAME_PATTERN = re.compile(HANDLER_PATTERN+"$", re.IGNORECASE) +HANDLER_PKG_EXT = ".zip" +HANDLER_PKG_PATTERN = re.compile(HANDLER_PATTERN+"\\"+HANDLER_PKG_EXT+"$", + re.IGNORECASE) + + def validate_has_key(obj, key, fullname): if key not in obj: raise ExtensionError("Missing: {0}".format(fullname)) @@ -163,6 +173,7 @@ def get_exthandlers_handler(): class ExtHandlersHandler(object): def __init__(self): self.protocol_util = get_protocol_util() + self.protocol = None self.ext_handlers = None self.last_etag = None self.log_report = False @@ -188,10 +199,74 @@ class ExtHandlersHandler(object): self.last_etag = etag self.report_ext_handlers_status() + self.cleanup_outdated_handlers() def run_status(self): self.report_ext_handlers_status() return + + def cleanup_outdated_handlers(self): + handlers = [] + pkgs = [] + + # Build a collection of uninstalled handlers and orphaned packages + # Note: + # -- An orphaned package is one without a corresponding handler + # directory + for item in os.listdir(conf.get_lib_dir()): + path = os.path.join(conf.get_lib_dir(), item) + + if version.is_agent_package(path) or version.is_agent_path(path): + continue + + if os.path.isdir(path): + if re.match(HANDLER_NAME_PATTERN, item) is None: + continue + try: + eh = ExtHandler() + + separator = item.rfind('-') + + eh.name = item[0:separator] + eh.properties.version = str(FlexibleVersion(item[separator+1:])) + + handler = ExtHandlerInstance(eh, self.protocol) + except Exception as e: + continue + if handler.get_handler_state() != ExtHandlerState.NotInstalled: + continue + handlers.append(handler) + + elif os.path.isfile(path) and \ + not os.path.isdir(path[0:-len(HANDLER_PKG_EXT)]): + if not re.match(HANDLER_PKG_PATTERN, item): + continue + pkgs.append(path) + + # Then, remove the orphaned packages + for pkg in pkgs: + try: + os.remove(pkg) + logger.verbose("Removed orphaned extension package " + "{0}".format(pkg)) + except Exception as e: + logger.warn("Failed to remove orphaned package: {0}".format( + pkg)) + + # Finally, remove the directories and packages of the + # uninstalled handlers + for handler in handlers: + handler.rm_ext_handler_dir() + pkg = os.path.join(conf.get_lib_dir(), + handler.get_full_name() + HANDLER_PKG_EXT) + if os.path.isfile(pkg): + try: + os.remove(pkg) + logger.verbose("Removed extension package " + "{0}".format(pkg)) + except Exception as e: + logger.warn("Failed to remove extension package: " + "{0}".format(pkg)) def handle_ext_handlers(self, etag=None): if self.ext_handlers.extHandlers is None or \ @@ -478,6 +553,14 @@ class ExtHandlerInstance(object): separator = path.rfind('-') version = FlexibleVersion(path[separator+1:]) + state_path = os.path.join(path, 'config', 'HandlerState') + + if not os.path.exists(state_path) or \ + fileutil.read_file(state_path) == \ + ExtHandlerState.NotInstalled: + logger.verbose("Ignoring version of uninstalled extension: " + "{0}".format(path)) + continue if lastest_version is None or lastest_version < version: lastest_version = version @@ -615,6 +698,7 @@ class ExtHandlerInstance(object): except IOError as e: message = "Failed to remove extension handler directory: {0}".format(e) self.report_event(message=message, is_success=False) + self.logger.warn(message) def update(self): self.set_operation(WALAEventOperation.Update) diff --git a/azurelinuxagent/ga/monitor.py b/azurelinuxagent/ga/monitor.py index 7ef7f04..dcfd6a4 100644 --- a/azurelinuxagent/ga/monitor.py +++ b/azurelinuxagent/ga/monitor.py @@ -178,11 +178,11 @@ class MonitorHandler(object): logger.error("{0}", e) def daemon(self): - last_heartbeat = datetime.datetime.min period = datetime.timedelta(minutes=30) + last_heartbeat = datetime.datetime.utcnow() - period while True: - if (datetime.datetime.now() - last_heartbeat) > period: - last_heartbeat = datetime.datetime.now() + if datetime.datetime.utcnow() >= (last_heartbeat + period): + last_heartbeat = datetime.datetime.utcnow() add_event( op=WALAEventOperation.HeartBeat, name=CURRENT_AGENT, diff --git a/azurelinuxagent/ga/update.py b/azurelinuxagent/ga/update.py index 203bb41..67eb785 100644 --- a/azurelinuxagent/ga/update.py +++ b/azurelinuxagent/ga/update.py @@ -30,13 +30,17 @@ import time import traceback import zipfile +from datetime import datetime, timedelta + import azurelinuxagent.common.conf as conf import azurelinuxagent.common.logger as logger import azurelinuxagent.common.utils.fileutil as fileutil import azurelinuxagent.common.utils.restutil as restutil import azurelinuxagent.common.utils.textutil as textutil -from azurelinuxagent.common.event import add_event, WALAEventOperation +from azurelinuxagent.common.event import add_event, \ + elapsed_milliseconds, \ + WALAEventOperation from azurelinuxagent.common.exception import UpdateError, ProtocolError from azurelinuxagent.common.future import ustr from azurelinuxagent.common.osutil import get_osutil @@ -53,6 +57,7 @@ from azurelinuxagent.ga.exthandlers import HandlerManifest AGENT_ERROR_FILE = "error.json" # File name for agent error record AGENT_MANIFEST_FILE = "HandlerManifest.json" +AGENT_SUPPORTED_FILE = "supported.json" CHILD_HEALTH_INTERVAL = 15 * 60 CHILD_LAUNCH_INTERVAL = 5 * 60 @@ -60,16 +65,14 @@ CHILD_LAUNCH_RESTART_MAX = 3 CHILD_POLL_INTERVAL = 60 MAX_FAILURE = 3 # Max failure allowed for agent before blacklisted -RETAIN_INTERVAL = 24 * 60 * 60 # Retain interval for black list -GOAL_STATE_INTERVAL = 25 +GOAL_STATE_INTERVAL = 3 REPORT_STATUS_INTERVAL = 15 ORPHAN_WAIT_INTERVAL = 15 * 60 * 60 AGENT_SENTINAL_FILE = "current_version" - def get_update_handler(): return UpdateHandler() @@ -98,7 +101,7 @@ class UpdateHandler(object): self.signal_handler = None return - def run_latest(self): + def run_latest(self, child_args=None): """ This method is called from the daemon to find and launch the most current, downloaded agent. @@ -127,6 +130,9 @@ class UpdateHandler(object): agent_name = latest_agent.name agent_version = latest_agent.version + if child_args is not None: + agent_cmd = "{0} {1}".format(agent_cmd, child_args) + try: # Launch the correct Python version for python-based agents @@ -198,7 +204,7 @@ class UpdateHandler(object): ret) logger.warn(msg) if latest_agent is not None: - latest_agent.mark_failure() + latest_agent.mark_failure(is_fatal=True) except Exception as e: msg = u"Agent {0} launched with command '{1}' failed with exception: {2}".format( @@ -237,10 +243,11 @@ class UpdateHandler(object): migrate_handler_state() try: + send_event_time = datetime.utcnow() + self._ensure_no_orphans() self._emit_restart_event() - # TODO: Add means to stop running while self.running: if self._is_orphaned: logger.info("Goal state agent {0} was orphaned -- exiting", CURRENT_AGENT) @@ -254,8 +261,29 @@ class UpdateHandler(object): self.agents[0].name) break + utc_start = datetime.utcnow() + + last_etag = exthandlers_handler.last_etag exthandlers_handler.run() - + + log_event = last_etag != exthandlers_handler.last_etag or \ + (datetime.utcnow() >= send_event_time) + add_event( + AGENT_NAME, + version=CURRENT_VERSION, + op=WALAEventOperation.ProcessGoalState, + is_success=True, + duration=elapsed_milliseconds(utc_start), + log_event=log_event) + if log_event: + send_event_time += timedelta(minutes=REPORT_STATUS_INTERVAL) + + test_agent = self.get_test_agent() + if test_agent is not None and test_agent.in_slice: + test_agent.enable() + logger.info(u"Enabled Agent {0} as test agent", test_agent.name) + break + time.sleep(GOAL_STATE_INTERVAL) except Exception as e: @@ -305,13 +333,21 @@ class UpdateHandler(object): if not conf.get_autoupdate_enabled(): return None - self._load_agents() + self._find_agents() available_agents = [agent for agent in self.agents if agent.is_available and agent.version > FlexibleVersion(AGENT_VERSION)] return available_agents[0] if len(available_agents) >= 1 else None + def get_test_agent(self): + agent = None + agents = [agent for agent in self._load_agents() if agent.is_test] + if len(agents) > 0: + agents.sort(key=lambda agent: agent.version, reverse=True) + agent = agents[0] + return agent + def _emit_restart_event(self): if not self._is_clean_start: msg = u"{0} did not terminate cleanly".format(CURRENT_AGENT) @@ -390,6 +426,7 @@ class UpdateHandler(object): host = None if protocol and protocol.client: host = protocol.client.get_host_plugin() + self._set_agents([GuestAgent(pkg=pkg, host=host) for pkg in pkg_list.versions]) self._purge_agents() self._filter_blacklisted_agents() @@ -457,6 +494,17 @@ class UpdateHandler(object): self.agents = [agent for agent in self.agents if not agent.is_blacklisted] return + def _find_agents(self): + """ + Load all non-blacklisted agents currently on disk. + """ + try: + self._set_agents(self._load_agents()) + self._filter_blacklisted_agents() + except Exception as e: + logger.warn(u"Exception occurred loading available agents: {0}", ustr(e)) + return + def _get_pid_files(self): pid_file = conf.get_agent_pid_file_path() @@ -502,17 +550,9 @@ class UpdateHandler(object): return fileutil.read_file(conf.get_agent_pid_file_path()) != ustr(parent_pid) def _load_agents(self): - """ - Load all non-blacklisted agents currently on disk. - """ - try: - path = os.path.join(conf.get_lib_dir(), "{0}-*".format(AGENT_NAME)) - self._set_agents([GuestAgent(path=agent_dir) - for agent_dir in glob.iglob(path) if os.path.isdir(agent_dir)]) - self._filter_blacklisted_agents() - except Exception as e: - logger.warn(u"Exception occurred loading available agents: {0}", ustr(e)) - return + path = os.path.join(conf.get_lib_dir(), "{0}-*".format(AGENT_NAME)) + return [GuestAgent(path=agent_dir) + for agent_dir in glob.iglob(path) if os.path.isdir(agent_dir)] def _purge_agents(self): """ @@ -610,7 +650,11 @@ class GuestAgent(object): logger.verbose(u"Instantiating Agent {0} from {1}", self.name, location) self.error = None + self.supported = None + self._load_error() + self._load_supported() + self._ensure_downloaded() return @@ -633,10 +677,19 @@ class GuestAgent(object): def get_agent_pkg_path(self): return ".".join((os.path.join(conf.get_lib_dir(), self.name), "zip")) + def get_agent_supported_file(self): + return os.path.join(conf.get_lib_dir(), self.name, AGENT_SUPPORTED_FILE) + def clear_error(self): self.error.clear() return + def enable(self): + if self.error.is_sentinel: + self.error.clear() + self.error.save() + return + @property def is_available(self): return self.is_downloaded and not self.is_blacklisted @@ -649,6 +702,14 @@ class GuestAgent(object): def is_downloaded(self): return self.is_blacklisted or os.path.isfile(self.get_agent_manifest_path()) + @property + def is_test(self): + return self.error.is_sentinel and self.supported.is_supported + + @property + def in_slice(self): + return self.is_test and self.supported.in_slice + def mark_failure(self, is_fatal=False): try: if not os.path.isdir(self.get_agent_dir()): @@ -666,7 +727,7 @@ class GuestAgent(object): logger.verbose(u"Ensuring Agent {0} is downloaded", self.name) if self.is_blacklisted: - logger.warn(u"Agent {0} is blacklisted - skipping download", self.name) + logger.info(u"Agent {0} is blacklisted - skipping download", self.name) return if self.is_downloaded: @@ -682,6 +743,7 @@ class GuestAgent(object): self._unpack() self._load_manifest() self._load_error() + self._load_supported() msg = u"Agent {0} downloaded successfully".format(self.name) logger.verbose(msg) @@ -770,6 +832,12 @@ class GuestAgent(object): logger.warn(u"Agent {0} failed loading error state: {1}", self.name, ustr(e)) return + def _load_supported(self): + try: + self.supported = Supported(self.get_agent_supported_file()) + except Exception as e: + self.supported = Supported() + def _load_manifest(self): path = self.get_agent_manifest_path() if not os.path.isfile(path): @@ -859,18 +927,15 @@ class GuestAgentError(object): self.failure_count = 0 self.was_fatal = False return - - def clear_old_failure(self): - if self.last_failure <= 0.0: - return - if self.last_failure < (time.time() - RETAIN_INTERVAL): - self.clear() - return @property def is_blacklisted(self): return self.was_fatal or self.failure_count >= MAX_FAILURE + @property + def is_sentinel(self): + return self.was_fatal and self.last_failure == 0.0 + def load(self): if self.path is not None and os.path.isfile(self.path): with open(self.path, 'r') as f: @@ -906,3 +971,61 @@ class GuestAgentError(object): self.last_failure, self.failure_count, self.was_fatal) + +class Supported(object): + def __init__(self, path): + if path is None: + raise UpdateError(u"Supported requires a path") + self.path = path + + self._load() + return + + @property + def is_supported(self): + return self._supported_distribution is not None + + @property + def in_slice(self): + d = self._supported_distribution + return d is not None and d.in_slice + + @property + def _supported_distribution(self): + for d in self.distributions: + dd = self.distributions[d] + if dd.is_supported: + return dd + return None + + def _load(self): + self.distributions = {} + try: + if self.path is not None and os.path.isfile(self.path): + j = json.loads(fileutil.read_file(self.path)) + for d in j: + self.distributions[d] = SupportedDistribution(j[d]) + except Exception as e: + logger.warn("Failed JSON parse of {0}: {1}".format(self.path, e)) + return + +class SupportedDistribution(object): + def __init__(self, s): + if s is None or not isinstance(s, dict): + raise UpdateError(u"SupportedDisribution requires a dictionary") + + self.slice = s['slice'] + self.versions = s['versions'] + + @property + def is_supported(self): + d = ','.join(platform.linux_distribution()) + for v in self.versions: + if re.match(v, d): + return True + return False + + @property + def in_slice(self): + n = int((60 * self.slice) / 100) + return (n - datetime.utcnow().second) > 0 diff --git a/azurelinuxagent/pa/deprovision/arch.py b/azurelinuxagent/pa/deprovision/arch.py new file mode 100644 index 0000000..e661f79 --- /dev/null +++ b/azurelinuxagent/pa/deprovision/arch.py @@ -0,0 +1,33 @@ +# Microsoft Azure Linux Agent +# +# Copyright 2014 Microsoft Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Requires Python 2.4+ and Openssl 1.0+ +# + +import azurelinuxagent.common.utils.fileutil as fileutil +from azurelinuxagent.pa.deprovision.default import DeprovisionHandler, \ + DeprovisionAction + +class ArchDeprovisionHandler(DeprovisionHandler): + def __init__(self): + super(ArchDeprovisionHandler, self).__init__() + + def setup(self, deluser): + warnings, actions = super(ArchDeprovisionHandler, self).setup(deluser) + warnings.append("WARNING! /etc/machine-id will be removed.") + files_to_del = ['/etc/machine-id'] + actions.append(DeprovisionAction(fileutil.rm_files, files_to_del)) + return warnings, actions diff --git a/azurelinuxagent/pa/deprovision/default.py b/azurelinuxagent/pa/deprovision/default.py index ced87ee..90d16c7 100644 --- a/azurelinuxagent/pa/deprovision/default.py +++ b/azurelinuxagent/pa/deprovision/default.py @@ -17,13 +17,17 @@ # Requires Python 2.4+ and Openssl 1.0+ # +import glob +import os.path import signal import sys + import azurelinuxagent.common.conf as conf -from azurelinuxagent.common.exception import ProtocolError -from azurelinuxagent.common.future import read_input import azurelinuxagent.common.utils.fileutil as fileutil import azurelinuxagent.common.utils.shellutil as shellutil + +from azurelinuxagent.common.exception import ProtocolError +from azurelinuxagent.common.future import read_input from azurelinuxagent.common.osutil import get_osutil from azurelinuxagent.common.protocol import get_protocol_util @@ -68,15 +72,19 @@ class DeprovisionHandler(object): def regen_ssh_host_key(self, warnings, actions): warnings.append("WARNING! All SSH host key pairs will be deleted.") actions.append(DeprovisionAction(fileutil.rm_files, - ['/etc/ssh/ssh_host_*key*'])) + [conf.get_ssh_key_glob()])) def stop_agent_service(self, warnings, actions): warnings.append("WARNING! The waagent service will be stopped.") actions.append(DeprovisionAction(self.osutil.stop_agent_service)) + def del_dirs(self, warnings, actions): + dirs = [conf.get_lib_dir(), conf.get_ext_log_dir()] + actions.append(DeprovisionAction(fileutil.rm_dirs, dirs)) + def del_files(self, warnings, actions): - files_to_del = ['/root/.bash_history', '/var/log/waagent.log'] - actions.append(DeprovisionAction(fileutil.rm_files, files_to_del)) + files = ['/root/.bash_history', '/var/log/waagent.log'] + actions.append(DeprovisionAction(fileutil.rm_files, files)) def del_resolv(self, warnings, actions): warnings.append("WARNING! /etc/resolv.conf will be deleted.") @@ -92,9 +100,63 @@ class DeprovisionHandler(object): actions.append(DeprovisionAction(fileutil.rm_files, ["/var/db/dhclient.leases.hn0", "/var/lib/NetworkManager/dhclient-*.lease"])) - def del_lib_dir(self, warnings, actions): - dirs_to_del = [conf.get_lib_dir()] - actions.append(DeprovisionAction(fileutil.rm_dirs, dirs_to_del)) + + def del_lib_dir_files(self, warnings, actions): + known_files = [ + 'HostingEnvironmentConfig.xml', + 'Incarnation', + 'Protocol', + 'SharedConfig.xml', + 'WireServerEndpoint' + ] + known_files_glob = [ + 'Extensions.*.xml', + 'ExtensionsConfig.*.xml', + 'GoalState.*.xml' + ] + + lib_dir = conf.get_lib_dir() + files = [f for f in \ + [os.path.join(lib_dir, kf) for kf in known_files] \ + if os.path.isfile(f)] + for p in known_files_glob: + files += glob.glob(os.path.join(lib_dir, p)) + + if len(files) > 0: + actions.append(DeprovisionAction(fileutil.rm_files, files)) + + def cloud_init_dirs(self, include_once=True): + dirs = [ + "/var/lib/cloud/instance", + "/var/lib/cloud/instances/", + "/var/lib/cloud/data" + ] + if include_once: + dirs += [ + "/var/lib/cloud/scripts/per-once" + ] + return dirs + + def cloud_init_files(self, include_once=True): + files = [ + "/etc/sudoers.d/90-cloud-init-users" + ] + if include_once: + files += [ + "/var/lib/cloud/sem/config_scripts_per_once.once" + ] + return files + + def del_cloud_init(self, warnings, actions, include_once=True): + dirs = [d for d in self.cloud_init_dirs(include_once=include_once) \ + if os.path.isdir(d)] + if len(dirs) > 0: + actions.append(DeprovisionAction(fileutil.rm_dirs, dirs)) + + files = [f for f in self.cloud_init_files(include_once=include_once) \ + if os.path.isfile(f)] + if len(files) > 0: + actions.append(DeprovisionAction(fileutil.rm_files, files)) def reset_hostname(self, warnings, actions): localhost = ["localhost.localdomain"] @@ -117,7 +179,8 @@ class DeprovisionHandler(object): if conf.get_delete_root_password(): self.del_root_password(warnings, actions) - self.del_lib_dir(warnings, actions) + self.del_cloud_init(warnings, actions) + self.del_dirs(warnings, actions) self.del_files(warnings, actions) self.del_resolv(warnings, actions) @@ -126,19 +189,55 @@ class DeprovisionHandler(object): return warnings, actions + def setup_changed_unique_id(self): + warnings = [] + actions = [] + + self.del_cloud_init(warnings, actions, include_once=False) + self.del_dhcp_lease(warnings, actions) + self.del_lib_dir_files(warnings, actions) + self.del_resolv(warnings, actions) + + return warnings, actions + def run(self, force=False, deluser=False): warnings, actions = self.setup(deluser) - for warning in warnings: - print(warning) - if not force: - confirm = read_input("Do you want to proceed (y/n)") - if not confirm.lower().startswith('y'): - return + self.do_warnings(warnings) + self.do_confirmation(force=force) + self.do_actions(actions) + + def run_changed_unique_id(self): + ''' + Clean-up files and directories that may interfere when the VM unique + identifier has changed. + While users *should* manually deprovision a VM, the files removed by + this routine will help keep the agent from getting confused + (since incarnation and extension settings, among other items, will + no longer be monotonically increasing). + ''' + warnings, actions = self.setup_changed_unique_id() + + self.do_warnings(warnings) + self.do_actions(actions) + + def do_actions(self, actions): self.actions_running = True for action in actions: action.invoke() + self.actions_running = False + + def do_confirmation(self, force=False): + if force: + return True + + confirm = read_input("Do you want to proceed (y/n)") + return True if confirm.lower().startswith('y') else False + + def do_warnings(self, warnings): + for warning in warnings: + print(warning) def handle_interrupt_signal(self, signum, frame): if not self.actions_running: diff --git a/azurelinuxagent/pa/deprovision/factory.py b/azurelinuxagent/pa/deprovision/factory.py index 72a5be1..5ac35a7 100644 --- a/azurelinuxagent/pa/deprovision/factory.py +++ b/azurelinuxagent/pa/deprovision/factory.py @@ -21,6 +21,7 @@ from azurelinuxagent.common.version import DISTRO_NAME, DISTRO_VERSION, \ DISTRO_FULL_NAME from .default import DeprovisionHandler +from .arch import ArchDeprovisionHandler from .clearlinux import ClearLinuxDeprovisionHandler from .coreos import CoreOSDeprovisionHandler from .ubuntu import UbuntuDeprovisionHandler @@ -28,6 +29,8 @@ from .ubuntu import UbuntuDeprovisionHandler def get_deprovision_handler(distro_name=DISTRO_NAME, distro_version=DISTRO_VERSION, distro_full_name=DISTRO_FULL_NAME): + if distro_name == "arch": + return ArchDeprovisionHandler() if distro_name == "ubuntu": return UbuntuDeprovisionHandler() if distro_name == "coreos": diff --git a/azurelinuxagent/pa/provision/cloudinit.py b/azurelinuxagent/pa/provision/cloudinit.py new file mode 100644 index 0000000..5789e9a --- /dev/null +++ b/azurelinuxagent/pa/provision/cloudinit.py @@ -0,0 +1,132 @@ +# Microsoft Azure Linux Agent +# +# Copyright 2014 Microsoft Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Requires Python 2.4+ and Openssl 1.0+ +# + +import os +import os.path +import time + +from datetime import datetime + +import azurelinuxagent.common.conf as conf +import azurelinuxagent.common.logger as logger +import azurelinuxagent.common.utils.fileutil as fileutil +import azurelinuxagent.common.utils.shellutil as shellutil + +from azurelinuxagent.common.event import elapsed_milliseconds +from azurelinuxagent.common.exception import ProvisionError, ProtocolError +from azurelinuxagent.common.future import ustr +from azurelinuxagent.common.protocol import OVF_FILE_NAME +from azurelinuxagent.common.protocol.ovfenv import OvfEnv +from azurelinuxagent.pa.provision.default import ProvisionHandler + + +class CloudInitProvisionHandler(ProvisionHandler): + def __init__(self): + super(CloudInitProvisionHandler, self).__init__() + + def run(self): + # If provision is enabled, run default provision handler + if conf.get_provision_enabled(): + logger.warn("Provisioning flag is enabled, which overrides using " + "cloud-init; running the default provisioning code") + super(CloudInitProvisionHandler, self).run() + return + + try: + if super(CloudInitProvisionHandler, self).is_provisioned(): + logger.info("Provisioning already completed, skipping.") + return + + utc_start = datetime.utcnow() + logger.info("Running CloudInit provisioning handler") + self.wait_for_ovfenv() + self.protocol_util.get_protocol() + self.report_not_ready("Provisioning", "Starting") + + thumbprint = self.wait_for_ssh_host_key() + self.write_provisioned() + logger.info("Finished provisioning") + + self.report_ready(thumbprint) + self.report_event("Provision succeed", + is_success=True, + duration=elapsed_milliseconds(utc_start)) + + except ProvisionError as e: + logger.error("Provisioning failed: {0}", ustr(e)) + self.report_not_ready("ProvisioningFailed", ustr(e)) + self.report_event(ustr(e)) + return + + def wait_for_ovfenv(self, max_retry=360, sleep_time=5): + """ + Wait for cloud-init to copy ovf-env.xml file from provision ISO + """ + ovf_file_path = os.path.join(conf.get_lib_dir(), OVF_FILE_NAME) + for retry in range(0, max_retry): + if os.path.isfile(ovf_file_path): + try: + OvfEnv(fileutil.read_file(ovf_file_path)) + return + except ProtocolError as pe: + raise ProvisionError("OVF xml could not be parsed " + "[{0}]: {1}".format(ovf_file_path, + ustr(pe))) + else: + if retry < max_retry - 1: + logger.info( + "Waiting for cloud-init to copy ovf-env.xml to {0} " + "[{1} retries remaining, " + "sleeping {2}s]".format(ovf_file_path, + max_retry - retry, + sleep_time)) + if not self.validate_cloud_init(): + logger.warn("cloud-init does not appear to be running") + time.sleep(sleep_time) + raise ProvisionError("Giving up, ovf-env.xml was not copied to {0} " + "after {1}s".format(ovf_file_path, + max_retry * sleep_time)) + + def wait_for_ssh_host_key(self, max_retry=360, sleep_time=5): + """ + Wait for cloud-init to generate ssh host key + """ + keypair_type = conf.get_ssh_host_keypair_type() + path = conf.get_ssh_key_public_path() + for retry in range(0, max_retry): + if os.path.isfile(path): + logger.info("ssh host key found at: {0}".format(path)) + try: + thumbprint = self.get_ssh_host_key_thumbprint(chk_err=False) + logger.info("Thumbprint obtained from : {0}".format(path)) + return thumbprint + except ProvisionError: + logger.warn("Could not get thumbprint from {0}".format(path)) + if retry < max_retry - 1: + logger.info("Waiting for ssh host key be generated at {0} " + "[{1} attempts remaining, " + "sleeping {2}s]".format(path, + max_retry - retry, + sleep_time)) + if not self.validate_cloud_init(): + logger.warn("cloud-init does not appear to be running") + time.sleep(sleep_time) + raise ProvisionError("Giving up, ssh host key was not found at {0} " + "after {1}s".format(path, + max_retry * sleep_time)) diff --git a/azurelinuxagent/pa/provision/default.py b/azurelinuxagent/pa/provision/default.py index 3a3f36f..d4870f1 100644 --- a/azurelinuxagent/pa/provision/default.py +++ b/azurelinuxagent/pa/provision/default.py @@ -20,19 +20,31 @@ Provision handler """ import os +import os.path +import re + +from datetime import datetime + +import azurelinuxagent.common.conf as conf import azurelinuxagent.common.logger as logger +import azurelinuxagent.common.utils.shellutil as shellutil +import azurelinuxagent.common.utils.fileutil as fileutil + from azurelinuxagent.common.future import ustr -import azurelinuxagent.common.conf as conf -from azurelinuxagent.common.event import add_event, WALAEventOperation +from azurelinuxagent.common.event import add_event, WALAEventOperation, \ + elapsed_milliseconds from azurelinuxagent.common.exception import ProvisionError, ProtocolError, \ OSUtilError -from azurelinuxagent.common.protocol.restapi import ProvisionStatus -import azurelinuxagent.common.utils.shellutil as shellutil -import azurelinuxagent.common.utils.fileutil as fileutil from azurelinuxagent.common.osutil import get_osutil +from azurelinuxagent.common.protocol.restapi import ProvisionStatus from azurelinuxagent.common.protocol import get_protocol_util +from azurelinuxagent.common.version import AGENT_NAME CUSTOM_DATA_FILE = "CustomData" +CLOUD_INIT_PATTERN = b".*/bin/cloud-init.*" +CLOUD_INIT_REGEX = re.compile(CLOUD_INIT_PATTERN) + +PROVISIONED_FILE = 'provisioned' class ProvisionHandler(object): @@ -41,54 +53,85 @@ class ProvisionHandler(object): self.protocol_util = get_protocol_util() def run(self): - # if provisioning is already done, return - provisioned = os.path.join(conf.get_lib_dir(), "provisioned") - if os.path.isfile(provisioned): - logger.info("Provisioning already completed, skipping.") - return - - thumbprint = None # If provision is not enabled, report ready and then return if not conf.get_provision_enabled(): logger.info("Provisioning is disabled, skipping.") - else: - logger.info("Running provisioning handler") - try: - logger.info("Copying ovf-env.xml") - ovf_env = self.protocol_util.copy_ovf_env() - self.protocol_util.get_protocol_by_file() - self.report_not_ready("Provisioning", "Starting") - logger.info("Starting provisioning") - self.provision(ovf_env) - thumbprint = self.reg_ssh_host_key() - self.osutil.restart_ssh_service() - self.report_event("Provision succeed", is_success=True) - except ProtocolError as e: - logger.error("[ProtocolError] Provisioning failed: {0}", e) - self.report_not_ready("ProvisioningFailed", ustr(e)) - self.report_event("Failed to copy ovf-env.xml: {0}".format(e)) - return - except ProvisionError as e: - logger.error("[ProvisionError] Provisioning failed: {0}", e) - self.report_not_ready("ProvisioningFailed", ustr(e)) - self.report_event(ustr(e)) + return + + try: + utc_start = datetime.utcnow() + thumbprint = None + + # if provisioning is already done, return + if self.is_provisioned(): + logger.info("Provisioning already completed, skipping.") return - # write out provisioned file and report Ready - fileutil.write_file(provisioned, "") - self.report_ready(thumbprint) - logger.info("Provisioning complete") + + logger.info("Running default provisioning handler") + + if not self.validate_cloud_init(is_expected=False): + raise ProvisionError("cloud-init appears to be running, " + "this is not expected, cannot continue") + + logger.info("Copying ovf-env.xml") + ovf_env = self.protocol_util.copy_ovf_env() + + self.protocol_util.get_protocol_by_file() + self.report_not_ready("Provisioning", "Starting") + logger.info("Starting provisioning") + + self.provision(ovf_env) + + thumbprint = self.reg_ssh_host_key() + self.osutil.restart_ssh_service() + + # write out provisioned file and report Ready + self.write_provisioned() + + self.report_event("Provision succeed", + is_success=True, + duration=elapsed_milliseconds(utc_start)) + + self.report_ready(thumbprint) + logger.info("Provisioning complete") + + except (ProtocolError, ProvisionError) as e: + self.report_not_ready("ProvisioningFailed", ustr(e)) + self.report_event(ustr(e)) + logger.error("Provisioning failed: {0}", ustr(e)) + return + + @staticmethod + def validate_cloud_init(is_expected=True): + pids = [pid for pid in os.listdir('/proc') if pid.isdigit()] + is_running = False + for pid in pids: + try: + pname = open(os.path.join('/proc', pid, 'cmdline'), 'rb').read() + if CLOUD_INIT_REGEX.match(pname): + is_running = True + msg = "cloud-init is running [PID {0}, {1}]".format(pid, + pname) + if is_expected: + logger.verbose(msg) + else: + logger.error(msg) + break + except IOError: + continue + return is_running == is_expected def reg_ssh_host_key(self): keypair_type = conf.get_ssh_host_keypair_type() if conf.get_regenerate_ssh_host_key(): - fileutil.rm_files("/etc/ssh/ssh_host_*key*") - keygen_cmd = "ssh-keygen -N '' -t {0} -f /etc/ssh/ssh_host_{1}_key" - shellutil.run(keygen_cmd.format(keypair_type, keypair_type)) - thumbprint = self.get_ssh_host_key_thumbprint(keypair_type) - return thumbprint - - def get_ssh_host_key_thumbprint(self, keypair_type, chk_err=True): - cmd = "ssh-keygen -lf /etc/ssh/ssh_host_{0}_key.pub".format(keypair_type) + fileutil.rm_files(conf.get_ssh_key_glob()) + keygen_cmd = "ssh-keygen -N '' -t {0} -f {1}" + shellutil.run(keygen_cmd.format(keypair_type, + conf.get_ssh_key_private_path())) + return self.get_ssh_host_key_thumbprint() + + def get_ssh_host_key_thumbprint(self, chk_err=True): + cmd = "ssh-keygen -lf {0}".format(conf.get_ssh_key_public_path()) ret = shellutil.run_get_output(cmd, chk_err=chk_err) if ret[0] == 0: return ret[1].rstrip().split()[1].replace(':', '') @@ -96,6 +139,45 @@ class ProvisionHandler(object): raise ProvisionError(("Failed to generate ssh host key: " "ret={0}, out= {1}").format(ret[0], ret[1])) + def provisioned_file_path(self): + return os.path.join(conf.get_lib_dir(), PROVISIONED_FILE) + + def is_provisioned(self): + ''' + A VM is considered provisionend *anytime* the provisioning + sentinel file exists and not provisioned *anytime* the file + is absent. + + If the VM was provisioned using an agent that did not record + the VM unique identifier, the provisioning file will be re-written + to include the identifier. + + A warning is logged *if* the VM unique identifier has changed + since VM was provisioned. + ''' + if not os.path.isfile(self.provisioned_file_path()): + return False + + s = fileutil.read_file(self.provisioned_file_path()).strip() + if s != self.osutil.get_instance_id(): + if len(s) > 0: + logger.warn("VM is provisioned, " + "but the VM unique identifier has changed -- " + "clearing cached state") + from azurelinuxagent.pa.deprovision \ + import get_deprovision_handler + deprovision_handler = get_deprovision_handler() + deprovision_handler.run_changed_unique_id() + + self.write_provisioned() + + return True + + def write_provisioned(self): + fileutil.write_file( + self.provisioned_file_path(), + get_osutil().get_instance_id()) + def provision(self, ovfenv): logger.info("Handle ovf-env.xml.") try: @@ -113,7 +195,7 @@ class ProvisionHandler(object): self.osutil.del_root_password() except OSUtilError as e: - raise ProvisionError("Failed to handle ovf-env.xml: {0}".format(e)) + raise ProvisionError("Failed to provision: {0}".format(ustr(e))) def config_user_account(self, ovfenv): logger.info("Create user account if not exists") @@ -141,11 +223,12 @@ class ProvisionHandler(object): if customdata is None: return - logger.info("Save custom data") lib_dir = conf.get_lib_dir() - if conf.get_decode_customdata(): + if conf.get_decode_customdata() or conf.get_execute_customdata(): + logger.info("Decode custom data") customdata = self.osutil.decode_customdata(customdata) + logger.info("Save custom data") customdata_file = os.path.join(lib_dir, CUSTOM_DATA_FILE) fileutil.write_file(customdata_file, customdata) @@ -164,9 +247,12 @@ class ProvisionHandler(object): logger.info("Deploy ssh key pairs.") self.osutil.deploy_ssh_keypair(ovfenv.username, keypair) - def report_event(self, message, is_success=False): - add_event(name="WALA", message=message, is_success=is_success, - op=WALAEventOperation.Provision) + def report_event(self, message, is_success=False, duration=0): + add_event(name=AGENT_NAME, + message=message, + duration=duration, + is_success=is_success, + op=WALAEventOperation.Provision) def report_not_ready(self, sub_status, description): status = ProvisionStatus(status="NotReady", subStatus=sub_status, diff --git a/azurelinuxagent/pa/provision/factory.py b/azurelinuxagent/pa/provision/factory.py index 9bbe35c..d87765f 100644 --- a/azurelinuxagent/pa/provision/factory.py +++ b/azurelinuxagent/pa/provision/factory.py @@ -15,18 +15,22 @@ # Requires Python 2.4+ and Openssl 1.0+ # +import azurelinuxagent.common.conf as conf import azurelinuxagent.common.logger as logger + from azurelinuxagent.common.utils.textutil import Version from azurelinuxagent.common.version import DISTRO_NAME, DISTRO_VERSION, \ DISTRO_FULL_NAME + from .default import ProvisionHandler -from .ubuntu import UbuntuProvisionHandler +from .cloudinit import CloudInitProvisionHandler def get_provision_handler(distro_name=DISTRO_NAME, distro_version=DISTRO_VERSION, distro_full_name=DISTRO_FULL_NAME): - if distro_name == "ubuntu": - return UbuntuProvisionHandler() + + if conf.get_provision_cloudinit(): + return CloudInitProvisionHandler() return ProvisionHandler() diff --git a/azurelinuxagent/pa/provision/ubuntu.py b/azurelinuxagent/pa/provision/ubuntu.py deleted file mode 100644 index 66866b2..0000000 --- a/azurelinuxagent/pa/provision/ubuntu.py +++ /dev/null @@ -1,102 +0,0 @@ -# Microsoft Azure Linux Agent -# -# Copyright 2014 Microsoft Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# Requires Python 2.4+ and Openssl 1.0+ -# - -import os -import time - -import azurelinuxagent.common.conf as conf -import azurelinuxagent.common.logger as logger -import azurelinuxagent.common.utils.fileutil as fileutil -from azurelinuxagent.common.exception import ProvisionError, ProtocolError -from azurelinuxagent.common.future import ustr -from azurelinuxagent.pa.provision.default import ProvisionHandler - -""" -On ubuntu image, provision could be disabled. -""" - - -class UbuntuProvisionHandler(ProvisionHandler): - def __init__(self): - super(UbuntuProvisionHandler, self).__init__() - - def run(self): - # If provision is enabled, run default provision handler - if conf.get_provision_enabled(): - super(UbuntuProvisionHandler, self).run() - return - - logger.info("run Ubuntu provision handler") - provisioned = os.path.join(conf.get_lib_dir(), "provisioned") - if os.path.isfile(provisioned): - return - - logger.info("Waiting cloud-init to copy ovf-env.xml.") - self.wait_for_ovfenv() - self.protocol_util.get_protocol() - self.report_not_ready("Provisioning", "Starting") - logger.info("Sleeping 1 second to avoid throttling.") - time.sleep(1) - try: - logger.info("Wait for ssh host key to be generated.") - thumbprint = self.wait_for_ssh_host_key() - fileutil.write_file(provisioned, "") - logger.info("Finished provisioning") - except ProvisionError as e: - logger.error("Provision failed: {0}", e) - self.report_not_ready("ProvisioningFailed", ustr(e)) - self.report_event(ustr(e)) - return - - self.report_ready(thumbprint) - self.report_event("Provision succeed", is_success=True) - - def wait_for_ovfenv(self, max_retry=60): - """ - Wait for cloud-init to copy ovf-env.xml file from provision ISO - """ - for retry in range(0, max_retry): - try: - self.protocol_util.get_ovf_env() - return - except ProtocolError: - if retry < max_retry - 1: - logger.info("Wait for cloud-init to copy ovf-env.xml") - time.sleep(5) - raise ProvisionError("ovf-env.xml is not copied") - - def wait_for_ssh_host_key(self, max_retry=60): - """ - Wait for cloud-init to generate ssh host key - """ - keypair_type = conf.get_ssh_host_keypair_type() - path = '/etc/ssh/ssh_host_{0}_key.pub'.format(keypair_type) - for retry in range(0, max_retry): - if os.path.isfile(path): - logger.info("ssh host key found at: {0}".format(path)) - try: - thumbprint = self.get_ssh_host_key_thumbprint(keypair_type, chk_err=False) - logger.info("Thumbprint obtained from : {0}".format(path)) - return thumbprint - except ProvisionError: - logger.warn("Could not get thumbprint from {0}".format(path)) - if retry < max_retry - 1: - logger.info("Wait for ssh host key be generated: {0}".format(path)) - time.sleep(5) - raise ProvisionError("ssh host key is not generated.") diff --git a/config/alpine/waagent.conf b/config/alpine/waagent.conf index 1161552..2e3f6a5 100644 --- a/config/alpine/waagent.conf +++ b/config/alpine/waagent.conf @@ -5,6 +5,9 @@ # Enable instance creation Provisioning.Enabled=y +# Rely on cloud-init to provision +Provisioning.UseCloudInit=n + # Password authentication for root account will be unavailable. Provisioning.DeleteRootPassword=y @@ -56,12 +59,18 @@ Logs.Verbose=n # Preferred network interface to communicate with Azure platform Network.Interface=eth0 +# Is FIPS enabled +OS.EnableFIPS=n + # Root device timeout in seconds. OS.RootDeviceScsiTimeout=300 # If "None", the system default version is used. OS.OpensslPath=None +# Set the path to SSH keys and configuration files +OS.SshDir=/etc/ssh + # Enable or disable goal state processing auto-update, default is enabled # AutoUpdate.Enabled=y diff --git a/config/arch/waagent.conf b/config/arch/waagent.conf new file mode 100644 index 0000000..686b90c --- /dev/null +++ b/config/arch/waagent.conf @@ -0,0 +1,109 @@ +# +# Microsoft Azure Linux Agent Configuration +# + +# Enable instance creation +Provisioning.Enabled=y + +# Rely on cloud-init to provision +Provisioning.UseCloudInit=n + +# Password authentication for root account will be unavailable. +Provisioning.DeleteRootPassword=n + +# Generate fresh host key pair. +Provisioning.RegenerateSshHostKeyPair=y + +# Supported values are "rsa", "dsa" and "ecdsa". +Provisioning.SshHostKeyPairType=rsa + +# Monitor host name changes and publish changes via DHCP requests. +Provisioning.MonitorHostName=y + +# Decode CustomData from Base64. +Provisioning.DecodeCustomData=n + +# Execute CustomData after provisioning. +Provisioning.ExecuteCustomData=n + +# Algorithm used by crypt when generating password hash. +#Provisioning.PasswordCryptId=6 + +# Length of random salt used when generating password hash. +#Provisioning.PasswordCryptSaltLength=10 + +# Allow reset password of sys user +Provisioning.AllowResetSysUser=n + +# Format if unformatted. If 'n', resource disk will not be mounted. +ResourceDisk.Format=y + +# File system on the resource disk +# Typically ext3 or ext4. FreeBSD images should use 'ufs2' here. +ResourceDisk.Filesystem=ext4 + +# Mount point for the resource disk +ResourceDisk.MountPoint=/mnt/resource + +# Create and use swapfile on resource disk. +ResourceDisk.EnableSwap=n + +# Size of the swapfile. +ResourceDisk.SwapSizeMB=0 + +# Comma-seperated list of mount options. See man(8) for valid options. +ResourceDisk.MountOptions=None + +# Respond to load balancer probes if requested by Windows Azure. +LBProbeResponder=y + +# Enable verbose logging (y|n) +Logs.Verbose=n + +# Is FIPS enabled +OS.EnableFIPS=n + +# Root device timeout in seconds. +OS.RootDeviceScsiTimeout=300 + +# If "None", the system default version is used. +OS.OpensslPath=None + +# Set the path to SSH keys and configuration files +OS.SshDir=/etc/ssh + +# If set, agent will use proxy server to access internet +#HttpProxy.Host=None +#HttpProxy.Port=None + +# Detect Scvmm environment, default is n +# DetectScvmmEnv=n + +# +# Lib.Dir=/var/lib/waagent + +# +# DVD.MountPoint=/mnt/cdrom/secure + +# +# Pid.File=/var/run/waagent.pid + +# +# Extension.LogDir=/var/log/azure + +# +# Home.Dir=/home + +# Enable RDMA management and set up, should only be used in HPC images +# OS.EnableRDMA=y + +# Enable or disable goal state processing auto-update, default is enabled +# AutoUpdate.Enabled=y + +# Determine the update family, this should not be changed +# AutoUpdate.GAFamily=Prod + +# Determine if the overprovisioning feature is enabled. If yes, hold extension +# handling until inVMArtifactsProfile.OnHold is false. +# Default is disabled +# EnableOverProvisioning=n diff --git a/config/bigip/waagent.conf b/config/bigip/waagent.conf index 06e6931..a6a380b 100644 --- a/config/bigip/waagent.conf +++ b/config/bigip/waagent.conf @@ -16,6 +16,9 @@ Role.TopologyConsumer=None # Enable instance creation Provisioning.Enabled=y +# Rely on cloud-init to provision +Provisioning.UseCloudInit=n + # Password authentication for root account will be unavailable. Provisioning.DeleteRootPassword=y @@ -51,12 +54,18 @@ LBProbeResponder=y # Enable verbose logging (y|n) Logs.Verbose=n +# Is FIPS enabled +OS.EnableFIPS=n + # Root device timeout in seconds. OS.RootDeviceScsiTimeout=300 # If "None", the system default version is used. OS.OpensslPath=None +# Set the path to SSH keys and configuration files +OS.SshDir=/etc/ssh + # Specify location of waagent lib dir on BIG-IP Lib.Dir=/shared/vadc/azure/waagent/ diff --git a/config/clearlinux/waagent.conf b/config/clearlinux/waagent.conf index c0d17a8..6606cd7 100644 --- a/config/clearlinux/waagent.conf +++ b/config/clearlinux/waagent.conf @@ -16,6 +16,9 @@ Role.TopologyConsumer=None # Enable instance creation Provisioning.Enabled=y +# Rely on cloud-init to provision +Provisioning.UseCloudInit=n + # Password authentication for root account will be unavailable. Provisioning.DeleteRootPassword=y @@ -56,12 +59,18 @@ ResourceDisk.SwapSizeMB=0 # Enable verbose logging (y|n) Logs.Verbose=n +# Is FIPS enabled +OS.EnableFIPS=n + # Root device timeout in seconds. OS.RootDeviceScsiTimeout=300 # If "None", the system default version is used. OS.OpensslPath=None +# Set the path to SSH keys and configuration files +OS.SshDir=/etc/ssh + # Enable or disable self-update, default is enabled AutoUpdate.Enabled=y AutoUpdate.GAFamily=Prod diff --git a/config/coreos/waagent.conf b/config/coreos/waagent.conf index 323331f..664d037 100644 --- a/config/coreos/waagent.conf +++ b/config/coreos/waagent.conf @@ -5,6 +5,9 @@ # Enable instance creation Provisioning.Enabled=y +# Rely on cloud-init to provision +Provisioning.UseCloudInit=n + # Password authentication for root account will be unavailable. Provisioning.DeleteRootPassword=n @@ -57,6 +60,12 @@ LBProbeResponder=y # Enable verbose logging (y|n) Logs.Verbose=n +# Is FIPS enabled +OS.EnableFIPS=n + +# Set the path to SSH keys and configuration files +OS.SshDir=/etc/ssh + # Root device timeout in seconds. OS.RootDeviceScsiTimeout=300 diff --git a/config/freebsd/waagent.conf b/config/freebsd/waagent.conf index 0589831..5149573 100644 --- a/config/freebsd/waagent.conf +++ b/config/freebsd/waagent.conf @@ -5,6 +5,9 @@ # Enable instance creation Provisioning.Enabled=y +# Rely on cloud-init to provision +Provisioning.UseCloudInit=n + # Password authentication for root account will be unavailable. Provisioning.DeleteRootPassword=y @@ -51,12 +54,18 @@ ResourceDisk.MountOptions=None # Enable verbose logging (y|n) Logs.Verbose=n +# Is FIPS enabled +OS.EnableFIPS=n + # Root device timeout in seconds. OS.RootDeviceScsiTimeout=300 # If "None", the system default version is used. OS.OpensslPath=None +# Set the path to SSH keys and configuration files +OS.SshDir=/etc/ssh + OS.PasswordPath=/etc/master.passwd OS.SudoersDir=/usr/local/etc/sudoers.d diff --git a/config/gaia/waagent.conf b/config/gaia/waagent.conf new file mode 100644 index 0000000..43ad35d --- /dev/null +++ b/config/gaia/waagent.conf @@ -0,0 +1,106 @@ +# +# Microsoft Azure Linux Agent Configuration +# + +# Enable instance creation +Provisioning.Enabled=y + +# Rely on cloud-init to provision +Provisioning.UseCloudInit=n + +# Password authentication for root account will be unavailable. +Provisioning.DeleteRootPassword=n + +# Generate fresh host key pair. +Provisioning.RegenerateSshHostKeyPair=n + +# Supported values are "rsa", "dsa" and "ecdsa". +Provisioning.SshHostKeyPairType=rsa + +# Monitor host name changes and publish changes via DHCP requests. +Provisioning.MonitorHostName=n + +# Decode CustomData from Base64. +Provisioning.DecodeCustomData=y + +# Execute CustomData after provisioning. +Provisioning.ExecuteCustomData=n + +# Algorithm used by crypt when generating password hash. +Provisioning.PasswordCryptId=1 + +# Length of random salt used when generating password hash. +#Provisioning.PasswordCryptSaltLength=10 + +# Allow reset password of sys user +Provisioning.AllowResetSysUser=y + +# Format if unformatted. If 'n', resource disk will not be mounted. +ResourceDisk.Format=y + +# File system on the resource disk +# Typically ext3 or ext4. FreeBSD images should use 'ufs2' here. +ResourceDisk.Filesystem=ext3 + +# Mount point for the resource disk +ResourceDisk.MountPoint=/mnt/resource + +# Create and use swapfile on resource disk. +ResourceDisk.EnableSwap=y + +# Size of the swapfile. +ResourceDisk.SwapSizeMB=1024 + +# Comma-seperated list of mount options. See man(8) for valid options. +ResourceDisk.MountOptions=None + +# Enable verbose logging (y|n) +Logs.Verbose=y + +# Is FIPS enabled +OS.EnableFIPS=n + +# Root device timeout in seconds. +OS.RootDeviceScsiTimeout=300 + +# If "None", the system default version is used. +OS.OpensslPath=/var/lib/waagent/openssl + +# Set the path to SSH keys and configuration files +OS.SshDir=/etc/ssh + +# If set, agent will use proxy server to access internet +#HttpProxy.Host=None +#HttpProxy.Port=None + +# Detect Scvmm environment, default is n +# DetectScvmmEnv=n + +# +# Lib.Dir=/var/lib/waagent + +# +# DVD.MountPoint=/mnt/cdrom/secure + +# +# Pid.File=/var/run/waagent.pid + +# +# Extension.LogDir=/var/log/azure + +# +# Home.Dir=/home + +# Enable RDMA management and set up, should only be used in HPC images +OS.EnableRDMA=n + +# Enable or disable goal state processing auto-update, default is enabled +AutoUpdate.Enabled=n + +# Determine the update family, this should not be changed +# AutoUpdate.GAFamily=Prod + +# Determine if the overprovisioning feature is enabled. If yes, hold extension +# handling until inVMArtifactsProfile.OnHold is false. +# Default is disabled +# EnableOverProvisioning=n diff --git a/config/suse/waagent.conf b/config/suse/waagent.conf index ca53acf..b2e90a8 100644 --- a/config/suse/waagent.conf +++ b/config/suse/waagent.conf @@ -5,6 +5,9 @@ # Enable instance creation Provisioning.Enabled=y +# Rely on cloud-init to provision +Provisioning.UseCloudInit=n + # Password authentication for root account will be unavailable. Provisioning.DeleteRootPassword=y @@ -57,12 +60,18 @@ LBProbeResponder=y # Enable verbose logging (y|n) Logs.Verbose=n +# Is FIPS enabled +OS.EnableFIPS=n + # Root device timeout in seconds. OS.RootDeviceScsiTimeout=300 # If "None", the system default version is used. OS.OpensslPath=None +# Set the path to SSH keys and configuration files +OS.SshDir=/etc/ssh + # If set, agent will use proxy server to access internet #HttpProxy.Host=None #HttpProxy.Port=None diff --git a/config/ubuntu/waagent.conf b/config/ubuntu/waagent.conf index dbc80fc..734a403 100644 --- a/config/ubuntu/waagent.conf +++ b/config/ubuntu/waagent.conf @@ -5,6 +5,9 @@ # Enable instance creation Provisioning.Enabled=n +# Rely on cloud-init to provision +Provisioning.UseCloudInit=y + # Password authentication for root account will be unavailable. Provisioning.DeleteRootPassword=y @@ -57,12 +60,18 @@ LBProbeResponder=y # Enable verbose logging (y|n) Logs.Verbose=n +# Is FIPS enabled +OS.EnableFIPS=n + # Root device timeout in seconds. OS.RootDeviceScsiTimeout=300 # If "None", the system default version is used. OS.OpensslPath=None +# Set the path to SSH keys and configuration files +OS.SshDir=/etc/ssh + # If set, agent will use proxy server to access internet #HttpProxy.Host=None #HttpProxy.Port=None diff --git a/config/waagent.conf b/config/waagent.conf index 8a69462..b1b1ba3 100644 --- a/config/waagent.conf +++ b/config/waagent.conf @@ -5,6 +5,9 @@ # Enable instance creation Provisioning.Enabled=y +# Rely on cloud-init to provision +Provisioning.UseCloudInit=n + # Password authentication for root account will be unavailable. Provisioning.DeleteRootPassword=y @@ -54,12 +57,18 @@ ResourceDisk.MountOptions=None # Enable verbose logging (y|n) Logs.Verbose=n +# Is FIPS enabled +OS.EnableFIPS=n + # Root device timeout in seconds. OS.RootDeviceScsiTimeout=300 # If "None", the system default version is used. OS.OpensslPath=None +# Set the path to SSH keys and configuration files +OS.SshDir=/etc/ssh + # If set, agent will use proxy server to access internet #HttpProxy.Host=None #HttpProxy.Port=None diff --git a/debian/changelog b/debian/changelog index bfb9d5f..0d09768 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,10 @@ +walinuxagent (2.2.12-0ubuntu1) artful; urgency=medium + + * New upstream release (LP: #1690854). + - Refreshed debian/patches/disable_import_test.patch. + + -- Łukasz 'sil2100' Zemczak <lukasz.zemczak@ubuntu.com> Thu, 18 May 2017 19:58:02 +0200 + walinuxagent (2.2.9-0ubuntu1) zesty; urgency=medium * New upstream release (LP: #1683521). diff --git a/debian/patches/disable_import_test.patch b/debian/patches/disable_import_test.patch index 14c0746..cc69b6b 100644 --- a/debian/patches/disable_import_test.patch +++ b/debian/patches/disable_import_test.patch @@ -1,14 +1,15 @@ -Index: walinuxagent-2.2.2/config/waagent.conf -=================================================================== ---- walinuxagent-2.2.2.orig/config/waagent.conf -+++ walinuxagent-2.2.2/config/waagent.conf -@@ -3,13 +3,13 @@ +--- a/config/waagent.conf ++++ b/config/waagent.conf +@@ -3,16 +3,16 @@ # # Enable instance creation -Provisioning.Enabled=y +Provisioning.Enabled=n + # Rely on cloud-init to provision + Provisioning.UseCloudInit=n + # Password authentication for root account will be unavailable. -Provisioning.DeleteRootPassword=y +Provisioning.DeleteRootPassword=n @@ -19,7 +20,7 @@ Index: walinuxagent-2.2.2/config/waagent.conf # Supported values are "rsa", "dsa" and "ecdsa". Provisioning.SshHostKeyPairType=rsa -@@ -33,14 +33,14 @@ Provisioning.ExecuteCustomData=n +@@ -36,14 +36,14 @@ Provisioning.AllowResetSysUser=n # Format if unformatted. If 'n', resource disk will not be mounted. diff --git a/init/arch/waagent.service b/init/arch/waagent.service new file mode 100644 index 0000000..d426eb2 --- /dev/null +++ b/init/arch/waagent.service @@ -0,0 +1,16 @@ +[Unit] +Description=Azure Linux Agent +Wants=network-online.target sshd.service sshd-keygen.service +After=network-online.target + +ConditionFileIsExecutable=/usr/bin/waagent +ConditionPathExists=/etc/waagent.conf + +[Service] +Type=simple +ExecStart=/usr/bin/python -u /usr/bin/waagent -daemon +Restart=always +RestartSec=5 + +[Install] +WantedBy=multi-user.target diff --git a/init/gaia/waagent b/init/gaia/waagent new file mode 100755 index 0000000..c706b85 --- /dev/null +++ b/init/gaia/waagent @@ -0,0 +1,56 @@ +#!/bin/bash +# +# Init file for AzureLinuxAgent. +# +# chkconfig: 2345 60 80 +# description: AzureLinuxAgent +# + +# source function library +. /etc/rc.d/init.d/functions + +RETVAL=0 +FriendlyName="AzureLinuxAgent" +WAZD_BIN=/usr/sbin/waagent.sh + +start() +{ + echo -n $"Starting $FriendlyName: " + $WAZD_BIN -start & + success + echo +} + +stop() +{ + echo -n $"Stopping $FriendlyName: " + killproc -p /var/run/waagent.pid $WAZD_BIN + RETVAL=$? + echo + return $RETVAL +} + +case "$1" in + start) + start + ;; + stop) + stop + ;; + restart) + stop + start + ;; + reload) + ;; + report) + ;; + status) + status $WAZD_BIN + RETVAL=$? + ;; + *) + echo $"Usage: $0 {start|stop|restart|status}" + RETVAL=1 +esac +exit $RETVAL @@ -88,6 +88,12 @@ def get_data_files(name, version, fullname): # TODO this is a mitigation to systemctl bug on 7.1 set_sysv_files(data_files) + elif name == 'arch': + set_bin_files(data_files, dest="/usr/bin") + set_conf_files(data_files, src=["config/arch/waagent.conf"]) + set_udev_files(data_files) + set_systemd_files(data_files, dest='/usr/lib/systemd/system', + src=["init/arch/waagent.service"]) elif name == 'coreos': set_bin_files(data_files, dest="/usr/share/oem/bin") set_conf_files(data_files, dest="/usr/share/oem", @@ -106,7 +112,7 @@ def get_data_files(name, version, fullname): set_bin_files(data_files) set_conf_files(data_files, src=["config/ubuntu/waagent.conf"]) set_logrotate_files(data_files) - set_udev_files(data_files, src=["config/99-azure-product-uuid.rules"]) + set_udev_files(data_files) if version.startswith("12") or version.startswith("14"): # Ubuntu12.04/14.04 - uses upstart set_files(data_files, dest="/etc/init", diff --git a/tests/common/osutil/test_default.py b/tests/common/osutil/test_default.py index 933787b..87acc60 100644 --- a/tests/common/osutil/test_default.py +++ b/tests/common/osutil/test_default.py @@ -18,8 +18,11 @@ import socket import glob import mock + import azurelinuxagent.common.osutil.default as osutil import azurelinuxagent.common.utils.shellutil as shellutil +from azurelinuxagent.common.exception import OSUtilError +from azurelinuxagent.common.future import ustr from azurelinuxagent.common.osutil import get_osutil from azurelinuxagent.common.utils import fileutil from tests.tools import * @@ -40,6 +43,50 @@ class TestOSUtil(AgentTestCase): self.assertEqual(run_patch.call_count, retries) self.assertEqual(run_patch.call_args_list[0][0][0], 'ifdown {0} && ifup {0}'.format(ifname)) + def test_get_dvd_device_success(self): + with patch.object(os, 'listdir', return_value=['cpu', 'cdrom0']): + osutil.DefaultOSUtil().get_dvd_device() + + def test_get_dvd_device_failure(self): + with patch.object(os, 'listdir', return_value=['cpu', 'notmatching']): + try: + osutil.DefaultOSUtil().get_dvd_device() + self.fail('OSUtilError was not raised') + except OSUtilError as ose: + self.assertTrue('notmatching' in ustr(ose)) + + @patch('time.sleep') + def test_mount_dvd_success(self, _): + msg = 'message' + with patch.object(osutil.DefaultOSUtil, + 'get_dvd_device', + return_value='/dev/cdrom'): + with patch.object(shellutil, + 'run_get_output', + return_value=(0, msg)) as patch_run: + with patch.object(os, 'makedirs'): + try: + osutil.DefaultOSUtil().mount_dvd() + except OSUtilError: + self.fail("mounting failed") + + @patch('time.sleep') + def test_mount_dvd_failure(self, _): + msg = 'message' + with patch.object(osutil.DefaultOSUtil, + 'get_dvd_device', + return_value='/dev/cdrom'): + with patch.object(shellutil, + 'run_get_output', + return_value=(1, msg)) as patch_run: + with patch.object(os, 'makedirs'): + try: + osutil.DefaultOSUtil().mount_dvd() + self.fail('OSUtilError was not raised') + except OSUtilError as ose: + self.assertTrue(msg in ustr(ose)) + self.assertTrue(patch_run.call_count == 6) + def test_get_first_if(self): ifname, ipaddr = osutil.DefaultOSUtil().get_first_if() self.assertTrue(ifname.startswith('eth')) @@ -315,5 +362,37 @@ Match host 192.168.1.2\n\ conf.get_sshd_conf_file_path(), expected_output) + @patch('os.path.isfile', return_value=True) + @patch('azurelinuxagent.common.utils.fileutil.read_file', + return_value="B9F3C233-9913-9F42-8EB3-BA656DF32502") + def test_get_instance_id_from_file(self, mock_read, mock_isfile): + util = osutil.DefaultOSUtil() + self.assertEqual( + "B9F3C233-9913-9F42-8EB3-BA656DF32502", + util.get_instance_id()) + + @patch('os.path.isfile', return_value=False) + @patch('azurelinuxagent.common.utils.shellutil.run_get_output', + return_value=[0, 'B9F3C233-9913-9F42-8EB3-BA656DF32502']) + def test_get_instance_id_from_dmidecode(self, mock_shell, mock_isfile): + util = osutil.DefaultOSUtil() + self.assertEqual( + "B9F3C233-9913-9F42-8EB3-BA656DF32502", + util.get_instance_id()) + + @patch('os.path.isfile', return_value=False) + @patch('azurelinuxagent.common.utils.shellutil.run_get_output', + return_value=[1, 'Error Value']) + def test_get_instance_id_missing(self, mock_shell, mock_isfile): + util = osutil.DefaultOSUtil() + self.assertEqual("", util.get_instance_id()) + + @patch('os.path.isfile', return_value=False) + @patch('azurelinuxagent.common.utils.shellutil.run_get_output', + return_value=[0, 'Unexpected Value']) + def test_get_instance_id_unexpected(self, mock_shell, mock_isfile): + util = osutil.DefaultOSUtil() + self.assertEqual("", util.get_instance_id()) + if __name__ == '__main__': unittest.main() diff --git a/tests/common/test_conf.py b/tests/common/test_conf.py new file mode 100644 index 0000000..1287b0d --- /dev/null +++ b/tests/common/test_conf.py @@ -0,0 +1,61 @@ +# Copyright 2014 Microsoft Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Requires Python 2.4+ and Openssl 1.0+ +# + +import mock +import os.path + +from azurelinuxagent.common.conf import * + +from tests.tools import * + + +class TestConf(AgentTestCase): + def setUp(self): + AgentTestCase.setUp(self) + self.conf = ConfigurationProvider() + load_conf_from_file( + os.path.join(data_dir, "test_waagent.conf"), + self.conf) + + def test_key_value_handling(self): + self.assertEqual("Value1", self.conf.get("FauxKey1", "Bad")) + self.assertEqual("Value2 Value2", self.conf.get("FauxKey2", "Bad")) + + def test_get_ssh_dir(self): + self.assertTrue(get_ssh_dir(self.conf).startswith("/notareal/path")) + + def test_get_sshd_conf_file_path(self): + self.assertTrue(get_sshd_conf_file_path( + self.conf).startswith("/notareal/path")) + + def test_get_ssh_key_glob(self): + self.assertTrue(get_ssh_key_glob( + self.conf).startswith("/notareal/path")) + + def test_get_ssh_key_private_path(self): + self.assertTrue(get_ssh_key_private_path( + self.conf).startswith("/notareal/path")) + + def test_get_ssh_key_public_path(self): + self.assertTrue(get_ssh_key_public_path( + self.conf).startswith("/notareal/path")) + + def test_get_fips_enabled(self): + self.assertTrue(get_fips_enabled(self.conf)) + + def test_get_provision_cloudinit(self): + self.assertTrue(get_provision_cloudinit(self.conf)) diff --git a/tests/daemon/test_daemon.py b/tests/daemon/test_daemon.py index dd31fd7..5694dc9 100644 --- a/tests/daemon/test_daemon.py +++ b/tests/daemon/test_daemon.py @@ -14,7 +14,9 @@ # # Requires Python 2.4+ and Openssl 1.0+ # -from azurelinuxagent.daemon import get_daemon_handler + +from azurelinuxagent.daemon import * +from azurelinuxagent.daemon.main import OPENSSL_FIPS_ENVIRONMENT from tests.tools import * @@ -30,8 +32,9 @@ class MockDaemonCall(object): self.daemon_handler.running = False raise Exception("Mock unhandled exception") -@patch("time.sleep") class TestDaemon(AgentTestCase): + + @patch("time.sleep") def test_daemon_restart(self, mock_sleep): #Mock daemon function daemon_handler = get_daemon_handler() @@ -45,6 +48,7 @@ class TestDaemon(AgentTestCase): mock_sleep.assert_any_call(15) self.assertEquals(2, daemon_handler.daemon.call_count) + @patch("time.sleep") @patch("azurelinuxagent.daemon.main.conf") @patch("azurelinuxagent.daemon.main.sys.exit") def test_check_pid(self, mock_exit, mock_conf, mock_sleep): @@ -58,6 +62,25 @@ class TestDaemon(AgentTestCase): daemon_handler.check_pid() mock_exit.assert_any_call(0) + + @patch("azurelinuxagent.daemon.main.DaemonHandler.check_pid") + @patch("azurelinuxagent.common.conf.get_fips_enabled", return_value=True) + def test_set_openssl_fips(self, mock_conf, mock_daemon): + daemon_handler = get_daemon_handler() + daemon_handler.running = False + with patch.dict("os.environ"): + daemon_handler.run() + self.assertTrue(OPENSSL_FIPS_ENVIRONMENT in os.environ) + self.assertEqual('1', os.environ[OPENSSL_FIPS_ENVIRONMENT]) + + @patch("azurelinuxagent.daemon.main.DaemonHandler.check_pid") + @patch("azurelinuxagent.common.conf.get_fips_enabled", return_value=False) + def test_does_not_set_openssl_fips(self, mock_conf, mock_daemon): + daemon_handler = get_daemon_handler() + daemon_handler.running = False + with patch.dict("os.environ"): + daemon_handler.run() + self.assertFalse(OPENSSL_FIPS_ENVIRONMENT in os.environ) if __name__ == '__main__': unittest.main() diff --git a/tests/data/events/1478123456789000.tld b/tests/data/events/1478123456789000.tld new file mode 100644 index 0000000..a689f4c --- /dev/null +++ b/tests/data/events/1478123456789000.tld @@ -0,0 +1 @@ +{"eventId": 1, "providerId": "69B669B9-4AF8-4C50-BDC4-6006FA76E975", "parameters": [{"name": "Name", "value": "Test Event"}, {"name": "Version", "value": "2.2.0"}, {"name": "IsInternal", "value": false}, {"name": "Operation", "value": "Some Operation"}, {"name": "OperationSuccess", "value": true}, {"name": "Message", "value": ""}, {"name": "Duration", "value": 0}, {"name": "ExtensionType", "value": ""}]}
\ No newline at end of file diff --git a/tests/data/events/1478123456789001.tld b/tests/data/events/1478123456789001.tld new file mode 100644 index 0000000..95460e3 --- /dev/null +++ b/tests/data/events/1478123456789001.tld @@ -0,0 +1 @@ +{"eventId": 1, "providerId": "69B669B9-4AF8-4C50-BDC4-6006FA76E975", "parameters": [{"name": "Name", "value": "Linux Event"}, {"name": "Version", "value": "2.2.0"}, {"name": "IsInternal", "value": false}, {"name": "Operation", "value": "Linux Operation"}, {"name": "OperationSuccess", "value": false}, {"name": "Message", "value": "Linux Message"}, {"name": "Duration", "value": 42}, {"name": "ExtensionType", "value": "Linux Event Type"}]}
\ No newline at end of file diff --git a/tests/data/events/1479766858966718.tld b/tests/data/events/1479766858966718.tld new file mode 100644 index 0000000..cc7ac67 --- /dev/null +++ b/tests/data/events/1479766858966718.tld @@ -0,0 +1 @@ +{"eventId": 1, "providerId": "69B669B9-4AF8-4C50-BDC4-6006FA76E975", "parameters": [{"name": "Name", "value": "WALinuxAgent"}, {"name": "Version", "value": "2.3.0.1"}, {"name": "IsInternal", "value": false}, {"name": "Operation", "value": "Enable"}, {"name": "OperationSuccess", "value": true}, {"name": "Message", "value": "Agent WALinuxAgent-2.3.0.1 launched with command 'python install.py' is successfully running"}, {"name": "Duration", "value": 0}, {"name": "ExtensionType", "value": ""}]}
\ No newline at end of file diff --git a/tests/data/ext/sample_ext.zip b/tests/data/ext/sample_ext-1.2.0.zip Binary files differindex 08cfaf7..08cfaf7 100644 --- a/tests/data/ext/sample_ext.zip +++ b/tests/data/ext/sample_ext-1.2.0.zip diff --git a/tests/data/ext/sample_ext/HandlerManifest.json b/tests/data/ext/sample_ext-1.2.0/HandlerManifest.json index 9890d0c..9890d0c 100644 --- a/tests/data/ext/sample_ext/HandlerManifest.json +++ b/tests/data/ext/sample_ext-1.2.0/HandlerManifest.json diff --git a/tests/data/ext/sample_ext/sample.py b/tests/data/ext/sample_ext-1.2.0/sample.py index 74bd839..74bd839 100755 --- a/tests/data/ext/sample_ext/sample.py +++ b/tests/data/ext/sample_ext-1.2.0/sample.py diff --git a/tests/data/ga/WALinuxAgent-2.2.11.zip b/tests/data/ga/WALinuxAgent-2.2.11.zip Binary files differnew file mode 100644 index 0000000..f018116 --- /dev/null +++ b/tests/data/ga/WALinuxAgent-2.2.11.zip diff --git a/tests/data/ga/WALinuxAgent-2.2.8.zip b/tests/data/ga/WALinuxAgent-2.2.8.zip Binary files differdeleted file mode 100644 index 04c60a8..0000000 --- a/tests/data/ga/WALinuxAgent-2.2.8.zip +++ /dev/null diff --git a/tests/data/ga/supported.json b/tests/data/ga/supported.json new file mode 100644 index 0000000..2ae3753 --- /dev/null +++ b/tests/data/ga/supported.json @@ -0,0 +1,8 @@ +{ + "ubuntu.16.10-x64": { + "versions": [ + "^Ubuntu,16.10,yakkety$" + ], + "slice": 10 + } +} diff --git a/tests/data/metadata/vmagent_manifest1.json b/tests/data/metadata/vmagent_manifest1.json new file mode 100644 index 0000000..544a708 --- /dev/null +++ b/tests/data/metadata/vmagent_manifest1.json @@ -0,0 +1,20 @@ +{ + "versions": [ + { + "version": "2.2.8", + "uris": [ + { + "uri": "https: //notused.com/ga/WALinuxAgent-2.2.8.zip" + } + ] + }, + { + "version": "2.2.9", + "uris": [ + { + "uri": "https: //notused.com/ga/WALinuxAgent-2.2.9.zip" + } + ] + } + ] +}
\ No newline at end of file diff --git a/tests/data/metadata/vmagent_manifest2.json b/tests/data/metadata/vmagent_manifest2.json new file mode 100644 index 0000000..544a708 --- /dev/null +++ b/tests/data/metadata/vmagent_manifest2.json @@ -0,0 +1,20 @@ +{ + "versions": [ + { + "version": "2.2.8", + "uris": [ + { + "uri": "https: //notused.com/ga/WALinuxAgent-2.2.8.zip" + } + ] + }, + { + "version": "2.2.9", + "uris": [ + { + "uri": "https: //notused.com/ga/WALinuxAgent-2.2.9.zip" + } + ] + } + ] +}
\ No newline at end of file diff --git a/tests/data/metadata/vmagent_manifests.json b/tests/data/metadata/vmagent_manifests.json new file mode 100644 index 0000000..2628f89 --- /dev/null +++ b/tests/data/metadata/vmagent_manifests.json @@ -0,0 +1,7 @@ +{ + "versionsManifestUris" : + [ + { "uri" : "https://notused.com/vmagent_manifest1.json" }, + { "uri" : "https://notused.com/vmagent_manifest2.json" } + ] +} diff --git a/tests/data/metadata/vmagent_manifests_invalid1.json b/tests/data/metadata/vmagent_manifests_invalid1.json new file mode 100644 index 0000000..55b08d1 --- /dev/null +++ b/tests/data/metadata/vmagent_manifests_invalid1.json @@ -0,0 +1,10 @@ +{ + "notTheRightKey": [ + { + "uri": "https://notused.com/vmagent_manifest1.json" + }, + { + "uri": "https://notused.com/vmagent_manifest2.json" + } + ] +}
\ No newline at end of file diff --git a/tests/data/metadata/vmagent_manifests_invalid2.json b/tests/data/metadata/vmagent_manifests_invalid2.json new file mode 100644 index 0000000..5df4252 --- /dev/null +++ b/tests/data/metadata/vmagent_manifests_invalid2.json @@ -0,0 +1,10 @@ +{ + "notTheRightKey": [ + { + "foo": "https://notused.com/vmagent_manifest1.json" + }, + { + "bar": "https://notused.com/vmagent_manifest2.json" + } + ] +}
\ No newline at end of file diff --git a/tests/data/test_waagent.conf b/tests/data/test_waagent.conf new file mode 100644 index 0000000..6368c39 --- /dev/null +++ b/tests/data/test_waagent.conf @@ -0,0 +1,111 @@ +# +# Microsoft Azure Linux Agent Configuration +# + +# Key / value handling test entries +=Value0 +FauxKey1= Value1 +FauxKey2=Value2 Value2 + +# Enable instance creation +Provisioning.Enabled=y + +# Rely on cloud-init to provision +Provisioning.UseCloudInit=y + +# Password authentication for root account will be unavailable. +Provisioning.DeleteRootPassword=y + +# Generate fresh host key pair. +Provisioning.RegenerateSshHostKeyPair=y + +# Supported values are "rsa", "dsa" and "ecdsa". +Provisioning.SshHostKeyPairType=rsa + +# Monitor host name changes and publish changes via DHCP requests. +Provisioning.MonitorHostName=y + +# Decode CustomData from Base64. +Provisioning.DecodeCustomData=n + +# Execute CustomData after provisioning. +Provisioning.ExecuteCustomData=n + +# Algorithm used by crypt when generating password hash. +#Provisioning.PasswordCryptId=6 + +# Length of random salt used when generating password hash. +#Provisioning.PasswordCryptSaltLength=10 + +# Allow reset password of sys user +Provisioning.AllowResetSysUser=n + +# Format if unformatted. If 'n', resource disk will not be mounted. +ResourceDisk.Format=y + +# File system on the resource disk +# Typically ext3 or ext4. FreeBSD images should use 'ufs2' here. +ResourceDisk.Filesystem=ext4 + +# Mount point for the resource disk +ResourceDisk.MountPoint=/mnt/resource + +# Create and use swapfile on resource disk. +ResourceDisk.EnableSwap=n + +# Size of the swapfile. +ResourceDisk.SwapSizeMB=0 + +# Comma-seperated list of mount options. See man(8) for valid options. +ResourceDisk.MountOptions=None + +# Enable verbose logging (y|n) +Logs.Verbose=n + +# Is FIPS enabled +OS.EnableFIPS=y + +# Root device timeout in seconds. +OS.RootDeviceScsiTimeout=300 + +# If "None", the system default version is used. +OS.OpensslPath=None + +# Set the path to SSH keys and configuration files +OS.SshDir=/notareal/path + +# If set, agent will use proxy server to access internet +#HttpProxy.Host=None +#HttpProxy.Port=None + +# Detect Scvmm environment, default is n +# DetectScvmmEnv=n + +# +# Lib.Dir=/var/lib/waagent + +# +# DVD.MountPoint=/mnt/cdrom/secure + +# +# Pid.File=/var/run/waagent.pid + +# +# Extension.LogDir=/var/log/azure + +# +# Home.Dir=/home + +# Enable RDMA management and set up, should only be used in HPC images +# OS.EnableRDMA=y + +# Enable or disable goal state processing auto-update, default is enabled +# AutoUpdate.Enabled=y + +# Determine the update family, this should not be changed +# AutoUpdate.GAFamily=Prod + +# Determine if the overprovisioning feature is enabled. If yes, hold extension +# handling until inVMArtifactsProfile.OnHold is false. +# Default is disabled +# EnableOverProvisioning=n diff --git a/tests/ga/test_extension.py b/tests/ga/test_extension.py index 1df0a04..2a60ea3 100644 --- a/tests/ga/test_extension.py +++ b/tests/ga/test_extension.py @@ -15,6 +15,14 @@ # Requires Python 2.4+ and Openssl 1.0+ # +import glob +import os +import os.path +import shutil +import tempfile +import zipfile + +import azurelinuxagent.common.conf as conf import azurelinuxagent.common.utils.fileutil as fileutil from tests.protocol.mockwiredata import * @@ -30,6 +38,103 @@ from azurelinuxagent.common.protocol.restapi import ExtHandlerStatus, \ from azurelinuxagent.ga.exthandlers import * from azurelinuxagent.common.protocol.wire import WireProtocol +class TestExtensionCleanup(AgentTestCase): + def setUp(self): + AgentTestCase.setUp(self) + self.ext_handlers = ExtHandlersHandler() + self.lib_dir = tempfile.mkdtemp() + + def _install_handlers(self, start=0, count=1, + handler_state=ExtHandlerState.Installed): + src = os.path.join(data_dir, "ext", "sample_ext-1.2.0.zip") + version = FlexibleVersion("1.2.0") + version += start - version.patch + + for i in range(start, start+count): + eh = ExtHandler() + eh.name = "sample_ext" + eh.properties.version = str(version) + handler = ExtHandlerInstance(eh, "unused") + + dst = os.path.join(self.lib_dir, + handler.get_full_name()+HANDLER_PKG_EXT) + shutil.copy(src, dst) + + if not handler_state is None: + zipfile.ZipFile(dst).extractall(handler.get_base_dir()) + handler.set_handler_state(handler_state) + + version += 1 + + def _count_packages(self): + return len(glob.glob(os.path.join(self.lib_dir, "*.zip"))) + + def _count_installed(self): + paths = os.listdir(self.lib_dir) + paths = [os.path.join(self.lib_dir, p) for p in paths] + return len([p for p in paths + if os.path.isdir(p) and self._is_installed(p)]) + + def _count_uninstalled(self): + paths = os.listdir(self.lib_dir) + paths = [os.path.join(self.lib_dir, p) for p in paths] + return len([p for p in paths + if os.path.isdir(p) and not self._is_installed(p)]) + + def _is_installed(self, path): + path = os.path.join(path, 'config', 'HandlerState') + return fileutil.read_file(path) != "NotInstalled" + + @patch("azurelinuxagent.common.conf.get_lib_dir") + def test_cleanup_leaves_installed_extensions(self, mock_conf): + mock_conf.return_value = self.lib_dir + + self._install_handlers(start=0, count=5, handler_state=ExtHandlerState.Installed) + self._install_handlers(start=5, count=5, handler_state=ExtHandlerState.Enabled) + + self.assertEqual(self._count_packages(), 10) + self.assertEqual(self._count_installed(), 10) + + self.ext_handlers.cleanup_outdated_handlers() + + self.assertEqual(self._count_packages(), 10) + self.assertEqual(self._count_installed(), 10) + self.assertEqual(self._count_uninstalled(), 0) + + @patch("azurelinuxagent.common.conf.get_lib_dir") + def test_cleanup_removes_uninstalled_extensions(self, mock_conf): + mock_conf.return_value = self.lib_dir + + self._install_handlers(start=0, count=5, handler_state=ExtHandlerState.Installed) + self._install_handlers(start=5, count=5, handler_state=ExtHandlerState.NotInstalled) + + self.assertEqual(self._count_packages(), 10) + self.assertEqual(self._count_installed(), 5) + self.assertEqual(self._count_uninstalled(), 5) + + self.ext_handlers.cleanup_outdated_handlers() + + self.assertEqual(self._count_packages(), 5) + self.assertEqual(self._count_installed(), 5) + self.assertEqual(self._count_uninstalled(), 0) + + @patch("azurelinuxagent.common.conf.get_lib_dir") + def test_cleanup_removes_orphaned_packages(self, mock_conf): + mock_conf.return_value = self.lib_dir + + self._install_handlers(start=0, count=5, handler_state=ExtHandlerState.Installed) + self._install_handlers(start=5, count=5, handler_state=None) + + self.assertEqual(self._count_packages(), 10) + self.assertEqual(self._count_installed(), 5) + self.assertEqual(self._count_uninstalled(), 0) + + self.ext_handlers.cleanup_outdated_handlers() + + self.assertEqual(self._count_packages(), 5) + self.assertEqual(self._count_installed(), 5) + self.assertEqual(self._count_uninstalled(), 0) + class TestHandlerStateMigration(AgentTestCase): def setUp(self): AgentTestCase.setUp(self) diff --git a/tests/ga/test_update.py b/tests/ga/test_update.py index e7a7af4..a83db95 100644 --- a/tests/ga/test_update.py +++ b/tests/ga/test_update.py @@ -17,9 +17,16 @@ from __future__ import print_function +from datetime import datetime + +import json +import shutil + from azurelinuxagent.common.protocol.hostplugin import * from azurelinuxagent.common.protocol.wire import * +from azurelinuxagent.common.utils.fileutil import * from azurelinuxagent.ga.update import * + from tests.tools import * NO_ERROR = { @@ -28,6 +35,18 @@ NO_ERROR = { "was_fatal" : False } +FATAL_ERROR = { + "last_failure" : 42.42, + "failure_count" : 2, + "was_fatal" : True +} + +SENTINEL_ERROR = { + "last_failure" : 0.0, + "failure_count" : 0, + "was_fatal" : True +} + WITH_ERROR = { "last_failure" : 42.42, "failure_count" : 2, @@ -137,19 +156,25 @@ class UpdateTestCase(AgentTestCase): fileutil.copy_file(agent, to_dir=self.tmp_dir) return - def expand_agents(self): + def expand_agents(self, mark_test=False): for agent in self.agent_pkgs(): - zipfile.ZipFile(agent).extractall(os.path.join( - self.tmp_dir, - fileutil.trim_ext(agent, "zip"))) + path = os.path.join(self.tmp_dir, fileutil.trim_ext(agent, "zip")) + zipfile.ZipFile(agent).extractall(path) + if mark_test: + src = os.path.join(data_dir, 'ga', 'supported.json') + dst = os.path.join(path, 'supported.json') + shutil.copy(src, dst) + + dst = os.path.join(path, 'error.json') + fileutil.write_file(dst, json.dumps(SENTINEL_ERROR)) return - def prepare_agent(self, version): + def prepare_agent(self, version, mark_test=False): """ Create a download for the current agent version, copied from test data """ self.copy_agents(get_agent_pkgs()[0]) - self.expand_agents() + self.expand_agents(mark_test=mark_test) versions = self.agent_versions() src_v = FlexibleVersion(str(versions[0])) @@ -214,6 +239,64 @@ class UpdateTestCase(AgentTestCase): return dst_v +class TestSupportedDistribution(UpdateTestCase): + def setUp(self): + UpdateTestCase.setUp(self) + self.sd = SupportedDistribution({ + 'slice':10, + 'versions': ['^Ubuntu,16.10,yakkety$']}) + + + def test_creation(self): + self.assertRaises(TypeError, SupportedDistribution) + self.assertRaises(UpdateError, SupportedDistribution, None) + + self.assertEqual(self.sd.slice, 10) + self.assertEqual(self.sd.versions, ['^Ubuntu,16.10,yakkety$']) + + @patch('platform.linux_distribution') + def test_is_supported(self, mock_dist): + mock_dist.return_value = ['Ubuntu', '16.10', 'yakkety'] + self.assertTrue(self.sd.is_supported) + + mock_dist.return_value = ['something', 'else', 'entirely'] + self.assertFalse(self.sd.is_supported) + + @patch('azurelinuxagent.ga.update.datetime') + def test_in_slice(self, mock_dt): + mock_dt.utcnow = Mock(return_value=datetime(2017, 1, 1, 0, 0, 5)) + self.assertTrue(self.sd.in_slice) + + mock_dt.utcnow = Mock(return_value=datetime(2017, 1, 1, 0, 0, 42)) + self.assertFalse(self.sd.in_slice) + + +class TestSupported(UpdateTestCase): + def setUp(self): + UpdateTestCase.setUp(self) + self.sp = Supported(os.path.join(data_dir, 'ga', 'supported.json')) + + def test_creation(self): + self.assertRaises(TypeError, Supported) + self.assertRaises(UpdateError, Supported, None) + + @patch('platform.linux_distribution') + def test_is_supported(self, mock_dist): + mock_dist.return_value = ['Ubuntu', '16.10', 'yakkety'] + self.assertTrue(self.sp.is_supported) + + mock_dist.return_value = ['something', 'else', 'entirely'] + self.assertFalse(self.sp.is_supported) + + @patch('platform.linux_distribution', return_value=['Ubuntu', '16.10', 'yakkety']) + @patch('azurelinuxagent.ga.update.datetime') + def test_in_slice(self, mock_dt, mock_dist): + mock_dt.utcnow = Mock(return_value=datetime(2017, 1, 1, 0, 0, 5)) + self.assertTrue(self.sp.in_slice) + + mock_dt.utcnow = Mock(return_value=datetime(2017, 1, 1, 0, 0, 42)) + self.assertFalse(self.sp.in_slice) + class TestGuestAgentError(UpdateTestCase): def test_creation(self): self.assertRaises(TypeError, GuestAgentError) @@ -241,6 +324,17 @@ class TestGuestAgentError(UpdateTestCase): self.assertEqual(NO_ERROR["was_fatal"], err.was_fatal) return + def test_is_sentinel(self): + with self.get_error_file(error_data=SENTINEL_ERROR) as path: + err = GuestAgentError(path.name) + self.assertTrue(err.is_blacklisted) + self.assertTrue(err.is_sentinel) + + with self.get_error_file(error_data=FATAL_ERROR) as path: + err = GuestAgentError(path.name) + self.assertTrue(err.is_blacklisted) + self.assertFalse(err.is_sentinel) + def test_load_preserves_error_state(self): with self.get_error_file(error_data=WITH_ERROR) as path: err = GuestAgentError(path.name) @@ -274,15 +368,6 @@ class TestGuestAgentError(UpdateTestCase): # Agent failed >= MAX_FAILURE, it should be blacklisted self.assertTrue(err.is_blacklisted) self.assertEqual(MAX_FAILURE, err.failure_count) - - # Clear old failure does not clear recent failure - err.clear_old_failure() - self.assertTrue(err.is_blacklisted) - - # Clear does remove old, outdated failures - err.last_failure -= RETAIN_INTERVAL * 2 - err.clear_old_failure() - self.assertFalse(err.is_blacklisted) return def test_mark_failure_permanent(self): @@ -334,6 +419,9 @@ class TestGuestAgent(UpdateTestCase): self.assertEqual(get_agent_name(), agent.name) self.assertEqual(get_agent_version(), agent.version) + self.assertFalse(agent.is_test) + self.assertFalse(agent.in_slice) + self.assertEqual(self.agent_path, agent.get_agent_dir()) path = os.path.join(self.agent_path, AGENT_MANIFEST_FILE) @@ -403,6 +491,48 @@ class TestGuestAgent(UpdateTestCase): self.assertTrue(agent.is_downloaded) return + @patch('platform.linux_distribution', return_value=['Ubuntu', '16.10', 'yakkety']) + def test_is_test(self, mock_dist): + self.expand_agents(mark_test=True) + agent = GuestAgent(path=self.agent_path) + + self.assertTrue(agent.is_blacklisted) + self.assertTrue(agent.is_test) + + @patch('platform.linux_distribution', return_value=['Ubuntu', '16.10', 'yakkety']) + @patch('azurelinuxagent.ga.update.datetime') + def test_in_slice(self, mock_dt, mock_dist): + self.expand_agents(mark_test=True) + agent = GuestAgent(path=self.agent_path) + + mock_dt.utcnow = Mock(return_value=datetime(2017, 1, 1, 0, 0, 5)) + self.assertTrue(agent.in_slice) + + mock_dt.utcnow = Mock(return_value=datetime(2017, 1, 1, 0, 0, 42)) + self.assertFalse(agent.in_slice) + + @patch('platform.linux_distribution', return_value=['Ubuntu', '16.10', 'yakkety']) + @patch('azurelinuxagent.ga.update.datetime') + def test_enable(self, mock_dt, mock_dist): + mock_dt.utcnow = Mock(return_value=datetime(2017, 1, 1, 0, 0, 5)) + + self.expand_agents(mark_test=True) + agent = GuestAgent(path=self.agent_path) + + self.assertTrue(agent.is_blacklisted) + self.assertTrue(agent.is_test) + self.assertTrue(agent.in_slice) + + agent.enable() + + self.assertFalse(agent.is_blacklisted) + self.assertFalse(agent.is_test) + + # Ensure the new state is preserved to disk + agent = GuestAgent(path=self.agent_path) + self.assertFalse(agent.is_blacklisted) + self.assertFalse(agent.is_test) + @patch("azurelinuxagent.ga.update.GuestAgent._ensure_downloaded") def test_mark_failure(self, mock_ensure): agent = GuestAgent(path=self.agent_path) @@ -933,6 +1063,14 @@ class TestUpdate(UpdateTestCase): self.assertEqual("1250_waagent.pid", os.path.basename(pid_file)) return + @patch('platform.linux_distribution', return_value=['Ubuntu', '16.10', 'yakkety']) + @patch('azurelinuxagent.ga.update.datetime') + def test_get_test_agent(self, mock_dt, mock_dist): + mock_dt.utcnow = Mock(return_value=datetime(2017, 1, 1, 0, 0, 5)) + self.prepare_agent(AGENT_VERSION, mark_test=True) + + self.assertNotEqual(None, self.update_handler.get_test_agent()) + def test_is_clean_start_returns_true_when_no_sentinal(self): self.assertFalse(os.path.isfile(self.update_handler._sentinal_file_path())) self.assertTrue(self.update_handler._is_clean_start) @@ -972,27 +1110,27 @@ class TestUpdate(UpdateTestCase): self.assertTrue(self.update_handler._is_orphaned) return - def test_load_agents(self): + def test_find_agents(self): self.prepare_agents() self.assertTrue(0 <= len(self.update_handler.agents)) - self.update_handler._load_agents() + self.update_handler._find_agents() self.assertEqual(len(get_agents(self.tmp_dir)), len(self.update_handler.agents)) return - def test_load_agents_does_reload(self): + def test_find_agents_does_reload(self): self.prepare_agents() - self.update_handler._load_agents() + self.update_handler._find_agents() agents = self.update_handler.agents - self.update_handler._load_agents() + self.update_handler._find_agents() self.assertNotEqual(agents, self.update_handler.agents) return - def test_load_agents_sorts(self): + def test_find_agents_sorts(self): self.prepare_agents() - self.update_handler._load_agents() + self.update_handler._find_agents() v = FlexibleVersion("100000") for a in self.update_handler.agents: @@ -1002,7 +1140,7 @@ class TestUpdate(UpdateTestCase): def test_purge_agents(self): self.prepare_agents() - self.update_handler._load_agents() + self.update_handler._find_agents() # Ensure at least three agents initially exist self.assertTrue(2 < len(self.update_handler.agents)) @@ -1014,7 +1152,7 @@ class TestUpdate(UpdateTestCase): # Reload and assert only the kept agents remain on disk self.update_handler.agents = kept_agents self.update_handler._purge_agents() - self.update_handler._load_agents() + self.update_handler._find_agents() self.assertEqual( [agent.version for agent in kept_agents], [agent.version for agent in self.update_handler.agents]) @@ -1032,7 +1170,7 @@ class TestUpdate(UpdateTestCase): self.assertTrue(os.path.exists(agent_path + ".zip")) return - def _test_run_latest(self, mock_child=None, mock_time=None): + def _test_run_latest(self, mock_child=None, mock_time=None, child_args=None): if mock_child is None: mock_child = ChildMock() if mock_time is None: @@ -1041,7 +1179,7 @@ class TestUpdate(UpdateTestCase): with patch('subprocess.Popen', return_value=mock_child) as mock_popen: with patch('time.time', side_effect=mock_time.time): with patch('time.sleep', side_effect=mock_time.sleep): - self.update_handler.run_latest() + self.update_handler.run_latest(child_args=child_args) self.assertEqual(1, mock_popen.call_count) return mock_popen.call_args @@ -1051,16 +1189,32 @@ class TestUpdate(UpdateTestCase): agent = self.update_handler.get_latest_agent() args, kwargs = self._test_run_latest() + args = args[0] cmds = textutil.safe_shlex_split(agent.get_agent_cmd()) if cmds[0].lower() == "python": cmds[0] = get_python_cmd() - self.assertEqual(args[0], cmds) + self.assertEqual(args, cmds) + self.assertTrue(len(args) > 1) + self.assertTrue(args[0].startswith("python")) + self.assertEqual("-run-exthandlers", args[len(args)-1]) self.assertEqual(True, 'cwd' in kwargs) self.assertEqual(agent.get_agent_dir(), kwargs['cwd']) self.assertEqual(False, '\x00' in cmds[0]) return + def test_run_latest_passes_child_args(self): + self.prepare_agents() + + agent = self.update_handler.get_latest_agent() + args, kwargs = self._test_run_latest(child_args="AnArgument") + args = args[0] + + self.assertTrue(len(args) > 1) + self.assertTrue(args[0].startswith("python")) + self.assertEqual("AnArgument", args[len(args)-1]) + return + def test_run_latest_polls_and_waits_for_success(self): mock_child = ChildMock(return_value=None) mock_time = TimeMock(time_increment=CHILD_HEALTH_INTERVAL/3) @@ -1147,7 +1301,8 @@ class TestUpdate(UpdateTestCase): with patch('azurelinuxagent.ga.update.UpdateHandler.get_latest_agent', return_value=latest_agent): self._test_run_latest(mock_child=ChildMock(return_value=1)) - self.assertTrue(latest_agent.is_available) + self.assertTrue(latest_agent.is_blacklisted) + self.assertFalse(latest_agent.is_available) self.assertNotEqual(0.0, latest_agent.error.last_failure) self.assertEqual(1, latest_agent.error.failure_count) return @@ -1198,8 +1353,6 @@ class TestUpdate(UpdateTestCase): self.update_handler.running = False return - calls = calls * invocations - fileutil.write_file(conf.get_agent_pid_file_path(), ustr(42)) with patch('azurelinuxagent.ga.exthandlers.get_exthandlers_handler') as mock_handler: @@ -1227,13 +1380,30 @@ class TestUpdate(UpdateTestCase): return def test_run_keeps_running(self): - self._test_run(invocations=15) + self._test_run(invocations=15, calls=[call.run()]*15) return def test_run_stops_if_update_available(self): self.update_handler._upgrade_available = Mock(return_value=True) self._test_run(invocations=0, calls=[], enable_updates=True) return + + @patch('platform.linux_distribution', return_value=['Ubuntu', '16.10', 'yakkety']) + @patch('azurelinuxagent.ga.update.datetime') + def test_run_stops_if_test_agent_available(self, mock_dt, mock_dist): + mock_dt.utcnow = Mock(return_value=datetime(2017, 1, 1, 0, 0, 5)) + self.prepare_agent(AGENT_VERSION, mark_test=True) + + agent = GuestAgent(path=self.agent_dir(AGENT_VERSION)) + agent.enable = Mock() + self.assertTrue(agent.is_test) + self.assertTrue(agent.in_slice) + + with patch('azurelinuxagent.ga.update.UpdateHandler.get_test_agent', + return_value=agent) as mock_test: + self._test_run(invocations=0) + self.assertEqual(mock_test.call_count, 1) + self.assertEqual(agent.enable.call_count, 1) def test_run_stops_if_orphaned(self): with patch('os.getppid', return_value=1): @@ -1417,6 +1587,5 @@ class TimeMock(Mock): self.next_time += self.time_increment return current_time - if __name__ == '__main__': unittest.main() diff --git a/tests/pa/test_deprovision.py b/tests/pa/test_deprovision.py index be34915..c4cd9b4 100644 --- a/tests/pa/test_deprovision.py +++ b/tests/pa/test_deprovision.py @@ -15,11 +15,117 @@ # Requires Python 2.4+ and Openssl 1.0+ # -from tests.tools import * +import tempfile + +import azurelinuxagent.common.utils.fileutil as fileutil + from azurelinuxagent.pa.deprovision import get_deprovision_handler +from azurelinuxagent.pa.deprovision.default import DeprovisionHandler +from tests.tools import * class TestDeprovision(AgentTestCase): + @patch("azurelinuxagent.pa.deprovision.default.DeprovisionHandler.cloud_init_dirs") + @patch("azurelinuxagent.pa.deprovision.default.DeprovisionHandler.cloud_init_files") + def test_del_cloud_init_without_once(self, + mock_files, + mock_dirs): + deprovision_handler = get_deprovision_handler("","","") + deprovision_handler.del_cloud_init([], [], include_once=False) + + mock_dirs.assert_called_with(include_once=False) + mock_files.assert_called_with(include_once=False) + + @patch("signal.signal") + @patch("azurelinuxagent.common.protocol.get_protocol_util") + @patch("azurelinuxagent.common.osutil.get_osutil") + @patch("azurelinuxagent.pa.deprovision.default.DeprovisionHandler.cloud_init_dirs") + @patch("azurelinuxagent.pa.deprovision.default.DeprovisionHandler.cloud_init_files") + def test_del_cloud_init(self, + mock_files, + mock_dirs, + mock_osutil, + mock_util, + mock_signal): + try: + with tempfile.NamedTemporaryFile() as f: + warnings = [] + actions = [] + + dirs = [tempfile.mkdtemp()] + mock_dirs.return_value = dirs + + files = [f.name] + mock_files.return_value = files + + deprovision_handler = get_deprovision_handler("","","") + deprovision_handler.del_cloud_init(warnings, actions) + + mock_dirs.assert_called_with(include_once=True) + mock_files.assert_called_with(include_once=True) + + self.assertEqual(len(warnings), 0) + self.assertEqual(len(actions), 2) + for da in actions: + if da.func == fileutil.rm_dirs: + self.assertEqual(da.args, dirs) + elif da.func == fileutil.rm_files: + self.assertEqual(da.args, files) + else: + self.assertTrue(False) + + try: + for da in actions: + da.invoke() + self.assertEqual(len([d for d in dirs if os.path.isdir(d)]), 0) + self.assertEqual(len([f for f in files if os.path.isfile(f)]), 0) + except Exception as e: + self.assertTrue(False, "Exception {0}".format(e)) + except OSError: + # Ignore the error caused by removing the file within the "with" + pass + + @distros("ubuntu") + @patch('azurelinuxagent.common.conf.get_lib_dir') + def test_del_lib_dir_files(self, + distro_name, + distro_version, + distro_full_name, + mock_conf): + files = [ + 'HostingEnvironmentConfig.xml', + 'Incarnation', + 'Protocol', + 'SharedConfig.xml', + 'WireServerEndpoint', + 'Extensions.1.xml', + 'ExtensionsConfig.1.xml', + 'GoalState.1.xml', + 'Extensions.2.xml', + 'ExtensionsConfig.2.xml', + 'GoalState.2.xml' + ] + + tmp = tempfile.mkdtemp() + mock_conf.return_value = tmp + for f in files: + fileutil.write_file(os.path.join(tmp, f), "Value") + + deprovision_handler = get_deprovision_handler(distro_name, + distro_version, + distro_full_name) + warnings = [] + actions = [] + deprovision_handler.del_lib_dir_files(warnings, actions) + + self.assertTrue(len(warnings) == 0) + self.assertTrue(len(actions) == 1) + self.assertEqual(fileutil.rm_files, actions[0].func) + self.assertTrue(len(actions[0].args) > 0) + for f in actions[0].args: + self.assertTrue(os.path.basename(f) in files) + + @distros("redhat") def test_deprovision(self, distro_name, diff --git a/tests/pa/test_provision.py b/tests/pa/test_provision.py index a98eacd..0446442 100644 --- a/tests/pa/test_provision.py +++ b/tests/pa/test_provision.py @@ -16,17 +16,22 @@ # import azurelinuxagent.common.utils.fileutil as fileutil + +from azurelinuxagent.common.exception import ProtocolError from azurelinuxagent.common.osutil.default import DefaultOSUtil from azurelinuxagent.common.protocol import OVF_FILE_NAME from azurelinuxagent.pa.provision import get_provision_handler +from azurelinuxagent.pa.provision.default import ProvisionHandler from tests.tools import * class TestProvision(AgentTestCase): @distros("redhat") - def test_provision(self, distro_name, distro_version, distro_full_name): - provision_handler = get_provision_handler(distro_name, distro_version, + @patch('azurelinuxagent.common.osutil.default.DefaultOSUtil.get_instance_id', + return_value='B9F3C233-9913-9F42-8EB3-BA656DF32502') + def test_provision(self, mock_util, distro_name, distro_version, distro_full_name): + provision_handler = get_provision_handler(distro_name, distro_version, distro_full_name) mock_osutil = MagicMock() mock_osutil.decode_customdata = Mock(return_value="") @@ -48,6 +53,45 @@ class TestProvision(AgentTestCase): data = DefaultOSUtil().decode_customdata(base64data) fileutil.write_file(tempfile.mktemp(), data) + @patch('os.path.isfile', return_value=False) + def test_is_provisioned_not_provisioned(self, mock_isfile): + ph = ProvisionHandler() + self.assertFalse(ph.is_provisioned()) + + @patch('os.path.isfile', return_value=True) + @patch('azurelinuxagent.common.utils.fileutil.read_file', + return_value="B9F3C233-9913-9F42-8EB3-BA656DF32502") + @patch('azurelinuxagent.pa.deprovision.get_deprovision_handler') + def test_is_provisioned_is_provisioned(self, + mock_deprovision, mock_read, mock_isfile): + ph = ProvisionHandler() + ph.osutil = Mock() + ph.osutil.get_instance_id = \ + Mock(return_value="B9F3C233-9913-9F42-8EB3-BA656DF32502") + ph.write_provisioned = Mock() + + deprovision_handler = Mock() + mock_deprovision.return_value = deprovision_handler + + self.assertTrue(ph.is_provisioned()) + deprovision_handler.run_changed_unique_id.assert_not_called() + + @patch('os.path.isfile', return_value=True) + @patch('azurelinuxagent.common.utils.fileutil.read_file', + side_effect=["Value"]) + @patch('azurelinuxagent.pa.deprovision.get_deprovision_handler') + def test_is_provisioned_not_deprovisioned(self, + mock_deprovision, mock_read, mock_isfile): + + ph = ProvisionHandler() + ph.osutil = Mock() + ph.write_provisioned = Mock() + + deprovision_handler = Mock() + mock_deprovision.return_value = deprovision_handler + + self.assertTrue(ph.is_provisioned()) + deprovision_handler.run_changed_unique_id.assert_called_once() if __name__ == '__main__': unittest.main() diff --git a/tests/protocol/mockwiredata.py b/tests/protocol/mockwiredata.py index c789de5..4e45623 100644 --- a/tests/protocol/mockwiredata.py +++ b/tests/protocol/mockwiredata.py @@ -30,7 +30,7 @@ DATA_FILE = { "ga_manifest" : "wire/ga_manifest.xml", "trans_prv": "wire/trans_prv", "trans_cert": "wire/trans_cert", - "test_ext": "ext/sample_ext.zip" + "test_ext": "ext/sample_ext-1.2.0.zip" } DATA_FILE_NO_EXT = DATA_FILE.copy() diff --git a/tests/protocol/test_hostplugin.py b/tests/protocol/test_hostplugin.py index ef91998..e203615 100644 --- a/tests/protocol/test_hostplugin.py +++ b/tests/protocol/test_hostplugin.py @@ -33,6 +33,7 @@ import azurelinuxagent.common.protocol.wire as wire import azurelinuxagent.common.protocol.hostplugin as hostplugin from azurelinuxagent.common import event +from azurelinuxagent.common.exception import ProtocolError, HttpError from azurelinuxagent.common.protocol.hostplugin import API_VERSION from azurelinuxagent.common.utils import restutil @@ -128,8 +129,10 @@ class TestHostPlugin(AgentTestCase): wire_protocol_client.get_goal_state = Mock(return_value=test_goal_state) wire_protocol_client.ext_conf = wire.ExtensionsConfig(None) wire_protocol_client.ext_conf.status_upload_blob = sas_url + wire_protocol_client.ext_conf.status_upload_blob_type = page_blob_type wire_protocol_client.status_blob.set_vm_status(status) wire_protocol_client.upload_status_blob() + self.assertEqual(patch_upload.call_count, 1) self.assertTrue(patch_put.call_count == 1, "Fallback was not engaged") self.assertTrue(patch_put.call_args[0][0] == sas_url) @@ -156,6 +159,7 @@ class TestHostPlugin(AgentTestCase): client.get_goal_state = Mock(return_value=test_goal_state) client.ext_conf = wire.ExtensionsConfig(None) client.ext_conf.status_upload_blob = sas_url + client.ext_conf.status_upload_blob_type = page_blob_type client.status_blob.set_vm_status(status) client.upload_status_blob() self.assertTrue(patch_put.call_count == 1, diff --git a/tests/protocol/test_metadata.py b/tests/protocol/test_metadata.py index f390f7a..ee4ba3e 100644 --- a/tests/protocol/test_metadata.py +++ b/tests/protocol/test_metadata.py @@ -15,14 +15,23 @@ # Requires Python 2.4+ and Openssl 1.0+ # -from tests.tools import * -from tests.protocol.mockmetadata import * +import json + +from azurelinuxagent.common.future import ustr + from azurelinuxagent.common.utils.restutil import httpclient -from azurelinuxagent.common.protocol.metadata import MetadataProtocol +from azurelinuxagent.common.protocol.metadata import * +from azurelinuxagent.common.protocol.restapi import * + +from tests.protocol.mockmetadata import * +from tests.tools import * -@patch("time.sleep") -@patch("azurelinuxagent.common.protocol.metadata.restutil") class TestMetadataProtocolGetters(AgentTestCase): + def load_json(self, path): + return json.loads(ustr(load_data(path)), encoding="utf-8") + + @patch("time.sleep") + @patch("azurelinuxagent.common.protocol.metadata.restutil") def _test_getters(self, test_data, mock_restutil ,_): mock_restutil.http_get.side_effect = test_data.mock_http_get @@ -43,3 +52,108 @@ class TestMetadataProtocolGetters(AgentTestCase): self._test_getters(test_data, *args) + @patch("azurelinuxagent.common.protocol.metadata.MetadataProtocol.update_goal_state") + @patch("azurelinuxagent.common.protocol.metadata.MetadataProtocol._get_data") + def test_get_vmagents_manifests(self, mock_get, mock_update): + data = self.load_json("metadata/vmagent_manifests.json") + mock_get.return_value = data, 42 + + protocol = MetadataProtocol() + manifests, etag = protocol.get_vmagent_manifests() + + self.assertEqual(mock_update.call_count, 1) + self.assertEqual(mock_get.call_count, 1) + + manifests_uri = BASE_URI.format( + METADATA_ENDPOINT, + "vmAgentVersions", + APIVERSION) + self.assertEqual(mock_get.call_args[0][0], manifests_uri) + + self.assertEqual(etag, 42) + self.assertNotEqual(None, manifests) + self.assertEqual(len(manifests.vmAgentManifests), 1) + + manifest = manifests.vmAgentManifests[0] + self.assertEqual(manifest.family, conf.get_autoupdate_gafamily()) + self.assertEqual(len(manifest.versionsManifestUris), 2) + + # Same etag returns the same data + data = self.load_json("metadata/vmagent_manifests_invalid1.json") + mock_get.return_value = data, 42 + next_manifests, etag = protocol.get_vmagent_manifests() + + self.assertEqual(etag, 42) + self.assertEqual(manifests, next_manifests) + + # New etag returns new data + mock_get.return_value = data, 43 + self.assertRaises(ProtocolError, protocol.get_vmagent_manifests) + + @patch("azurelinuxagent.common.protocol.metadata.MetadataProtocol.update_goal_state") + @patch("azurelinuxagent.common.protocol.metadata.MetadataProtocol._get_data") + def test_get_vmagents_manifests_raises(self, mock_get, mock_update): + data = self.load_json("metadata/vmagent_manifests_invalid1.json") + mock_get.return_value = data, 42 + + protocol = MetadataProtocol() + self.assertRaises(ProtocolError, protocol.get_vmagent_manifests) + + data = self.load_json("metadata/vmagent_manifests_invalid2.json") + mock_get.return_value = data, 43 + self.assertRaises(ProtocolError, protocol.get_vmagent_manifests) + + @patch("azurelinuxagent.common.protocol.metadata.MetadataProtocol.update_goal_state") + @patch("azurelinuxagent.common.protocol.metadata.MetadataProtocol._get_data") + def test_get_vmagent_pkgs(self, mock_get, mock_update): + data = self.load_json("metadata/vmagent_manifests.json") + mock_get.return_value = data, 42 + + protocol = MetadataProtocol() + manifests, etag = protocol.get_vmagent_manifests() + manifest = manifests.vmAgentManifests[0] + + data = self.load_json("metadata/vmagent_manifest1.json") + mock_get.return_value = data, 42 + pkgs = protocol.get_vmagent_pkgs(manifest) + + self.assertNotEqual(None, pkgs) + self.assertEqual(len(pkgs.versions), 2) + + for pkg in pkgs.versions: + self.assertNotEqual(None, pkg.version) + self.assertTrue(len(pkg.uris) > 0) + + for uri in pkg.uris: + self.assertTrue(uri.uri.endswith("zip")) + + @patch("azurelinuxagent.common.protocol.metadata.MetadataProtocol._post_data") + def test_report_event(self, mock_post): + events = TelemetryEventList() + + data = self.load_json("events/1478123456789000.tld") + event = TelemetryEvent() + set_properties("event", event, data) + events.events.append(event) + + data = self.load_json("events/1478123456789001.tld") + event = TelemetryEvent() + set_properties("event", event, data) + events.events.append(event) + + data = self.load_json("events/1479766858966718.tld") + event = TelemetryEvent() + set_properties("event", event, data) + events.events.append(event) + + protocol = MetadataProtocol() + protocol.report_event(events) + + events_uri = BASE_URI.format( + METADATA_ENDPOINT, + "status/telemetry", + APIVERSION) + + self.assertEqual(mock_post.call_count, 1) + self.assertEqual(mock_post.call_args[0][0], events_uri) + self.assertEqual(mock_post.call_args[0][1], get_properties(events)) diff --git a/tests/protocol/test_wire.py b/tests/protocol/test_wire.py index dda7a2b..ba9fc7d 100644 --- a/tests/protocol/test_wire.py +++ b/tests/protocol/test_wire.py @@ -152,11 +152,12 @@ class TestWireProtocolGetters(AgentTestCase): wire_protocol_client = WireProtocol(wireserver_url).client wire_protocol_client.ext_conf = ExtensionsConfig(None) wire_protocol_client.ext_conf.status_upload_blob = testurl + wire_protocol_client.ext_conf.status_upload_blob_type = testtype wire_protocol_client.status_blob.vm_status = vmstatus with patch.object(WireClient, "get_goal_state") as patch_get_goal_state: with patch.object(HostPluginProtocol, "put_vm_status") as patch_host_ga_plugin_upload: - with patch.object(StatusBlob, "upload", return_value=True) as patch_default_upload: + with patch.object(StatusBlob, "upload") as patch_default_upload: HostPluginProtocol.set_default_channel(False) wire_protocol_client.upload_status_blob() @@ -190,7 +191,25 @@ class TestWireProtocolGetters(AgentTestCase): self.assertTrue(HostPluginProtocol.is_default_channel()) HostPluginProtocol.set_default_channel(False) - def test_upload_status_blob_error_reporting(self, *args): + def test_upload_status_blob_unknown_type_assumes_block(self, *args): + vmstatus = VMStatus(message="Ready", status="Ready") + wire_protocol_client = WireProtocol(wireserver_url).client + wire_protocol_client.ext_conf = ExtensionsConfig(None) + wire_protocol_client.ext_conf.status_upload_blob = testurl + wire_protocol_client.ext_conf.status_upload_blob_type = "NotALegalType" + wire_protocol_client.status_blob.vm_status = vmstatus + + with patch.object(WireClient, "get_goal_state") as patch_get_goal_state: + with patch.object(StatusBlob, "prepare") as patch_prepare: + with patch.object(StatusBlob, "upload") as patch_default_upload: + HostPluginProtocol.set_default_channel(False) + wire_protocol_client.upload_status_blob() + + patch_prepare.assert_called_once_with("BlockBlob") + patch_default_upload.assert_called_once_with(testurl) + patch_get_goal_state.assert_not_called() + + def test_upload_status_blob_reports_prepare_error(self, *args): vmstatus = VMStatus(message="Ready", status="Ready") wire_protocol_client = WireProtocol(wireserver_url).client wire_protocol_client.ext_conf = ExtensionsConfig(None) @@ -199,29 +218,13 @@ class TestWireProtocolGetters(AgentTestCase): wire_protocol_client.status_blob.vm_status = vmstatus goal_state = GoalState(WireProtocolData(DATA_FILE).goal_state) - with patch.object(HostPluginProtocol, - "ensure_initialized", - return_value=True): - with patch.object(StatusBlob, - "put_block_blob", - side_effect=HttpError("error")): - with patch.object(StatusBlob, - "get_blob_type", - return_value='BlockBlob'): - with patch.object(HostPluginProtocol, - "put_vm_status"): - with patch.object(WireClient, - "report_blob_type", - side_effect=MagicMock()): - with patch.object(event, - "add_event") as patch_add_event: - HostPluginProtocol.set_default_channel(False) - wire_protocol_client.get_goal_state = Mock(return_value=goal_state) - wire_protocol_client.upload_status_blob() - wire_protocol_client.get_goal_state.assert_called_once() - self.assertTrue(patch_add_event.call_count == 1) - self.assertTrue(patch_add_event.call_args_list[0][1]['op'] == 'ReportStatus') - self.assertFalse(HostPluginProtocol.is_default_channel()) + with patch.object(StatusBlob, "prepare", + side_effect=Exception) as mock_prepare: + with patch.object(WireClient, "report_status_event") as mock_event: + wire_protocol_client.upload_status_blob() + + mock_prepare.assert_called_once() + mock_event.assert_called_once() def test_get_in_vm_artifacts_profile_blob_not_available(self, *args): wire_protocol_client = WireProtocol(wireserver_url).client diff --git a/tests/test_agent.py b/tests/test_agent.py new file mode 100644 index 0000000..1b35933 --- /dev/null +++ b/tests/test_agent.py @@ -0,0 +1,92 @@ +# Copyright 2014 Microsoft Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Requires Python 2.4+ and Openssl 1.0+ +# + +import mock +import os.path + +from azurelinuxagent.agent import * +from azurelinuxagent.common.conf import * + +from tests.tools import * + + +class TestAgent(AgentTestCase): + + def test_accepts_configuration_path(self): + conf_path = os.path.join(data_dir, "test_waagent.conf") + c, f, v, cfp = parse_args(["-configuration-path:" + conf_path]) + self.assertEqual(cfp, conf_path) + + @patch("os.path.exists", return_value=True) + def test_checks_configuration_path(self, mock_exists): + conf_path = "/foo/bar-baz/something.conf" + c, f, v, cfp = parse_args(["-configuration-path:"+conf_path]) + self.assertEqual(cfp, conf_path) + self.assertEqual(mock_exists.call_count, 1) + + @patch("sys.stderr") + @patch("os.path.exists", return_value=False) + @patch("sys.exit", side_effect=Exception) + def test_rejects_missing_configuration_path(self, mock_exit, mock_exists, mock_stderr): + try: + c, f, v, cfp = parse_args(["-configuration-path:/foo/bar.conf"]) + self.assertTrue(False) + except Exception: + self.assertEqual(mock_exit.call_count, 1) + + def test_configuration_path_defaults_to_none(self): + c, f, v, cfp = parse_args([]) + self.assertEqual(cfp, None) + + def test_agent_accepts_configuration_path(self): + Agent(False, + conf_file_path=os.path.join(data_dir, "test_waagent.conf")) + self.assertTrue(conf.get_fips_enabled()) + + @patch("azurelinuxagent.common.conf.load_conf_from_file") + def test_agent_uses_default_configuration_path(self, mock_load): + Agent(False) + mock_load.assert_called_once_with("/etc/waagent.conf") + + @patch("azurelinuxagent.daemon.get_daemon_handler") + @patch("azurelinuxagent.common.conf.load_conf_from_file") + def test_agent_does_not_pass_configuration_path(self, + mock_load, mock_handler): + + mock_daemon = Mock() + mock_daemon.run = Mock() + mock_handler.return_value = mock_daemon + + agent = Agent(False) + agent.daemon() + + mock_daemon.run.assert_called_once_with(child_args=None) + mock_load.assert_called_once() + + @patch("azurelinuxagent.daemon.get_daemon_handler") + @patch("azurelinuxagent.common.conf.load_conf_from_file") + def test_agent_passes_configuration_path(self, mock_load, mock_handler): + + mock_daemon = Mock() + mock_daemon.run = Mock() + mock_handler.return_value = mock_daemon + + agent = Agent(False, conf_file_path="/foo/bar.conf") + agent.daemon() + + mock_daemon.run.assert_called_once_with(child_args="-configuration-path:/foo/bar.conf") + mock_load.assert_called_once() diff --git a/tests/utils/test_file_util.py b/tests/utils/test_file_util.py index 76fb15b..0b92513 100644 --- a/tests/utils/test_file_util.py +++ b/tests/utils/test_file_util.py @@ -15,9 +15,14 @@ # Requires Python 2.4+ and Openssl 1.0+ # +import glob +import random +import string +import tempfile import uuid import azurelinuxagent.common.utils.fileutil as fileutil + from azurelinuxagent.common.future import ustr from tests.tools import * @@ -69,9 +74,6 @@ class TestFileOperations(AgentTestCase): self.assertEquals('abc', filename) def test_remove_files(self): - import random - import string - import glob random_word = lambda : ''.join(random.choice(string.ascii_uppercase + string.digits) for _ in range(5)) #Create 10 test files @@ -90,9 +92,27 @@ class TestFileOperations(AgentTestCase): self.assertEqual(0, len(glob.glob(os.path.join(self.tmp_dir, test_file_pattern)))) self.assertEqual(0, len(glob.glob(os.path.join(self.tmp_dir, test_file_pattern2)))) + def test_remove_dirs(self): + dirs = [] + for n in range(0,5): + dirs.append(tempfile.mkdtemp()) + for d in dirs: + for n in range(0, random.choice(range(0,10))): + fileutil.write_file(os.path.join(d, "test"+str(n)), "content") + for n in range(0, random.choice(range(0,10))): + dd = os.path.join(d, "testd"+str(n)) + os.mkdir(dd) + for nn in range(0, random.choice(range(0,10))): + os.symlink(dd, os.path.join(dd, "sym"+str(nn))) + for n in range(0, random.choice(range(0,10))): + os.symlink(d, os.path.join(d, "sym"+str(n))) + + fileutil.rm_dirs(*dirs) + + for d in dirs: + self.assertEqual(len(os.listdir(d)), 0) + def test_get_all_files(self): - import random - import string random_word = lambda: ''.join(random.choice(string.ascii_uppercase + string.digits) for _ in range(5)) # Create 10 test files at the root dir and 10 other in the sub dir |