diff options
37 files changed, 1177 insertions, 239 deletions
@@ -2,9 +2,9 @@ ### INTRODUCTION -The Microsoft Azure Linux Agent (waagent) manages Linux & FreeBSD provisioning, +The Microsoft Azure Linux Agent (waagent) manages Linux & BSD provisioning, and VM interaction with the Azure Fabric Controller. It provides the following -functionality for Linux and FreeBSD IaaS deployments: +functionality for Linux and BSD IaaS deployments: * Image Provisioning - Creation of a user account @@ -59,6 +59,7 @@ of supported systems on the Microsoft Azure Platform as described here: http://support.microsoft.com/kb/2805216 Supported Linux Distributions: + * Archlinux * CoreOS * CentOS 6.2+ * Red Hat Enterprise Linux 6.7+ @@ -70,6 +71,7 @@ Supported Linux Distributions: Other Supported Systems: * FreeBSD 10+ (Azure Linux Agent v2.0.10+) + * OpenBSD 6+ (Azure Linux Agent v2.2.11+) Waagent depends on some system packages in order to function properly: @@ -303,8 +305,8 @@ _Default: ext4_ This specifies the filesystem type for the resource disk. Supported values vary by Linux distribution. If the string is X, then mkfs.X should be present on the -Linux image. SLES 11 images should typically use 'ext3'. FreeBSD images should -use 'ufs2' here. +Linux image. SLES 11 images should typically use 'ext3'. BSD images should use +'ufs2' here. * __ResourceDisk.MountPoint__ _Type: String_ diff --git a/azurelinuxagent/agent.py b/azurelinuxagent/agent.py index 90b4253..d1ac354 100644 --- a/azurelinuxagent/agent.py +++ b/azurelinuxagent/agent.py @@ -129,7 +129,7 @@ def main(args=[]): elif command == "help": usage() elif command == "start": - start() + start(conf_file_path=conf_file_path) else: try: agent = Agent(verbose, conf_file_path=conf_file_path) @@ -217,13 +217,16 @@ def usage(): "").format(sys.argv[0]))) print("") -def start(): +def start(conf_file_path=None): """ Start agent daemon in a background process and set stdout/stderr to /dev/null """ devnull = open(os.devnull, 'w') - subprocess.Popen([sys.argv[0], '-daemon'], stdout=devnull, stderr=devnull) + args = [sys.argv[0], '-daemon'] + if conf_file_path is not None: + args.append('-configuration-path:{0}'.format(conf_file_path)) + subprocess.Popen(args, stdout=devnull, stderr=devnull) if __name__ == '__main__' : main() diff --git a/azurelinuxagent/common/event.py b/azurelinuxagent/common/event.py index 116478b..723b8bf 100644 --- a/azurelinuxagent/common/event.py +++ b/azurelinuxagent/common/event.py @@ -25,7 +25,7 @@ import datetime import threading import platform -from datetime import datetime +from datetime import datetime, timedelta import azurelinuxagent.common.logger as logger @@ -39,6 +39,7 @@ from azurelinuxagent.common.version import DISTRO_NAME, DISTRO_VERSION, \ DISTRO_CODE_NAME, AGENT_VERSION, \ CURRENT_AGENT, CURRENT_VERSION +_EVENT_MSG = "Event: name={0}, op={1}, message={2}" class WALAEventOperation: ActivateResourceDisk = "ActivateResourceDisk" @@ -47,6 +48,7 @@ class WALAEventOperation: Enable = "Enable" HealthCheck = "HealthCheck" HeartBeat = "HeartBeat" + HostPlugin = "HostPlugin" Install = "Install" InitializeHostPlugin = "InitializeHostPlugin" ProcessGoalState = "ProcessGoalState" @@ -58,10 +60,19 @@ class WALAEventOperation: Upgrade = "Upgrade" Update = "Update" +def _log_event(name, op, message, is_success=True): + global _EVENT_MSG + + if not is_success: + logger.error(_EVENT_MSG, name, op, message) + else: + logger.info(_EVENT_MSG, name, op, message) + class EventLogger(object): def __init__(self): self.event_dir = None + self.periodic_events = {} def save_event(self, data): if self.event_dir is None: @@ -92,9 +103,33 @@ class EventLogger(object): except IOError as e: raise EventError("Failed to write events to file:{0}", e) + def reset_periodic(self): + self.periodic_messages = {} + + def is_period_elapsed(self, delta, h): + return h not in self.periodic_messages or \ + (self.periodic_messages[h] + delta) <= datetime.now() + + def add_periodic(self, + delta, name, op="", is_success=True, duration=0, + version=CURRENT_VERSION, message="", evt_type="", + is_internal=False, log_event=True, force=False): + + h = hash(name+op+ustr(is_success)+message) + + if force or self.is_period_elapsed(delta, h): + self.add_event(name, + op=op, is_success=is_success, duration=duration, + version=version, message=message, evt_type=evt_type, + is_internal=is_internal, log_event=log_event) + self.periodic_messages[h] = datetime.now() + def add_event(self, name, op="", is_success=True, duration=0, version=CURRENT_VERSION, - message="", evt_type="", is_internal=False): + message="", evt_type="", is_internal=False, log_event=True): + if not is_success or log_event: + _log_event(name, op, message, is_success=is_success) + event = TelemetryEvent(1, "69B669B9-4AF8-4C50-BDC4-6006FA76E975") event.parameters.append(TelemetryEventParam('Name', name)) event.parameters.append(TelemetryEventParam('Version', str(version))) @@ -129,21 +164,40 @@ def report_event(op, is_success=True, message=''): message=message, op=op) +def report_periodic(delta, op, is_success=True, message=''): + from azurelinuxagent.common.version import AGENT_NAME, CURRENT_VERSION + add_periodic(delta, AGENT_NAME, + version=CURRENT_VERSION, + is_success=is_success, + message=message, + op=op) def add_event(name, op="", is_success=True, duration=0, version=CURRENT_VERSION, message="", evt_type="", is_internal=False, log_event=True, reporter=__event_logger__): - if log_event or not is_success: - log = logger.info if is_success else logger.error - log("Event: name={0}, op={1}, message={2}", name, op, message) + if reporter.event_dir is None: + logger.warn("Event reporter is not initialized.") + _log_event(name, op, message, is_success=is_success) + return + + reporter.add_event( + name, op=op, is_success=is_success, duration=duration, + version=str(version), message=message, evt_type=evt_type, + is_internal=is_internal, log_event=log_event) +def add_periodic( + delta, name, op="", is_success=True, duration=0, version=CURRENT_VERSION, + message="", evt_type="", is_internal=False, log_event=True, force=False, + reporter=__event_logger__): if reporter.event_dir is None: logger.warn("Event reporter is not initialized.") + _log_event(name, op, message, is_success=is_success) return - reporter.add_event(name, op=op, is_success=is_success, duration=duration, - version=str(version), message=message, evt_type=evt_type, - is_internal=is_internal) + reporter.add_periodic( + delta, name, op=op, is_success=is_success, duration=duration, + version=str(version), message=message, evt_type=evt_type, + is_internal=is_internal, log_event=log_event, force=force) def init_event_logger(event_dir, reporter=__event_logger__): reporter.event_dir = event_dir diff --git a/azurelinuxagent/common/future.py b/azurelinuxagent/common/future.py index 8509732..8d5b70b 100644 --- a/azurelinuxagent/common/future.py +++ b/azurelinuxagent/common/future.py @@ -13,8 +13,6 @@ if sys.version_info[0]== 3: bytebuffer = memoryview - read_input = input - elif sys.version_info[0] == 2: import httplib as httpclient from urlparse import urlparse @@ -24,8 +22,6 @@ elif sys.version_info[0] == 2: bytebuffer = buffer - read_input = raw_input - else: raise ImportError("Unknown python version:{0}".format(sys.version_info)) diff --git a/azurelinuxagent/common/logger.py b/azurelinuxagent/common/logger.py index c1eb18f..bfdc73a 100644 --- a/azurelinuxagent/common/logger.py +++ b/azurelinuxagent/common/logger.py @@ -20,7 +20,13 @@ Log utils import os import sys from azurelinuxagent.common.future import ustr -from datetime import datetime +from datetime import datetime, timedelta + +EVERY_DAY = timedelta(days=1) +EVERY_HALF_DAY = timedelta(hours=12) +EVERY_HOUR = timedelta(hours=1) +EVERY_HALF_HOUR = timedelta(minutes=30) +EVERY_FIFTEEN_MINUTES = timedelta(minutes=15) class Logger(object): """ @@ -28,10 +34,23 @@ class Logger(object): """ def __init__(self, logger=None, prefix=None): self.appenders = [] - if logger is not None: - self.appenders.extend(logger.appenders) + self.logger = self if logger is None else logger + self.periodic_messages = {} self.prefix = prefix + def reset_periodic(self): + self.logger.periodic_messages = {} + + def is_period_elapsed(self, delta, h): + return h not in self.logger.periodic_messages or \ + (self.logger.periodic_messages[h] + delta) <= datetime.now() + + def periodic(self, delta, msg_format, *args): + h = hash(msg_format) + if self.is_period_elapsed(delta, h): + self.info(msg_format, *args) + self.logger.periodic_messages[h] = datetime.now() + def verbose(self, msg_format, *args): self.log(LogLevel.VERBOSE, msg_format, *args) @@ -62,8 +81,12 @@ class Logger(object): log_item = ustr(log_item.encode('ascii', "backslashreplace"), encoding="ascii") + for appender in self.appenders: appender.write(level, log_item) + if self.logger != self: + for appender in self.logger.appenders: + appender.write(level, log_item) def add_appender(self, appender_type, level, path): appender = _create_logger_appender(appender_type, level, path) @@ -129,6 +152,12 @@ class AppenderType(object): def add_logger_appender(appender_type, level=LogLevel.INFO, path=None): DEFAULT_LOGGER.add_appender(appender_type, level, path) +def reset_periodic(): + DEFAULT_LOGGER.reset_periodic() + +def periodic(delta, msg_format, *args): + DEFAULT_LOGGER.periodic(delta, msg_format, *args) + def verbose(msg_format, *args): DEFAULT_LOGGER.verbose(msg_format, *args) diff --git a/azurelinuxagent/common/osutil/bigip.py b/azurelinuxagent/common/osutil/bigip.py index fea7aff..8f6570f 100644 --- a/azurelinuxagent/common/osutil/bigip.py +++ b/azurelinuxagent/common/osutil/bigip.py @@ -258,57 +258,6 @@ class BigIpOSUtil(DefaultOSUtil): """ logger.warn("Eject is not supported on this platform") - def set_admin_access_to_ip(self, dest_ip): - """Sets admin access to an IP address - - This method is primarily used to limit which user account is allowed to - communicate with the Azure(Stack) metadata service. This service is at - the address 169.254.169.254 and includes information about the device - that "normal" users should not be allowed to see. - - We cannot use this iptables command that comes with the default class - because we do not ship the 'ipt_owner' iptables extension with BIG-IP. - - This should not be a problem though as the only people who should have - access to BIG-IP are people who are root anyways. Our system is not - a "general purpose" user system. So for those reasons I am dropping - that requirement from our implementation. - - :param dest_ip: The IP address that you want to allow admin access for - """ - self._set_accept_admin_access_to_ip(dest_ip) - self._set_drop_admin_access_to_ip(dest_ip) - - def _set_accept_admin_access_to_ip(self, dest_ip): - """Sets the "accept" IP Tables rules - - I broke this out to a separate function so that I could more easily - test it in the tests/common/osutil/test_default.py code - - :param dest_ip: - :return: - """ - # This allows root to access dest_ip - rm_old = "iptables -D OUTPUT -d {0} -j ACCEPT" - rule = "iptables -A OUTPUT -d {0} -j ACCEPT" - shellutil.run(rm_old.format(dest_ip), chk_err=False) - shellutil.run(rule.format(dest_ip)) - - def _set_drop_admin_access_to_ip(self, dest_ip): - """Sets the "drop" IP Tables rules - - I broke this out to a separate function so that I could more easily - test it in the tests/common/osutil/test_default.py code - - :param dest_ip: - :return: - """ - # This blocks all other users to access dest_ip - rm_old = "iptables -D OUTPUT -d {0} -j DROP" - rule = "iptables -A OUTPUT -d {0} -j DROP" - shellutil.run(rm_old.format(dest_ip), chk_err=False) - shellutil.run(rule.format(dest_ip)) - def get_first_if(self): """Return the interface name, and ip addr of the management interface. diff --git a/azurelinuxagent/common/osutil/default.py b/azurelinuxagent/common/osutil/default.py index 20dc1f3..58c0ef8 100644 --- a/azurelinuxagent/common/osutil/default.py +++ b/azurelinuxagent/common/osutil/default.py @@ -841,18 +841,5 @@ class DefaultOSUtil(object): def get_processor_cores(self): return multiprocessing.cpu_count() - def set_admin_access_to_ip(self, dest_ip): - #This allows root to access dest_ip - rm_old= "iptables -D OUTPUT -d {0} -j ACCEPT -m owner --uid-owner 0" - rule = "iptables -A OUTPUT -d {0} -j ACCEPT -m owner --uid-owner 0" - shellutil.run(rm_old.format(dest_ip), chk_err=False) - shellutil.run(rule.format(dest_ip)) - - #This blocks all other users to access dest_ip - rm_old = "iptables -D OUTPUT -d {0} -j DROP" - rule = "iptables -A OUTPUT -d {0} -j DROP" - shellutil.run(rm_old.format(dest_ip), chk_err=False) - shellutil.run(rule.format(dest_ip)) - def check_pid_alive(self, pid): return pid is not None and os.path.isdir(os.path.join('/proc', pid)) diff --git a/azurelinuxagent/common/osutil/factory.py b/azurelinuxagent/common/osutil/factory.py index 3447651..2be90ab 100644 --- a/azurelinuxagent/common/osutil/factory.py +++ b/azurelinuxagent/common/osutil/factory.py @@ -24,6 +24,7 @@ from .clearlinux import ClearLinuxUtil from .coreos import CoreOSUtil from .debian import DebianOSUtil from .freebsd import FreeBSDOSUtil +from .openbsd import OpenBSDOSUtil from .redhat import RedhatOSUtil, Redhat6xOSUtil from .suse import SUSEOSUtil, SUSE11OSUtil from .ubuntu import UbuntuOSUtil, Ubuntu12OSUtil, Ubuntu14OSUtil, UbuntuSnappyOSUtil @@ -87,6 +88,9 @@ def get_osutil(distro_name=DISTRO_NAME, elif distro_name == "freebsd": return FreeBSDOSUtil() + elif distro_name == "openbsd": + return OpenBSDOSUtil() + elif distro_name == "bigip": return BigIpOSUtil() diff --git a/azurelinuxagent/common/osutil/freebsd.py b/azurelinuxagent/common/osutil/freebsd.py index 0f465a9..39d1760 100644 --- a/azurelinuxagent/common/osutil/freebsd.py +++ b/azurelinuxagent/common/osutil/freebsd.py @@ -229,17 +229,21 @@ class FreeBSDOSUtil(DefaultOSUtil): err, output = shellutil.run_get_output(cmd_search_blkvsc) if err == 0: output = output.rstrip() - cmd_search_dev="camcontrol devlist | grep {0} | awk -F \( '{{print $2}}'|awk -F , '{{print $1}}'".format(output) + cmd_search_dev="camcontrol devlist | grep {0} | awk -F \( '{{print $2}}'|sed -e 's/.*(//'| sed -e 's/).*//'".format(output) err, output = shellutil.run_get_output(cmd_search_dev) if err == 0: - return output.rstrip() + for possible in output.rstrip().split(','): + if not possible.startswith('pass'): + return possible cmd_search_storvsc = "camcontrol devlist -b | grep storvsc{0} | awk '{{print $1}}'".format(output) err, output = shellutil.run_get_output(cmd_search_storvsc) if err == 0: output = output.rstrip() - cmd_search_dev="camcontrol devlist | grep {0} | awk -F \( '{{print $2}}'|awk -F , '{{print $1}}'".format(output) + cmd_search_dev="camcontrol devlist | grep {0} | awk -F \( '{{print $2}}'|sed -e 's/.*(//'| sed -e 's/).*//'".format(output) err, output = shellutil.run_get_output(cmd_search_dev) if err == 0: - return output.rstrip() + for possible in output.rstrip().split(','): + if not possible.startswith('pass'): + return possible return None diff --git a/azurelinuxagent/common/osutil/gaia.py b/azurelinuxagent/common/osutil/gaia.py index a1069d3..6a87b6b 100644 --- a/azurelinuxagent/common/osutil/gaia.py +++ b/azurelinuxagent/common/osutil/gaia.py @@ -16,15 +16,20 @@ # Requires Python 2.4+ and Openssl 1.0+ # +import base64 import socket import struct import time -import azurelinuxagent.common.logger as logger +import azurelinuxagent.common.conf as conf from azurelinuxagent.common.exception import OSUtilError +from azurelinuxagent.common.future import ustr, bytebuffer +import azurelinuxagent.common.logger as logger +from azurelinuxagent.common.osutil.default import DefaultOSUtil +from azurelinuxagent.common.utils.cryptutil import CryptUtil +import azurelinuxagent.common.utils.fileutil as fileutil import azurelinuxagent.common.utils.shellutil as shellutil import azurelinuxagent.common.utils.textutil as textutil -from azurelinuxagent.common.osutil.default import DefaultOSUtil class GaiaOSUtil(DefaultOSUtil): @@ -64,12 +69,11 @@ class GaiaOSUtil(DefaultOSUtil): if ret != 0: raise OSUtilError("Failed to delete root password") - def _replace_user(path, username): + def _replace_user(self, path, username): + if path.startswith('$HOME'): + path = '/home' + path[5:] parts = path.split('/') - for i in xrange(len(parts)): - if parts[i] == '$HOME': - parts[i + 1] = username - break + parts[2] = username return '/'.join(parts) def deploy_ssh_keypair(self, username, keypair): @@ -80,13 +84,57 @@ class GaiaOSUtil(DefaultOSUtil): super(GaiaOSUtil, self).deploy_ssh_keypair( username, (path, thumbprint)) + def openssl_to_openssh(self, input_file, output_file): + cryptutil = CryptUtil(conf.get_openssl_cmd()) + ret, out = shellutil.run_get_output( + conf.get_openssl_cmd() + + " rsa -pubin -noout -text -in '" + input_file + "'") + if ret != 0: + raise OSUtilError('openssl failed with {0}'.format(ret)) + + modulus = [] + exponent = [] + buf = None + for line in out.split('\n'): + if line.startswith('Modulus:'): + buf = modulus + buf.append(line) + continue + if line.startswith('Exponent:'): + buf = exponent + buf.append(line) + continue + if buf and line: + buf.append(line.strip().replace(':', '')) + + def text_to_num(buf): + if len(buf) == 1: + return int(buf[0].split()[1]) + return long(''.join(buf[1:]), 16) + + n = text_to_num(modulus) + e = text_to_num(exponent) + + keydata = bytearray() + keydata.extend(struct.pack('>I', len('ssh-rsa'))) + keydata.extend(b'ssh-rsa') + keydata.extend(struct.pack('>I', len(cryptutil.num_to_bytes(e)))) + keydata.extend(cryptutil.num_to_bytes(e)) + keydata.extend(struct.pack('>I', len(cryptutil.num_to_bytes(n)) + 1)) + keydata.extend(b'\0') + keydata.extend(cryptutil.num_to_bytes(n)) + keydata_base64 = base64.b64encode(bytebuffer(keydata)) + fileutil.write_file(output_file, + ustr(b'ssh-rsa ' + keydata_base64 + b'\n', + encoding='utf-8')) + def deploy_ssh_pubkey(self, username, pubkey): logger.info('deploy_ssh_pubkey') username = 'admin' path, thumbprint, value = pubkey path = self._replace_user(path, username) super(GaiaOSUtil, self).deploy_ssh_pubkey( - 'admin', (path, thumbprint, value)) + username, (path, thumbprint, value)) def eject_dvd(self, chk_err=True): logger.warn('eject is not supported on GAiA') @@ -114,7 +162,7 @@ class GaiaOSUtil(DefaultOSUtil): def restart_ssh_service(self): return shellutil.run('/sbin/service sshd condrestart', chk_err=False) - def _address_to_string(addr): + def _address_to_string(self, addr): return socket.inet_ntoa(struct.pack("!I", addr)) def _get_prefix(self, mask): @@ -146,6 +194,3 @@ class GaiaOSUtil(DefaultOSUtil): def del_account(self, username): logger.warn('del_account is ignored on GAiA') - - def set_admin_access_to_ip(self, dest_ip): - logger.warn('set_admin_access_to_ip is ignored on GAiA') diff --git a/azurelinuxagent/common/osutil/openbsd.py b/azurelinuxagent/common/osutil/openbsd.py new file mode 100644 index 0000000..9bfe6de --- /dev/null +++ b/azurelinuxagent/common/osutil/openbsd.py @@ -0,0 +1,345 @@ +# Microsoft Azure Linux Agent +# +# Copyright 2014 Microsoft Corporation +# Copyright 2017 Reyk Floeter <reyk@openbsd.org> +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Requires Python 2.4+ and OpenSSL 1.0+ + +import os +import re +import time +import glob +import datetime + +import azurelinuxagent.common.utils.fileutil as fileutil +import azurelinuxagent.common.utils.shellutil as shellutil +import azurelinuxagent.common.logger as logger +import azurelinuxagent.common.conf as conf + +from azurelinuxagent.common.exception import OSUtilError +from azurelinuxagent.common.osutil.default import DefaultOSUtil + +UUID_PATTERN = re.compile( + r'^\s*[A-F0-9]{8}(?:\-[A-F0-9]{4}){3}\-[A-F0-9]{12}\s*$', + re.IGNORECASE) + +class OpenBSDOSUtil(DefaultOSUtil): + def __init__(self): + super(OpenBSDOSUtil, self).__init__() + self._scsi_disks_timeout_set = False + + def get_instance_id(self): + ret, output = shellutil.run_get_output("sysctl -n hw.uuid") + if ret != 0 or UUID_PATTERN.match(output) is None: + return "" + return output.strip() + + def set_hostname(self, hostname): + fileutil.write_file("/etc/myname", "{}\n".format(hostname)) + shellutil.run("hostname {0}".format(hostname), chk_err=False) + + def restart_ssh_service(self): + return shellutil.run('rcctl restart sshd', chk_err=False) + + def start_agent_service(self): + return shellutil.run('rcctl start waagent', chk_err=False) + + def stop_agent_service(self): + return shellutil.run('rcctl stop waagent', chk_err=False) + + def register_agent_service(self): + shellutil.run('chmod 0555 /etc/rc.d/waagent', chk_err=False) + return shellutil.run('rcctl enable waagent', chk_err=False) + + def unregister_agent_service(self): + return shellutil.run('rcctl disable waagent', chk_err=False) + + def del_account(self, username): + if self.is_sys_user(username): + logger.error("{0} is a system user. Will not delete it.", + username) + shellutil.run("> /var/run/utmp") + shellutil.run("userdel -r " + username) + self.conf_sudoer(username, remove=True) + + def conf_sudoer(self, username, nopasswd=False, remove=False): + doas_conf = "/etc/doas.conf" + doas = None + if not remove: + if not os.path.isfile(doas_conf): + # always allow root to become root + doas = "permit keepenv nopass root\n" + fileutil.append_file(doas_conf, doas) + if nopasswd: + doas = "permit keepenv nopass {0}\n".format(username) + else: + doas = "permit keepenv persist {0}\n".format(username) + fileutil.append_file(doas_conf, doas) + fileutil.chmod(doas_conf, 0o644) + else: + # Remove user from doas.conf + if os.path.isfile(doas_conf): + try: + content = fileutil.read_file(doas_conf) + doas = content.split("\n") + doas = [x for x in doas if username not in x] + fileutil.write_file(doas_conf, "\n".join(doas)) + except IOError as err: + raise OSUtilError("Failed to remove sudoer: " + "{0}".format(err)) + + def chpasswd(self, username, password, crypt_id=6, salt_len=10): + if self.is_sys_user(username): + raise OSUtilError(("User {0} is a system user. " + "Will not set passwd.").format(username)) + cmd = "echo -n {0}|encrypt".format(password) + ret, output = shellutil.run_get_output(cmd, log_cmd=False) + if ret != 0: + raise OSUtilError(("Failed to encrypt password for {0}: {1}" + "").format(username, output)) + passwd_hash = output.strip() + cmd = "usermod -p '{0}' {1}".format(passwd_hash, username) + ret, output = shellutil.run_get_output(cmd, log_cmd=False) + if ret != 0: + raise OSUtilError(("Failed to set password for {0}: {1}" + "").format(username, output)) + + def del_root_password(self): + ret, output = shellutil.run_get_output('usermod -p "*" root') + if ret: + raise OSUtilError("Failed to delete root password: " + "{0}".format(output)) + + def get_if_mac(self, ifname): + data = self._get_net_info() + if data[0] == ifname: + return data[2].replace(':', '').upper() + return None + + def get_first_if(self): + return self._get_net_info()[:2] + + def route_add(self, net, mask, gateway): + cmd = 'route add {0} {1} {2}'.format(net, gateway, mask) + return shellutil.run(cmd, chk_err=False) + + def is_missing_default_route(self): + ret = shellutil.run("route -n get default", chk_err=False) + if ret == 0: + return False + return True + + def is_dhcp_enabled(self): + pass + + def start_dhcp_service(self): + pass + + def stop_dhcp_service(self): + pass + + def get_dhcp_lease_endpoint(self): + """ + OpenBSD has a sligthly different lease file format. + """ + endpoint = None + pathglob = '/var/db/dhclient.leases.{}'.format(self.get_first_if()[0]) + + HEADER_LEASE = "lease" + HEADER_OPTION = "option option-245" + HEADER_EXPIRE = "expire" + FOOTER_LEASE = "}" + FORMAT_DATETIME = "%Y/%m/%d %H:%M:%S %Z" + + logger.info("looking for leases in path [{0}]".format(pathglob)) + for lease_file in glob.glob(pathglob): + leases = open(lease_file).read() + if HEADER_OPTION in leases: + cached_endpoint = None + has_option_245 = False + expired = True # assume expired + for line in leases.splitlines(): + if line.startswith(HEADER_LEASE): + cached_endpoint = None + has_option_245 = False + expired = True + elif HEADER_OPTION in line: + try: + ipaddr = line.split(" ")[-1].strip(";").split(":") + cached_endpoint = \ + ".".join(str(int(d, 16)) for d in ipaddr) + has_option_245 = True + except ValueError: + logger.error("could not parse '{0}'".format(line)) + elif HEADER_EXPIRE in line: + if "never" in line: + expired = False + else: + try: + expire_string = line.split( + " ", 4)[-1].strip(";") + expire_date = datetime.datetime.strptime( + expire_string, FORMAT_DATETIME) + if expire_date > datetime.datetime.utcnow(): + expired = False + except ValueError: + logger.error("could not parse expiry token " + "'{0}'".format(line)) + elif FOOTER_LEASE in line: + logger.info("dhcp entry:{0}, 245:{1}, expired: {2}" + .format(cached_endpoint, has_option_245, expired)) + if not expired and cached_endpoint is not None and has_option_245: + endpoint = cached_endpoint + logger.info("found endpoint [{0}]".format(endpoint)) + # we want to return the last valid entry, so + # keep searching + if endpoint is not None: + logger.info("cached endpoint found [{0}]".format(endpoint)) + else: + logger.info("cached endpoint not found") + return endpoint + + def allow_dhcp_broadcast(self): + pass + + def set_route_for_dhcp_broadcast(self, ifname): + return shellutil.run("route add 255.255.255.255 -iface " + "{0}".format(ifname), chk_err=False) + + def remove_route_for_dhcp_broadcast(self, ifname): + shellutil.run("route delete 255.255.255.255 -iface " + "{0}".format(ifname), chk_err=False) + + def get_dhcp_pid(self): + ret, output = shellutil.run_get_output("pgrep -n dhclient", + chk_err=False) + return output if ret == 0 else None + + def get_dvd_device(self, dev_dir='/dev'): + pattern = r'cd[0-9]c' + for dvd in [re.match(pattern, dev) for dev in os.listdir(dev_dir)]: + if dvd is not None: + return "/dev/{0}".format(dvd.group(0)) + raise OSUtilError("Failed to get DVD device") + + def mount_dvd(self, + max_retry=6, + chk_err=True, + dvd_device=None, + mount_point=None, + sleep_time=5): + if dvd_device is None: + dvd_device = self.get_dvd_device() + if mount_point is None: + mount_point = conf.get_dvd_mount_point() + if not os.path.isdir(mount_point): + os.makedirs(mount_point) + + for retry in range(0, max_retry): + retcode = self.mount(dvd_device, mount_point, option="-o ro -t udf", + chk_err=chk_err) + if retcode == 0: + logger.info("Successfully mounted DVD") + return + if retry < max_retry - 1: + mountlist = shellutil.run_get_output("/sbin/mount")[1] + existing = self.get_mount_point(mountlist, dvd_device) + if existing is not None: + logger.info("{0} is mounted at {1}", dvd_device, existing) + return + logger.warn("Mount DVD failed: retry={0}, ret={1}", retry, + retcode) + time.sleep(sleep_time) + if chk_err: + raise OSUtilError("Failed to mount DVD.") + + def eject_dvd(self, chk_err=True): + dvd = self.get_dvd_device() + retcode = shellutil.run("cdio eject {0}".format(dvd)) + if chk_err and retcode != 0: + raise OSUtilError("Failed to eject DVD: ret={0}".format(retcode)) + + def restart_if(self, ifname, retries=3, wait=5): + # Restart dhclient only to publish hostname + shellutil.run("/sbin/dhclient {0}".format(ifname), chk_err=False) + + def get_total_mem(self): + ret, output = shellutil.run_get_output("sysctl -n hw.physmem") + if ret: + raise OSUtilError("Failed to get total memory: {0}".format(output)) + try: + return int(output)/1024/1024 + except ValueError: + raise OSUtilError("Failed to get total memory: {0}".format(output)) + + def get_processor_cores(self): + ret, output = shellutil.run_get_output("sysctl -n hw.ncpu") + if ret: + raise OSUtilError("Failed to get processor cores.") + + try: + return int(output) + except ValueError: + raise OSUtilError("Failed to get total memory: {0}".format(output)) + + def set_scsi_disks_timeout(self, timeout): + pass + + def check_pid_alive(self, pid): + if not pid: + return + return shellutil.run('ps -p {0}'.format(pid), chk_err=False) == 0 + + @staticmethod + def _get_net_info(): + """ + There is no SIOCGIFCONF + on OpenBSD - just parse ifconfig. + Returns strings: iface, inet4_addr, and mac + or 'None,None,None' if unable to parse. + We will sleep and retry as the network must be up. + """ + iface = '' + inet = '' + mac = '' + + ret, output = shellutil.run_get_output( + 'ifconfig hvn | grep -E "^hvn.:" | sed "s/:.*//g"', chk_err=False) + if ret: + raise OSUtilError("Can't find ether interface:{0}".format(output)) + ifaces = output.split() + if not ifaces: + raise OSUtilError("Can't find ether interface.") + iface = ifaces[0] + + ret, output = shellutil.run_get_output( + 'ifconfig ' + iface, chk_err=False) + if ret: + raise OSUtilError("Can't get info for interface:{0}".format(iface)) + + for line in output.split('\n'): + if line.find('inet ') != -1: + inet = line.split()[1] + elif line.find('lladdr ') != -1: + mac = line.split()[1] + logger.verbose("Interface info: ({0},{1},{2})", iface, inet, mac) + + return iface, inet, mac + + def device_for_ide_port(self, port_id): + """ + Return device name attached to ide port 'n'. + """ + return "wd{0}".format(port_id) diff --git a/azurelinuxagent/common/protocol/hostplugin.py b/azurelinuxagent/common/protocol/hostplugin.py index 464fd35..9af8a97 100644 --- a/azurelinuxagent/common/protocol/hostplugin.py +++ b/azurelinuxagent/common/protocol/hostplugin.py @@ -70,7 +70,7 @@ class HostPluginProtocol(object): if not self.is_initialized: self.api_versions = self.get_api_versions() self.is_available = API_VERSION in self.api_versions - self.is_initialized = True + self.is_initialized = self.is_available from azurelinuxagent.common.event import WALAEventOperation, report_event report_event(WALAEventOperation.InitializeHostPlugin, is_success=self.is_available) @@ -143,7 +143,9 @@ class HostPluginProtocol(object): headers = {"x-ms-vmagentlog-deploymentid": self.deployment_id, "x-ms-vmagentlog-containerid": self.container_id} - logger.info("HostGAPlugin: Put VM log to [{0}]".format(url)) + logger.periodic( + logger.EVERY_FIFTEEN_MINUTES, + "HostGAPlugin: Put VM log to [{0}]".format(url)) try: response = restutil.http_put(url, content, headers) if response.status != httpclient.OK: @@ -175,7 +177,7 @@ class HostPluginProtocol(object): self._put_page_blob_status(sas_url, status_blob) if not HostPluginProtocol.is_default_channel(): - logger.info("HostGAPlugin: Setting host plugin as default channel") + logger.verbose("HostGAPlugin: Setting host plugin as default channel") HostPluginProtocol.set_default_channel(True) except Exception as e: message = "HostGAPlugin: Exception Put VM status: {0}, {1}".format(e, traceback.format_exc()) @@ -288,12 +290,23 @@ class HostPluginProtocol(object): @staticmethod def read_response_error(response): - if response is None: - return '' - body = remove_bom(response.read()) - if PY_VERSION_MAJOR < 3 and body is not None: - body = ustr(body, encoding='utf-8') - return "{0}, {1}, {2}".format( - response.status, - response.reason, - body) + result = '' + if response is not None: + try: + body = remove_bom(response.read()) + result = "[{0}: {1}] {2}".format(response.status, + response.reason, + body) + + # this result string is passed upstream to several methods + # which do a raise HttpError() or a format() of some kind; + # as a result it cannot have any unicode characters + if PY_VERSION_MAJOR < 3: + result = ustr(result, encoding='ascii', errors='ignore') + else: + result = result\ + .encode(encoding='ascii', errors='ignore')\ + .decode(encoding='ascii', errors='ignore') + except Exception: + logger.warn(traceback.format_exc()) + return result diff --git a/azurelinuxagent/common/protocol/metadata.py b/azurelinuxagent/common/protocol/metadata.py index c50b3dd..b0b6f67 100644 --- a/azurelinuxagent/common/protocol/metadata.py +++ b/azurelinuxagent/common/protocol/metadata.py @@ -113,7 +113,7 @@ class MetadataProtocol(Protocol): except HttpError as e: raise ProtocolError(ustr(e)) if resp.status != httpclient.CREATED: - raise ProtocolError("{0} - POST: {1}".format(resp.status, url)) + logger.warn("{0} for POST {1}".format(resp.status, url)) def _get_trans_cert(self): trans_crt_file = os.path.join(conf.get_lib_dir(), @@ -236,14 +236,14 @@ class MetadataProtocol(Protocol): return ext_list, etag def get_ext_handler_pkgs(self, ext_handler): - logger.info("Get extension handler packages") + logger.verbose("Get extension handler packages") pkg_list = ExtHandlerPackageList() manifest = None for version_uri in ext_handler.versionUris: try: manifest, etag = self._get_data(version_uri.uri) - logger.info("Successfully downloaded manifest") + logger.verbose("Successfully downloaded manifest") break except ProtocolError as e: logger.warn("Failed to fetch manifest: {0}", e) diff --git a/azurelinuxagent/common/protocol/util.py b/azurelinuxagent/common/protocol/util.py index 0ba03ec..bb3500a 100644 --- a/azurelinuxagent/common/protocol/util.py +++ b/azurelinuxagent/common/protocol/util.py @@ -162,12 +162,7 @@ class ProtocolUtil(object): def _detect_metadata_protocol(self): protocol = MetadataProtocol() protocol.detect() - - # only allow root access METADATA_ENDPOINT - self.osutil.set_admin_access_to_ip(METADATA_ENDPOINT) - self.save_protocol("MetadataProtocol") - return protocol def _detect_protocol(self, protocols): diff --git a/azurelinuxagent/common/protocol/wire.py b/azurelinuxagent/common/protocol/wire.py index 936be8c..d731e11 100644 --- a/azurelinuxagent/common/protocol/wire.py +++ b/azurelinuxagent/common/protocol/wire.py @@ -597,9 +597,9 @@ class WireClient(object): Call storage service, handle SERVICE_UNAVAILABLE(503) """ - # force the chk_proxy arg to True, since all calls to storage should - # use a configured proxy - kwargs['chk_proxy'] = True + # Default to use the configured HTTP proxy + if not 'chk_proxy' in kwargs or kwargs['chk_proxy'] is None: + kwargs['chk_proxy'] = True for retry in range(0, 3): resp = http_req(*args, **kwargs) @@ -626,7 +626,7 @@ class WireClient(object): logger.verbose("Manifest could not be downloaded, falling back to host plugin") host = self.get_host_plugin() uri, headers = host.get_artifact_request(version.uri) - response = self.fetch(uri, headers) + response = self.fetch(uri, headers, chk_proxy=False) if not response: host = self.get_host_plugin(force_update=True) logger.info("Retry fetch in {0} seconds", @@ -642,14 +642,15 @@ class WireClient(object): return response raise ProtocolError("Failed to fetch manifest from all sources") - def fetch(self, uri, headers=None): + def fetch(self, uri, headers=None, chk_proxy=None): logger.verbose("Fetch [{0}] with headers [{1}]", uri, headers) return_value = None try: resp = self.call_storage_service( restutil.http_get, uri, - headers) + headers, + chk_proxy=chk_proxy) if resp.status == httpclient.OK: return_value = self.decode_config(resp.read()) else: @@ -831,7 +832,7 @@ class WireClient(object): if not blob_type in ["BlockBlob", "PageBlob"]: blob_type = "BlockBlob" - logger.info("Status Blob type is unspecified " + logger.verbose("Status Blob type is unspecified " "-- assuming it is a BlockBlob") try: @@ -998,17 +999,17 @@ class WireClient(object): artifacts_profile = None if self.has_artifacts_profile_blob(): blob = self.ext_conf.artifacts_profile_blob - logger.info("Getting the artifacts profile") + logger.verbose("Getting the artifacts profile") profile = self.fetch(blob) if profile is None: logger.warn("Download failed, falling back to host plugin") host = self.get_host_plugin() uri, headers = host.get_artifact_request(blob) - profile = self.decode_config(self.fetch(uri, headers)) + profile = self.decode_config(self.fetch(uri, headers, chk_proxy=False)) if not textutil.is_str_none_or_whitespace(profile): - logger.info("Artifacts profile downloaded successfully") + logger.verbose("Artifacts profile downloaded successfully") artifacts_profile = InVMArtifactsProfile(profile) return artifacts_profile diff --git a/azurelinuxagent/common/version.py b/azurelinuxagent/common/version.py index dc3592b..d1d4c62 100644 --- a/azurelinuxagent/common/version.py +++ b/azurelinuxagent/common/version.py @@ -77,6 +77,9 @@ def get_distro(): if 'FreeBSD' in platform.system(): release = re.sub('\-.*\Z', '', ustr(platform.release())) osinfo = ['freebsd', release, '', 'freebsd'] + elif 'OpenBSD' in platform.system(): + release = re.sub('\-.*\Z', '', ustr(platform.release())) + osinfo = ['openbsd', release, '', 'openbsd'] elif 'linux_distribution' in dir(platform): supported = platform._supported_dists + ('alpine',) osinfo = list(platform.linux_distribution(full_distribution_name=0, @@ -110,7 +113,7 @@ def get_distro(): AGENT_NAME = "WALinuxAgent" AGENT_LONG_NAME = "Azure Linux Agent" -AGENT_VERSION = '2.2.12' +AGENT_VERSION = '2.2.14' AGENT_LONG_VERSION = "{0}-{1}".format(AGENT_NAME, AGENT_VERSION) AGENT_DESCRIPTION = """ The Azure Linux Agent supports the provisioning and running of Linux @@ -160,7 +163,10 @@ CURRENT_AGENT, CURRENT_VERSION = set_current_agent() def set_goal_state_agent(): agent = None - pids = [pid for pid in os.listdir('/proc') if pid.isdigit()] + if os.path.isdir("/proc"): + pids = [pid for pid in os.listdir('/proc') if pid.isdigit()] + else: + pids = [] for pid in pids: try: pname = open(os.path.join('/proc', pid, 'cmdline'), 'rb').read() diff --git a/azurelinuxagent/daemon/main.py b/azurelinuxagent/daemon/main.py index 5b8db2f..e8dbb37 100644 --- a/azurelinuxagent/daemon/main.py +++ b/azurelinuxagent/daemon/main.py @@ -79,7 +79,7 @@ class DaemonHandler(object): err_msg = traceback.format_exc() add_event(name=AGENT_NAME, is_success=False, message=ustr(err_msg), op=WALAEventOperation.UnhandledError) - logger.info("Sleep 15 seconds and restart daemon") + logger.warn("Daemon ended with exception -- Sleep 15 seconds and restart daemon") time.sleep(15) def check_pid(self): diff --git a/azurelinuxagent/daemon/resourcedisk/factory.py b/azurelinuxagent/daemon/resourcedisk/factory.py index 76e5a23..41a0cba 100644 --- a/azurelinuxagent/daemon/resourcedisk/factory.py +++ b/azurelinuxagent/daemon/resourcedisk/factory.py @@ -22,6 +22,7 @@ from azurelinuxagent.common.version import DISTRO_NAME, \ DISTRO_FULL_NAME from .default import ResourceDiskHandler from .freebsd import FreeBSDResourceDiskHandler +from .openbsd import OpenBSDResourceDiskHandler def get_resourcedisk_handler(distro_name=DISTRO_NAME, distro_version=DISTRO_VERSION, @@ -29,5 +30,8 @@ def get_resourcedisk_handler(distro_name=DISTRO_NAME, if distro_name == "freebsd": return FreeBSDResourceDiskHandler() + if distro_name == "openbsd": + return OpenBSDResourceDiskHandler() + return ResourceDiskHandler() diff --git a/azurelinuxagent/daemon/resourcedisk/freebsd.py b/azurelinuxagent/daemon/resourcedisk/freebsd.py index e43d9c4..35ae06b 100644 --- a/azurelinuxagent/daemon/resourcedisk/freebsd.py +++ b/azurelinuxagent/daemon/resourcedisk/freebsd.py @@ -59,7 +59,7 @@ class FreeBSDResourceDiskHandler(ResourceDiskHandler): disks = self.parse_gpart_list(output) device = self.osutil.device_for_ide_port(1) - if device is None: + if device is None or not device in disks: # fallback logic to find device err, output = shellutil.run_get_output('camcontrol periphlist 2:1:0') if err: diff --git a/azurelinuxagent/daemon/resourcedisk/openbsd.py b/azurelinuxagent/daemon/resourcedisk/openbsd.py new file mode 100644 index 0000000..1454f6f --- /dev/null +++ b/azurelinuxagent/daemon/resourcedisk/openbsd.py @@ -0,0 +1,113 @@ +# Microsoft Azure Linux Agent +# +# Copyright 2014 Microsoft Corporation +# Copyright 2017 Reyk Floeter <reyk@openbsd.org> +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Requires Python 2.4+ and OpenSSL 1.0+ +# +import azurelinuxagent.common.logger as logger +import azurelinuxagent.common.utils.fileutil as fileutil +import azurelinuxagent.common.utils.shellutil as shellutil +import azurelinuxagent.common.conf as conf +from azurelinuxagent.common.exception import ResourceDiskError +from azurelinuxagent.daemon.resourcedisk.default import ResourceDiskHandler + +class OpenBSDResourceDiskHandler(ResourceDiskHandler): + def __init__(self): + super(OpenBSDResourceDiskHandler, self).__init__() + # Fase File System (FFS) is UFS + if self.fs == 'ufs' or self.fs == 'ufs2': + self.fs = 'ffs' + + def create_swap_space(self, mount_point, size_mb): + pass + + def enable_swap(self, mount_point): + size_mb = conf.get_resourcedisk_swap_size_mb() + if size_mb: + logger.info("Enable swap") + device = self.osutil.device_for_ide_port(1) + err, output = shellutil.run_get_output("swapctl -a /dev/" + "{0}b".format(device), + chk_err=False) + if err: + logger.error("Failed to enable swap, error {0}", output) + + def mount_resource_disk(self, mount_point): + fs = self.fs + if fs != 'ffs': + raise ResourceDiskError("Unsupported filesystem type: {0}, only " + "ufs/ffs is supported.".format(fs)) + + # 1. Get device + device = self.osutil.device_for_ide_port(1) + + if not device: + raise ResourceDiskError("Unable to detect resource disk device.") + logger.info('Resource disk device {0} found.', device) + + # 2. Get partition + partition = "/dev/{0}a".format(device) + + # 3. Mount partition + mount_list = shellutil.run_get_output("mount")[1] + existing = self.osutil.get_mount_point(mount_list, partition) + + if existing: + logger.info("Resource disk {0} is already mounted", partition) + return existing + + fileutil.mkdir(mount_point, mode=0o755) + mount_cmd = 'mount -t {0} {1} {2}'.format(self.fs, + partition, mount_point) + err = shellutil.run(mount_cmd, chk_err=False) + if err: + logger.info('Creating {0} filesystem on {1}'.format(fs, device)) + + fdisk_cmd = "/sbin/fdisk -yi {0}".format(device) + err, output = shellutil.run_get_output(fdisk_cmd, chk_err=False) + if err: + raise ResourceDiskError("Failed to create new MBR on {0}, " + "error: {1}".format(device, output)) + + size_mb = conf.get_resourcedisk_swap_size_mb() + if size_mb: + if size_mb > 512 * 1024: + size_mb = 512 * 1024 + disklabel_cmd = ("echo -e '{0} 1G-* 50%\nswap 1-{1}M 50%' " + "| disklabel -w -A -T /dev/stdin " + "{2}").format(mount_point, size_mb, device) + ret, output = shellutil.run_get_output( + disklabel_cmd, chk_err=False) + if ret: + raise ResourceDiskError("Failed to create new disklabel " + "on {0}, error " + "{1}".format(device, output)) + + err, output = shellutil.run_get_output("newfs -O2 {0}a" + "".format(device)) + if err: + raise ResourceDiskError("Failed to create new filesystem on " + "partition {0}, error " + "{1}".format(partition, output)) + + err, output = shellutil.run_get_output(mount_cmd, chk_err=False) + if err: + raise ResourceDiskError("Failed to mount partition {0}, " + "error {1}".format(partition, output)) + + logger.info("Resource disk partition {0} is mounted at {1} with fstype " + "{2}", partition, mount_point, fs) + return mount_point diff --git a/azurelinuxagent/ga/exthandlers.py b/azurelinuxagent/ga/exthandlers.py index b44ed6d..4324d92 100644 --- a/azurelinuxagent/ga/exthandlers.py +++ b/azurelinuxagent/ga/exthandlers.py @@ -781,7 +781,6 @@ class ExtHandlerInstance(object): heartbeat_file = os.path.join(conf.get_lib_dir(), self.get_heartbeat_file()) - self.logger.info("Collect heart beat") if not os.path.isfile(heartbeat_file): raise ExtensionError("Failed to get heart beat file") if not self.is_responsive(heartbeat_file): diff --git a/azurelinuxagent/ga/update.py b/azurelinuxagent/ga/update.py index 67eb785..10eac82 100644 --- a/azurelinuxagent/ga/update.py +++ b/azurelinuxagent/ga/update.py @@ -38,7 +38,7 @@ import azurelinuxagent.common.utils.fileutil as fileutil import azurelinuxagent.common.utils.restutil as restutil import azurelinuxagent.common.utils.textutil as textutil -from azurelinuxagent.common.event import add_event, \ +from azurelinuxagent.common.event import add_event, add_periodic, \ elapsed_milliseconds, \ WALAEventOperation from azurelinuxagent.common.exception import UpdateError, ProtocolError @@ -46,6 +46,7 @@ from azurelinuxagent.common.future import ustr from azurelinuxagent.common.osutil import get_osutil from azurelinuxagent.common.protocol import get_protocol_util from azurelinuxagent.common.protocol.hostplugin import HostPluginProtocol +from azurelinuxagent.common.protocol.wire import WireProtocol from azurelinuxagent.common.utils.flexible_version import FlexibleVersion from azurelinuxagent.common.version import AGENT_NAME, AGENT_VERSION, AGENT_LONG_VERSION, \ AGENT_DIR_GLOB, AGENT_PKG_GLOB, \ @@ -67,7 +68,6 @@ CHILD_POLL_INTERVAL = 60 MAX_FAILURE = 3 # Max failure allowed for agent before blacklisted GOAL_STATE_INTERVAL = 3 -REPORT_STATUS_INTERVAL = 15 ORPHAN_WAIT_INTERVAL = 15 * 60 * 60 @@ -207,19 +207,21 @@ class UpdateHandler(object): latest_agent.mark_failure(is_fatal=True) except Exception as e: - msg = u"Agent {0} launched with command '{1}' failed with exception: {2}".format( - agent_name, - agent_cmd, - ustr(e)) - logger.warn(msg) - add_event( - AGENT_NAME, - version=agent_version, - op=WALAEventOperation.Enable, - is_success=False, - message=msg) - if latest_agent is not None: - latest_agent.mark_failure(is_fatal=True) + # Ignore child errors during termination + if self.running: + msg = u"Agent {0} launched with command '{1}' failed with exception: {2}".format( + agent_name, + agent_cmd, + ustr(e)) + logger.warn(msg) + add_event( + AGENT_NAME, + version=agent_version, + op=WALAEventOperation.Enable, + is_success=False, + message=msg) + if latest_agent is not None: + latest_agent.mark_failure(is_fatal=True) self.child_process = None return @@ -266,17 +268,14 @@ class UpdateHandler(object): last_etag = exthandlers_handler.last_etag exthandlers_handler.run() - log_event = last_etag != exthandlers_handler.last_etag or \ - (datetime.utcnow() >= send_event_time) - add_event( - AGENT_NAME, - version=CURRENT_VERSION, - op=WALAEventOperation.ProcessGoalState, - is_success=True, - duration=elapsed_milliseconds(utc_start), - log_event=log_event) - if log_event: - send_event_time += timedelta(minutes=REPORT_STATUS_INTERVAL) + if last_etag != exthandlers_handler.last_etag: + add_event( + AGENT_NAME, + version=CURRENT_VERSION, + op=WALAEventOperation.ProcessGoalState, + is_success=True, + duration=elapsed_milliseconds(utc_start), + log_event=True) test_agent = self.get_test_agent() if test_agent is not None and test_agent.in_slice: @@ -423,10 +422,7 @@ class UpdateHandler(object): # The code leaves on disk available, but blacklisted, agents so as to # preserve the state. Otherwise, those agents could be again # downloaded and inappropriately retried. - host = None - if protocol and protocol.client: - host = protocol.client.get_host_plugin() - + host = self._get_host_plugin(protocol=protocol) self._set_agents([GuestAgent(pkg=pkg, host=host) for pkg in pkg_list.versions]) self._purge_agents() self._filter_blacklisted_agents() @@ -505,6 +501,13 @@ class UpdateHandler(object): logger.warn(u"Exception occurred loading available agents: {0}", ustr(e)) return + def _get_host_plugin(self, protocol=None): + return protocol.client.get_host_plugin() \ + if protocol and \ + type(protocol) is WireProtocol and \ + protocol.client \ + else None + def _get_pid_files(self): pid_file = conf.get_agent_pid_file_path() @@ -602,6 +605,8 @@ class UpdateHandler(object): return os.path.join(conf.get_lib_dir(), AGENT_SENTINAL_FILE) def _shutdown(self): + self.running = False + if not os.path.isfile(self._sentinal_file_path()): return @@ -647,7 +652,7 @@ class GuestAgent(object): self.version = FlexibleVersion(version) location = u"disk" if path is not None else u"package" - logger.verbose(u"Instantiating Agent {0} from {1}", self.name, location) + logger.verbose(u"Loading Agent {0} from package {1}", self.name, location) self.error = None self.supported = None @@ -716,7 +721,7 @@ class GuestAgent(object): os.makedirs(self.get_agent_dir()) self.error.mark_failure(is_fatal=is_fatal) self.error.save() - if is_fatal: + if self.error.is_blacklisted: logger.warn(u"Agent {0} is permanently blacklisted", self.name) except Exception as e: logger.warn(u"Agent {0} failed recording error state: {1}", self.name, ustr(e)) @@ -727,7 +732,7 @@ class GuestAgent(object): logger.verbose(u"Ensuring Agent {0} is downloaded", self.name) if self.is_blacklisted: - logger.info(u"Agent {0} is blacklisted - skipping download", self.name) + logger.verbose(u"Agent {0} is blacklisted - skipping download", self.name) return if self.is_downloaded: diff --git a/azurelinuxagent/pa/deprovision/default.py b/azurelinuxagent/pa/deprovision/default.py index 90d16c7..e2c5613 100644 --- a/azurelinuxagent/pa/deprovision/default.py +++ b/azurelinuxagent/pa/deprovision/default.py @@ -27,10 +27,15 @@ import azurelinuxagent.common.utils.fileutil as fileutil import azurelinuxagent.common.utils.shellutil as shellutil from azurelinuxagent.common.exception import ProtocolError -from azurelinuxagent.common.future import read_input from azurelinuxagent.common.osutil import get_osutil from azurelinuxagent.common.protocol import get_protocol_util +def read_input(message): + if sys.version_info[0] >= 3: + return input(message) + else: + return raw_input(message) + class DeprovisionAction(object): def __init__(self, func, args=[], kwargs={}): self.func = func @@ -86,6 +91,15 @@ class DeprovisionHandler(object): files = ['/root/.bash_history', '/var/log/waagent.log'] actions.append(DeprovisionAction(fileutil.rm_files, files)) + # For OpenBSD + actions.append(DeprovisionAction(fileutil.rm_files, + ["/etc/random.seed", + "/var/db/host.random", + "/etc/isakmpd/local.pub", + "/etc/isakmpd/private/local.key", + "/etc/iked/private/local.key", + "/etc/iked/local.pub"])) + def del_resolv(self, warnings, actions): warnings.append("WARNING! /etc/resolv.conf will be deleted.") files_to_del = ["/etc/resolv.conf"] @@ -96,9 +110,13 @@ class DeprovisionHandler(object): dirs_to_del = ["/var/lib/dhclient", "/var/lib/dhcpcd", "/var/lib/dhcp"] actions.append(DeprovisionAction(fileutil.rm_dirs, dirs_to_del)) - # For Freebsd, NM controlled - actions.append(DeprovisionAction(fileutil.rm_files, ["/var/db/dhclient.leases.hn0", - "/var/lib/NetworkManager/dhclient-*.lease"])) + # For FreeBSD and OpenBSD + actions.append(DeprovisionAction(fileutil.rm_files, + ["/var/db/dhclient.leases.*"])) + + # For FreeBSD, NM controlled + actions.append(DeprovisionAction(fileutil.rm_files, + ["/var/lib/NetworkManager/dhclient-*.lease"])) def del_lib_dir_files(self, warnings, actions): @@ -137,23 +155,28 @@ class DeprovisionHandler(object): ] return dirs - def cloud_init_files(self, include_once=True): - files = [ - "/etc/sudoers.d/90-cloud-init-users" - ] + def cloud_init_files(self, include_once=True, deluser=False): + files = [] + if deluser: + files += [ + "/etc/sudoers.d/90-cloud-init-users" + ] if include_once: files += [ "/var/lib/cloud/sem/config_scripts_per_once.once" ] return files - def del_cloud_init(self, warnings, actions, include_once=True): + def del_cloud_init(self, warnings, actions, + include_once=True, deluser=False): dirs = [d for d in self.cloud_init_dirs(include_once=include_once) \ if os.path.isdir(d)] if len(dirs) > 0: actions.append(DeprovisionAction(fileutil.rm_dirs, dirs)) - files = [f for f in self.cloud_init_files(include_once=include_once) \ + files = [f for f in self.cloud_init_files( + include_once=include_once, + deluser=deluser) \ if os.path.isfile(f)] if len(files) > 0: actions.append(DeprovisionAction(fileutil.rm_files, files)) @@ -179,7 +202,7 @@ class DeprovisionHandler(object): if conf.get_delete_root_password(): self.del_root_password(warnings, actions) - self.del_cloud_init(warnings, actions) + self.del_cloud_init(warnings, actions, deluser=deluser) self.del_dirs(warnings, actions) self.del_files(warnings, actions) self.del_resolv(warnings, actions) @@ -193,10 +216,10 @@ class DeprovisionHandler(object): warnings = [] actions = [] - self.del_cloud_init(warnings, actions, include_once=False) + self.del_cloud_init(warnings, actions, + include_once=False, deluser=False) self.del_dhcp_lease(warnings, actions) self.del_lib_dir_files(warnings, actions) - self.del_resolv(warnings, actions) return warnings, actions @@ -204,8 +227,8 @@ class DeprovisionHandler(object): warnings, actions = self.setup(deluser) self.do_warnings(warnings) - self.do_confirmation(force=force) - self.do_actions(actions) + if self.do_confirmation(force=force): + self.do_actions(actions) def run_changed_unique_id(self): ''' @@ -246,5 +269,3 @@ class DeprovisionHandler(object): print ('Deprovisioning may not be interrupted.') return - - diff --git a/azurelinuxagent/pa/provision/default.py b/azurelinuxagent/pa/provision/default.py index d4870f1..959a2fe 100644 --- a/azurelinuxagent/pa/provision/default.py +++ b/azurelinuxagent/pa/provision/default.py @@ -103,8 +103,11 @@ class ProvisionHandler(object): @staticmethod def validate_cloud_init(is_expected=True): - pids = [pid for pid in os.listdir('/proc') if pid.isdigit()] is_running = False + if os.path.isdir("/proc"): + pids = [pid for pid in os.listdir('/proc') if pid.isdigit()] + else: + pids = [] for pid in pids: try: pname = open(os.path.join('/proc', pid, 'cmdline'), 'rb').read() diff --git a/config/gaia/waagent.conf b/config/gaia/waagent.conf index 43ad35d..75550a6 100644 --- a/config/gaia/waagent.conf +++ b/config/gaia/waagent.conf @@ -55,7 +55,7 @@ ResourceDisk.SwapSizeMB=1024 ResourceDisk.MountOptions=None # Enable verbose logging (y|n) -Logs.Verbose=y +Logs.Verbose=n # Is FIPS enabled OS.EnableFIPS=n diff --git a/config/openbsd/waagent.conf b/config/openbsd/waagent.conf new file mode 100644 index 0000000..09e7db7 --- /dev/null +++ b/config/openbsd/waagent.conf @@ -0,0 +1,105 @@ +# +# Microsoft Azure Linux Agent Configuration +# + +# Enable instance creation +Provisioning.Enabled=y + +# Rely on cloud-init to provision +Provisioning.UseCloudInit=n + +# Password authentication for root account will be unavailable. +Provisioning.DeleteRootPassword=y + +# Generate fresh host key pair. +Provisioning.RegenerateSshHostKeyPair=y + +# Supported values are "rsa", "dsa", "ecdsa", and "ed25519". +Provisioning.SshHostKeyPairType=ed25519 + +# Monitor host name changes and publish changes via DHCP requests. +Provisioning.MonitorHostName=y + +# Decode CustomData from Base64. +Provisioning.DecodeCustomData=n + +# Execute CustomData after provisioning. +Provisioning.ExecuteCustomData=n + +# Algorithm used by crypt when generating password hash. +#Provisioning.PasswordCryptId=6 + +# Length of random salt used when generating password hash. +#Provisioning.PasswordCryptSaltLength=10 + +# Format if unformatted. If 'n', resource disk will not be mounted. +ResourceDisk.Format=y + +# File system on the resource disk +# Typically ext3 or ext4. OpenBSD images should use 'ufs2' here. +ResourceDisk.Filesystem=ufs2 + +# Mount point for the resource disk +ResourceDisk.MountPoint=/mnt/resource + +# Create and use swapfile on resource disk. +ResourceDisk.EnableSwap=y + +# Max size of the swap partition in MB +ResourceDisk.SwapSizeMB=65536 + +# Comma-seperated list of mount options. See man(8) for valid options. +ResourceDisk.MountOptions=None + +# Enable verbose logging (y|n) +Logs.Verbose=n + +# Is FIPS enabled +OS.EnableFIPS=n + +# Root device timeout in seconds. +OS.RootDeviceScsiTimeout=300 + +# If "None", the system default version is used. +OS.OpensslPath=/usr/local/bin/eopenssl + +# Set the path to SSH keys and configuration files +OS.SshDir=/etc/ssh + +OS.PasswordPath=/etc/master.passwd + +# If set, agent will use proxy server to access internet +#HttpProxy.Host=None +#HttpProxy.Port=None + +# Detect Scvmm environment, default is n +# DetectScvmmEnv=n + +# +# Lib.Dir=/var/lib/waagent + +# +# DVD.MountPoint=/mnt/cdrom/secure + +# +# Pid.File=/var/run/waagent.pid + +# +# Extension.LogDir=/var/log/azure + +# +# Home.Dir=/home + +# Enable RDMA management and set up, should only be used in HPC images +# OS.EnableRDMA=y + +# Enable or disable goal state processing auto-update, default is enabled +# AutoUpdate.Enabled=y + +# Determine the update family, this should not be changed +# AutoUpdate.GAFamily=Prod + +# Determine if the overprovisioning feature is enabled. If yes, hold extension +# handling until inVMArtifactsProfile.OnHold is false. +# Default is disabled +# EnableOverProvisioning=n diff --git a/debian/changelog b/debian/changelog index 0d09768..453beee 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,11 @@ +walinuxagent (2.2.14-0ubuntu1) artful; urgency=medium + + * New upstream release (LP: #1701350). + * debian/copyright: + - Refreshed copyright content. + + -- Łukasz 'sil2100' Zemczak <lukasz.zemczak@ubuntu.com> Mon, 03 Jul 2017 13:44:00 +0200 + walinuxagent (2.2.12-0ubuntu1) artful; urgency=medium * New upstream release (LP: #1690854). diff --git a/debian/copyright b/debian/copyright index fba1f44..1447fe2 100644 --- a/debian/copyright +++ b/debian/copyright @@ -4,16 +4,11 @@ Upstream-Contact: Microsoft Corporation <walinuxagent@microsoft.com> Source: https://github.com/Windows-Azure/WALinuxAgent/ Files: * -Copyright: 2012, Microsoft Corporation <walinuxagent@microsoft.com> -License: Apache-2.0 - -Files: waaagent -Copyright: 2012, Microsoft Corporation <walinuxagent@microsoft.com> - 2012, Ben Howard <ben.howard@canonical.com> +Copyright: 2012-2017, Microsoft Corporation <walinuxagent@microsoft.com> License: Apache-2.0 Files: debian/* -Copyright: 2012, Ben Howard <ben.howard@canonical.com> +Copyright: 2012-2017, Canonical Group, Ltd License: Apache-2.0 License: Apache-2.0 diff --git a/init/openbsd/waagent b/init/openbsd/waagent new file mode 100644 index 0000000..43f8948 --- /dev/null +++ b/init/openbsd/waagent @@ -0,0 +1,10 @@ +#!/bin/sh + +daemon="python2.7 /usr/local/sbin/waagent -start" + +. /etc/rc.d/rc.subr + +pexp="python /usr/local/sbin/waagent -daemon" +rc_reload=NO + +rc_cmd $1 @@ -58,7 +58,11 @@ def set_systemd_files(data_files, dest="/lib/systemd/system", data_files.append((dest, src)) -def set_rc_files(data_files, dest="/etc/rc.d/", src=["init/freebsd/waagent"]): +def set_freebsd_rc_files(data_files, dest="/etc/rc.d/", src=["init/freebsd/waagent"]): + data_files.append((dest, src)) + + +def set_openbsd_rc_files(data_files, dest="/etc/rc.d/", src=["init/openbsd/waagent"]): data_files.append((dest, src)) @@ -143,7 +147,11 @@ def get_data_files(name, version, fullname): elif name == 'freebsd': set_bin_files(data_files, dest="/usr/local/sbin") set_conf_files(data_files, src=["config/freebsd/waagent.conf"]) - set_rc_files(data_files) + set_freebsd_rc_files(data_files) + elif name == 'openbsd': + set_bin_files(data_files, dest="/usr/local/sbin") + set_conf_files(data_files, src=["config/openbsd/waagent.conf"]) + set_openbsd_rc_files(data_files) else: # Use default setting set_bin_files(data_files) diff --git a/tests/common/osutil/test_bigip.py b/tests/common/osutil/test_bigip.py index 4d1b006..f5958cf 100644 --- a/tests/common/osutil/test_bigip.py +++ b/tests/common/osutil/test_bigip.py @@ -282,35 +282,6 @@ class TestBigIpOSUtil_mount_dvd(AgentTestCase): self.assertEqual(args[1].call_count, 1) -class TestBigIpOSUtil_set_admin_access_to_ip(AgentTestCase): - - @patch.object(shellutil, "run", return_value=0) - @patch.object(osutil.BigIpOSUtil, - '_set_accept_admin_access_to_ip', return_value=None) - @patch.object(osutil.BigIpOSUtil, - '_set_drop_admin_access_to_ip', return_value=None) - def test_success(self, *args): - osutil.BigIpOSUtil.set_admin_access_to_ip( - osutil.BigIpOSUtil(), '192.168.10.10' - ) - self.assertEqual(args[0].call_count, 1) - self.assertEqual(args[1].call_count, 1) - - @patch.object(shellutil, "run", return_value=0) - def test_accept_access(self, *args): - osutil.BigIpOSUtil._set_accept_admin_access_to_ip( - osutil.BigIpOSUtil(), '192.168.10.10' - ) - self.assertEqual(args[0].call_count, 2) - - @patch.object(shellutil, "run", return_value=0) - def test_drop_access(self, *args): - osutil.BigIpOSUtil._set_drop_admin_access_to_ip( - osutil.BigIpOSUtil(), '192.168.10.10' - ) - self.assertEqual(args[0].call_count, 2) - - class TestBigIpOSUtil_route_add(AgentTestCase): @patch.object(shellutil, "run", return_value=0) diff --git a/tests/common/test_event.py b/tests/common/test_event.py index f535411..a485edf 100644 --- a/tests/common/test_event.py +++ b/tests/common/test_event.py @@ -17,12 +17,78 @@ from __future__ import print_function +from datetime import datetime + +import azurelinuxagent.common.event as event +import azurelinuxagent.common.logger as logger + from azurelinuxagent.common.event import init_event_logger, add_event from azurelinuxagent.common.future import ustr +from azurelinuxagent.common.version import CURRENT_VERSION + from tests.tools import * class TestEvent(AgentTestCase): + + @patch('azurelinuxagent.common.event.EventLogger.add_event') + def test_periodic_emits_if_not_previously_sent(self, mock_event): + init_event_logger(tempfile.mkdtemp()) + event.__event_logger__.reset_periodic() + + event.add_periodic(logger.EVERY_DAY, "FauxEvent") + mock_event.assert_called_once() + + @patch('azurelinuxagent.common.event.EventLogger.add_event') + def test_periodic_does_not_emit_if_previously_sent(self, mock_event): + init_event_logger(tempfile.mkdtemp()) + event.__event_logger__.reset_periodic() + + event.add_periodic(logger.EVERY_DAY, "FauxEvent") + self.assertEqual(1, mock_event.call_count) + + event.add_periodic(logger.EVERY_DAY, "FauxEvent") + self.assertEqual(1, mock_event.call_count) + + @patch('azurelinuxagent.common.event.EventLogger.add_event') + def test_periodic_emits_if_forced(self, mock_event): + init_event_logger(tempfile.mkdtemp()) + event.__event_logger__.reset_periodic() + + event.add_periodic(logger.EVERY_DAY, "FauxEvent") + self.assertEqual(1, mock_event.call_count) + + event.add_periodic(logger.EVERY_DAY, "FauxEvent", force=True) + self.assertEqual(2, mock_event.call_count) + + @patch('azurelinuxagent.common.event.EventLogger.add_event') + def test_periodic_emits_after_elapsed_delta(self, mock_event): + init_event_logger(tempfile.mkdtemp()) + event.__event_logger__.reset_periodic() + + event.add_periodic(logger.EVERY_DAY, "FauxEvent") + self.assertEqual(1, mock_event.call_count) + + event.add_periodic(logger.EVERY_DAY, "FauxEvent") + self.assertEqual(1, mock_event.call_count) + + h = hash("FauxEvent"+""+ustr(True)+"") + event.__event_logger__.periodic_messages[h] = \ + datetime.now() - logger.EVERY_DAY - logger.EVERY_HOUR + event.add_periodic(logger.EVERY_DAY, "FauxEvent") + self.assertEqual(2, mock_event.call_count) + + @patch('azurelinuxagent.common.event.EventLogger.add_event') + def test_periodic_forwards_args(self, mock_event): + init_event_logger(tempfile.mkdtemp()) + event.__event_logger__.reset_periodic() + + event.add_periodic(logger.EVERY_DAY, "FauxEvent") + mock_event.assert_called_once_with( + "FauxEvent", + duration=0, evt_type='', is_internal=False, is_success=True, + log_event=True, message='', op='', version=str(CURRENT_VERSION)) + def test_save_event(self): tmp_evt = tempfile.mkdtemp() init_event_logger(tmp_evt) diff --git a/tests/common/test_logger.py b/tests/common/test_logger.py new file mode 100644 index 0000000..9e298b3 --- /dev/null +++ b/tests/common/test_logger.py @@ -0,0 +1,66 @@ +# Copyright 2016 Microsoft Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Requires Python 2.4+ and Openssl 1.0+ +# + +from datetime import datetime + +import azurelinuxagent.common.logger as logger + +from tests.tools import * + +_MSG = "This is our test logging message {0} {1}" +_DATA = ["arg1", "arg2"] + +class TestLogger(AgentTestCase): + + @patch('azurelinuxagent.common.logger.Logger.info') + def test_periodic_emits_if_not_previously_sent(self, mock_info): + logger.reset_periodic() + + logger.periodic(logger.EVERY_DAY, _MSG, *_DATA) + mock_info.assert_called_once() + + @patch('azurelinuxagent.common.logger.Logger.info') + def test_periodic_does_not_emit_if_previously_sent(self, mock_info): + logger.reset_periodic() + + logger.periodic(logger.EVERY_DAY, _MSG, *_DATA) + self.assertEqual(1, mock_info.call_count) + + logger.periodic(logger.EVERY_DAY, _MSG, *_DATA) + self.assertEqual(1, mock_info.call_count) + + @patch('azurelinuxagent.common.logger.Logger.info') + def test_periodic_emits_after_elapsed_delta(self, mock_info): + logger.reset_periodic() + + logger.periodic(logger.EVERY_DAY, _MSG, *_DATA) + self.assertEqual(1, mock_info.call_count) + + logger.periodic(logger.EVERY_DAY, _MSG, *_DATA) + self.assertEqual(1, mock_info.call_count) + + logger.DEFAULT_LOGGER.periodic_messages[hash(_MSG)] = \ + datetime.now() - logger.EVERY_DAY - logger.EVERY_HOUR + logger.periodic(logger.EVERY_DAY, _MSG, *_DATA) + self.assertEqual(2, mock_info.call_count) + + @patch('azurelinuxagent.common.logger.Logger.info') + def test_periodic_forwards_message_and_args(self, mock_info): + logger.reset_periodic() + + logger.periodic(logger.EVERY_DAY, _MSG, *_DATA) + mock_info.assert_called_once_with(_MSG, *_DATA) diff --git a/tests/ga/test_update.py b/tests/ga/test_update.py index a83db95..0c8642c 100644 --- a/tests/ga/test_update.py +++ b/tests/ga/test_update.py @@ -23,6 +23,7 @@ import json import shutil from azurelinuxagent.common.protocol.hostplugin import * +from azurelinuxagent.common.protocol.metadata import * from azurelinuxagent.common.protocol.wire import * from azurelinuxagent.common.utils.fileutil import * from azurelinuxagent.ga.update import * @@ -1019,6 +1020,21 @@ class TestUpdate(UpdateTestCase): self.assertEqual(kept_agents, self.update_handler.agents) return + @patch('azurelinuxagent.common.protocol.wire.WireClient.get_host_plugin') + def test_get_host_plugin_returns_host_for_wireserver(self, mock_get_host): + protocol = WireProtocol('12.34.56.78') + mock_get_host.return_value = "faux host" + host = self.update_handler._get_host_plugin(protocol=protocol) + mock_get_host.assert_called_once() + self.assertEqual("faux host", host) + + @patch('azurelinuxagent.common.protocol.wire.WireClient.get_host_plugin') + def test_get_host_plugin_returns_none_otherwise(self, mock_get_host): + protocol = MetadataProtocol() + host = self.update_handler._get_host_plugin(protocol=protocol) + mock_get_host.assert_not_called() + self.assertEqual(None, host) + def test_get_latest_agent(self): latest_version = self.prepare_agents() @@ -1324,6 +1340,24 @@ class TestUpdate(UpdateTestCase): self.assertEqual(1, latest_agent.error.failure_count) return + def test_run_latest_exception_does_not_blacklist_if_terminating(self): + self.prepare_agents() + + latest_agent = self.update_handler.get_latest_agent() + self.assertTrue(latest_agent.is_available) + self.assertEqual(0.0, latest_agent.error.last_failure) + self.assertEqual(0, latest_agent.error.failure_count) + + with patch('azurelinuxagent.ga.update.UpdateHandler.get_latest_agent', return_value=latest_agent): + self.update_handler.running = False + self._test_run_latest(mock_child=ChildMock(side_effect=Exception("Attempt blacklisting"))) + + self.assertTrue(latest_agent.is_available) + self.assertFalse(latest_agent.error.is_blacklisted) + self.assertEqual(0.0, latest_agent.error.last_failure) + self.assertEqual(0, latest_agent.error.failure_count) + return + @patch('signal.signal') def test_run_latest_captures_signals(self, mock_signal): self._test_run_latest() @@ -1462,12 +1496,14 @@ class TestUpdate(UpdateTestCase): def test_shutdown(self): self.update_handler._set_sentinal() self.update_handler._shutdown() + self.assertFalse(self.update_handler.running) self.assertFalse(os.path.isfile(self.update_handler._sentinal_file_path())) return def test_shutdown_ignores_missing_sentinal_file(self): self.assertFalse(os.path.isfile(self.update_handler._sentinal_file_path())) self.update_handler._shutdown() + self.assertFalse(self.update_handler.running) self.assertFalse(os.path.isfile(self.update_handler._sentinal_file_path())) return diff --git a/tests/pa/test_deprovision.py b/tests/pa/test_deprovision.py index c4cd9b4..b2f7f0c 100644 --- a/tests/pa/test_deprovision.py +++ b/tests/pa/test_deprovision.py @@ -15,6 +15,7 @@ # Requires Python 2.4+ and Openssl 1.0+ # +import signal import tempfile import azurelinuxagent.common.utils.fileutil as fileutil @@ -25,16 +26,44 @@ from tests.tools import * class TestDeprovision(AgentTestCase): + @patch('signal.signal') + @patch('azurelinuxagent.common.osutil.get_osutil') + @patch('azurelinuxagent.common.protocol.get_protocol_util') + @patch('azurelinuxagent.pa.deprovision.default.read_input') + def test_confirmation(self, + mock_read, mock_protocol, mock_util, mock_signal): + dh = DeprovisionHandler() + + dh.setup = Mock() + dh.setup.return_value = ([], []) + dh.do_actions = Mock() + + # Do actions if confirmed + mock_read.return_value = "y" + dh.run() + self.assertEqual(1, dh.do_actions.call_count) + + # Skip actions if not confirmed + mock_read.return_value = "n" + dh.run() + self.assertEqual(1, dh.do_actions.call_count) + + # Do actions if forced + mock_read.return_value = "n" + dh.run(force=True) + self.assertEqual(2, dh.do_actions.call_count) + @patch("azurelinuxagent.pa.deprovision.default.DeprovisionHandler.cloud_init_dirs") @patch("azurelinuxagent.pa.deprovision.default.DeprovisionHandler.cloud_init_files") def test_del_cloud_init_without_once(self, mock_files, mock_dirs): deprovision_handler = get_deprovision_handler("","","") - deprovision_handler.del_cloud_init([], [], include_once=False) + deprovision_handler.del_cloud_init([], [], + include_once=False, deluser=False) mock_dirs.assert_called_with(include_once=False) - mock_files.assert_called_with(include_once=False) + mock_files.assert_called_with(include_once=False, deluser=False) @patch("signal.signal") @patch("azurelinuxagent.common.protocol.get_protocol_util") @@ -59,10 +88,11 @@ class TestDeprovision(AgentTestCase): mock_files.return_value = files deprovision_handler = get_deprovision_handler("","","") - deprovision_handler.del_cloud_init(warnings, actions) + deprovision_handler.del_cloud_init(warnings, actions, + deluser=True) mock_dirs.assert_called_with(include_once=True) - mock_files.assert_called_with(include_once=True) + mock_files.assert_called_with(include_once=True, deluser=True) self.assertEqual(len(warnings), 0) self.assertEqual(len(actions), 2) diff --git a/tests/protocol/test_hostplugin.py b/tests/protocol/test_hostplugin.py index e203615..b18b691 100644 --- a/tests/protocol/test_hostplugin.py +++ b/tests/protocol/test_hostplugin.py @@ -19,7 +19,7 @@ import base64 import json import sys - +from azurelinuxagent.common.future import ustr if sys.version_info[0] == 3: import http.client as httpclient @@ -224,6 +224,61 @@ class TestHostPlugin(AgentTestCase): test_goal_state, exp_method, exp_url, exp_data) + def test_read_response_error(self): + """ + Validate the read_response_error method handles encoding correctly + """ + responses = ['message', b'message', '\x80message\x80'] + response = MagicMock() + response.status = 'status' + response.reason = 'reason' + with patch.object(response, 'read') as patch_response: + for s in responses: + patch_response.return_value = s + result = hostplugin.HostPluginProtocol.read_response_error(response) + self.assertTrue('[status: reason]' in result) + self.assertTrue('message' in result) + + def test_read_response_bytes(self): + response_bytes = '7b:0a:20:20:20:20:22:65:72:72:6f:72:43:6f:64:65:22:' \ + '3a:20:22:54:68:65:20:62:6c:6f:62:20:74:79:70:65:20:' \ + '69:73:20:69:6e:76:61:6c:69:64:20:66:6f:72:20:74:68:' \ + '69:73:20:6f:70:65:72:61:74:69:6f:6e:2e:22:2c:0a:20:' \ + '20:20:20:22:6d:65:73:73:61:67:65:22:3a:20:22:c3:af:' \ + 'c2:bb:c2:bf:3c:3f:78:6d:6c:20:76:65:72:73:69:6f:6e:' \ + '3d:22:31:2e:30:22:20:65:6e:63:6f:64:69:6e:67:3d:22:' \ + '75:74:66:2d:38:22:3f:3e:3c:45:72:72:6f:72:3e:3c:43:' \ + '6f:64:65:3e:49:6e:76:61:6c:69:64:42:6c:6f:62:54:79:' \ + '70:65:3c:2f:43:6f:64:65:3e:3c:4d:65:73:73:61:67:65:' \ + '3e:54:68:65:20:62:6c:6f:62:20:74:79:70:65:20:69:73:' \ + '20:69:6e:76:61:6c:69:64:20:66:6f:72:20:74:68:69:73:' \ + '20:6f:70:65:72:61:74:69:6f:6e:2e:0a:52:65:71:75:65:' \ + '73:74:49:64:3a:63:37:34:32:39:30:63:62:2d:30:30:30:' \ + '31:2d:30:30:62:35:2d:30:36:64:61:2d:64:64:36:36:36:' \ + '61:30:30:30:22:2c:0a:20:20:20:20:22:64:65:74:61:69:' \ + '6c:73:22:3a:20:22:22:0a:7d'.split(':') + expected_response = '[status: reason] {\n "errorCode": "The blob ' \ + 'type is invalid for this operation.",\n ' \ + '"message": "<?xml version="1.0" ' \ + 'encoding="utf-8"?>' \ + '<Error><Code>InvalidBlobType</Code><Message>The ' \ + 'blob type is invalid for this operation.\n' \ + 'RequestId:c74290cb-0001-00b5-06da-dd666a000",' \ + '\n "details": ""\n}' + + response_string = ''.join(chr(int(b, 16)) for b in response_bytes) + response = MagicMock() + response.status = 'status' + response.reason = 'reason' + with patch.object(response, 'read') as patch_response: + patch_response.return_value = response_string + result = hostplugin.HostPluginProtocol.read_response_error(response) + self.assertEqual(result, expected_response) + try: + raise HttpError("{0}".format(result)) + except HttpError as e: + self.assertTrue(result in ustr(e)) + def test_no_fallback(self): """ Validate fallback to upload status using HostGAPlugin is not happening diff --git a/tests/protocol/test_wire.py b/tests/protocol/test_wire.py index ba9fc7d..02976ca 100644 --- a/tests/protocol/test_wire.py +++ b/tests/protocol/test_wire.py @@ -80,26 +80,36 @@ class TestWireProtocolGetters(AgentTestCase): url = testurl headers = {} - # no kwargs + # no kwargs -- Default to True WireClient.call_storage_service(http_req) - # kwargs, no chk_proxy + + # kwargs, no chk_proxy -- Default to True WireClient.call_storage_service(http_req, url, headers) - # kwargs, chk_proxy False + + # kwargs, chk_proxy None -- Default to True + WireClient.call_storage_service(http_req, + url, + headers, + chk_proxy=None) + + # kwargs, chk_proxy False -- Keep False WireClient.call_storage_service(http_req, url, headers, chk_proxy=False) - # kwargs, chk_proxy True + + # kwargs, chk_proxy True -- Keep True WireClient.call_storage_service(http_req, url, headers, chk_proxy=True) # assert - self.assertTrue(http_patch.call_count == 4) - for c in http_patch.call_args_list: - self.assertTrue(c[-1]['chk_proxy'] == True) + self.assertTrue(http_patch.call_count == 5) + for i in range(0,5): + c = http_patch.call_args_list[i][-1]['chk_proxy'] + self.assertTrue(c == (True if i != 3 else False)) def test_status_blob_parsing(self, *args): wire_protocol_client = WireProtocol(wireserver_url).client |