diff options
| author | Ben Howard <ben.howard@ubuntu.com> | 2015-12-07 16:48:51 -0700 |
|---|---|---|
| committer | usd-importer <ubuntu-server@lists.ubuntu.com> | 2015-12-08 16:10:11 +0000 |
| commit | 83432149e212155469b1e9f06eb0095121377356 (patch) | |
| tree | 9029a2e7836dce3025d5c5a6527d74bbf8d799e5 | |
| parent | 04946cba49f19c0b6b876bccdbb36d47334af002 (diff) | |
| download | vyos-walinuxagent-83432149e212155469b1e9f06eb0095121377356.tar.gz vyos-walinuxagent-83432149e212155469b1e9f06eb0095121377356.zip | |
Import patches-unapplied version 2.1.2-0ubuntu1 to ubuntu/xenial-proposed
Imported using git-ubuntu import.
Changelog parent: 04946cba49f19c0b6b876bccdbb36d47334af002
New changelog entries:
* New upstream release (LP: #1523715):
- Bug fixes for Ubuntu 15.10 on Azure
- Enablement for Azure Stack
- Dropped patch for systemd job as upstream now includes it.
94 files changed, 2025 insertions, 941 deletions
@@ -1,4 +1,4 @@ -Windows Azure Linux Agent +Microsoft Azure Linux Agent Copyright 2012 Microsoft Corporation This product includes software developed at @@ -55,7 +55,7 @@ REQUIREMENTS The following systems have been tested and are known to work with the Azure Linux Agent. Please note that this list may differ from the official list -of supported systems on the Windows Azure Platform as described here: +of supported systems on the Microsoft Azure Platform as described here: http://support.microsoft.com/kb/2805216 Supported Linux Distributions: @@ -199,6 +199,8 @@ Provisioning.SshHostKeyPairType=rsa Provisioning.MonitorHostName=y Provisioning.DecodeCustomData=n Provisioning.ExecuteCustomData=n +Provisioning.PasswordCryptId=6 +Provisioning.PasswordCryptSaltLength=10 ResourceDisk.Format=y ResourceDisk.Filesystem=ext4 ResourceDisk.MountPoint=/mnt/resource @@ -300,6 +302,20 @@ Type: Boolean Default: n If set, waagent will execute CustomData after provisioning. +Provisioning.PasswordCryptId: +Type:String Default:6 + +Algorithm used by crypt when generating password hash. + 1 - MD5 + 2a - Blowfish + 5 - SHA-256 + 6 - SHA-512 + +Provisioning.PasswordCryptSaltLength +Type:String Default:10 + +Length of random salt used when generating password hash. + ResourceDisk.Format: Type: Boolean Default: y diff --git a/azurelinuxagent/agent.py b/azurelinuxagent/agent.py index 5e61a6c..849a192 100644 --- a/azurelinuxagent/agent.py +++ b/azurelinuxagent/agent.py @@ -1,4 +1,4 @@ -# Windows Azure Linux Agent +# Microsoft Azure Linux Agent # # Copyright 2014 Microsoft Corporation # diff --git a/azurelinuxagent/conf.py b/azurelinuxagent/conf.py index 3185d99..2b0eb01 100644 --- a/azurelinuxagent/conf.py +++ b/azurelinuxagent/conf.py @@ -1,4 +1,4 @@ -# Windows Azure Linux Agent +# Microsoft Azure Linux Agent # # Copyright 2014 Microsoft Corporation # diff --git a/azurelinuxagent/distro/__init__.py b/azurelinuxagent/distro/__init__.py index 4b2b9e1..d9b82f5 100644 --- a/azurelinuxagent/distro/__init__.py +++ b/azurelinuxagent/distro/__init__.py @@ -1,4 +1,4 @@ -# Windows Azure Linux Agent +# Microsoft Azure Linux Agent # # Copyright 2014 Microsoft Corporation # diff --git a/azurelinuxagent/distro/centos/__init__.py b/azurelinuxagent/distro/centos/__init__.py index 4b2b9e1..d9b82f5 100644 --- a/azurelinuxagent/distro/centos/__init__.py +++ b/azurelinuxagent/distro/centos/__init__.py @@ -1,4 +1,4 @@ -# Windows Azure Linux Agent +# Microsoft Azure Linux Agent # # Copyright 2014 Microsoft Corporation # diff --git a/azurelinuxagent/distro/centos/loader.py b/azurelinuxagent/distro/centos/loader.py index 379f027..9dc428f 100644 --- a/azurelinuxagent/distro/centos/loader.py +++ b/azurelinuxagent/distro/centos/loader.py @@ -1,4 +1,4 @@ -# Windows Azure Linux Agent +# Microsoft Azure Linux Agent # # Copyright 2014 Microsoft Corporation # diff --git a/azurelinuxagent/distro/coreos/__init__.py b/azurelinuxagent/distro/coreos/__init__.py index 7a4980e..8c1bbdb 100644 --- a/azurelinuxagent/distro/coreos/__init__.py +++ b/azurelinuxagent/distro/coreos/__init__.py @@ -1,4 +1,4 @@ -# Windows Azure Linux Agent +# Microsoft Azure Linux Agent # # Copyright 2014 Microsoft Corporation # diff --git a/azurelinuxagent/distro/coreos/deprovision.py b/azurelinuxagent/distro/coreos/deprovision.py index f0ff604..99d3a40 100644 --- a/azurelinuxagent/distro/coreos/deprovision.py +++ b/azurelinuxagent/distro/coreos/deprovision.py @@ -1,4 +1,4 @@ -# Windows Azure Linux Agent +# Microsoft Azure Linux Agent # # Copyright 2014 Microsoft Corporation # diff --git a/azurelinuxagent/distro/coreos/handlerFactory.py b/azurelinuxagent/distro/coreos/handlerFactory.py index f0490e8..58f476c 100644 --- a/azurelinuxagent/distro/coreos/handlerFactory.py +++ b/azurelinuxagent/distro/coreos/handlerFactory.py @@ -1,4 +1,4 @@ -# Windows Azure Linux Agent +# Microsoft Azure Linux Agent # # Copyright 2014 Microsoft Corporation # diff --git a/azurelinuxagent/distro/coreos/loader.py b/azurelinuxagent/distro/coreos/loader.py index ec009ef..802f276 100644 --- a/azurelinuxagent/distro/coreos/loader.py +++ b/azurelinuxagent/distro/coreos/loader.py @@ -1,4 +1,4 @@ -# Windows Azure Linux Agent +# Microsoft Azure Linux Agent # # Copyright 2014 Microsoft Corporation # diff --git a/azurelinuxagent/distro/coreos/osutil.py b/azurelinuxagent/distro/coreos/osutil.py index 6dfba64..c244311 100644 --- a/azurelinuxagent/distro/coreos/osutil.py +++ b/azurelinuxagent/distro/coreos/osutil.py @@ -37,7 +37,7 @@ class CoreOSUtil(DefaultOSUtil): super(CoreOSUtil, self).__init__() self.waagent_path='/usr/share/oem/bin/waagent' self.python_path='/usr/share/oem/python/bin' - self.conf_path = '/usr/share/oem/waagent.conf' + self.conf_file_path = '/usr/share/oem/waagent.conf' if 'PATH' in os.environ: path = "{0}:{1}".format(os.environ['PATH'], self.python_path) else: @@ -55,7 +55,7 @@ class CoreOSUtil(DefaultOSUtil): #User 'core' is not a sysuser if username == 'core': return False - return super(CoreOSUtil, self).IsSysUser(username) + return super(CoreOSUtil, self).is_sys_user(username) def is_dhcp_enabled(self): return True @@ -88,3 +88,11 @@ class CoreOSUtil(DefaultOSUtil): def decode_customdata(self, data): return base64.b64decode(data) + def set_ssh_client_alive_interval(self): + #In CoreOS, /etc/sshd_config is mount readonly. Skip the setting + pass + + def conf_sshd(self, disable_password): + #In CoreOS, /etc/sshd_config is mount readonly. Skip the setting + pass + diff --git a/azurelinuxagent/distro/debian/__init__.py b/azurelinuxagent/distro/debian/__init__.py index 4b2b9e1..d9b82f5 100644 --- a/azurelinuxagent/distro/debian/__init__.py +++ b/azurelinuxagent/distro/debian/__init__.py @@ -1,4 +1,4 @@ -# Windows Azure Linux Agent +# Microsoft Azure Linux Agent # # Copyright 2014 Microsoft Corporation # diff --git a/azurelinuxagent/distro/debian/loader.py b/azurelinuxagent/distro/debian/loader.py index 0787758..cc0c06f 100644 --- a/azurelinuxagent/distro/debian/loader.py +++ b/azurelinuxagent/distro/debian/loader.py @@ -1,4 +1,4 @@ -# Windows Azure Linux Agent +# Microsoft Azure Linux Agent # # Copyright 2014 Microsoft Corporation # diff --git a/azurelinuxagent/distro/default/__init__.py b/azurelinuxagent/distro/default/__init__.py index 4b2b9e1..d9b82f5 100644 --- a/azurelinuxagent/distro/default/__init__.py +++ b/azurelinuxagent/distro/default/__init__.py @@ -1,4 +1,4 @@ -# Windows Azure Linux Agent +# Microsoft Azure Linux Agent # # Copyright 2014 Microsoft Corporation # diff --git a/azurelinuxagent/distro/default/deprovision.py b/azurelinuxagent/distro/default/deprovision.py index 231f4eb..b62c5f6 100644 --- a/azurelinuxagent/distro/default/deprovision.py +++ b/azurelinuxagent/distro/default/deprovision.py @@ -1,4 +1,4 @@ -# Windows Azure Linux Agent +# Microsoft Azure Linux Agent # # Copyright 2014 Microsoft Corporation # @@ -19,6 +19,7 @@ import azurelinuxagent.conf as conf from azurelinuxagent.utils.osutil import OSUTIL +from azurelinuxagent.future import read_input import azurelinuxagent.protocol as prot import azurelinuxagent.protocol.ovfenv as ovf import azurelinuxagent.utils.fileutil as fileutil @@ -58,8 +59,6 @@ class DeprovisionHandler(object): def regen_ssh_host_key(self, warnings, actions): warnings.append("WARNING! All SSH host key pairs will be deleted.") - actions.append(DeprovisionAction(OSUTIL.set_hostname, - ['localhost.localdomain'])) actions.append(DeprovisionAction(shellutil.run, ['rm -f /etc/ssh/ssh_host_*key*'])) @@ -80,6 +79,11 @@ class DeprovisionHandler(object): dirs_to_del = [OSUTIL.get_lib_dir()] actions.append(DeprovisionAction(fileutil.rm_dirs, dirs_to_del)) + def reset_hostname(self, warnings, actions): + localhost = ["localhost.localdomain"] + actions.append(DeprovisionAction(OSUTIL.set_hostname, localhost)) + actions.append(DeprovisionAction(OSUTIL.set_dhcp_hostname, localhost)) + def setup(self, deluser): warnings = [] actions = [] @@ -89,6 +93,7 @@ class DeprovisionHandler(object): self.regen_ssh_host_key(warnings, actions) self.del_dhcp_lease(warnings, actions) + self.reset_hostname(warnings, actions) if conf.get_switch("Provisioning.DeleteRootPassword", False): self.del_root_password(warnings, actions) @@ -107,7 +112,7 @@ class DeprovisionHandler(object): print(warning) if not force: - confirm = input("Do you want to proceed (y/n)") + confirm = read_input("Do you want to proceed (y/n)") if not confirm.lower().startswith('y'): return diff --git a/azurelinuxagent/distro/default/dhcp.py b/azurelinuxagent/distro/default/dhcp.py index 574ebd4..4fd23ef 100644 --- a/azurelinuxagent/distro/default/dhcp.py +++ b/azurelinuxagent/distro/default/dhcp.py @@ -1,4 +1,4 @@ -# Windows Azure Linux Agent +# Microsoft Azure Linux Agent # # Copyright 2014 Microsoft Corporation # diff --git a/azurelinuxagent/distro/default/env.py b/azurelinuxagent/distro/default/env.py index 6a67113..28bf718 100644 --- a/azurelinuxagent/distro/default/env.py +++ b/azurelinuxagent/distro/default/env.py @@ -1,4 +1,4 @@ -# Windows Azure Linux Agent +# Microsoft Azure Linux Agent # # Copyright 2014 Microsoft Corporation # diff --git a/azurelinuxagent/distro/default/extension.py b/azurelinuxagent/distro/default/extension.py index 58ba84e..f6c02aa 100644 --- a/azurelinuxagent/distro/default/extension.py +++ b/azurelinuxagent/distro/default/extension.py @@ -1,4 +1,4 @@ -# Windows Azure Linux Agent +# Microsoft Azure Linux Agent # # Copyright 2014 Microsoft Corporation # @@ -21,18 +21,35 @@ import zipfile import time import json import subprocess +import shutil import azurelinuxagent.logger as logger from azurelinuxagent.future import text from azurelinuxagent.utils.osutil import OSUTIL import azurelinuxagent.protocol as prot +from azurelinuxagent.metadata import AGENT_VERSION from azurelinuxagent.event import add_event, WALAEventOperation from azurelinuxagent.exception import ExtensionError import azurelinuxagent.utils.fileutil as fileutil import azurelinuxagent.utils.restutil as restutil import azurelinuxagent.utils.shellutil as shellutil +from azurelinuxagent.utils.textutil import Version + +#HandlerEnvironment.json schema version +HANDLER_ENVIRONMENT_VERSION = 1.0 VALID_EXTENSION_STATUS = ['transitioning', 'error', 'success', 'warning'] +VALID_HANDLER_STATUS = ['Ready', 'NotReady', "Installing", "Unresponsive"] + +def handler_state_to_status(handler_state): + if handler_state == "Enabled": + return "Ready" + elif handler_state in VALID_HANDLER_STATUS: + return handler_state + else: + return "NotReady" + + def validate_has_key(obj, key, fullname): if key not in obj: raise ExtensionError("Missing: {0}".format(fullname)) @@ -41,103 +58,52 @@ def validate_in_range(val, valid_range, name): if val not in valid_range: raise ExtensionError("Invalid {0}: {1}".format(name, val)) -def try_get(dictionary, key, default=None): - try: - return dictionary[key] - except KeyError: - return default +def parse_formatted_message(formatted_message): + if formatted_message is None: + return None + validate_has_key(formatted_message, 'lang', 'formattedMessage/lang') + validate_has_key(formatted_message, 'message', 'formattedMessage/message') + return formatted_message.get('message') + -def extension_sub_status_to_v2(substatus): +def parse_ext_substatus(substatus): #Check extension sub status format - validate_has_key(substatus, 'name', 'substatus/name') validate_has_key(substatus, 'status', 'substatus/status') - validate_has_key(substatus, 'code', 'substatus/code') - validate_has_key(substatus, 'formattedMessage', 'substatus/formattedMessage') - validate_has_key(substatus['formattedMessage'], 'lang', - 'substatus/formattedMessage/lang') - validate_has_key(substatus['formattedMessage'], 'message', - 'substatus/formattedMessage/message') - validate_in_range(substatus['status'], VALID_EXTENSION_STATUS, 'substatus/status') status = prot.ExtensionSubStatus() - status.name = try_get(substatus, 'name') - status.status = try_get(substatus, 'status') - status.code = try_get(substatus, 'code') - status.message = try_get(substatus['formattedMessage'], 'message') + status.name = substatus.get('name') + status.status = substatus.get('status') + status.code = substatus.get('code', 0) + formatted_message = substatus.get('formattedMessage') + status.message = parse_formatted_message(formatted_message) return status -def ext_status_to_v2(ext_status, seq_no): +def parse_ext_status(ext_status, data): + if data is None or len(data) is None: + return + #Currently, only the first status will be reported + data = data[0] #Check extension status format - validate_has_key(ext_status, 'status', 'status') - validate_has_key(ext_status['status'], 'status', 'status/status') - validate_has_key(ext_status['status'], 'operation', 'status/operation') - validate_has_key(ext_status['status'], 'code', 'status/code') - validate_has_key(ext_status['status'], 'name', 'status/name') - validate_has_key(ext_status['status'], 'formattedMessage', - 'status/formattedMessage') - validate_has_key(ext_status['status']['formattedMessage'], 'lang', - 'status/formattedMessage/lang') - validate_has_key(ext_status['status']['formattedMessage'], 'message', - 'status/formattedMessage/message') - - validate_in_range(ext_status['status']['status'], VALID_EXTENSION_STATUS, + validate_has_key(data, 'status', 'status') + status_data = data['status'] + validate_has_key(status_data, 'status', 'status/status') + + validate_in_range(status_data['status'], VALID_EXTENSION_STATUS, 'status/status') - status = prot.ExtensionStatus() - status.name = try_get(ext_status['status'], 'name') - status.configurationAppliedTime = try_get(ext_status['status'], - 'configurationAppliedTime') - status.operation = try_get(ext_status['status'], 'operation') - status.status = try_get(ext_status['status'], 'status') - status.code = try_get(ext_status['status'], 'code') - status.message = try_get(ext_status['status']['formattedMessage'], 'message') - status.sequenceNumber = seq_no - - substatus_list = try_get(ext_status['status'], 'substatus', []) + applied_time = status_data.get('configurationAppliedTime') + ext_status.configurationAppliedTime = applied_time + ext_status.operation = status_data.get('operation') + ext_status.status = status_data.get('status') + ext_status.code = status_data.get('code', 0) + formatted_message = status_data.get('formattedMessage') + ext_status.message = parse_formatted_message(formatted_message) + substatus_list = status_data.get('substatus') + if substatus_list is None: + return for substatus in substatus_list: - status.substatusList.extend(extension_sub_status_to_v2(substatus)) - return status - -class ExtensionsHandler(object): - - def process(self): - protocol = prot.FACTORY.get_default_protocol() - ext_list = protocol.get_extensions() - - h_status_list = [] - for extension in ext_list.extensions: - #TODO handle extension in parallel - pkg_list = protocol.get_extension_pkgs(extension) - h_status = self.process_extension(extension, pkg_list) - h_status_list.append(h_status) - - return h_status_list - - def process_extension(self, extension, pkg_list): - installed_version = get_installed_version(extension.name) - if installed_version is not None: - ext = ExtensionInstance(extension, pkg_list, - installed_version, installed=True) - else: - ext = ExtensionInstance(extension, pkg_list, - extension.properties.version) - try: - ext.init_logger() - ext.handle() - status = ext.collect_handler_status() - except ExtensionError as e: - logger.error("Failed to handle extension: {0}-{1}\n {2}", - ext.get_name(), ext.get_version(), e) - add_event(name=ext.get_name(), is_success=False, - op=ext.get_curr_op(), message = text(e)) - ext_status = prot.ExtensionStatus(status='error', code='-1', - operation = ext.get_curr_op(), - message = text(e), - seq_no = ext.get_seq_no()) - status = ext.create_handler_status(ext_status) - status.status = "Ready" - return status + ext_status.substatusList.append(parse_ext_substatus(substatus)) def parse_extension_dirname(dirname): """ @@ -160,67 +126,204 @@ def get_installed_version(target_name): name, version = parse_extension_dirname(dir_name) #Here we need to ensure names are exactly the same. if name == target_name: - if installed_version is None or installed_version < version: + if installed_version is None or \ + Version(installed_version) < Version(version): installed_version = version return installed_version -class ExtensionInstance(object): - def __init__(self, extension, pkg_list, curr_version, installed=False): - self.extension = extension +class ExtHandlerState(object): + Enabled = "Enabled" + Disabled = "Disabled" + Failed = "Failed" + + +class ExtHandlersHandler(object): + + def process(self): + try: + protocol = prot.FACTORY.get_default_protocol() + ext_handlers = protocol.get_ext_handlers() + except prot.ProtocolError as e: + add_event(name="WALA", is_success=False, message = text(e)) + return + + + vm_status = prot.VMStatus() + vm_status.vmAgent.version = AGENT_VERSION + vm_status.vmAgent.status = "Ready" + vm_status.vmAgent.message = "Guest Agent is running" + + if ext_handlers.extHandlers is None or \ + len(ext_handlers.extHandlers) == 0: + logger.verb("No extensions to handle") + else: + for ext_handler in ext_handlers.extHandlers: + #TODO handle extension in parallel + try: + pkg_list = protocol.get_ext_handler_pkgs(ext_handler) + except prot.ProtocolError as e: + add_event(name="WALA", is_success=False, message=text(e)) + continue + + handler_status = self.process_extension(ext_handler, pkg_list) + if handler_status is not None: + vm_status.vmAgent.extensionHandlers.append(handler_status) + + try: + logger.verb("Report vm agent status") + protocol.report_vm_status(vm_status) + except prot.ProtocolError as e: + add_event(name="WALA", is_success=False, message = text(e)) + + def process_extension(self, ext_handler, pkg_list): + installed_version = get_installed_version(ext_handler.name) + if installed_version is not None: + handler = ExtHandlerInstance(ext_handler, pkg_list, + installed_version, installed=True) + else: + handler = ExtHandlerInstance(ext_handler, pkg_list, + ext_handler.properties.version) + handler.handle() + + if handler.ext_status is not None: + try: + protocol = prot.FACTORY.get_default_protocol() + protocol.report_ext_status(handler.name, handler.ext.name, + handler.ext_status) + except prot.ProtocolError as e: + add_event(name="WALA", is_success=False, message=text(e)) + + return handler.handler_status + +class ExtHandlerInstance(object): + def __init__(self, ext_handler, pkg_list, curr_version, installed=False): + self.ext_handler = ext_handler + self.name = ext_handler.name + self.version = ext_handler.properties.version self.pkg_list = pkg_list + self.state = ext_handler.properties.state + self.update_policy = ext_handler.properties.upgradePolicy + self.curr_version = curr_version - self.lib_dir = OSUTIL.get_lib_dir() self.installed = installed - self.settings = None - - #Extension will have no more than 1 settings instance - if len(extension.properties.extensions) > 0: - self.settings = extension.properties.extensions[0] - self.enabled = False - self.curr_op = None + self.handler_state = None + self.lib_dir = OSUTIL.get_lib_dir() + + self.ext_status = prot.ExtensionStatus() + self.handler_status = prot.ExtHandlerStatus() + self.handler_status.name = self.name + self.handler_status.version = self.curr_version + + #Currently, extension settings will have no more than 1 instance + if len(ext_handler.properties.extensions) > 0: + self.ext = ext_handler.properties.extensions[0] + self.handler_status.extensions = [self.ext.name] + else: + #When no extension settings, set sequenceNumber to 0 + self.ext = prot.Extension(sequenceNumber=0) + self.ext_status.sequenceNumber = self.ext.sequenceNumber prefix = "[{0}]".format(self.get_full_name()) self.logger = logger.Logger(logger.DEFAULT_LOGGER, prefix) def init_logger(self): #Init logger appender for extension - fileutil.mkdir(self.get_log_dir(), mode=0o700) + fileutil.mkdir(self.get_log_dir(), mode=0o644) log_file = os.path.join(self.get_log_dir(), "CommandExecution.log") self.logger.add_appender(logger.AppenderType.FILE, - logger.LogLevel.INFO, log_file) + logger.LogLevel.INFO, log_file) def handle(self): - self.logger.info("Process extension settings:") - self.logger.info(" Name: {0}", self.get_name()) - self.logger.info(" Version: {0}", self.get_version()) + self.init_logger() + self.logger.verb("Start processing extension handler") + + try: + self.handle_state() + except ExtensionError as e: + self.set_state_err(text(e)) + self.report_event(is_success=False, message=text(e)) + self.logger.error("Failed to process extension handler") + return + try: + if self.installed: + self.collect_ext_status() + self.collect_handler_status() + except ExtensionError as e: + self.report_event(is_success=False, message=text(e)) + self.logger.error("Failed to get extension handler status") + return + + self.logger.verb("Finished processing extension handler") + + def handle_state(self): if self.installed: - self.logger.info("Installed version:{0}", self.curr_version) - h_status = self.get_handler_status() - self.enabled = (h_status == "Ready") - - state = self.get_state() - if state == 'enabled': - self.handle_enable() - elif state == 'disabled': - self.handle_disable() - elif state == 'uninstall': - self.handle_disable() - self.handle_uninstall() + self.handler_state = self.get_state() + + self.handler_status.status = handler_state_to_status(self.handler_state) + self.logger.verb("Handler state: {0}", self.handler_state) + self.logger.verb("Sequence number: {0}", self.ext.sequenceNumber) + + if self.state == 'enabled': + if self.handler_state == ExtHandlerState.Failed: + self.logger.verb("Found previous failure, quit handle_enable") + return + + if self.handler_state == ExtHandlerState.Enabled: + self.logger.verb("Already enabled with sequenceNumber: {0}", + self.ext.sequenceNumber) + self.logger.verb("Quit handle_enable") + return + + try: + new = self.handle_enable() + if new is not None: + #Upgrade happened + new.set_state(ExtHandlerState.Enabled) + else: + self.set_state(ExtHandlerState.Enabled) + + except ExtensionError as e: + self.set_state(ExtHandlerState.Failed) + raise e + elif self.state == 'disabled': + if self.handler_state == ExtHandlerState.Failed: + self.logger.verb("Found previous failure, quit handle_disable") + return + + if self.handler_state == ExtHandlerState.Disabled: + self.logger.verb("Already disabled with sequenceNumber: {0}", + self.ext.sequenceNumber) + self.logger.verb("Quit handle_disable") + return + + try: + self.handle_disable() + self.set_state(ExtHandlerState.Disabled) + except ExtensionError as e: + self.set_state(ExtHandlerState.Failed) + raise e + elif self.state == 'uninstall': + try: + self.handle_uninstall() + except ExtensionError as e: + self.set_state(ExtHandlerState.Failed) + raise e else: - raise ExtensionError("Unknown extension state:{0}".format(state)) + raise ExtensionError("Unknown state:{0}".format(self.state)) def handle_enable(self): target_version = self.get_target_version() + self.logger.info("Target version: {0}", target_version) if self.installed: - if target_version > self.curr_version: - self.upgrade(target_version) - elif target_version == self.curr_version: + if Version(target_version) > Version(self.curr_version): + return self.upgrade(target_version) + elif Version(target_version) == Version(self.curr_version): self.enable() else: - raise ExtensionError("A newer version has already been installed") + raise ExtensionError("A newer version is already installed") else: - if target_version > self.get_version(): + if Version(target_version) > Version(self.version): #This will happen when auto upgrade policy is enabled self.logger.info("Auto upgrade to new version:{0}", target_version) @@ -231,21 +334,45 @@ class ExtensionInstance(object): self.enable() def handle_disable(self): - if not self.installed or not self.enabled: + if not self.installed: + self.logger.verb("Not installed, quit disable") return + self.disable() def handle_uninstall(self): if not self.installed: + self.logger.verb("Not installed, quit unistall") + self.handler_status = None + self.ext_status = None return + self.disable() self.uninstall() + def report_event(self, is_success=True, message=""): + if self.ext_status is not None: + if not is_success: + self.ext_status.status = "error" + self.ext_status.code = -1 + if self.handler_status is not None: + self.handler_status.message = message + if not is_success: + self.handler_status.status = "NotReady" + add_event(name=self.name, op=self.ext_status.operation, + is_success=is_success, message=message) + + def set_operation(self, operation): + if self.ext_status.operation != WALAEventOperation.Upgrade: + self.ext_status.operation = operation + def upgrade(self, target_version): self.logger.info("Upgrade from: {0} to {1}", self.curr_version, target_version) - self.curr_op=WALAEventOperation.Upgrade + self.set_operation(WALAEventOperation.Upgrade) + old = self - new = ExtensionInstance(self.extension, self.pkg_list, target_version) + new = ExtHandlerInstance(self.ext_handler, self.pkg_list, + target_version) self.logger.info("Download new extension package") new.init_logger() new.download() @@ -262,19 +389,20 @@ class ExtensionInstance(object): new.install() self.logger.info("Enable new extension") new.enable() - add_event(name=self.get_name(), is_success=True, - op=self.curr_op, message="") + return new def download(self): self.logger.info("Download extension package") - self.curr_op=WALAEventOperation.Download + self.set_operation(WALAEventOperation.Download) + uris = self.get_package_uris() package = None for uri in uris: try: resp = restutil.http_get(uri.uri, chk_proxy=True) - package = resp.read() - break + if resp.status == restutil.httpclient.OK: + package = resp.read() + break except restutil.HttpError as e: self.logger.warn("Failed download extension from: {0}", uri.uri) @@ -287,14 +415,13 @@ class ExtensionInstance(object): zipfile.ZipFile(pkg_file).extractall(self.get_base_dir()) chmod = "find {0} -type f | xargs chmod u+x".format(self.get_base_dir()) shellutil.run(chmod) - add_event(name=self.get_name(), is_success=True, - op=self.curr_op, message="") + self.report_event(message="Download succeeded") def init_dir(self): self.logger.info("Initialize extension directory") #Save HandlerManifest.json man_file = fileutil.search_file(self.get_base_dir(), - 'HandlerManifest.json') + 'HandlerManifest.json') man = fileutil.read_file(man_file, remove_bom=True) fileutil.write_file(self.get_manifest_file(), man) @@ -303,106 +430,141 @@ class ExtensionInstance(object): fileutil.mkdir(status_dir, mode=0o700) conf_dir = self.get_conf_dir() fileutil.mkdir(conf_dir, mode=0o700) - - #Init handler state to uninstall - self.set_handler_status("NotReady") + + self.make_handler_state_dir() #Save HandlerEnvironment.json self.create_handler_env() def enable(self): self.logger.info("Enable extension.") - self.curr_op=WALAEventOperation.Enable + self.set_operation(WALAEventOperation.Enable) + man = self.load_manifest() self.launch_command(man.get_enable_command()) - self.set_handler_status("Ready") - add_event(name=self.get_name(), is_success=True, - op=self.curr_op, message="") def disable(self): self.logger.info("Disable extension.") - self.curr_op=WALAEventOperation.Disable + self.set_operation(WALAEventOperation.Disable) + man = self.load_manifest() self.launch_command(man.get_disable_command(), timeout=900) - self.set_handler_status("Ready") - add_event(name=self.get_name(), is_success=True, - op=self.curr_op, message="") def install(self): self.logger.info("Install extension.") - self.curr_op=WALAEventOperation.Install + self.set_operation(WALAEventOperation.Install) + man = self.load_manifest() - self.set_handler_status("Installing") self.launch_command(man.get_install_command(), timeout=900) - self.set_handler_status("Ready") - add_event(name=self.get_name(), is_success=True, - op=self.curr_op, message="") + self.installed = True def uninstall(self): self.logger.info("Uninstall extension.") - self.curr_op=WALAEventOperation.UnInstall + self.set_operation(WALAEventOperation.UnInstall) + man = self.load_manifest() self.launch_command(man.get_uninstall_command()) - self.set_handler_status("NotReady") - add_event(name=self.get_name(), is_success=True, - op=self.curr_op, message="") + + self.logger.info("Remove ext handler dir: {0}", self.get_base_dir()) + try: + shutil.rmtree(self.get_base_dir()) + except IOError as e: + raise ExtensionError("Failed to rm ext handler dir: {0}".format(e)) + + self.installed = False + self.handler_status = None + self.ext_status = None def update(self): self.logger.info("Update extension.") - self.curr_op=WALAEventOperation.Update + self.set_operation(WALAEventOperation.Update) + man = self.load_manifest() self.launch_command(man.get_update_command(), timeout=900) - add_event(name=self.get_name(), is_success=True, - op=self.curr_op, message="") - - def create_handler_status(self, ext_status, heartbeat=None): - status = prot.ExtensionHandlerStatus() - status.handlerName = self.get_name() - status.handlerVersion = self.get_version() - status.status = self.get_handler_status() - status.extensionStatusList.append(ext_status) - return status def collect_handler_status(self): + self.logger.verb("Collect extension handler status") + if self.handler_status is None: + return + + handler_state = self.get_state() + self.handler_status.status = handler_state_to_status(handler_state) + self.handler_status.message = self.get_state_err() man = self.load_manifest() - heartbeat=None if man.is_report_heartbeat(): heartbeat = self.collect_heartbeat() - ext_status = self.collect_extension_status() - status= self.create_handler_status(ext_status, heartbeat) - status.status = self.get_handler_status() - if heartbeat is not None: - status.status = heartbeat['status'] - status.extensionStatusList.append(ext_status) - return status - - def collect_extension_status(self): + if heartbeat is not None: + self.handler_status.status = heartbeat['status'] + + def collect_ext_status(self): + self.logger.verb("Collect extension status") + if self.handler_status is None: + return + + if self.ext is None: + return + ext_status_file = self.get_status_file() try: - ext_status_str = fileutil.read_file(ext_status_file) - ext_status = json.loads(ext_status_str) + data_str = fileutil.read_file(ext_status_file) + data = json.loads(data_str) + parse_ext_status(self.ext_status, data) except IOError as e: raise ExtensionError("Failed to get status file: {0}".format(e)) except ValueError as e: raise ExtensionError("Malformed status file: {0}".format(e)) - return ext_status_to_v2(ext_status[0], - self.settings.sequenceNumber) + + def make_handler_state_dir(self): + handler_state_dir = self.get_handler_state_dir() + fileutil.mkdir(handler_state_dir, 0o600) + if not os.path.exists(handler_state_dir): + os.makedirs(handler_state_dir) - def get_handler_status(self): - h_status = "uninstalled" - h_status_file = self.get_handler_state_file() + def get_state(self): + handler_state_file = self.get_handler_state_file() + if not os.path.isfile(handler_state_file): + return None try: - h_status = fileutil.read_file(h_status_file) - return h_status + handler_state = fileutil.read_file(handler_state_file) + if handler_state is not None: + handler_state = handler_state.rstrip() + return handler_state except IOError as e: - raise ExtensionError("Failed to get handler status: {0}".format(e)) + err = "Failed to get handler state: {0}".format(e) + add_event(name=self.name, is_success=False, message=err) - def set_handler_status(self, status): - h_status_file = self.get_handler_state_file() + def set_state(self, state): + handler_state_file = self.get_handler_state_file() + if not os.path.isfile(handler_state_file): + self.make_handler_state_dir() + try: + fileutil.write_file(handler_state_file, state) + except IOError as e: + err = "Failed to set handler state: {0}".format(e) + add_event(name=self.name, is_success=False, message=err) + + def get_state_err(self): + """Get handler error message""" + handler_state_err_file= self.get_handler_state_err_file() + if not os.path.isfile(handler_state_err_file): + return None try: - fileutil.write_file(h_status_file, status) + message = fileutil.read_file(handler_state_err_file) + return message except IOError as e: - raise ExtensionError("Failed to set handler status: {0}".format(e)) + err = "Failed to get handler state message: {0}".format(e) + add_event(name=self.name, is_success=False, message=err) + + def set_state_err(self, message): + """Set handler error message""" + handler_state_err_file = self.get_handler_state_err_file() + if not os.path.isfile(handler_state_err_file): + self.make_handler_state_dir() + try: + fileutil.write_file(handler_state_err_file, message) + except IOError as e: + err = "Failed to set handler state message: {0}".format(e) + add_event(name=self.name, is_success=False, message=err) def collect_heartbeat(self): self.logger.info("Collect heart beat") @@ -426,7 +588,7 @@ class ExtensionInstance(object): return heartbeat def is_responsive(self, heartbeat_file): - last_update=int(time.time()-os.stat(heartbeat_file).st_mtime) + last_update=int(time.time() - os.stat(heartbeat_file).st_mtime) return last_update > 600 # not updated for more than 10 min def launch_command(self, cmd, timeout=300): @@ -452,6 +614,7 @@ class ExtensionInstance(object): ret = child.wait() if ret == None or ret != 0: raise ExtensionError("Non-zero exit code: {0}, {1}".format(ret, cmd)) + self.report_event(message="Launch command succeeded: {0}".format(cmd)) def load_manifest(self): man_file = self.get_manifest_file() @@ -464,16 +627,15 @@ class ExtensionInstance(object): return HandlerManifest(data[0]) - def update_settings(self): - if self.settings is None: - self.logger.verbose("Extension has no settings") + if self.ext is None: + self.logger.verb("Extension has no settings") return settings = { - 'publicSettings': self.settings.publicSettings, - 'protectedSettings': self.settings.privateSettings, - 'protectedSettingsCertThumbprint': self.settings.certificateThumbprint + 'publicSettings': self.ext.publicSettings, + 'protectedSettings': self.ext.privateSettings, + 'protectedSettingsCertThumbprint': self.ext.certificateThumbprint } ext_settings = { "runtimeSettings":[{ @@ -482,13 +644,10 @@ class ExtensionInstance(object): } fileutil.write_file(self.get_settings_file(), json.dumps(ext_settings)) - latest = os.path.join(self.get_conf_dir(), "latest") - fileutil.write_file(latest, self.settings.sequenceNumber) - def create_handler_env(self): env = [{ - "name": self.get_name(), - "version" : self.get_version(), + "name": self.name, + "version" : HANDLER_ENVIRONMENT_VERSION, "handlerEnvironment" : { "logFolder" : self.get_log_dir(), "configFolder" : self.get_conf_dir(), @@ -500,8 +659,8 @@ class ExtensionInstance(object): json.dumps(env)) def get_target_version(self): - version = self.get_version() - update_policy = self.get_upgrade_policy() + version = self.version + update_policy = self.update_policy if update_policy is None or update_policy.lower() != 'auto': return version @@ -509,45 +668,29 @@ class ExtensionInstance(object): if major is None: raise ExtensionError("Wrong version format: {0}".format(version)) - packages = [x for x in self.pkg_list.versions if x.version.startswith(major + ".")] - packages = sorted(packages, key=lambda x: x.version, reverse=True) + packages = [x for x in self.pkg_list.versions \ + if x.version.startswith(major + ".")] + packages = sorted(packages, key=lambda x: Version(x.version), + reverse=True) if len(packages) <= 0: raise ExtensionError("Can't find version: {0}.*".format(major)) return packages[0].version def get_package_uris(self): - version = self.get_version() + version = self.curr_version packages = self.pkg_list.versions if packages is None: raise ExtensionError("Package uris is None.") for package in packages: - if package.version == version: + if Version(package.version) == Version(version): return package.uris raise ExtensionError("Can't get package uris for {0}.".format(version)) - - def get_curr_op(self): - return self.curr_op - - def get_name(self): - return self.extension.name - - def get_version(self): - return self.extension.properties.version - - def get_state(self): - return self.extension.properties.state - - def get_seq_no(self): - return self.settings.sequenceNumber - - def get_upgrade_policy(self): - return self.extension.properties.upgradePolicy - + def get_full_name(self): - return "{0}-{1}".format(self.get_name(), self.curr_version) + return "{0}-{1}".format(self.name, self.curr_version) def get_base_dir(self): return os.path.join(OSUTIL.get_lib_dir(), self.get_full_name()) @@ -557,17 +700,27 @@ class ExtensionInstance(object): def get_status_file(self): return os.path.join(self.get_status_dir(), - "{0}.status".format(self.settings.sequenceNumber)) + "{0}.status".format(self.ext.sequenceNumber)) def get_conf_dir(self): return os.path.join(self.get_base_dir(), 'config') def get_settings_file(self): return os.path.join(self.get_conf_dir(), - "{0}.settings".format(self.settings.sequenceNumber)) + "{0}.settings".format(self.ext.sequenceNumber)) + + def get_handler_state_dir(self): + return os.path.join(OSUTIL.get_lib_dir(), "handler_state", + self.get_full_name()) def get_handler_state_file(self): - return os.path.join(self.get_conf_dir(), 'HandlerState') + return os.path.join(self.get_handler_state_dir(), + '{0}.state'.format(self.ext.sequenceNumber)) + + def get_handler_state_err_file(self): + return os.path.join(self.get_handler_state_dir(), + '{0}.error'.format(self.ext.sequenceNumber)) + def get_heartbeat_file(self): return os.path.join(self.get_base_dir(), 'heartbeat.log') @@ -579,7 +732,7 @@ class ExtensionInstance(object): return os.path.join(self.get_base_dir(), 'HandlerEnvironment.json') def get_log_dir(self): - return os.path.join(OSUTIL.get_ext_log_dir(), self.get_name(), + return os.path.join(OSUTIL.get_ext_log_dir(), self.name, self.curr_version) class HandlerEnvironment(object): diff --git a/azurelinuxagent/distro/default/handlerFactory.py b/azurelinuxagent/distro/default/handlerFactory.py index 98b2380..dceb2a3 100644 --- a/azurelinuxagent/distro/default/handlerFactory.py +++ b/azurelinuxagent/distro/default/handlerFactory.py @@ -1,4 +1,4 @@ -# Windows Azure Linux Agent +# Microsoft Azure Linux Agent # # Copyright 2014 Microsoft Corporation # @@ -23,7 +23,7 @@ from .dhcp import DhcpHandler from .env import EnvHandler from .provision import ProvisionHandler from .resourceDisk import ResourceDiskHandler -from .extension import ExtensionsHandler +from .extension import ExtHandlersHandler from .deprovision import DeprovisionHandler class DefaultHandlerFactory(object): @@ -35,6 +35,6 @@ class DefaultHandlerFactory(object): self.env_handler = EnvHandler(self) self.provision_handler = ProvisionHandler() self.resource_disk_handler = ResourceDiskHandler() - self.extension_handler = ExtensionsHandler() + self.ext_handlers_handler = ExtHandlersHandler() self.deprovision_handler = DeprovisionHandler() diff --git a/azurelinuxagent/distro/default/init.py b/azurelinuxagent/distro/default/init.py index 337fdea..db74fef 100644 --- a/azurelinuxagent/distro/default/init.py +++ b/azurelinuxagent/distro/default/init.py @@ -1,4 +1,4 @@ -# Windows Azure Linux Agent +# Microsoft Azure Linux Agent # # Copyright 2014 Microsoft Corporation # diff --git a/azurelinuxagent/distro/default/loader.py b/azurelinuxagent/distro/default/loader.py index d7dbe87..55a51e0 100644 --- a/azurelinuxagent/distro/default/loader.py +++ b/azurelinuxagent/distro/default/loader.py @@ -1,4 +1,4 @@ -# Windows Azure Linux Agent +# Microsoft Azure Linux Agent # # Copyright 2014 Microsoft Corporation # diff --git a/azurelinuxagent/distro/default/osutil.py b/azurelinuxagent/distro/default/osutil.py index 8e3fb77..00a57cc 100644 --- a/azurelinuxagent/distro/default/osutil.py +++ b/azurelinuxagent/distro/default/osutil.py @@ -117,22 +117,16 @@ class DefaultOSUtil(object): "retcode:{1}, " "output:{2}").format(username, retcode, out)) - def chpasswd(self, username, password, use_salt=True, salt_type=6, - salt_len=10): + def chpasswd(self, username, password, crypt_id=6, salt_len=10): if self.is_sys_user(username): raise OSUtilError(("User {0} is a system user. " "Will not set passwd.").format(username)) - passwd_hash = textutil.gen_password_hash(password, use_salt, salt_type, - salt_len) - try: - passwd_content = fileutil.read_file(self.passwd_file_path) - passwd = passwd_content.split("\n") - new_passwd = [x for x in passwd if not x.startswith(username)] - new_passwd.append("{0}:{1}:14600::::::".format(username, passwd_hash)) - fileutil.write_file(self.passwd_file_path, "\n".join(new_passwd)) - except IOError as e: + passwd_hash = textutil.gen_password_hash(password, crypt_id, salt_len) + cmd = "usermod -p '{0}' {1}".format(passwd_hash, username) + ret, output = shellutil.run_get_output(cmd, log_cmd=False) + if ret != 0: raise OSUtilError(("Failed to set password for {0}: {1}" - "").format(username, e)) + "").format(username, output)) def conf_sudoer(self, username, nopasswd): # for older distros create sudoers.d @@ -344,9 +338,10 @@ class DefaultOSUtil(object): raise OSUtilError("Failed to umount dvd.") def eject_dvd(self, chk_err=True): - retcode = shellutil.run("eject") + dvd = self.get_dvd_device() + retcode = shellutil.run("eject {0}".format(dvd)) if chk_err and retcode != 0: - raise OSUtilError("Failed to eject dvd") + raise OSUtilError("Failed to eject dvd: ret={0}".format(retcode)) def load_atappix_mod(self): if self.is_atapiix_mod_loaded(): diff --git a/azurelinuxagent/distro/default/provision.py b/azurelinuxagent/distro/default/provision.py index 1e9c459..424f083 100644 --- a/azurelinuxagent/distro/default/provision.py +++ b/azurelinuxagent/distro/default/provision.py @@ -49,8 +49,13 @@ class ProvisionHandler(object): protocol = prot.FACTORY.get_default_protocol() try: status = prot.ProvisionStatus(status="NotReady", - subStatus="Provision started") - protocol.report_provision_status(status) + subStatus="Provisioning", + description="Starting") + try: + protocol.report_provision_status(status) + except prot.ProtocolError as e: + add_event(name="WALA", is_success=False, message=text(e), + op=WALAEventOperation.Provision) self.provision() fileutil.write_file(provisioned, "") @@ -59,17 +64,28 @@ class ProvisionHandler(object): logger.info("Finished provisioning") status = prot.ProvisionStatus(status="Ready") status.properties.certificateThumbprint = thumbprint - protocol.report_provision_status(status) + + try: + protocol.report_provision_status(status) + except prot.ProtocolError as pe: + add_event(name="WALA", is_success=False, message=text(pe), + op=WALAEventOperation.Provision) add_event(name="WALA", is_success=True, message="", - op=WALAEventOperation.Provision) + op=WALAEventOperation.Provision) except ProvisionError as e: logger.error("Provision failed: {0}", e) status = prot.ProvisionStatus(status="NotReady", - subStatus= text(e)) - protocol.report_provision_status(status) + subStatus="ProvisioningFailed", + description= text(e)) + try: + protocol.report_provision_status(status) + except prot.ProtocolError as pe: + add_event(name="WALA", is_success=False, message=text(pe), + op=WALAEventOperation.Provision) + add_event(name="WALA", is_success=False, message=text(e), - op=WALAEventOperation.Provision) + op=WALAEventOperation.Provision) def reg_ssh_host_key(self): keypair_type = conf.get("Provisioning.SshHostKeyPairType", "rsa") @@ -120,10 +136,10 @@ class ProvisionHandler(object): if ovfenv.user_password is not None: logger.info("Set user password.") - use_salt = conf.get_switch("Provision.UseSalt", True) - salt_type = conf.get_switch("Provision.SaltType", 6) - OSUTIL.chpasswd(ovfenv.username, ovfenv.user_password, - use_salt,salt_type) + crypt_id = conf.get("Provision.PasswordCryptId", "6") + salt_len = conf.get_int("Provision.PasswordCryptSaltLength", 10) + OSUTIL.chpasswd(ovfenv.username, ovfenv.user_password, + crypt_id=crypt_id, salt_len=salt_len) logger.info("Configure sudoer") OSUTIL.conf_sudoer(ovfenv.username, ovfenv.user_password is None) diff --git a/azurelinuxagent/distro/default/resourceDisk.py b/azurelinuxagent/distro/default/resourceDisk.py index d4ef1c9..734863c 100644 --- a/azurelinuxagent/distro/default/resourceDisk.py +++ b/azurelinuxagent/distro/default/resourceDisk.py @@ -1,4 +1,4 @@ -# Windows Azure Linux Agent +# Microsoft Azure Linux Agent # # Copyright 2014 Microsoft Corporation # diff --git a/azurelinuxagent/distro/default/run.py b/azurelinuxagent/distro/default/run.py index 13880b4..dfd3b03 100644 --- a/azurelinuxagent/distro/default/run.py +++ b/azurelinuxagent/distro/default/run.py @@ -1,4 +1,4 @@ -# Windows Azure Linux Agent +# Microsoft Azure Linux Agent # # Copyright 2014 Microsoft Corporation # @@ -27,8 +27,8 @@ from azurelinuxagent.metadata import AGENT_LONG_NAME, AGENT_VERSION, \ DISTRO_NAME, DISTRO_VERSION, \ DISTRO_FULL_NAME, PY_VERSION_MAJOR, \ PY_VERSION_MINOR, PY_VERSION_MICRO -import azurelinuxagent.protocol as prot import azurelinuxagent.event as event +import azurelinuxagent.protocol as prot from azurelinuxagent.utils.osutil import OSUTIL import azurelinuxagent.utils.fileutil as fileutil @@ -65,22 +65,7 @@ class MainHandler(object): protocol = prot.FACTORY.get_default_protocol() while True: - #Handle extensions - h_status_list = self.handlers.extension_handler.process() - - #Report status - vm_status = prot.VMStatus() - vm_status.vmAgent.agentVersion = AGENT_LONG_NAME - vm_status.vmAgent.status = "Ready" - vm_status.vmAgent.message = "Guest Agent is running" - for h_status in h_status_list: - vm_status.extensionHandlers.append(h_status) - try: - logger.info("Report vm status") - protocol.report_status(vm_status) - except prot.ProtocolError as e: - logger.error("Failed to report vm status: {0}", e) - + self.handlers.ext_handlers_handler.process() time.sleep(25) diff --git a/azurelinuxagent/distro/default/scvmm.py b/azurelinuxagent/distro/default/scvmm.py index 18fad4b..680c04b 100644 --- a/azurelinuxagent/distro/default/scvmm.py +++ b/azurelinuxagent/distro/default/scvmm.py @@ -1,4 +1,4 @@ -# Windows Azure Linux Agent +# Microsoft Azure Linux Agent # # Copyright 2014 Microsoft Corporation # diff --git a/azurelinuxagent/distro/loader.py b/azurelinuxagent/distro/loader.py index 0060a7f..375abd2 100644 --- a/azurelinuxagent/distro/loader.py +++ b/azurelinuxagent/distro/loader.py @@ -25,7 +25,7 @@ def get_distro_loader(): logger.verb("Loading distro implemetation from: {0}", DISTRO_NAME) pkg_name = "azurelinuxagent.distro.{0}.loader".format(DISTRO_NAME) return __import__(pkg_name, fromlist="loader") - except ImportError as e: + except (ImportError, ValueError): logger.warn("Unable to load distro implemetation for {0}.", DISTRO_NAME) logger.warn("Use default distro implemetation instead.") return default_loader diff --git a/azurelinuxagent/distro/oracle/__init__.py b/azurelinuxagent/distro/oracle/__init__.py index 4b2b9e1..d9b82f5 100644 --- a/azurelinuxagent/distro/oracle/__init__.py +++ b/azurelinuxagent/distro/oracle/__init__.py @@ -1,4 +1,4 @@ -# Windows Azure Linux Agent +# Microsoft Azure Linux Agent # # Copyright 2014 Microsoft Corporation # diff --git a/azurelinuxagent/distro/oracle/loader.py b/azurelinuxagent/distro/oracle/loader.py index 379f027..9dc428f 100644 --- a/azurelinuxagent/distro/oracle/loader.py +++ b/azurelinuxagent/distro/oracle/loader.py @@ -1,4 +1,4 @@ -# Windows Azure Linux Agent +# Microsoft Azure Linux Agent # # Copyright 2014 Microsoft Corporation # diff --git a/azurelinuxagent/distro/redhat/__init__.py b/azurelinuxagent/distro/redhat/__init__.py index 4b2b9e1..d9b82f5 100644 --- a/azurelinuxagent/distro/redhat/__init__.py +++ b/azurelinuxagent/distro/redhat/__init__.py @@ -1,4 +1,4 @@ -# Windows Azure Linux Agent +# Microsoft Azure Linux Agent # # Copyright 2014 Microsoft Corporation # diff --git a/azurelinuxagent/distro/redhat/loader.py b/azurelinuxagent/distro/redhat/loader.py index 911e74d..8d3c75b 100644 --- a/azurelinuxagent/distro/redhat/loader.py +++ b/azurelinuxagent/distro/redhat/loader.py @@ -1,4 +1,4 @@ -# Windows Azure Linux Agent +# Microsoft Azure Linux Agent # # Copyright 2014 Microsoft Corporation # diff --git a/azurelinuxagent/distro/redhat/osutil.py b/azurelinuxagent/distro/redhat/osutil.py index c6c3016..7478867 100644 --- a/azurelinuxagent/distro/redhat/osutil.py +++ b/azurelinuxagent/distro/redhat/osutil.py @@ -122,15 +122,14 @@ class Redhat6xOSUtil(DefaultOSUtil): ret= shellutil.run_get_output("pidof dhclient") return ret[1] if ret[0] == 0 else None -class RedhatOSUtil(Redhat6xOSUtil): - def __init__(self): - super(RedhatOSUtil, self).__init__() - def set_hostname(self, hostname): - super(RedhatOSUtil, self).set_hostname(hostname) + """ + Set /etc/sysconfig/network + """ fileutil.update_conf_file('/etc/sysconfig/network', 'HOSTNAME', 'HOSTNAME={0}'.format(hostname)) + shellutil.run("hostname {0}".format(hostname), chk_err=False) def set_dhcp_hostname(self, hostname): ifname = self.get_if_name() @@ -139,6 +138,24 @@ class RedhatOSUtil(Redhat6xOSUtil): 'DHCP_HOSTNAME', 'DHCP_HOSTNAME={0}'.format(hostname)) +class RedhatOSUtil(Redhat6xOSUtil): + def __init__(self): + super(RedhatOSUtil, self).__init__() + + def set_hostname(self, hostname): + """ + Set /etc/hostname + Unlike redhat 6.x, redhat 7.x will set hostname to /etc/hostname + """ + DefaultOSUtil.set_hostname(self, hostname) + + def publish_hostname(self, hostname): + """ + Restart NetworkManager first before publishing hostname + """ + shellutil.run("service NetworkManager restart") + super(RedhatOSUtil, self).publish_hostname(hostname) + def register_agent_service(self): return shellutil.run("systemctl enable waagent", chk_err=False) diff --git a/azurelinuxagent/distro/suse/__init__.py b/azurelinuxagent/distro/suse/__init__.py index 4b2b9e1..d9b82f5 100644 --- a/azurelinuxagent/distro/suse/__init__.py +++ b/azurelinuxagent/distro/suse/__init__.py @@ -1,4 +1,4 @@ -# Windows Azure Linux Agent +# Microsoft Azure Linux Agent # # Copyright 2014 Microsoft Corporation # diff --git a/azurelinuxagent/distro/suse/loader.py b/azurelinuxagent/distro/suse/loader.py index e38aa17..b01384b 100644 --- a/azurelinuxagent/distro/suse/loader.py +++ b/azurelinuxagent/distro/suse/loader.py @@ -1,4 +1,4 @@ -# Windows Azure Linux Agent +# Microsoft Azure Linux Agent # # Copyright 2014 Microsoft Corporation # diff --git a/azurelinuxagent/distro/suse/osutil.py b/azurelinuxagent/distro/suse/osutil.py index 870e0b7..8d6f5bf 100644 --- a/azurelinuxagent/distro/suse/osutil.py +++ b/azurelinuxagent/distro/suse/osutil.py @@ -79,6 +79,26 @@ class SUSEOSUtil(SUSE11OSUtil): super(SUSEOSUtil, self).__init__() self.dhclient_name = 'wickedd-dhcp4' + def stop_dhcp_service(self): + cmd = "systemctl stop {0}".format(self.dhclient_name) + return shellutil.run(cmd, chk_err=False) + + def start_dhcp_service(self): + cmd = "systemctl start {0}".format(self.dhclient_name) + return shellutil.run(cmd, chk_err=False) + + def start_network(self) : + return shellutil.run("systemctl start network", chk_err=False) + + def restart_ssh_service(self): + return shellutil.run("systemctl restart sshd", chk_err=False) + + def stop_agent_service(self): + return shellutil.run("systemctl stop waagent", chk_err=False) + + def start_agent_service(self): + return shellutil.run("systemctl start waagent", chk_err=False) + def register_agent_service(self): return shellutil.run("systemctl enable waagent", chk_err=False) diff --git a/azurelinuxagent/distro/ubuntu/__init__.py b/azurelinuxagent/distro/ubuntu/__init__.py index 4b2b9e1..d9b82f5 100644 --- a/azurelinuxagent/distro/ubuntu/__init__.py +++ b/azurelinuxagent/distro/ubuntu/__init__.py @@ -1,4 +1,4 @@ -# Windows Azure Linux Agent +# Microsoft Azure Linux Agent # # Copyright 2014 Microsoft Corporation # diff --git a/azurelinuxagent/distro/ubuntu/deprovision.py b/azurelinuxagent/distro/ubuntu/deprovision.py index 10fa123..0c3c4e5 100644 --- a/azurelinuxagent/distro/ubuntu/deprovision.py +++ b/azurelinuxagent/distro/ubuntu/deprovision.py @@ -1,4 +1,4 @@ -# Windows Azure Linux Agent +# Microsoft Azure Linux Agent # # Copyright 2014 Microsoft Corporation # diff --git a/azurelinuxagent/distro/ubuntu/handlerFactory.py b/azurelinuxagent/distro/ubuntu/handlerFactory.py index c8d0906..11f7f04 100644 --- a/azurelinuxagent/distro/ubuntu/handlerFactory.py +++ b/azurelinuxagent/distro/ubuntu/handlerFactory.py @@ -1,4 +1,4 @@ -# Windows Azure Linux Agent +# Microsoft Azure Linux Agent # # Copyright 2014 Microsoft Corporation # diff --git a/azurelinuxagent/distro/ubuntu/loader.py b/azurelinuxagent/distro/ubuntu/loader.py index 26db4fa..3fe2239 100644 --- a/azurelinuxagent/distro/ubuntu/loader.py +++ b/azurelinuxagent/distro/ubuntu/loader.py @@ -1,4 +1,4 @@ -# Windows Azure Linux Agent +# Microsoft Azure Linux Agent # # Copyright 2014 Microsoft Corporation # @@ -17,16 +17,20 @@ # Requires Python 2.4+ and Openssl 1.0+ # -from azurelinuxagent.metadata import DISTRO_NAME, DISTRO_VERSION +from azurelinuxagent.metadata import DISTRO_NAME, DISTRO_VERSION, DISTRO_FULL_NAME def get_osutil(): from azurelinuxagent.distro.ubuntu.osutil import Ubuntu1204OSUtil, \ UbuntuOSUtil, \ - Ubuntu14xOSUtil + Ubuntu14xOSUtil, \ + UbuntuSnappyOSUtil + if DISTRO_VERSION == "12.04": return Ubuntu1204OSUtil() elif DISTRO_VERSION == "14.04" or DISTRO_VERSION == "14.10": return Ubuntu14xOSUtil() + elif DISTRO_FULL_NAME == "Snappy Ubuntu Core": + return UbuntuSnappyOSUtil() else: return UbuntuOSUtil() diff --git a/azurelinuxagent/distro/ubuntu/osutil.py b/azurelinuxagent/distro/ubuntu/osutil.py index 1e51c2a..adf7660 100644 --- a/azurelinuxagent/distro/ubuntu/osutil.py +++ b/azurelinuxagent/distro/ubuntu/osutil.py @@ -63,3 +63,13 @@ class UbuntuOSUtil(Ubuntu14xOSUtil): def unregister_agent_service(self): return shellutil.run("systemctl mask walinuxagent", chk_err=False) +class UbuntuSnappyOSUtil(Ubuntu14xOSUtil): + def __init__(self): + super(UbuntuSnappyOSUtil, self).__init__() + self.conf_file_path = '/apps/walinuxagent/current/waagent.conf' + + def remove_rules_files(self, rules_files=""): + pass + + def restore_rules_files(self, rules_files=""): + pass diff --git a/azurelinuxagent/distro/ubuntu/provision.py b/azurelinuxagent/distro/ubuntu/provision.py index 7551074..a68fe4d 100644 --- a/azurelinuxagent/distro/ubuntu/provision.py +++ b/azurelinuxagent/distro/ubuntu/provision.py @@ -1,4 +1,4 @@ -# Windows Azure Linux Agent +# Microsoft Azure Linux Agent # # Copyright 2014 Microsoft Corporation # @@ -23,6 +23,7 @@ import azurelinuxagent.logger as logger from azurelinuxagent.future import text import azurelinuxagent.conf as conf import azurelinuxagent.protocol as prot +from azurelinuxagent.event import add_event, WALAEventOperation from azurelinuxagent.exception import * from azurelinuxagent.utils.osutil import OSUTIL import azurelinuxagent.utils.shellutil as shellutil @@ -54,11 +55,25 @@ class UbuntuProvisionHandler(ProvisionHandler): logger.info("Finished provisioning") status = prot.ProvisionStatus(status="Ready") status.properties.certificateThumbprint = thumbprint - protocol.report_provision_status(status) + try: + protocol.report_provision_status(status) + except prot.ProtocolError as pe: + add_event(name="WALA", is_success=False, message=text(pe), + op=WALAEventOperation.Provision) except ProvisionError as e: logger.error("Provision failed: {0}", e) - protocol.report_provision_status(status="NotReady", subStatus=text(e)) + status = prot.ProvisionStatus(status="NotReady", + subStatus="ProvisioningFailed", + description= text(e)) + try: + protocol.report_provision_status(status) + except prot.ProtocolError as pe: + add_event(name="WALA", is_success=False, message=text(pe), + op=WALAEventOperation.Provision) + + add_event(name="WALA", is_success=False, message=text(e), + op=WALAEventOperation.Provision) def wait_for_ssh_host_key(self, max_retry=60): kepair_type = conf.get("Provisioning.SshHostKeyPairType", "rsa") diff --git a/azurelinuxagent/event.py b/azurelinuxagent/event.py index f866a22..02e8017 100644 --- a/azurelinuxagent/event.py +++ b/azurelinuxagent/event.py @@ -82,9 +82,11 @@ class EventMonitor(object): def collect_event(self, evt_file_name): try: + logger.verb("Found event file: {0}", evt_file_name) with open(evt_file_name, "rb") as evt_file: #if fail to open or delete the file, throw exception json_str = evt_file.read().decode("utf-8",'ignore') + logger.verb("Processed event file: {0}", evt_file_name) os.remove(evt_file_name) return json_str except IOError as e: @@ -107,11 +109,11 @@ class EventMonitor(object): data = json.loads(data_str) except ValueError as e: logger.verb(data_str) - logger.error("Failed to decode json event file{0}", e) + logger.error("Failed to decode json event file: {0}", e) continue event = prot.TelemetryEvent() - prot.set_properties(event, data) + prot.set_properties("event", event, data) event.parameters.extend(self.sysinfo) event_list.events.append(event) if len(event_list.events) == 0: @@ -151,8 +153,10 @@ def save_event(data): except IOError as e: raise EventError("Failed to write events to file:{0}", e) -def add_event(name, op, is_success, duration=0, version="1.0", +def add_event(name, op="", is_success=True, duration=0, version="1.0", message="", evt_type="", is_internal=False): + log = logger.info if is_success else logger.error + log("Event: name={0}, op={1}, message={2}", name, op, message) event = prot.TelemetryEvent(1, "69B669B9-4AF8-4C50-BDC4-6006FA76E975") event.parameters.append(prot.TelemetryEventParam('Name', name)) event.parameters.append(prot.TelemetryEventParam('Version', version)) @@ -179,7 +183,6 @@ def dump_unhandled_err(name): error = traceback.format_exception(last_type, last_value, last_traceback) message= "".join(error) - logger.error(message) add_event(name, is_success=False, message=message, op=WALAEventOperation.UnhandledError) diff --git a/azurelinuxagent/exception.py b/azurelinuxagent/exception.py index 7c31394..d7d9b0a 100644 --- a/azurelinuxagent/exception.py +++ b/azurelinuxagent/exception.py @@ -1,4 +1,4 @@ -# Windows Azure Linux Agent +# Microsoft Azure Linux Agent # # Copyright 2014 Microsoft Corporation # diff --git a/azurelinuxagent/future.py b/azurelinuxagent/future.py index 8186fcf..8451345 100644 --- a/azurelinuxagent/future.py +++ b/azurelinuxagent/future.py @@ -9,11 +9,13 @@ if sys.version_info[0]== 3: from urllib.parse import urlparse text = str bytebuffer = memoryview + read_input = input elif sys.version_info[0] == 2: import httplib as httpclient from urlparse import urlparse text = unicode bytebuffer = buffer + read_input = raw_input else: raise ImportError("Unknown python version:{0}".format(sys.version_info)) diff --git a/azurelinuxagent/handler.py b/azurelinuxagent/handler.py index c180112..538ee30 100644 --- a/azurelinuxagent/handler.py +++ b/azurelinuxagent/handler.py @@ -1,4 +1,4 @@ -# Windows Azure Linux Agent +# Microsoft Azure Linux Agent # # Copyright 2014 Microsoft Corporation # diff --git a/azurelinuxagent/logger.py b/azurelinuxagent/logger.py index 126d6bc..21c02a6 100644 --- a/azurelinuxagent/logger.py +++ b/azurelinuxagent/logger.py @@ -20,10 +20,9 @@ """ Log utils """ - +import os import sys from azurelinuxagent.future import text -import azurelinuxagent.utils.textutil as textutil from datetime import datetime class Logger(object): @@ -36,7 +35,7 @@ class Logger(object): self.appenders.extend(logger.appenders) self.prefix = prefix - def verbose(self, msg_format, *args): + def verb(self, msg_format, *args): self.log(LogLevel.VERBOSE, msg_format, *args) def info(self, msg_format, *args): @@ -49,6 +48,9 @@ class Logger(object): self.log(LogLevel.ERROR, msg_format, *args) def log(self, level, msg_format, *args): + #if msg_format is not unicode convert it to unicode + if type(msg_format) is not text: + msg_format = text(msg_format, errors="backslashreplace") if len(args) > 0: msg = msg_format.format(*args) else: @@ -60,7 +62,9 @@ class Logger(object): msg) else: log_item = u"{0} {1} {2}\n".format(time, level_str, msg) - log_item = text(log_item.encode("ascii", "backslashreplace"), encoding='ascii') + + log_item = text(log_item.encode('ascii', "backslashreplace"), + encoding="ascii") for appender in self.appenders: appender.write(level, log_item) @@ -107,7 +111,6 @@ class StdoutAppender(object): except IOError: pass - #Initialize logger instance DEFAULT_LOGGER = Logger() @@ -132,7 +135,7 @@ def add_logger_appender(appender_type, level=LogLevel.INFO, path=None): DEFAULT_LOGGER.add_appender(appender_type, level, path) def verb(msg_format, *args): - DEFAULT_LOGGER.verbose(msg_format, *args) + DEFAULT_LOGGER.verb(msg_format, *args) def info(msg_format, *args): DEFAULT_LOGGER.info(msg_format, *args) diff --git a/azurelinuxagent/metadata.py b/azurelinuxagent/metadata.py index 83d4676..5cf4902 100644 --- a/azurelinuxagent/metadata.py +++ b/azurelinuxagent/metadata.py @@ -1,4 +1,4 @@ -# Windows Azure Linux Agent +# Microsoft Azure Linux Agent # # Copyright 2014 Microsoft Corporation # @@ -21,6 +21,7 @@ import os import re import platform import sys +import azurelinuxagent.utils.fileutil as fileutil from azurelinuxagent.future import text def get_distro(): @@ -46,7 +47,7 @@ def get_distro(): AGENT_NAME = "WALinuxAgent" AGENT_LONG_NAME = "Azure Linux Agent" -AGENT_VERSION = '2.1.1' +AGENT_VERSION = '2.1.2' AGENT_LONG_VERSION = "{0}-{1}".format(AGENT_NAME, AGENT_VERSION) AGENT_DESCRIPTION = """\ The Azure Linux Agent supports the provisioning and running of Linux @@ -70,24 +71,12 @@ PY_VERSION_MICRO = sys.version_info[2] Add this walk arround for detecting Snappy Ubuntu Core temporarily, until ubuntu fixed this bug: https://bugs.launchpad.net/snappy/+bug/1481086 """ -def which(program): - # Return path of program for execution if found in path - def is_exe(fpath): - return os.path.isfile(fpath) and os.access(fpath, os.X_OK) - _fpath, _ = os.path.split(program) - if _fpath: - if is_exe(program): - return program - else: - for path in os.environ.get("PATH", "").split(os.pathsep): - path = path.strip('"') - exe_file = os.path.join(path, program) - if is_exe(exe_file): - return exe_file - return None - def is_snappy(): - return which("snappy") + if os.path.exists("/etc/motd"): + motd = fileutil.read_file("/etc/motd") + if "snappy" in motd: + return True + return False if is_snappy(): DISTRO_FULL_NAME = "Snappy Ubuntu Core" diff --git a/azurelinuxagent/protocol/__init__.py b/azurelinuxagent/protocol/__init__.py index 65d8a5d..a4572e6 100644 --- a/azurelinuxagent/protocol/__init__.py +++ b/azurelinuxagent/protocol/__init__.py @@ -1,4 +1,4 @@ -# Windows Azure Linux Agent +# Microsoft Azure Linux Agent # # Copyright 2014 Microsoft Corporation # diff --git a/azurelinuxagent/protocol/common.py b/azurelinuxagent/protocol/common.py index 77247ab..367794f 100644 --- a/azurelinuxagent/protocol/common.py +++ b/azurelinuxagent/protocol/common.py @@ -1,4 +1,4 @@ -# Windows Azure Linux Agent +# Microsoft Azure Linux Agent # # Copyright 2014 Microsoft Corporation # @@ -22,6 +22,7 @@ import re import json import xml.dom.minidom import azurelinuxagent.logger as logger +from azurelinuxagent.future import text import azurelinuxagent.utils.fileutil as fileutil class ProtocolError(Exception): @@ -32,51 +33,49 @@ class ProtocolNotFound(Exception): def validata_param(name, val, expected_type): if val is None: - raise ProtocolError("Param {0} is None".format(name)) + raise ProtocolError("{0} is None".format(name)) if not isinstance(val, expected_type): - raise ProtocolError("Param {0} type should be {1}".format(name, - expected_type)) - -def set_properties(obj, data): - validata_param("obj", obj, DataContract) - validata_param("data", data, dict) - - props = vars(obj) - for name, val in list(props.items()): - try: - new_val = data[name] - except KeyError: - continue - - if isinstance(new_val, dict): - set_properties(val, new_val) - elif isinstance(new_val, list): - validata_param("list", val, DataContractList) - for data_item in new_val: - item = val.item_cls() - set_properties(item, data_item) - val.append(item) - else: - setattr(obj, name, new_val) + raise ProtocolError(("{0} type should be {1} not {2}" + "").format(name, expected_type, type(val))) + +def set_properties(name, obj, data): + if isinstance(obj, DataContract): + validata_param("Property '{0}'".format(name), data, dict) + for prob_name, prob_val in data.items(): + prob_full_name = "{0}.{1}".format(name, prob_name) + try: + prob = getattr(obj, prob_name) + except AttributeError: + logger.warn("Unknown property: {0}", prob_full_name) + continue + prob = set_properties(prob_full_name, prob, prob_val) + setattr(obj, prob_name, prob) + return obj + elif isinstance(obj, DataContractList): + validata_param("List '{0}'".format(name), data, list) + for item_data in data: + item = obj.item_cls() + item = set_properties(name, item, item_data) + obj.append(item) + return obj + else: + return data def get_properties(obj): - validata_param("obj", obj, DataContract) - - data = {} - props = vars(obj) - for name, val in list(props.items()): - if isinstance(val, DataContract): - data[name] = get_properties(val) - elif isinstance(val, DataContractList): - if len(val) == 0: - continue - data[name] = [] - for item in val: - date_item = get_properties(item) - data[name].append(date_item) - elif val is not None: - data[name] = val - return data + if isinstance(obj, DataContract): + data = {} + props = vars(obj) + for prob_name, prob in list(props.items()): + data[prob_name] = get_properties(prob) + return data + elif isinstance(obj, DataContractList): + data = [] + for item in obj: + item_data = get_properties(item) + data.append(item_data) + return data + else: + return obj class DataContract(object): pass @@ -85,6 +84,9 @@ class DataContractList(list): def __init__(self, item_cls): self.item_cls = item_cls +""" +Data contract between guest and host +""" class VMInfo(DataContract): def __init__(self, subscriptionId=None, vmName=None): self.subscriptionId = subscriptionId @@ -100,7 +102,7 @@ class CertList(DataContract): def __init__(self): self.certificates = DataContractList(Cert) -class ExtensionSettings(DataContract): +class Extension(DataContract): def __init__(self, name=None, sequenceNumber=None, publicSettings=None, privateSettings=None, certificateThumbprint=None): self.name = name @@ -109,47 +111,39 @@ class ExtensionSettings(DataContract): self.privateSettings = privateSettings self.certificateThumbprint = certificateThumbprint -class ExtensionProperties(DataContract): +class ExtHandlerProperties(DataContract): def __init__(self): self.version = None self.upgradePolicy = None self.state = None - self.extensions = DataContractList(ExtensionSettings) + self.extensions = DataContractList(Extension) -class ExtensionVersionUri(DataContract): +class ExtHandlerVersionUri(DataContract): def __init__(self): self.uri = None -class Extension(DataContract): +class ExtHandler(DataContract): def __init__(self, name=None): self.name = name - self.properties = ExtensionProperties() - self.version_uris = DataContractList(ExtensionVersionUri) + self.properties = ExtHandlerProperties() + self.versionUris = DataContractList(ExtHandlerVersionUri) -class ExtensionList(DataContract): +class ExtHandlerList(DataContract): def __init__(self): - self.extensions = DataContractList(Extension) + self.extHandlers = DataContractList(ExtHandler) -class ExtensionPackageUri(DataContract): +class ExtHandlerPackageUri(DataContract): def __init__(self, uri=None): self.uri = uri -class ExtensionPackage(DataContract): +class ExtHandlerPackage(DataContract): def __init__(self, version = None): self.version = version - self.uris = DataContractList(ExtensionPackageUri) + self.uris = DataContractList(ExtHandlerPackageUri) -class ExtensionPackageList(DataContract): +class ExtHandlerPackageList(DataContract): def __init__(self): - self.versions = DataContractList(ExtensionPackage) - -class InstanceMetadata(DataContract): - def __init__(self, deploymentName=None, roleName=None, roleInstanceId=None, - containerId=None): - self.deploymentName = deploymentName - self.roleName = roleName - self.roleInstanceId = roleInstanceId - self.containerId = containerId + self.versions = DataContractList(ExtHandlerPackage) class VMProperties(DataContract): def __init__(self, certificateThumbprint=None): @@ -163,12 +157,6 @@ class ProvisionStatus(DataContract): self.description = description self.properties = VMProperties() -class VMAgentStatus(DataContract): - def __init__(self, agentVersion=None, status=None, message=None): - self.agentVersion = agentVersion - self.status = status - self.message = message - class ExtensionSubStatus(DataContract): def __init__(self, name=None, status=None, code=None, message=None): self.name = name @@ -177,30 +165,34 @@ class ExtensionSubStatus(DataContract): self.message = message class ExtensionStatus(DataContract): - def __init__(self, name=None, configurationAppliedTime=None, operation=None, - status=None, code=None, message=None, seq_no=None): - self.name = name + def __init__(self, configurationAppliedTime=None, operation=None, + status=None, seq_no=None, code=None, message=None): self.configurationAppliedTime = configurationAppliedTime self.operation = operation self.status = status + self.sequenceNumber = seq_no self.code = code self.message = message - self.sequenceNumber = seq_no self.substatusList = DataContractList(ExtensionSubStatus) -class ExtensionHandlerStatus(DataContract): - def __init__(self, handlerName=None, handlerVersion=None, status=None, - message=None): - self.handlerName = handlerName - self.handlerVersion = handlerVersion +class ExtHandlerStatus(DataContract): + def __init__(self, name=None, version=None, status=None, message=None): + self.name = name + self.version = version self.status = status self.message = message - self.extensionStatusList = DataContractList(ExtensionStatus) + self.extensions = DataContractList(text) + +class VMAgentStatus(DataContract): + def __init__(self, version=None, status=None, message=None): + self.version = version + self.status = status + self.message = message + self.extensionHandlers = DataContractList(ExtHandlerStatus) class VMStatus(DataContract): def __init__(self): self.vmAgent = VMAgentStatus() - self.extensionHandlers = DataContractList(ExtensionHandlerStatus) class TelemetryEventParam(DataContract): def __init__(self, name=None, value=None): @@ -228,16 +220,19 @@ class Protocol(DataContract): def get_certs(self): raise NotImplementedError() - def get_extensions(self): + def get_ext_handlers(self): + raise NotImplementedError() + + def get_ext_handler_pkgs(self, extension): raise NotImplementedError() - def get_extension_pkgs(self, extension): + def report_provision_status(self, provision_status): raise NotImplementedError() - def report_provision_status(self, status): + def report_vm_status(self, vm_status): raise NotImplementedError() - def report_status(self, status): + def report_ext_status(self, ext_handler_name, ext_name, ext_status): raise NotImplementedError() def report_event(self, event): diff --git a/azurelinuxagent/protocol/ovfenv.py b/azurelinuxagent/protocol/ovfenv.py index 2e0411d..9c845ee 100644 --- a/azurelinuxagent/protocol/ovfenv.py +++ b/azurelinuxagent/protocol/ovfenv.py @@ -1,4 +1,4 @@ -# Windows Azure Linux Agent +# Microsoft Azure Linux Agent # # Copyright 2014 Microsoft Corporation # @@ -58,12 +58,17 @@ def copy_ovf_env(): ovfxml = re.sub("<UserPassword>.*?<", "<UserPassword>*<", ovfxml) ovf_file_path = os.path.join(OSUTIL.get_lib_dir(), OVF_FILE_NAME) fileutil.write_file(ovf_file_path, ovfxml) - OSUTIL.umount_dvd() - OSUTIL.eject_dvd() except IOError as e: raise ProtocolError(text(e)) except OSUtilError as e: raise ProtocolError(text(e)) + + try: + OSUTIL.umount_dvd() + OSUTIL.eject_dvd() + except OSUtilError as e: + logger.warn(text(e)) + return ovfenv def _validate_ovf(val, msg): diff --git a/azurelinuxagent/protocol/protocolFactory.py b/azurelinuxagent/protocol/protocolFactory.py index d2ca201..0bf6e52 100644 --- a/azurelinuxagent/protocol/protocolFactory.py +++ b/azurelinuxagent/protocol/protocolFactory.py @@ -1,4 +1,4 @@ -# Windows Azure Linux Agent +# Microsoft Azure Linux Agent # # Copyright 2014 Microsoft Corporation # diff --git a/azurelinuxagent/protocol/v1.py b/azurelinuxagent/protocol/v1.py index 54a80b6..92fcc06 100644 --- a/azurelinuxagent/protocol/v1.py +++ b/azurelinuxagent/protocol/v1.py @@ -1,4 +1,4 @@ -# Windows Azure Linux Agent +# Microsoft Azure Linux Agent # # Copyright 2014 Microsoft Corporation # @@ -24,7 +24,7 @@ import traceback import xml.sax.saxutils as saxutils import xml.etree.ElementTree as ET import azurelinuxagent.logger as logger -from azurelinuxagent.future import text, httpclient +from azurelinuxagent.future import text, httpclient, bytebuffer import azurelinuxagent.utils.restutil as restutil from azurelinuxagent.utils.textutil import parse_doc, findall, find, findtext, \ getattrib, gettext, remove_bom @@ -54,6 +54,9 @@ TRANSPORT_PRV_FILE_NAME = "TransportPrivate.pem" PROTOCOL_VERSION = "2012-11-30" +SHORT_WAITING_INTERVAL = 1 # 1 second +LONG_WAITING_INTERVAL = 15 # 15 seconds + class WireProtocolResourceGone(ProtocolError): pass @@ -77,113 +80,89 @@ class WireProtocol(Protocol): certificates = self.client.get_certs() return certificates.cert_list - def get_extensions(self): + def get_ext_handlers(self): #Update goal state to get latest extensions config self.client.update_goal_state() ext_conf = self.client.get_ext_conf() - return ext_conf.ext_list + return ext_conf.ext_handlers - def get_extension_pkgs(self, extension): + def get_ext_handler_pkgs(self, ext_handler): goal_state = self.client.get_goal_state() - man = self.client.get_ext_manifest(extension, goal_state) + man = self.client.get_ext_manifest(ext_handler, goal_state) return man.pkg_list - def report_provision_status(self, provisionStatus): - validata_param("provisionStatus", provisionStatus, ProvisionStatus) - - if provisionStatus.status is not None: - self.client.report_health(provisionStatus.status, - provisionStatus.subStatus, - provisionStatus.description) - if provisionStatus.properties.certificateThumbprint is not None: - thumbprint = provisionStatus.properties.certificateThumbprint + def report_provision_status(self, provision_status): + validata_param("provision_status", provision_status, ProvisionStatus) + + if provision_status.status is not None: + self.client.report_health(provision_status.status, + provision_status.subStatus, + provision_status.description) + if provision_status.properties.certificateThumbprint is not None: + thumbprint = provision_status.properties.certificateThumbprint self.client.report_role_prop(thumbprint) - def report_status(self, vmStatus): - validata_param("vmStatus", vmStatus, VMStatus) - self.client.upload_status_blob(vmStatus) + def report_vm_status(self, vm_status): + validata_param("vm_status", vm_status, VMStatus) + self.client.status_blob.set_vm_status(vm_status) + self.client.upload_status_blob() + + def report_ext_status(self, ext_handler_name, ext_name, ext_status): + validata_param("ext_status", ext_status, ExtensionStatus) + self.client.status_blob.set_ext_status(ext_handler_name, ext_status) def report_event(self, events): validata_param("events", events, TelemetryEventList) self.client.report_event(events) -def _fetch_cache(local_file): - if not os.path.isfile(local_file): - raise ProtocolError("{0} is missing.".format(local_file)) - return fileutil.read_file(local_file) - -def _fetch_uri(uri, headers, chk_proxy=False): - try: - resp = restutil.http_get(uri, headers, chk_proxy=chk_proxy) - except restutil.HttpError as e: - raise ProtocolError(text(e)) - - if(resp.status == httpclient.GONE): - raise WireProtocolResourceGone(uri) - if(resp.status != httpclient.OK): - raise ProtocolError("{0} - {1}".format(resp.status, uri)) - data = resp.read() - if data is None: - return None - data = remove_bom(data) - xml_text = text(data, encoding='utf-8') - return xml_text - -def _fetch_manifest(version_uris): - for version_uri in version_uris: - try: - xml_text = _fetch_uri(version_uri.uri, None, chk_proxy=True) - return xml_text - except IOError as e: - logger.warn("Failed to fetch ExtensionManifest: {0}, {1}", e, - version_uri.uri) - raise ProtocolError("Failed to fetch ExtensionManifest from all sources") - def _build_role_properties(container_id, role_instance_id, thumbprint): - xml = ("<?xml version=\"1.0\" encoding=\"utf-8\"?>" - "<RoleProperties>" - "<Container>" - "<ContainerId>{0}</ContainerId>" - "<RoleInstances>" - "<RoleInstance>" - "<Id>{1}</Id>" - "<Properties>" - "<Property name=\"CertificateThumbprint\" value=\"{2}\" />" - "</Properties>" - "</RoleInstance>" - "</RoleInstances>" - "</Container>" - "</RoleProperties>" - "").format(container_id, role_instance_id, thumbprint) + xml = (u"<?xml version=\"1.0\" encoding=\"utf-8\"?>" + u"<RoleProperties>" + u"<Container>" + u"<ContainerId>{0}</ContainerId>" + u"<RoleInstances>" + u"<RoleInstance>" + u"<Id>{1}</Id>" + u"<Properties>" + u"<Property name=\"CertificateThumbprint\" value=\"{2}\" />" + u"</Properties>" + u"</RoleInstance>" + u"</RoleInstances>" + u"</Container>" + u"</RoleProperties>" + u"").format(container_id, role_instance_id, thumbprint) return xml def _build_health_report(incarnation, container_id, role_instance_id, status, substatus, description): - detail = '' + #Escape '&', '<' and '>' + description = saxutils.escape(text(description)) + detail = u'' if substatus is not None: - detail = ("<Details>" - "<SubStatus>{0}</SubStatus>" - "<Description>{1}</Description>" - "</Details>").format(substatus, description) - xml = ("<?xml version=\"1.0\" encoding=\"utf-8\"?>" - "<Health " - "xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\"" - " xmlns:xsd=\"http://www.w3.org/2001/XMLSchema\">" - "<GoalStateIncarnation>{0}</GoalStateIncarnation>" - "<Container>" - "<ContainerId>{1}</ContainerId>" - "<RoleInstanceList>" - "<Role>" - "<InstanceId>{2}</InstanceId>" - "<Health>" - "<State>{3}</State>" - "{4}" - "</Health>" - "</Role>" - "</RoleInstanceList>" - "</Container>" - "</Health>" - "").format(incarnation, + substatus = saxutils.escape(text(substatus)) + detail = (u"<Details>" + u"<SubStatus>{0}</SubStatus>" + u"<Description>{1}</Description>" + u"</Details>").format(substatus, description) + xml = (u"<?xml version=\"1.0\" encoding=\"utf-8\"?>" + u"<Health " + u"xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\"" + u" xmlns:xsd=\"http://www.w3.org/2001/XMLSchema\">" + u"<GoalStateIncarnation>{0}</GoalStateIncarnation>" + u"<Container>" + u"<ContainerId>{1}</ContainerId>" + u"<RoleInstanceList>" + u"<Role>" + u"<InstanceId>{2}</InstanceId>" + u"<Health>" + u"<State>{3}</State>" + u"{4}" + u"</Health>" + u"</Role>" + u"</RoleInstanceList>" + u"</Container>" + u"</Health>" + u"").format(incarnation, container_id, role_instance_id, status, @@ -193,19 +172,19 @@ def _build_health_report(incarnation, container_id, role_instance_id, """ Convert VMStatus object to status blob format """ -def guest_agent_status_to_v1(ga_status): +def ga_status_to_v1(ga_status): formatted_msg = { 'lang' : 'en-US', 'message' : ga_status.message } v1_ga_status = { - 'version' : ga_status.agentVersion, + 'version' : ga_status.version, 'status' : ga_status.status, 'formattedMessage' : formatted_msg } return v1_ga_status -def extension_substatus_to_v1(sub_status_list): +def ext_substatus_to_v1(sub_status_list): status_list = [] for substatus in sub_status_list: status = { @@ -220,14 +199,14 @@ def extension_substatus_to_v1(sub_status_list): status_list.append(status) return status_list -def extension_handler_status_to_v1(handler_status, timestamp): - if handler_status is None or len(handler_status.extensionStatusList) == 0: - return - ext_status = handler_status.extensionStatusList[0] - sub_status = extension_substatus_to_v1(ext_status.substatusList) - ext_in_status = { +def ext_status_to_v1(ext_name, ext_status): + if ext_status is None: + return None + timestamp = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()) + v1_sub_status = ext_substatus_to_v1(ext_status.substatusList) + v1_ext_status = { "status":{ - "name": ext_status.name, + "name": ext_name, "configurationAppliedTime": ext_status.configurationAppliedTime, "operation": ext_status.operation, "status": ext_status.status, @@ -237,33 +216,47 @@ def extension_handler_status_to_v1(handler_status, timestamp): "message": ext_status.message } }, + "version": 1.0, "timestampUTC": timestamp } - - if len(sub_status) != 0: - ext_in_status['substatus'] = sub_status - + if len(v1_sub_status) != 0: + v1_ext_status['substatus'] = v1_sub_status + return v1_ext_status + +def ext_handler_status_to_v1(handler_status, ext_statuses, timestamp): v1_handler_status = { - 'handlerVersion' : handler_status.handlerVersion, - 'handlerName' : handler_status.handlerName, + 'handlerVersion' : handler_status.version, + 'handlerName' : handler_status.name, 'status' : handler_status.status, - 'runtimeSettingsStatus' : { - 'settingsStatus' : ext_in_status, - 'sequenceNumber' : ext_status.sequenceNumber - } } - return v1_handler_status + if handler_status.message is not None: + v1_handler_status["formattedMessage"] = { + "lang":"en-US", + "message": handler_status.message + } + if len(handler_status.extensions) > 0: + #Currently, no more than one extension per handler + ext_name = handler_status.extensions[0] + ext_status = ext_statuses.get(ext_name) + v1_ext_status = ext_status_to_v1(ext_name, ext_status) + if ext_status is not None and v1_ext_status is not None: + v1_handler_status["runtimeSettingsStatus"] = { + 'settingsStatus' : v1_ext_status, + 'sequenceNumber' : ext_status.sequenceNumber + } + return v1_handler_status -def vm_status_to_v1(vm_status): +def vm_status_to_v1(vm_status, ext_statuses): timestamp = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()) - v1_ga_status = guest_agent_status_to_v1(vm_status.vmAgent) + v1_ga_status = ga_status_to_v1(vm_status.vmAgent) v1_handler_status_list = [] - for handler_status in vm_status.extensionHandlers: - v1_handler_status = extension_handler_status_to_v1(handler_status, - timestamp) - v1_handler_status_list.append(v1_handler_status) + for handler_status in vm_status.vmAgent.extensionHandlers: + v1_handler_status = ext_handler_status_to_v1(handler_status, + ext_statuses, timestamp) + if v1_handler_status is not None: + v1_handler_status_list.append(v1_handler_status) v1_agg_status = { 'guestAgentStatus': v1_ga_status, @@ -278,35 +271,53 @@ def vm_status_to_v1(vm_status): class StatusBlob(object): - def __init__(self, vm_status): - self.vm_status = vm_status + def __init__(self, client): + self.vm_status = None + self.ext_statuses = {} + self.client = client + def set_vm_status(self, vm_status): + validata_param("vmAgent", vm_status, VMStatus) + self.vm_status = vm_status + + def set_ext_status(self, ext_handler_name, ext_status): + validata_param("extensionStatus", ext_status, ExtensionStatus) + self.ext_statuses[ext_handler_name]= ext_status + def to_json(self): - report = vm_status_to_v1(self.vm_status) + report = vm_status_to_v1(self.vm_status, self.ext_statuses) return json.dumps(report) __storage_version__ = "2014-02-14" def upload(self, url): - logger.info("Upload status blob") + #TODO upload extension only if content has changed + logger.verb("Upload status blob") blob_type = self.get_blob_type(url) data = self.to_json() - if blob_type == "BlockBlob": - self.put_block_blob(url, data) - elif blob_type == "PageBlob": - self.put_page_blob(url, data) - else: - raise ProtocolError("Unknown blob type: {0}".format(blob_type)) + try: + if blob_type == "BlockBlob": + self.put_block_blob(url, data) + elif blob_type == "PageBlob": + self.put_page_blob(url, data) + else: + raise ProtocolError("Unknown blob type: {0}".format(blob_type)) + except restutil.HttpError as e: + raise ProtocolError("Failed to upload status blob: {0}".format(e)) def get_blob_type(self, url): #Check blob type logger.verb("Check blob type.") timestamp = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()) - resp = restutil.http_head(url, { - "x-ms-date" : timestamp, - 'x-ms-version' : self.__class__.__storage_version__ - }) + try: + resp = self.client.call_storage_service(restutil.http_head, url, { + "x-ms-date" : timestamp, + 'x-ms-version' : self.__class__.__storage_version__ + }) + except restutil.HttpError as e: + raise ProtocolError((u"Failed to get status blob type: {0}" + u"").format(e)) if resp is None or resp.status != httpclient.OK: raise ProtocolError(("Failed to get status blob type: {0}" "").format(resp.status)) @@ -318,33 +329,47 @@ class StatusBlob(object): def put_block_blob(self, url, data): logger.verb("Upload block blob") timestamp = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()) - resp = restutil.http_put(url, data, { - "x-ms-date" : timestamp, - "x-ms-blob-type" : "BlockBlob", - "Content-Length": text(len(data)), - "x-ms-version" : self.__class__.__storage_version__ - }) - if resp is None or resp.status != httpclient.CREATED: + try: + resp = self.client.call_storage_service(restutil.http_put, url, + data, { + "x-ms-date" : timestamp, + "x-ms-blob-type" : "BlockBlob", + "Content-Length": text(len(data)), + "x-ms-version" : self.__class__.__storage_version__ + }) + except restutil.HttpError as e: + raise ProtocolError((u"Failed to upload block blob: {0}" + u"").format(e)) + if resp.status != httpclient.CREATED: raise ProtocolError(("Failed to upload block blob: {0}" "").format(resp.status)) def put_page_blob(self, url, data): logger.verb("Replace old page blob") + + #Convert string into bytes + data=bytearray(data, encoding='utf-8') timestamp = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()) + #Align to 512 bytes - page_blob_size = ((len(data) + 511) / 512) * 512 - resp = restutil.http_put(url, "", { - "x-ms-date" : timestamp, - "x-ms-blob-type" : "PageBlob", - "Content-Length": "0", - "x-ms-blob-content-length" : text(page_blob_size), - "x-ms-version" : self.__class__.__storage_version__ - }) - if resp is None or resp.status != httpclient.CREATED: + page_blob_size = int((len(data) + 511) / 512) * 512 + try: + resp = self.client.call_storage_service(restutil.http_put, url, + "", { + "x-ms-date" : timestamp, + "x-ms-blob-type" : "PageBlob", + "Content-Length": "0", + "x-ms-blob-content-length" : text(page_blob_size), + "x-ms-version" : self.__class__.__storage_version__ + }) + except restutil.HttpError as e: + raise ProtocolError((u"Failed to clean up page blob: {0}" + u"").format(e)) + if resp.status != httpclient.CREATED: raise ProtocolError(("Failed to clean up page blob: {0}" "").format(resp.status)) - if '?' in url < 0: + if url.count("?") < 0: url = "{0}?comp=page".format(url) else: url = "{0}&comp=page".format(url) @@ -359,15 +384,20 @@ class StatusBlob(object): #Align to 512 bytes page_end = int((end + 511) / 512) * 512 buf_size = page_end - start - buf = bytearray(source=data[start:end], encoding="utf-8") - #TODO buffer is not defined in python3, however we need this to make httplib to work on python 2.6 - resp = restutil.http_put(url, buf, { - "x-ms-date" : timestamp, - "x-ms-range" : "bytes={0}-{1}".format(start, page_end - 1), - "x-ms-page-write" : "update", - "x-ms-version" : self.__class__.__storage_version__, - "Content-Length": text(page_end - start) - }) + buf = bytearray(buf_size) + buf[0: content_size] = data[start: end] + try: + resp = self.client.call_storage_service(restutil.http_put, url, + bytebuffer(buf), { + "x-ms-date" : timestamp, + "x-ms-range" : "bytes={0}-{1}".format(start, page_end - 1), + "x-ms-page-write" : "update", + "x-ms-version" : self.__class__.__storage_version__, + "Content-Length": text(page_end - start) + }) + except restutil.HttpError as e: + raise ProtocolError((u"Failed to upload page blob: {0}" + u"").format(e)) if resp is None or resp.status != httpclient.CREATED: raise ProtocolError(("Failed to upload page blob: {0}" "").format(resp.status)) @@ -408,59 +438,176 @@ class WireClient(object): self.shared_conf = None self.certs = None self.ext_conf = None + self.last_request = 0 self.req_count = 0 + self.status_blob = StatusBlob(self) + + def prevent_throttling(self): + """ + Try to avoid throttling of wire server + """ + now = time.time() + if now - self.last_request < 1: + logger.info("Last request issued less than 1 second ago") + logger.info("Sleep {0} second to avoid throttling.", + SHORT_WAITING_INTERVAL) + time.sleep(SHORT_WAITING_INTERVAL) + self.last_request = now + + self.req_count += 1 + if self.req_count % 3 == 0: + logger.info("Sleep {0} second to avoid throttling.", + SHORT_WAITING_INTERVAL) + time.sleep(SHORT_WAITING_INTERVAL) + self.req_count = 0 + + def call_wireserver(self, http_req, *args, **kwargs): + """ + Call wire server. Handle throttling(403) and Resource Gone(410) + """ + self.prevent_throttling() + for retry in range(0, 3): + resp = http_req(*args, **kwargs) + if resp.status == httpclient.FORBIDDEN: + logger.warn("Sending too much request to wire server") + logger.info("Sleep {0} second to avoid throttling.", + LONG_WAITING_INTERVAL) + time.sleep(LONG_WAITING_INTERVAL) + elif resp.status == httpclient.GONE: + msg = args[0] if len(args) > 0 else "" + raise WireProtocolResourceGone(msg) + else: + return resp + raise ProtocolError(("Calling wire server failed: {0}" + "").format(resp.status)) + + def decode_config(self, data): + if data is None: + return None + data = remove_bom(data) + xml_text = text(data, encoding='utf-8') + return xml_text + + def fetch_config(self, uri, headers): + try: + resp = self.call_wireserver(restutil.http_get, uri, + headers=headers) + except restutil.HttpError as e: + raise ProtocolError(text(e)) + + if(resp.status != httpclient.OK): + raise ProtocolError("{0} - {1}".format(resp.status, uri)) + + return self.decode_config(resp.read()) + + def fetch_cache(self, local_file): + if not os.path.isfile(local_file): + raise ProtocolError("{0} is missing.".format(local_file)) + try: + return fileutil.read_file(local_file) + except IOError as e: + raise ProtocolError("Failed to read cache: {0}".format(e)) + + def save_cache(self, local_file, data): + try: + fileutil.write_file(local_file, data) + except IOError as e: + raise ProtocolError("Failed to write cache: {0}".format(e)) + + def call_storage_service(self, http_req, *args, **kwargs): + """ + Call storage service, handle SERVICE_UNAVAILABLE(503) + """ + for retry in range(0, 3): + resp = http_req(*args, **kwargs) + if resp.status == httpclient.SERVICE_UNAVAILABLE: + logger.warn("Storage service is not avaible temporaryly") + logger.info("Will retry later, in {0} seconds", + LONG_WAITING_INTERVAL) + time.sleep(LONG_WAITING_INTERVAL) + else: + return resp + raise ProtocolError(("Calling storage endpoint failed: {0}" + "").format(resp.status)) + + def fetch_manifest(self, version_uris): + for version_uri in version_uris: + try: + resp = self.call_storage_service(restutil.http_get, + version_uri.uri, None, + chk_proxy=True) + except restutil.HttpError as e: + raise ProtocolError(text(e)) + + if resp.status == httpclient.OK: + return self.decode_config(resp.read()) + logger.warn("Failed to fetch ExtensionManifest: {0}, {1}", + resp.status, version_uri.uri) + logger.info("Will retry later, in {0} seconds", + LONG_WAITING_INTERVAL) + time.sleep(LONG_WAITING_INTERVAL) + raise ProtocolError(("Failed to fetch ExtensionManifest from " + "all sources")) + def update_hosting_env(self, goal_state): if goal_state.hosting_env_uri is None: raise ProtocolError("HostingEnvironmentConfig uri is empty") local_file = HOSTING_ENV_FILE_NAME - xml_text = _fetch_uri(goal_state.hosting_env_uri, self.get_header()) - fileutil.write_file(local_file, xml_text) + xml_text = self.fetch_config(goal_state.hosting_env_uri, + self.get_header()) + self.save_cache(local_file, xml_text) self.hosting_env = HostingEnv(xml_text) def update_shared_conf(self, goal_state): if goal_state.shared_conf_uri is None: raise ProtocolError("SharedConfig uri is empty") local_file = SHARED_CONF_FILE_NAME - xml_text = _fetch_uri(goal_state.shared_conf_uri, self.get_header()) - fileutil.write_file(local_file, xml_text) + xml_text = self.fetch_config(goal_state.shared_conf_uri, + self.get_header()) + self.save_cache(local_file, xml_text) self.shared_conf = SharedConfig(xml_text) def update_certs(self, goal_state): if goal_state.certs_uri is None: return local_file = CERTS_FILE_NAME - xml_text = _fetch_uri(goal_state.certs_uri, self.get_header_for_cert()) - fileutil.write_file(local_file, xml_text) - self.certs = Certificates(xml_text) + xml_text = self.fetch_config(goal_state.certs_uri, + self.get_header_for_cert()) + self.save_cache(local_file, xml_text) + self.certs = Certificates(self, xml_text) def update_ext_conf(self, goal_state): if goal_state.ext_uri is None: - raise ProtocolError("ExtensionsConfig uri is empty") + logger.info("ExtensionsConfig.xml uri is empty") + self.ext_conf = ExtensionsConfig(None) + return incarnation = goal_state.incarnation local_file = EXT_CONF_FILE_NAME.format(incarnation) - xml_text = _fetch_uri(goal_state.ext_uri, - self.get_header()) - fileutil.write_file(local_file, xml_text) + xml_text = self.fetch_config(goal_state.ext_uri, self.get_header()) + self.save_cache(local_file, xml_text) self.ext_conf = ExtensionsConfig(xml_text) - for extension in self.ext_conf.ext_list.extensions: - self.update_ext_manifest(extension, goal_state) + for ext_handler in self.ext_conf.ext_handlers.extHandlers: + self.update_ext_handler_manifest(ext_handler, goal_state) - def update_ext_manifest(self, extension, goal_state): - local_file = MANIFEST_FILE_NAME.format(extension.name, - goal_state.incarnation) - xml_text = _fetch_manifest(extension.version_uris) - fileutil.write_file(local_file, xml_text) + def update_ext_handler_manifest(self, ext_handler, goal_state): + local_file = MANIFEST_FILE_NAME.format(ext_handler.name, + goal_state.incarnation) + xml_text = self.fetch_manifest(ext_handler.versionUris) + self.save_cache(local_file, xml_text) def update_goal_state(self, forced=False, max_retry=3): uri = GOAL_STATE_URI.format(self.endpoint) - xml_text = _fetch_uri(uri, self.get_header()) + xml_text = self.fetch_config(uri, self.get_header()) goal_state = GoalState(xml_text) + incarnation_file = os.path.join(OSUTIL.get_lib_dir(), + INCARNATION_FILE_NAME) + if not forced: last_incarnation = None - if(os.path.isfile(INCARNATION_FILE_NAME)): - last_incarnation = fileutil.read_file(INCARNATION_FILE_NAME) + if(os.path.isfile(incarnation_file)): + last_incarnation = fileutil.read_file(incarnation_file) new_incarnation = goal_state.incarnation if last_incarnation is not None and \ last_incarnation == new_incarnation: @@ -471,10 +618,10 @@ class WireClient(object): for retry in range(0, max_retry): try: self.goal_state = goal_state - goal_state_file = GOAL_STATE_FILE_NAME.format(goal_state.incarnation) - fileutil.write_file(goal_state_file, xml_text) - fileutil.write_file(INCARNATION_FILE_NAME, - goal_state.incarnation) + file_name = GOAL_STATE_FILE_NAME.format(goal_state.incarnation) + goal_state_file = os.path.join(OSUTIL.get_lib_dir(), file_name) + self.save_cache(goal_state_file, xml_text) + self.save_cache(incarnation_file, goal_state.incarnation) self.update_hosting_env(goal_state) self.update_shared_conf(goal_state) self.update_certs(goal_state) @@ -482,35 +629,35 @@ class WireClient(object): return except WireProtocolResourceGone: logger.info("Incarnation is out of date. Update goalstate.") - xml_text = _fetch_uri(GOAL_STATE_URI, self.get_header()) + xml_text = self.fetch_config(uri, self.get_header()) goal_state = GoalState(xml_text) raise ProtocolError("Exceeded max retry updating goal state") def get_goal_state(self): if(self.goal_state is None): - incarnation = _fetch_cache(INCARNATION_FILE_NAME) + incarnation = self.fetch_cache(INCARNATION_FILE_NAME) goal_state_file = GOAL_STATE_FILE_NAME.format(incarnation) - xml_text = _fetch_cache(goal_state_file) + xml_text = self.fetch_cache(goal_state_file) self.goal_state = GoalState(xml_text) return self.goal_state def get_hosting_env(self): if(self.hosting_env is None): - xml_text = _fetch_cache(HOSTING_ENV_FILE_NAME) + xml_text = self.fetch_cache(HOSTING_ENV_FILE_NAME) self.hosting_env = HostingEnv(xml_text) return self.hosting_env def get_shared_conf(self): if(self.shared_conf is None): - xml_text = _fetch_cache(SHARED_CONF_FILE_NAME) + xml_text = self.fetch_cache(SHARED_CONF_FILE_NAME) self.shared_conf = SharedConfig(xml_text) return self.shared_conf def get_certs(self): if(self.certs is None): - xml_text = _fetch_cache(Certificates) - self.certs = Certificates(xml_text) + xml_text = self.fetch_cache(CERTS_FILE_NAME) + self.certs = Certificates(self, xml_text) if self.certs is None: return None return self.certs @@ -518,20 +665,23 @@ class WireClient(object): def get_ext_conf(self): if(self.ext_conf is None): goal_state = self.get_goal_state() - local_file = EXT_CONF_FILE_NAME.format(goal_state.incarnation) - xml_text = _fetch_cache(local_file) - self.ext_conf = ExtensionsConfig(xml_text) + if goal_state.ext_uri is None: + self.ext_conf = ExtensionsConfig(None) + else: + local_file = EXT_CONF_FILE_NAME.format(goal_state.incarnation) + xml_text = self.fetch_cache(local_file) + self.ext_conf = ExtensionsConfig(xml_text) return self.ext_conf def get_ext_manifest(self, extension, goal_state): local_file = MANIFEST_FILE_NAME.format(extension.name, goal_state.incarnation) - xml_text = _fetch_cache(local_file) + xml_text = self.fetch_cache(local_file) return ExtensionManifest(xml_text) def check_wire_protocol_version(self): uri = VERSION_INFO_URI.format(self.endpoint) - version_info_xml = _fetch_uri(uri, None) + version_info_xml = self.fetch_config(uri, None) version_info = VersionInfo(version_info_xml) preferred = version_info.get_preferred() @@ -544,42 +694,50 @@ class WireClient(object): error = ("Agent supported wire protocol version: {0} was not " "advised by Fabric.").format(PROTOCOL_VERSION) raise ProtocolNotFound(error) - - def upload_status_blob(self, vm_status): + + def upload_status_blob(self): ext_conf = self.get_ext_conf() - status_blob = StatusBlob(vm_status) - status_blob.upload(ext_conf.status_upload_blob) + if ext_conf.status_upload_blob is not None: + self.status_blob.upload(ext_conf.status_upload_blob) def report_role_prop(self, thumbprint): goal_state = self.get_goal_state() role_prop = _build_role_properties(goal_state.container_id, goal_state.role_instance_id, thumbprint) + role_prop = role_prop.encode("utf-8") role_prop_uri = ROLE_PROP_URI.format(self.endpoint) - ret = restutil.http_post(role_prop_uri, - role_prop, - headers=self.get_header_for_xml_content()) - + headers = self.get_header_for_xml_content() + try: + resp = self.call_wireserver(restutil.http_post, role_prop_uri, + role_prop, headers = headers) + except restutil.HttpError as e: + raise ProtocolError((u"Failed to send role properties: {0}" + u"").format(e)) + if resp.status != httpclient.ACCEPTED: + raise ProtocolError((u"Failed to send role properties: {0}" + u", {1}").format(resp.status, resp.read())) def report_health(self, status, substatus, description): goal_state = self.get_goal_state() health_report = _build_health_report(goal_state.incarnation, - goal_state.container_id, - goal_state.role_instance_id, - status, - substatus, - description) + goal_state.container_id, + goal_state.role_instance_id, + status, + substatus, + description) + health_report = health_report.encode("utf-8") health_report_uri = HEALTH_REPORT_URI.format(self.endpoint) headers = self.get_header_for_xml_content() - resp = restutil.http_post(health_report_uri, - health_report, - headers=headers) - def prevent_throttling(self): - self.req_count += 1 - if self.req_count % 3 == 0: - logger.info("Sleep 15 before sending event to avoid throttling.") - self.req_count = 0 - time.sleep(15) + try: + resp = self.call_wireserver(restutil.http_post, health_report_uri, + health_report, headers = headers) + except restutil.HttpError as e: + raise ProtocolError((u"Failed to send provision status: {0}" + u"").format(e)) + if resp.status != httpclient.OK: + raise ProtocolError((u"Failed to send provision status: {0}" + u", {1}").format(resp.status, resp.read())) def send_event(self, provider_id, event_str): uri = TELEMETRY_URI.format(self.endpoint) @@ -590,9 +748,8 @@ class WireClient(object): '</TelemetryData>') data = data_format.format(provider_id, event_str) try: - self.prevent_throttling() header = self.get_header_for_xml_content() - resp = restutil.http_post(uri, data, header) + resp = self.call_wireserver(restutil.http_post, uri, data, header) except restutil.HttpError as e: raise ProtocolError("Failed to send events:{0}".format(e)) @@ -635,7 +792,7 @@ class WireClient(object): def get_header_for_cert(self): cert = "" - content = _fetch_cache(TRANSPORT_CERT_FILE_NAME) + content = self.fetch_cache(TRANSPORT_CERT_FILE_NAME) for line in content.split('\n'): if "CERTIFICATE" not in line: cert += line.rstrip() @@ -762,10 +919,9 @@ class Certificates(object): """ Object containing certificates of host and provisioned user. """ - def __init__(self, xml_text=None): - if xml_text is None: - raise ValueError("Certificates.xml is None") + def __init__(self, client, xml_text): logger.verb("Load Certificates.xml") + self.client = client self.lib_dir = OSUTIL.get_lib_dir() self.openssl_cmd = OSUTIL.get_openssl_cmd() self.cert_list = CertList() @@ -787,7 +943,7 @@ class Certificates(object): "\n" "{2}").format(P7M_FILE_NAME, P7M_FILE_NAME, data) - fileutil.write_file(os.path.join(self.lib_dir, P7M_FILE_NAME), p7m) + self.client.save_cache(os.path.join(self.lib_dir, P7M_FILE_NAME), p7m) #decrypt certificates cmd = ("{0} cms -decrypt -in {1} -inkey {2} -recip {3}" "| {4} pkcs12 -nodes -password pass: -out {5}" @@ -844,13 +1000,12 @@ class Certificates(object): for v1_cert in v1_cert_list: cert = Cert() - set_properties(cert, v1_cert) + set_properties("certs", cert, v1_cert) self.cert_list.certificates.append(cert) def write_to_tmp_file(self, index, suffix, buf): file_name = os.path.join(self.lib_dir, "{0}.{1}".format(index, suffix)) - with open(file_name, 'w') as tmp: - tmp.writelines(buf) + self.client.save_cache(file_name, "".join(buf)) return file_name @@ -861,12 +1016,11 @@ class ExtensionsConfig(object): """ def __init__(self, xml_text): - if xml_text is None: - raise ValueError("ExtensionsConfig is None") logger.verb("Load ExtensionsConfig.xml") - self.ext_list = ExtensionList() + self.ext_handlers = ExtHandlerList() self.status_upload_blob = None - self.parse(xml_text) + if xml_text is not None: + self.parse(xml_text) def parse(self, xml_text): """ @@ -879,38 +1033,38 @@ class ExtensionsConfig(object): plugin_settings = findall(plugin_settings_list, "Plugin") for plugin in plugins: - ext = self.parse_ext(plugin) - self.ext_list.extensions.append(ext) - self.parse_ext_settings(ext, plugin_settings) + ext_handler = self.parse_plugin(plugin) + self.ext_handlers.extHandlers.append(ext_handler) + self.parse_plugin_settings(ext_handler, plugin_settings) self.status_upload_blob = findtext(xml_doc, "StatusUploadBlob") - def parse_ext(self, plugin): - ext = Extension() - ext.name = getattrib(plugin, "name") - ext.properties.version = getattrib(plugin, "version") - ext.properties.state = getattrib(plugin, "state") + def parse_plugin(self, plugin): + ext_handler = ExtHandler() + ext_handler.name = getattrib(plugin, "name") + ext_handler.properties.version = getattrib(plugin, "version") + ext_handler.properties.state = getattrib(plugin, "state") auto_upgrade = getattrib(plugin, "autoUpgrade") if auto_upgrade is not None and auto_upgrade.lower() == "true": - ext.properties.upgradePolicy = "auto" + ext_handler.properties.upgradePolicy = "auto" else: - ext.properties.upgradePolicy = "manual" + ext_handler.properties.upgradePolicy = "manual" location = getattrib(plugin, "location") failover_location = getattrib(plugin, "failoverlocation") for uri in [location, failover_location]: - version_uri = ExtensionVersionUri() + version_uri = ExtHandlerVersionUri() version_uri.uri = uri - ext.version_uris.append(version_uri) - return ext + ext_handler.versionUris.append(version_uri) + return ext_handler - def parse_ext_settings(self, ext, plugin_settings): + def parse_plugin_settings(self, ext_handler, plugin_settings): if plugin_settings is None: return - name = ext.name - version = ext.properties.version + name = ext_handler.name + version = ext_handler.properties.version settings = [x for x in plugin_settings \ if getattrib(x, "name") == name and \ getattrib(x ,"version") == version] @@ -930,20 +1084,23 @@ class ExtensionsConfig(object): for plugin_settings_list in runtime_settings["runtimeSettings"]: handler_settings = plugin_settings_list["handlerSettings"] - ext_settings = ExtensionSettings() - ext_settings.sequenceNumber = seqNo - ext_settings.publicSettings = handler_settings.get("publicSettings", None) - ext_settings.privateSettings = handler_settings.get("protectedSettings", None) - thumbprint = handler_settings.get("protectedSettingsCertThumbprint", None) - ext_settings.certificateThumbprint = thumbprint - ext.properties.extensions.append(ext_settings) + ext = Extension() + #There is no "extension name" in wire protocol. + #Put + ext.name = ext_handler.name + ext.sequenceNumber = seqNo + ext.publicSettings = handler_settings.get("publicSettings") + ext.privateSettings = handler_settings.get("protectedSettings") + thumbprint = handler_settings.get("protectedSettingsCertThumbprint") + ext.certificateThumbprint = thumbprint + ext_handler.properties.extensions.append(ext) class ExtensionManifest(object): def __init__(self, xml_text): if xml_text is None: raise ValueError("ExtensionManifest is None") logger.verb("Load ExtensionManifest.xml") - self.pkg_list = ExtensionPackageList() + self.pkg_list = ExtHandlerPackageList() self.parse(xml_text) def parse(self, xml_text): @@ -954,10 +1111,10 @@ class ExtensionManifest(object): uris = find(package, "Uris") uri_list = findall(uris, "Uri") uri_list = [gettext(x) for x in uri_list] - package = ExtensionPackage() + package = ExtHandlerPackage() package.version = version for uri in uri_list: - pkg_uri = ExtensionPackageUri() + pkg_uri = ExtHandlerVersionUri() pkg_uri.uri = uri package.uris.append(pkg_uri) self.pkg_list.versions.append(package) diff --git a/azurelinuxagent/protocol/v2.py b/azurelinuxagent/protocol/v2.py index d7c9143..34102b7 100644 --- a/azurelinuxagent/protocol/v2.py +++ b/azurelinuxagent/protocol/v2.py @@ -1,4 +1,4 @@ -# Windows Azure Linux Agent +# Microsoft Azure Linux Agent # # Copyright 2014 Microsoft Corporation # @@ -25,7 +25,7 @@ ENDPOINT='169.254.169.254' #TODO use http for azure pack test #ENDPOINT='localhost' APIVERSION='2015-05-01-preview' -BASE_URI = "http://{0}/Microsoft.Compute/{1}?api-version={{{2}}}{3}" +BASE_URI = "http://{0}/Microsoft.Compute/{1}?api-version={2}{3}" def _add_content_type(headers): if headers is None: @@ -47,12 +47,15 @@ class MetadataProtocol(Protocol): self.provision_status_uri = BASE_URI.format(self.endpoint, "provisioningStatus", self.apiversion, "") - self.status_uri = BASE_URI.format(self.endpoint, "status", - self.apiversion, "") + self.vm_status_uri = BASE_URI.format(self.endpoint, "status/vmagent", + self.apiversion, "") + self.ext_status_uri = BASE_URI.format(self.endpoint, + "status/extensions/{0}", + self.apiversion, "") self.event_uri = BASE_URI.format(self.endpoint, "status/telemetry", self.apiversion, "") - def _get_data(self, data_type, url, headers=None): + def _get_data(self, url, headers=None): try: resp = restutil.http_get(url, headers=headers) except restutil.HttpError as e: @@ -60,20 +63,15 @@ class MetadataProtocol(Protocol): if resp.status != httpclient.OK: raise ProtocolError("{0} - GET: {1}".format(resp.status, url)) - try: - data = resp.read() - if data is None: - return None - data = json.loads(text(data, encoding="utf-8")) - except ValueError as e: - raise ProtocolError(text(e)) - obj = data_type() - set_properties(obj, data) - return obj - def _put_data(self, url, obj, headers=None): + data = resp.read() + if data is None: + return None + data = json.loads(text(data, encoding="utf-8")) + return data + + def _put_data(self, url, data, headers=None): headers = _add_content_type(headers) - data = get_properties(obj) try: resp = restutil.http_put(url, json.dumps(data), headers=headers) except restutil.HttpError as e: @@ -81,9 +79,8 @@ class MetadataProtocol(Protocol): if resp.status != httpclient.OK: raise ProtocolError("{0} - PUT: {1}".format(resp.status, url)) - def _post_data(self, url, obj, headers=None): + def _post_data(self, url, data, headers=None): headers = _add_content_type(headers) - data = get_properties(obj) try: resp = restutil.http_post(url, json.dumps(data), headers=headers) except restutil.HttpError as e: @@ -95,28 +92,54 @@ class MetadataProtocol(Protocol): pass def get_vminfo(self): - return self._get_data(VMInfo, self.identity_uri) + vminfo = VMInfo() + data = self._get_data(self.identity_uri) + set_properties("vminfo", vminfo, data) + return vminfo def get_certs(self): - #TODO walk arround for azure pack test + #TODO download and save certs return CertList() - certs = self._get_data(CertList, self.cert_uri) - #TODO download pfx and convert to pem - return certs - - def get_extensions(self): - return self._get_data(ExtensionList, self.ext_uri) - - def report_provision_status(self, status): - validata_param('status', status, ProvisionStatus) - self._put_data(self.provision_status_uri, status) - - def report_status(self, status): - validata_param('status', status, VMStatus) - self._put_data(self.status_uri, status) + def get_ext_handlers(self): + ext_list = ExtHandlerList() + data = self._get_data(self.ext_uri) + set_properties("extensionHandlers", ext_list.extHandlers, data) + return ext_list + + def get_ext_handler_pkgs(self, ext_handler): + ext_handler_pkgs = ExtHandlerPackageList() + data = None + for version_uri in ext_handler.versionUris: + try: + data = self._get_data(version_uri.uri) + break + except ProtocolError as e: + logger.warn("Failed to get version uris: {0}", e) + logger.info("Retry getting version uris") + set_properties("extensionPackages", ext_handler_pkgs, data) + return ext_handler_pkgs + + def report_provision_status(self, provision_status): + validata_param('provisionStatus', provision_status, ProvisionStatus) + data = get_properties(provision_status) + self._put_data(self.provision_status_uri, data) + + def report_vm_status(self, vm_status): + validata_param('vmStatus', vm_status, VMStatus) + data = get_properties(vm_status) + self._put_data(self.vm_status_uri, data) + + def report_ext_status(self, ext_handler_name, ext_name, ext_status): + validata_param('extensionStatus', ext_status, ExtensionStatus) + data = get_properties(ext_status) + uri = self.ext_status_uri.format(ext_name) + self._put_data(uri, data) def report_event(self, events): - validata_param('events', events, TelemetryEventList) - self._post_data(self.event_uri, events) + #TODO disable telemetry for azure stack test + #validata_param('events', events, TelemetryEventList) + #data = get_properties(events) + #self._post_data(self.event_uri, data) + pass diff --git a/azurelinuxagent/utils/__init__.py b/azurelinuxagent/utils/__init__.py index 4b2b9e1..d9b82f5 100644 --- a/azurelinuxagent/utils/__init__.py +++ b/azurelinuxagent/utils/__init__.py @@ -1,4 +1,4 @@ -# Windows Azure Linux Agent +# Microsoft Azure Linux Agent # # Copyright 2014 Microsoft Corporation # diff --git a/azurelinuxagent/utils/fileutil.py b/azurelinuxagent/utils/fileutil.py index 5e7fecf..08592bc 100644 --- a/azurelinuxagent/utils/fileutil.py +++ b/azurelinuxagent/utils/fileutil.py @@ -1,4 +1,4 @@ -# Windows Azure Linux Agent +# Microsoft Azure Linux Agent # # Copyright 2014 Microsoft Corporation # diff --git a/azurelinuxagent/utils/osutil.py b/azurelinuxagent/utils/osutil.py index 756400c..9de47e7 100644 --- a/azurelinuxagent/utils/osutil.py +++ b/azurelinuxagent/utils/osutil.py @@ -1,4 +1,4 @@ -# Windows Azure Linux Agent +# Microsoft Azure Linux Agent # # Copyright 2014 Microsoft Corporation # diff --git a/azurelinuxagent/utils/restutil.py b/azurelinuxagent/utils/restutil.py index 1015f71..2acfa57 100644 --- a/azurelinuxagent/utils/restutil.py +++ b/azurelinuxagent/utils/restutil.py @@ -1,4 +1,4 @@ -# Windows Azure Linux Agent +# Microsoft Azure Linux Agent # # Copyright 2014 Microsoft Corporation # @@ -66,7 +66,7 @@ def _http_request(method, host, rel_uri, port=None, data=None, secure=False, #If proxy is used, full url is needed. url = "https://{0}:{1}{2}".format(host, port, rel_uri) else: - conn = httpclient.HTTPSConnection(host, port) + conn = httpclient.HTTPSConnection(host, port, timeout=10) url = rel_uri else: port = 80 if port is None else port @@ -75,7 +75,7 @@ def _http_request(method, host, rel_uri, port=None, data=None, secure=False, #If proxy is used, full url is needed. url = "http://{0}:{1}{2}".format(host, port, rel_uri) else: - conn = httpclient.HTTPConnection(host, port) + conn = httpclient.HTTPConnection(host, port, timeout=10) url = rel_uri if headers == None: conn.request(method, url, data) diff --git a/azurelinuxagent/utils/shellutil.py b/azurelinuxagent/utils/shellutil.py index f4305d9..372c78a 100644 --- a/azurelinuxagent/utils/shellutil.py +++ b/azurelinuxagent/utils/shellutil.py @@ -1,4 +1,4 @@ -# Windows Azure Linux Agent +# Microsoft Azure Linux Agent # # Copyright 2014 Microsoft Corporation # @@ -65,21 +65,25 @@ def run(cmd, chk_err=True): retcode,out=run_get_output(cmd,chk_err) return retcode -def run_get_output(cmd, chk_err=True): +def run_get_output(cmd, chk_err=True, log_cmd=True): """ Wrapper for subprocess.check_output. Execute 'cmd'. Returns return code and STDOUT, trapping expected exceptions. Reports exceptions to Error if chk_err parameter is True """ - logger.verb("run cmd '{0}'", cmd) + if log_cmd: + logger.verb(u"run cmd '{0}'", cmd) try: output=subprocess.check_output(cmd,stderr=subprocess.STDOUT,shell=True) + output = text(output, encoding='utf-8', errors="backslashreplace") except subprocess.CalledProcessError as e : - if chk_err : - logger.error("run cmd '{0}' failed", e.cmd) - logger.error("Error Code:{0}", e.returncode) - logger.error("Result:{0}", e.output[:-1].decode('latin-1')) - return e.returncode, e.output.decode('latin-1') - return 0, text(output, encoding="utf-8") + output = text(e.output, encoding='utf-8', errors="backslashreplace") + if chk_err: + if log_cmd: + logger.error(u"run cmd '{0}' failed", e.cmd) + logger.error(u"Error Code:{0}", e.returncode) + logger.error(u"Result:{0}", output) + return e.returncode, output + return 0, output #End shell command util functions diff --git a/azurelinuxagent/utils/textutil.py b/azurelinuxagent/utils/textutil.py index 2e66b0e..e0f1395 100644 --- a/azurelinuxagent/utils/textutil.py +++ b/azurelinuxagent/utils/textutil.py @@ -1,4 +1,4 @@ -# Windows Azure Linux Agent +# Microsoft Azure Linux Agent # # Copyright 2014 Microsoft Corporation # @@ -22,6 +22,7 @@ import string import struct import xml.dom.minidom as minidom import sys +from distutils.version import LooseVersion def parse_doc(xml_text): """ @@ -217,12 +218,11 @@ def remove_bom(c): c = c[3:] return c -def gen_password_hash(password, use_salt, salt_type, salt_len): - salt="$6$" - if use_salt: - collection = string.ascii_letters + string.digits - salt = ''.join(random.choice(collection) for _ in range(salt_len)) - salt = "${0}${1}".format(salt_type, salt) +def gen_password_hash(password, crypt_id, salt_len): + collection = string.ascii_letters + string.digits + salt = ''.join(random.choice(collection) for _ in range(salt_len)) + salt = "${0}${1}".format(crypt_id, salt) return crypt.crypt(password, salt) +Version = LooseVersion diff --git a/bin/waagent2.0 b/bin/waagent2.0 index 52e022f..94c8ac8 100755 --- a/bin/waagent2.0 +++ b/bin/waagent2.0 @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# Windows Azure Linux Agent +# Microsoft Azure Linux Agent # # Copyright 2015 Microsoft Corporation # @@ -79,7 +79,7 @@ if not hasattr(subprocess,'check_output'): subprocess.CalledProcessError=CalledProcessError GuestAgentName = "WALinuxAgent" -GuestAgentLongName = "Windows Azure Linux Agent" +GuestAgentLongName = "Microsoft Azure Linux Agent" GuestAgentVersion = "WALinuxAgent-2.0.15-pre" ProtocolVersion = "2012-11-30" #WARNING this value is used to confirm the correct fabric protocol. @@ -106,7 +106,7 @@ HandlerStatusToAggStatus = {"installed":"Installing", "enabled":"Ready", "uninta WaagentConf = """\ # -# Windows Azure Linux Agent Configuration +# Microsoft Azure Linux Agent Configuration # Role.StateConsumer=None # Specified program is invoked with the argument "Ready" when we report ready status @@ -126,7 +126,7 @@ ResourceDisk.MountPoint=/mnt/resource # ResourceDisk.EnableSwap=n # Create and use swapfile on resource disk. ResourceDisk.SwapSizeMB=0 # Size of the swapfile. -LBProbeResponder=y # Respond to load balancer probes if requested by Windows Azure. +LBProbeResponder=y # Respond to load balancer probes if requested by Microsoft Azure. Logs.Verbose=n # Enable verbose logs @@ -656,7 +656,7 @@ command=/usr/sbin/waagent pidfile=/var/run/waagent.pid command_args=-daemon command_background=true -name="Windows Azure Linux Agent" +name="Microsoft Azure Linux Agent" depend() { @@ -725,7 +725,7 @@ class gentooDistro(AbstractDistro): suse_init_file = """\ #! /bin/sh # -# Windows Azure Linux Agent sysV init script +# Microsoft Azure Linux Agent sysV init script # # Copyright 2013 Microsoft Corporation # Copyright SUSE LLC @@ -751,12 +751,12 @@ suse_init_file = """\ # System startup script for the waagent # ### BEGIN INIT INFO -# Provides: WindowsAzureLinuxAgent +# Provides: MicrosoftAzureLinuxAgent # Required-Start: $network sshd # Required-Stop: $network sshd # Default-Start: 3 5 # Default-Stop: 0 1 2 6 -# Description: Start the WindowsAzureLinuxAgent +# Description: Start the MicrosoftAzureLinuxAgent ### END INIT INFO PYTHON=/usr/bin/python @@ -789,14 +789,14 @@ rc_reset case "$1" in start) - echo -n "Starting WindowsAzureLinuxAgent" + echo -n "Starting MicrosoftAzureLinuxAgent" ## Start daemon with startproc(8). If this fails ## the echo return value is set appropriate. startproc -f ${PYTHON} ${WAZD_BIN} -daemon rc_status -v ;; stop) - echo -n "Shutting down WindowsAzureLinuxAgent" + echo -n "Shutting down MicrosoftAzureLinuxAgent" ## Stop daemon with killproc(8) and if this fails ## set echo the echo return value. killproc -p ${WAZD_PIDFILE} ${PYTHON} ${WAZD_BIN} @@ -820,7 +820,7 @@ case "$1" in rc_status ;; status) - echo -n "Checking for service WindowsAzureLinuxAgent " + echo -n "Checking for service MicrosoftAzureLinuxAgent " ## Check status with checkproc(8), if process is running ## checkproc will return with exit status 0. @@ -902,17 +902,17 @@ class SuSEDistro(AbstractDistro): redhat_init_file= """\ #!/bin/bash # -# Init file for WindowsAzureLinuxAgent. +# Init file for MicrosoftAzureLinuxAgent. # # chkconfig: 2345 60 80 -# description: WindowsAzureLinuxAgent +# description: MicrosoftAzureLinuxAgent # # source function library . /etc/rc.d/init.d/functions RETVAL=0 -FriendlyName="WindowsAzureLinuxAgent" +FriendlyName="MicrosoftAzureLinuxAgent" WAZD_BIN=/usr/sbin/waagent start() @@ -1194,15 +1194,15 @@ class CoreOSDistro(AbstractDistro): debian_init_file = """\ #!/bin/sh ### BEGIN INIT INFO -# Provides: WindowsAzureLinuxAgent +# Provides: MicrosoftAzureLinuxAgent # Required-Start: $network $syslog # Required-Stop: $network $syslog # Should-Start: $network $syslog # Should-Stop: $network $syslog # Default-Start: 2 3 4 5 # Default-Stop: 0 1 6 -# Short-Description: WindowsAzureLinuxAgent -# Description: WindowsAzureLinuxAgent +# Short-Description: MicrosoftAzureLinuxAgent +# Description: MicrosoftAzureLinuxAgent ### END INIT INFO . /lib/lsb/init-functions @@ -1213,7 +1213,7 @@ WAZD_PID=/var/run/waagent.pid case "$1" in start) - log_begin_msg "Starting WindowsAzureLinuxAgent..." + log_begin_msg "Starting MicrosoftAzureLinuxAgent..." pid=$( pidofproc $WAZD_BIN ) if [ -n "$pid" ] ; then log_begin_msg "Already running." @@ -1225,7 +1225,7 @@ case "$1" in ;; stop) - log_begin_msg "Stopping WindowsAzureLinuxAgent..." + log_begin_msg "Stopping MicrosoftAzureLinuxAgent..." start-stop-daemon --stop --quiet --oknodo --pidfile $WAZD_PID ret=$? rm -f $WAZD_PID @@ -1350,7 +1350,7 @@ class KaliDistro(debianDistro): # UbuntuDistro ############################################################ ubuntu_upstart_file = """\ -#walinuxagent - start Windows Azure agent +#walinuxagent - start Microsoft Azure agent description "walinuxagent" author "Ben Howard <ben.howard@canonical.com>" @@ -1474,7 +1474,7 @@ class LinuxMintDistro(UbuntuDistro): ############################################################ fedora_systemd_service = """\ [Unit] -Description=Windows Azure Linux Agent +Description=Microsoft Azure Linux Agent After=network.target After=sshd.service ConditionFileIsExecutable=/usr/sbin/waagent @@ -1599,7 +1599,7 @@ class fedoraDistro(redhatDistro): ############################################################ FreeBSDWaagentConf = """\ # -# Windows Azure Linux Agent Configuration +# Microsoft Azure Linux Agent Configuration # Role.StateConsumer=None # Specified program is invoked with the argument "Ready" when we report ready status @@ -1619,7 +1619,7 @@ ResourceDisk.MountPoint=/mnt/resource # ResourceDisk.EnableSwap=n # Create and use swapfile on resource disk. ResourceDisk.SwapSizeMB=0 # Size of the swapfile. -LBProbeResponder=y # Respond to load balancer probes if requested by Windows Azure. +LBProbeResponder=y # Respond to load balancer probes if requested by Microsoft Azure. Logs.Verbose=n # Enable verbose logs @@ -5086,7 +5086,7 @@ class Agent(Util): name = "DefaultGateway" else: endpoint = IpAddress - name = "Windows Azure wire protocol endpoint" + name = "Microsoft Azure wire protocol endpoint" LogIfVerbose(name + ": " + IpAddress + " at " + hex(i)) else: Error("HandleDhcpResponse: Data too small for option " + str(option)) @@ -5098,7 +5098,7 @@ class Agent(Util): def DoDhcpWork(self): """ Discover the wire server via DHCP option 245. - And workaround incompatibility with Windows Azure DHCP servers. + And workaround incompatibility with Microsoft Azure DHCP servers. """ ShortSleep = False # Sleep 1 second before retrying DHCP queries. ifname=None @@ -5212,7 +5212,7 @@ class Agent(Util): if not goalStateXml: Error("UpdateGoalState failed.") return - Log("Retrieved GoalState from Windows Azure Fabric.") + Log("Retrieved GoalState from Microsoft Azure Fabric.") self.GoalState = GoalState(self).Parse(goalStateXml) return self.GoalState @@ -5524,15 +5524,15 @@ class Agent(Util): except: pass - Log("Probing for Windows Azure environment.") + Log("Probing for Microsoft Azure environment.") self.Endpoint = self.DoDhcpWork() if self.Endpoint == None: - Log("Windows Azure environment not detected.") + Log("Microsoft Azure environment not detected.") while True: time.sleep(60) - Log("Discovered Windows Azure endpoint: " + self.Endpoint) + Log("Discovered Microsoft Azure endpoint: " + self.Endpoint) if not self.CheckVersions(): Error("Agent.CheckVersions failed") sys.exit(1) diff --git a/config/suse/waagent.conf b/config/suse/waagent.conf index e429c74..b4f4798 100644 --- a/config/suse/waagent.conf +++ b/config/suse/waagent.conf @@ -1,5 +1,5 @@ # -# Windows Azure Linux Agent Configuration +# Microsoft Azure Linux Agent Configuration # # Specified program is invoked with the argument "Ready" when we report ready status @@ -34,6 +34,12 @@ Provisioning.DecodeCustomData=n # Execute CustomData after provisioning. Provisioning.ExecuteCustomData=n +# Algorithm used by crypt when generating password hash. +#Provisioning.PasswordCryptId=6 + +# Length of random salt used when generating password hash. +#Provisioning.PasswordCryptSaltLength=10 + # Format if unformatted. If 'n', resource disk will not be mounted. ResourceDisk.Format=y @@ -50,7 +56,7 @@ ResourceDisk.EnableSwap=n # Size of the swapfile. ResourceDisk.SwapSizeMB=0 -# Respond to load balancer probes if requested by Windows Azure. +# Respond to load balancer probes if requested by Microsoft Azure. LBProbeResponder=y # Enable verbose logging (y|n) diff --git a/config/ubuntu/waagent.conf b/config/ubuntu/waagent.conf index ab50418..db29d80 100644 --- a/config/ubuntu/waagent.conf +++ b/config/ubuntu/waagent.conf @@ -1,5 +1,5 @@ # -# Windows Azure Linux Agent Configuration +# Microsoft Azure Linux Agent Configuration # # Specified program is invoked with the argument "Ready" when we report ready status @@ -34,6 +34,12 @@ Provisioning.DecodeCustomData=n # Execute CustomData after provisioning. Provisioning.ExecuteCustomData=n +# Algorithm used by crypt when generating password hash. +#Provisioning.PasswordCryptId=6 + +# Length of random salt used when generating password hash. +#Provisioning.PasswordCryptSaltLength=10 + # Format if unformatted. If 'n', resource disk will not be mounted. ResourceDisk.Format=n @@ -42,7 +48,7 @@ ResourceDisk.Format=n ResourceDisk.Filesystem=ext4 # Mount point for the resource disk -ResourceDisk.MountPoint=/mnt/resource +ResourceDisk.MountPoint=/mnt # Create and use swapfile on resource disk. ResourceDisk.EnableSwap=n @@ -50,7 +56,7 @@ ResourceDisk.EnableSwap=n # Size of the swapfile. ResourceDisk.SwapSizeMB=0 -# Respond to load balancer probes if requested by Windows Azure. +# Respond to load balancer probes if requested by Microsoft Azure. LBProbeResponder=y # Enable verbose logging (y|n) diff --git a/config/waagent.conf b/config/waagent.conf index 4435d56..639e723 100644 --- a/config/waagent.conf +++ b/config/waagent.conf @@ -1,5 +1,5 @@ # -# Windows Azure Linux Agent Configuration +# Microsoft Azure Linux Agent Configuration # # Specified program is invoked with the argument "Ready" when we report ready status @@ -34,6 +34,12 @@ Provisioning.DecodeCustomData=n # Execute CustomData after provisioning. Provisioning.ExecuteCustomData=n +# Algorithm used by crypt when generating password hash. +#Provisioning.PasswordCryptId=6 + +# Length of random salt used when generating password hash. +#Provisioning.PasswordCryptSaltLength=10 + # Format if unformatted. If 'n', resource disk will not be mounted. ResourceDisk.Format=y diff --git a/debian/changelog b/debian/changelog index 858f787..e0dee1e 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,12 @@ +walinuxagent (2.1.2-0ubuntu1) xenial; urgency=medium + + * New upstream release (LP: #1523715): + - Bug fixes for Ubuntu 15.10 on Azure + - Enablement for Azure Stack + - Dropped patch for systemd job as upstream now includes it. + + -- Ben Howard <ben.howard@ubuntu.com> Mon, 07 Dec 2015 16:48:51 -0700 + walinuxagent (2.1.1-0ubuntu5) xenial; urgency=medium * Disable RH testing. diff --git a/debian/control b/debian/control index 8310d8c..3e561e0 100644 --- a/debian/control +++ b/debian/control @@ -3,7 +3,12 @@ Section: python Priority: extra Maintainer: Ben Howard <ben.howard@ubuntu.com> XSBC-Original-Maintainer: Microsoft Corporation <walinuxagent@microsoft.com> -Build-Depends: debhelper (>= 8), python3-all, python3-setuptools, dh-systemd, python3-pyasn1 +Build-Depends: debhelper (>= 8), + dh-systemd, + python3-all, + python3-setuptools, + python3-pyasn1, + openssl (>=1.0) Standards-Version: 3.9.6 X-Python3-Version: >= 3.2 Homepage: http://go.microsoft.com/fwlink/?LinkId=250998 diff --git a/debian/patches/disable-provisioning.patch b/debian/patches/disable-provisioning.patch deleted file mode 100644 index 716ae12..0000000 --- a/debian/patches/disable-provisioning.patch +++ /dev/null @@ -1,11 +0,0 @@ ---- a/config/ubuntu/waagent.conf -+++ b/config/ubuntu/waagent.conf -@@ -42,7 +42,7 @@ - ResourceDisk.Filesystem=ext4 - - # Mount point for the resource disk --ResourceDisk.MountPoint=/mnt/resource -+ResourceDisk.MountPoint=/mnt - - # Create and use swapfile on resource disk. - ResourceDisk.EnableSwap=n diff --git a/debian/patches/disable_import_test.patch b/debian/patches/disable_import_test.patch index 21b15e0..f7e6b64 100644 --- a/debian/patches/disable_import_test.patch +++ b/debian/patches/disable_import_test.patch @@ -15,3 +15,39 @@ if __name__ == '__main__': unittest.main() +--- a/config/waagent.conf ++++ b/config/waagent.conf +@@ -14,13 +14,13 @@ + Role.TopologyConsumer=None + + # Enable instance creation +-Provisioning.Enabled=y ++Provisioning.Enabled=n + + # Password authentication for root account will be unavailable. +-Provisioning.DeleteRootPassword=y ++Provisioning.DeleteRootPassword=n + + # Generate fresh host key pair. +-Provisioning.RegenerateSshHostKeyPair=y ++Provisioning.RegenerateSshHostKeyPair=n + + # Supported values are "rsa", "dsa" and "ecdsa". + Provisioning.SshHostKeyPairType=rsa +@@ -41,14 +41,14 @@ + #Provisioning.PasswordCryptSaltLength=10 + + # Format if unformatted. If 'n', resource disk will not be mounted. +-ResourceDisk.Format=y ++ResourceDisk.Format=n + + # File system on the resource disk + # Typically ext3 or ext4. FreeBSD images should use 'ufs2' here. + ResourceDisk.Filesystem=ext4 + + # Mount point for the resource disk +-ResourceDisk.MountPoint=/mnt/resource ++ResourceDisk.MountPoint=/mnt + + # Create and use swapfile on resource disk. + ResourceDisk.EnableSwap=n diff --git a/debian/patches/series b/debian/patches/series index 70a6cde..d32f0ed 100644 --- a/debian/patches/series +++ b/debian/patches/series @@ -1,6 +1,4 @@ cloud-init-default-cfg.patch -start-after-cloudinit.patch disable-udev-rules-removal.patch -disable-provisioning.patch disable_rhel_tests.patch disable_import_test.patch diff --git a/debian/patches/start-after-cloudinit.patch b/debian/patches/start-after-cloudinit.patch deleted file mode 100644 index 235b70e..0000000 --- a/debian/patches/start-after-cloudinit.patch +++ /dev/null @@ -1,12 +0,0 @@ ---- a/init/ubuntu/walinuxagent.service -+++ b/init/ubuntu/walinuxagent.service -@@ -1,7 +1,7 @@ - [Unit] - Description=Azure Linux Agent --After=network.target --After=sshd.service -+After=network-online.target cloud-final.service -+Wants=network-online.target sshd.service sshd-keygen.service cloud-final.service - ConditionFileIsExecutable=/usr/sbin/waagent - ConditionPathExists=/etc/waagent.conf - diff --git a/init/suse/waagent b/init/suse/waagent new file mode 100755 index 0000000..b77b0fa --- /dev/null +++ b/init/suse/waagent @@ -0,0 +1,112 @@ +#! /bin/sh +# +# Microsoft Azure Linux Agent sysV init script +# +# Copyright 2013 Microsoft Corporation +# Copyright SUSE LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# /etc/init.d/waagent +# +# and symbolic link +# +# /usr/sbin/rcwaagent +# +# System startup script for the waagent +# +### BEGIN INIT INFO +# Provides: MicrosoftAzureLinuxAgent +# Required-Start: $network sshd +# Required-Stop: $network sshd +# Default-Start: 3 5 +# Default-Stop: 0 1 2 6 +# Description: Start the MicrosoftAzureLinuxAgent +### END INIT INFO + +PYTHON=/usr/bin/python +WAZD_BIN=/usr/sbin/waagent +WAZD_CONF=/etc/waagent.conf +WAZD_PIDFILE=/var/run/waagent.pid + +test -x "$WAZD_BIN" || { echo "$WAZD_BIN not installed"; exit 5; } +test -e "$WAZD_CONF" || { echo "$WAZD_CONF not found"; exit 6; } + +. /etc/rc.status + +# First reset status of this service +rc_reset + +# Return values acc. to LSB for all commands but status: +# 0 - success +# 1 - misc error +# 2 - invalid or excess args +# 3 - unimplemented feature (e.g. reload) +# 4 - insufficient privilege +# 5 - program not installed +# 6 - program not configured +# +# Note that starting an already running service, stopping +# or restarting a not-running service as well as the restart +# with force-reload (in case signalling is not supported) are +# considered a success. + + +case "$1" in + start) + echo -n "Starting MicrosoftAzureLinuxAgent" + ## Start daemon with startproc(8). If this fails + ## the echo return value is set appropriate. + startproc -f ${PYTHON} ${WAZD_BIN} -start + rc_status -v + ;; + stop) + echo -n "Shutting down MicrosoftAzureLinuxAgent" + ## Stop daemon with killproc(8) and if this fails + ## set echo the echo return value. + killproc -p ${WAZD_PIDFILE} ${PYTHON} ${WAZD_BIN} + rc_status -v + ;; + try-restart) + ## Stop the service and if this succeeds (i.e. the + ## service was running before), start it again. + $0 status >/dev/null && $0 restart + rc_status + ;; + restart) + ## Stop the service and regardless of whether it was + ## running or not, start it again. + $0 stop + sleep 1 + $0 start + rc_status + ;; + force-reload|reload) + rc_status + ;; + status) + echo -n "Checking for service MicrosoftAzureLinuxAgent " + ## Check status with checkproc(8), if process is running + ## checkproc will return with exit status 0. + + checkproc -p ${WAZD_PIDFILE} ${PYTHON} ${WAZD_BIN} + rc_status -v + ;; + probe) + ;; + *) + echo "Usage: $0 {start|stop|status|try-restart|restart|force-reload|reload}" + exit 1 + ;; +esac +rc_exit diff --git a/init/ubuntu/walinuxagent b/init/ubuntu/walinuxagent index a17174c..0253202 100644 --- a/init/ubuntu/walinuxagent +++ b/init/ubuntu/walinuxagent @@ -1,2 +1,2 @@ -# To disable the Windows Azure Agent, set WALINUXAGENT_ENABLED=0 +# To disable the Microsoft Azure Agent, set WALINUXAGENT_ENABLED=0 WALINUXAGENT_ENABLED=1 diff --git a/init/ubuntu/walinuxagent.conf b/init/ubuntu/walinuxagent.conf index 2ce6608..331125f 100644 --- a/init/ubuntu/walinuxagent.conf +++ b/init/ubuntu/walinuxagent.conf @@ -1,4 +1,4 @@ -description "Windows Azure Linux agent" +description "Microsoft Azure Linux agent" author "Ben Howard <ben.howard@canonical.com>" start on runlevel [2345] diff --git a/init/ubuntu/walinuxagent.service b/init/ubuntu/walinuxagent.service index 1a67835..681435e 100755 --- a/init/ubuntu/walinuxagent.service +++ b/init/ubuntu/walinuxagent.service @@ -1,7 +1,9 @@ [Unit] Description=Azure Linux Agent -After=network.target -After=sshd.service + +After=network-online.target cloud-final.service +Wants=network-online.target sshd.service sshd-keygen.service cloud-final.service + ConditionFileIsExecutable=/usr/sbin/waagent ConditionPathExists=/etc/waagent.conf @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# Windows Azure Linux Agent setup.py +# Microsoft Azure Linux Agent setup.py # # Copyright 2013 Microsoft Corporation # @@ -22,7 +22,6 @@ from azurelinuxagent.metadata import AGENT_NAME, AGENT_VERSION, \ AGENT_DESCRIPTION, \ DISTRO_NAME, DISTRO_VERSION, DISTRO_FULL_NAME -from azurelinuxagent.utils.osutil import OSUTIL import azurelinuxagent.agent as agent import setuptools from setuptools import find_packages @@ -62,11 +61,14 @@ def get_data_files(name, version, fullname): set_bin_files(data_files) set_conf_files(data_files) set_logrotate_files(data_files) - if version >= "7.0": - #redhat7.0+ uses systemd - set_systemd_files(data_files, dest="/var/lib/systemd/system") - else: + if version.startswith("6"): set_sysv_files(data_files) + else: + #redhat7.0+ use systemd + set_systemd_files(data_files, dest="/usr/lib/systemd/system") + if version.startswith("7.1"): + #TODO this is a mitigation to systemctl bug on 7.1 + set_sysv_files(data_files) elif name == 'coreos': set_bin_files(data_files, dest="/usr/share/oem/bin") @@ -78,8 +80,8 @@ def get_data_files(name, version, fullname): set_bin_files(data_files) set_conf_files(data_files, src=["config/ubuntu/waagent.conf"]) set_logrotate_files(data_files) - if version < "15.04": - #Ubuntu15.04- uses upstart + if version.startswith("12") or version.startswith("14"): + #Ubuntu12.04/14.04 - uses upstart set_files(data_files, dest="/etc/init", src=["init/ubuntu/walinuxagent.conf"]) set_files(data_files, dest='/etc/default', @@ -88,17 +90,21 @@ def get_data_files(name, version, fullname): set_files(data_files, dest="<TODO>", src=["init/ubuntu/snappy/walinuxagent.yml"]) else: + #Ubuntu15.04+ uses systemd set_systemd_files(data_files, src=["init/ubuntu/walinuxagent.service"]) elif name == 'suse': set_bin_files(data_files) set_conf_files(data_files, src=["config/suse/waagent.conf"]) set_logrotate_files(data_files) - if fullname == 'SUSE Linux Enterprise Server' and version >= '12' or \ - fullname == 'openSUSE' and version >= '13.2': - set_systemd_files(data_files, dest='/var/lib/systemd/system') + if fullname == 'SUSE Linux Enterprise Server' and \ + version.startswith('11') or \ + fullname == 'openSUSE' and version.startswith('13.1'): + set_sysv_files(data_files, dest='/etc/init.d', + src=["init/suse/waagent"]) else: - set_sysv_files(data_files, dest='/etc/init.d') + #sles 12+ and openSUSE 13.2+ use systemd + set_systemd_files(data_files, dest='/usr/lib/systemd/system') else: #Use default setting set_bin_files(data_files) diff --git a/snappy/bin/waagent b/snappy/bin/waagent new file mode 100755 index 0000000..e65bc0c --- /dev/null +++ b/snappy/bin/waagent @@ -0,0 +1,49 @@ +#!/usr/bin/env python +# +# Azure Linux Agent +# +# Copyright 2015 Microsoft Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Requires Python 2.6+ and Openssl 1.0+ +# +# Implements parts of RFC 2131, 1541, 1497 and +# http://msdn.microsoft.com/en-us/library/cc227282%28PROT.10%29.aspx +# http://msdn.microsoft.com/en-us/library/cc227259%28PROT.13%29.aspx +# + +import os +import imp +import sys + +if __name__ == '__main__' : + import azurelinuxagent.agent as agent + """ + Invoke main method of agent + """ + agent.main() + +if __name__ == 'waagent': + """ + Load waagent2.0 to support old version of extensions + """ + if sys.version_info[0] == 3: + raise ImportError("waagent2.0 doesn't support python3") + bin_path = os.path.dirname(os.path.abspath(__file__)) + agent20_path = os.path.join(bin_path, "waagent2.0") + if not os.path.isfile(agent20_path): + raise ImportError("Can't load waagent") + agent20 = imp.load_source('waagent', agent20_path) + __all__ = dir(agent20) + diff --git a/snappy/bin/waagent.start b/snappy/bin/waagent.start new file mode 100755 index 0000000..ec71224 --- /dev/null +++ b/snappy/bin/waagent.start @@ -0,0 +1,5 @@ +#!/bin/bash + +export PYTHONPATH="${PATH}:${SNAP_APP_PATH}/lib" + +python3 ${SNAP_APP_PATH}/bin/waagent -daemon diff --git a/snappy/lib/readme.md b/snappy/lib/readme.md new file mode 100644 index 0000000..c91a79c --- /dev/null +++ b/snappy/lib/readme.md @@ -0,0 +1 @@ +Copy the azurelinuxagent folder here diff --git a/snappy/meta/package.yaml b/snappy/meta/package.yaml new file mode 100644 index 0000000..85e4835 --- /dev/null +++ b/snappy/meta/package.yaml @@ -0,0 +1,10 @@ +name: walinuxagent +version: 2.1.1 +vendor: Microsoft Corporation <lizzha@microsoft.com> +type: framework +services: + - name: walinuxagent + start: bin/waagent.start + security-policy: + apparmor: meta/walinuxagent.apparmor + seccomp: meta/walinuxagent.seccomp diff --git a/snappy/meta/readme.md b/snappy/meta/readme.md new file mode 100644 index 0000000..33a7768 --- /dev/null +++ b/snappy/meta/readme.md @@ -0,0 +1,2 @@ +Microsoft Azure Linux Agent Snap Framework + diff --git a/snappy/meta/walinuxagent.apparmor b/snappy/meta/walinuxagent.apparmor new file mode 100644 index 0000000..8315713 --- /dev/null +++ b/snappy/meta/walinuxagent.apparmor @@ -0,0 +1,85 @@ +# AppArmor confinement for waagent + +#include <tunables/global> + +# Specified profile variables +###VAR### + +###PROFILEATTACH### flags=(attach_disconnected) { + #include <abstractions/base> + #include <abstractions/ssl_certs> + #include <abstractions/openssl> + #include <abstractions/python> + + # Executable binaries + /usr/{,s}bin/* ixr, + /{,s}bin/* ixr, + + # Capabilities + capability net_bind_service, + capability net_raw, + capability net_admin, + capability dac_override, + capability sys_module, + capability sys_admin, + capability sys_ptrace, + + ptrace (read), + ptrace (trace), + + mount, + umount, + network, + + # Log path + /var/log/waagent.log rw, + /var/log/azure/ rw, + /var/log/azure/** rw, + + # Lib path + /var/lib/waagent/ rw, + /var/lib/waagent/** mrwlk, + # Enable VM extensions to execute unconfined + /var/lib/waagent/** PUx, + /{,usr/}lib/ r, + /{,usr/}lib/** r, + + /etc/ r, + /etc/** r, + /etc/udev/rules.d/** w, + + /usr/share/ r, + /usr/share/** r, + /usr/local/{,s}bin/ r, + /usr/{,s}bin/ r, + /{,s}bin/ r, + + /dev/ r, + /dev/sr0 r, + /dev/null w, + /dev/console rw, + /dev/tty rw, + + /run/ r, + /run/** r, + /run/mount/utab w, + /run/waagent.pid w, + + @{PROC}/ r, + @{PROC}/** r, + + /sys/module/ r, + /sys/module/** r, + /sys/firmware/acpi/tables/** r, + /sys/block/ r, + /sys/block/sd*/device/timeout rw, + /sys/devices/** rw, + + /mnt/cdrom/ rw, + /mnt/cdrom/secure/ rw, + + # Writable for the install directory + @{CLICK_DIR}/@{APP_PKGNAME}/ r, + @{CLICK_DIR}/@{APP_PKGNAME}/@{APP_VERSION}/ r, + @{CLICK_DIR}/@{APP_PKGNAME}/@{APP_VERSION}/** mrwklix, +} diff --git a/snappy/meta/walinuxagent.seccomp b/snappy/meta/walinuxagent.seccomp new file mode 100644 index 0000000..90fbc81 --- /dev/null +++ b/snappy/meta/walinuxagent.seccomp @@ -0,0 +1 @@ +@unrestricted diff --git a/snappy/readme.md b/snappy/readme.md new file mode 100644 index 0000000..348895e --- /dev/null +++ b/snappy/readme.md @@ -0,0 +1,17 @@ +### Building the walinuxagent snap package + +1. You will need the snappy developer tools on your Ubuntu Developer Desktop: + + $ sudo add-apt-repository ppa:snappy-dev/tools + $ sudo apt-get update + $ sudo apt-get upgrade + $ sudo apt-get install snappy-tools + +2. Copy the azurelinuxagent folder to snappy/lib + + $ cp -rf azurelinuxagent snappy/lib + +3. Build the snap package under the snappy folder + + $ cd snappy + $ snappy build
\ No newline at end of file diff --git a/snappy/waagent.conf b/snappy/waagent.conf new file mode 100644 index 0000000..74dccb8 --- /dev/null +++ b/snappy/waagent.conf @@ -0,0 +1,70 @@ +# +# Microsoft Azure Linux Agent Configuration +# + +# Specified program is invoked with the argument "Ready" when we report ready status +# to the endpoint server. +Role.StateConsumer=None + +# Specified program is invoked with XML file argument specifying role +# configuration. +Role.ConfigurationConsumer=None + +# Specified program is invoked with XML file argument specifying role topology. +Role.TopologyConsumer=None + +# Enable instance creation +Provisioning.Enabled=n + +# Password authentication for root account will be unavailable. +Provisioning.DeleteRootPassword=y + +# Generate fresh host key pair. +Provisioning.RegenerateSshHostKeyPair=n + +# Supported values are "rsa", "dsa" and "ecdsa". +Provisioning.SshHostKeyPairType=rsa + +# Monitor host name changes and publish changes via DHCP requests. +Provisioning.MonitorHostName=n + +# Decode CustomData from Base64. +Provisioning.DecodeCustomData=n + +# Execute CustomData after provisioning. +Provisioning.ExecuteCustomData=n + +# Format if unformatted. If 'n', resource disk will not be mounted. +ResourceDisk.Format=n + +# File system on the resource disk +# Typically ext3 or ext4. FreeBSD images should use 'ufs2' here. +ResourceDisk.Filesystem=ext4 + +# Mount point for the resource disk +ResourceDisk.MountPoint=/mnt/resource + +# Create and use swapfile on resource disk. +ResourceDisk.EnableSwap=n + +# Size of the swapfile. +ResourceDisk.SwapSizeMB=0 + +# Respond to load balancer probes if requested by Microsoft Azure. +LBProbeResponder=y + +# Enable verbose logging (y|n) +Logs.Verbose=n + +# Root device timeout in seconds. +OS.RootDeviceScsiTimeout=300 + +# If "None", the system default version is used. +OS.OpensslPath=None + +# If set, agent will use proxy server to access internet +#HttpProxy.Host=None +#HttpProxy.Port=None + +# Detect Scvmm environment, default is n +# DetectScvmmEnv=n diff --git a/tests/test_certificates.py b/tests/test_certificates.py index 2950c6b..18f28c4 100644 --- a/tests/test_certificates.py +++ b/tests/test_certificates.py @@ -184,7 +184,8 @@ class TestCertificates(unittest.TestCase): transport_cert) fileutil.write_file(os.path.join('/tmp', "TransportPrivate.pem"), transport_private) - config = v1.Certificates(certs_sample) + client = v1.WireClient("http://foo.bar") + config = v1.Certificates(client, certs_sample) self.assertNotEquals(None, config) self.assertTrue(os.path.isfile(crt1)) self.assertTrue(os.path.isfile(crt2)) diff --git a/tests/test_datacontract.py b/tests/test_datacontract.py new file mode 100644 index 0000000..4d4edd7 --- /dev/null +++ b/tests/test_datacontract.py @@ -0,0 +1,54 @@ +# Copyright 2014 Microsoft Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Requires Python 2.4+ and Openssl 1.0+ +# +# Implements parts of RFC 2131, 1541, 1497 and +# http://msdn.microsoft.com/en-us/library/cc227282%28PROT.10%29.aspx +# http://msdn.microsoft.com/en-us/library/cc227259%28PROT.13%29.aspx + +import tests.env +from tests.tools import * +import uuid +import unittest +import os +import shutil +import time +from azurelinuxagent.protocol.common import * + +class SampleDataContract(DataContract): + def __init__(self): + self.foo = None + self.bar = DataContractList(int) + +class TestDataContract(unittest.TestCase): + def test_get_properties(self): + obj = SampleDataContract() + obj.foo = "foo" + obj.bar.append(1) + data = get_properties(obj) + self.assertEquals("foo", data["foo"]) + self.assertEquals(list, type(data["bar"])) + + def test_set_properties(self): + obj = SampleDataContract() + data = { + 'foo' : 1, + 'baz': 'a' + } + set_properties('sample', obj, data) + self.assertFalse(hasattr(obj, 'baz')) + +if __name__ == '__main__': + unittest.main() diff --git a/tests/test_ext.py b/tests/test_ext.py index 126c0ec..a68a851 100644 --- a/tests/test_ext.py +++ b/tests/test_ext.py @@ -51,8 +51,8 @@ ext_sample_json = { }] } } -ext_sample = prot.Extension() -prot.set_properties(ext_sample, ext_sample_json) +ext_sample = prot.ExtHandler() +prot.set_properties("extensions", ext_sample, ext_sample_json) pkd_list_sample_str={ "versions": [{ @@ -67,8 +67,8 @@ pkd_list_sample_str={ }] }] } -pkg_list_sample = prot.ExtensionPackageList() -prot.set_properties(pkg_list_sample, pkd_list_sample_str) +pkg_list_sample = prot.ExtHandlerPackageList() +prot.set_properties("packages", pkg_list_sample, pkd_list_sample_str) manifest_sample_str = { "handlerManifest":{ @@ -81,11 +81,50 @@ manifest_sample_str = { } manifest_sample = ext.HandlerManifest(manifest_sample_str) +ext_status_sample=""" +[{ + "version": 1.0, + "timestampUTC": "2015-11-12T06:59:48Z", + "status": { + "name": "<Handler workload name>", + "operation": "<name of the operation being performed>", + "configurationAppliedTime": "2015-11-12T06:59:48Z", + "status": "error", + "code": 0, + "formattedMessage": { + "lang": "en-US", + "message": "formatted user message" + }, + "substatus": [{ + "name": "<Handler workload subcomponent name>", + "status": "error", + "code": 0 , + "formattedMessage": { + "lang": "lang[-locale]", + "message": "formatted user message" + } + },{ + "status": "error" + }] + } +}] +""" + +ext_status_sample_min=""" +[{ + "version": 1.0, + "timestampUTC": "2015-11-12T06:59:48Z", + "status": { + "status": "error" + } +}] +""" + def mock_load_manifest(self): return manifest_sample mock_launch_command = MockFunc() -mock_set_handler_status = MockFunc() +mock_set_state = MockFunc() def mock_download(self): fileutil.mkdir(self.get_base_dir()) @@ -106,14 +145,16 @@ class TestExtensions(unittest.TestCase): self.assertEqual('2.1', test_ext) def test_getters(self): - test_ext = ext.ExtensionInstance(ext_sample, pkg_list_sample, - ext_sample.properties.version, False) + test_ext = ext.ExtHandlerInstance(ext_sample, pkg_list_sample, + ext_sample.properties.version, False) self.assertEqual("/tmp/TestExt-2.0", test_ext.get_base_dir()) self.assertEqual("/tmp/TestExt-2.0/status", test_ext.get_status_dir()) self.assertEqual("/tmp/TestExt-2.0/status/0.status", test_ext.get_status_file()) - self.assertEqual("/tmp/TestExt-2.0/config/HandlerState", + self.assertEqual("/tmp/handler_state/TestExt-2.0/0.state", test_ext.get_handler_state_file()) + self.assertEqual("/tmp/handler_state/TestExt-2.0/0.error", + test_ext.get_handler_state_err_file()) self.assertEqual("/tmp/TestExt-2.0/config", test_ext.get_conf_dir()) self.assertEqual("/tmp/TestExt-2.0/config/0.settings", test_ext.get_settings_file()) @@ -125,53 +166,57 @@ class TestExtensions(unittest.TestCase): test_ext.get_env_file()) self.assertEqual("/tmp/log/TestExt/2.0", test_ext.get_log_dir()) - test_ext = ext.ExtensionInstance(ext_sample, pkg_list_sample, "2.1", False) + test_ext = ext.ExtHandlerInstance(ext_sample, pkg_list_sample, + "2.1", False) self.assertEqual("/tmp/TestExt-2.1", test_ext.get_base_dir()) self.assertEqual("2.1", test_ext.get_target_version()) - @mock(ext.ExtensionInstance, 'load_manifest', mock_load_manifest) - @mock(ext.ExtensionInstance, 'launch_command', mock_launch_command) - @mock(ext.ExtensionInstance, 'set_handler_status', mock_set_handler_status) + @mock(ext.ExtHandlerInstance, 'load_manifest', mock_load_manifest) + @mock(ext.ExtHandlerInstance, 'launch_command', mock_launch_command) + @mock(ext.ExtHandlerInstance, 'set_state', mock_set_state) def test_handle_uninstall(self): mock_launch_command.args = None - mock_set_handler_status.args = None - test_ext = ext.ExtensionInstance(ext_sample, pkg_list_sample, - ext_sample.properties.version, False) + mock_set_state.args = None + test_ext = ext.ExtHandlerInstance(ext_sample, pkg_list_sample, + ext_sample.properties.version, False) + if not os.path.isdir(test_ext.get_base_dir()): + os.makedirs(test_ext.get_base_dir()) test_ext.handle_uninstall() self.assertEqual(None, mock_launch_command.args) - self.assertEqual(None, mock_set_handler_status.args) - self.assertEqual(None, test_ext.get_curr_op()) + self.assertEqual(None, mock_set_state.args) - test_ext = ext.ExtensionInstance(ext_sample, pkg_list_sample, - ext_sample.properties.version, True) + test_ext = ext.ExtHandlerInstance(ext_sample, pkg_list_sample, + ext_sample.properties.version, True) + if not os.path.isdir(test_ext.get_base_dir()): + os.makedirs(test_ext.get_base_dir()) test_ext.handle_uninstall() - self.assertEqual(manifest_sample.get_uninstall_command(), mock_launch_command.args[0]) - self.assertEqual("UnInstall", test_ext.get_curr_op()) - self.assertEqual("NotReady", mock_set_handler_status.args[0]) - - @mock(ext.ExtensionInstance, 'load_manifest', mock_load_manifest) - @mock(ext.ExtensionInstance, 'launch_command', mock_launch_command) - @mock(ext.ExtensionInstance, 'download', mock_download) - @mock(ext.ExtensionInstance, 'get_handler_status', MockFunc(retval="enabled")) - @mock(ext.ExtensionInstance, 'set_handler_status', mock_set_handler_status) - def test_handle(self): + self.assertEqual(manifest_sample.get_uninstall_command(), + mock_launch_command.args[0]) + + @mock(ext.ExtHandlerInstance, 'upgrade', MockFunc()) + @mock(ext.ExtHandlerInstance, 'enable', MockFunc()) + @mock(ext.ExtHandlerInstance, 'download', MockFunc()) + @mock(ext.ExtHandlerInstance, 'init_dir', MockFunc()) + @mock(ext.ExtHandlerInstance, 'install', MockFunc()) + def test_handle_enable(self): #Test enable - test_ext = ext.ExtensionInstance(ext_sample, pkg_list_sample, - ext_sample.properties.version, False) - test_ext.init_logger() - self.assertEqual(1, len(test_ext.logger.appenders) - len(logger.DEFAULT_LOGGER.appenders)) - test_ext.handle() - + test_ext = ext.ExtHandlerInstance(ext_sample, pkg_list_sample, + ext_sample.properties.version, False) + test_ext.handle_enable() + #Test upgrade - test_ext = ext.ExtensionInstance(ext_sample, pkg_list_sample, - ext_sample.properties.version, False) - test_ext.init_logger() - self.assertEqual(1, len(test_ext.logger.appenders) - len(logger.DEFAULT_LOGGER.appenders)) - test_ext.handle() + test_ext = ext.ExtHandlerInstance(ext_sample, pkg_list_sample, + "2.0" , True) + test_ext.handle_enable() def test_status_convert(self): - ext_status = json.loads('[{"status": {"status": "success", "formattedMessage": {"lang": "en-US", "message": "Script is finished"}, "operation": "Enable", "code": "0", "name": "Microsoft.OSTCExtensions.CustomScriptForLinux"}, "version": "1.0", "timestampUTC": "2015-06-27T08:34:50Z"}]') - ext.ext_status_to_v2(ext_status[0], 0) + data = json.loads(ext_status_sample) + ext_status = prot.ExtensionStatus() + ext.parse_ext_status(ext_status, data) + + data = json.loads(ext_status_sample_min) + ext_status = prot.ExtensionStatus() + ext.parse_ext_status(ext_status, data) if __name__ == '__main__': diff --git a/tests/test_extensionsconfig.py b/tests/test_extensionsconfig.py index 4dbf30d..505c73d 100644 --- a/tests/test_extensionsconfig.py +++ b/tests/test_extensionsconfig.py @@ -128,7 +128,7 @@ EmptyPublicSettings=u"""\ class TestExtensionsConfig(unittest.TestCase): def test_extensions_config(self): config = v1.ExtensionsConfig(ext_conf_sample) - extensions = config.ext_list.extensions + extensions = config.ext_handlers.extHandlers self.assertNotEquals(None, extensions) self.assertEquals(1, len(extensions)) self.assertNotEquals(None, extensions[0]) diff --git a/tests/test_logger.py b/tests/test_logger.py index 121a8fe..20e9259 100644 --- a/tests/test_logger.py +++ b/tests/test_logger.py @@ -1,3 +1,4 @@ +# -*- coding: utf-8 -*- # Copyright 2014 Microsoft Corporation # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -30,7 +31,7 @@ class TestLogger(unittest.TestCase): def test_no_appender(self): #The logger won't throw exception even if no appender. _logger = logger.Logger() - _logger.verbose("Assert no exception") + _logger.verb("Assert no exception") _logger.info("Assert no exception") _logger.warn("Assert no exception") _logger.error("Assert no exception") @@ -40,14 +41,14 @@ class TestLogger(unittest.TestCase): _logger.info("This is an exception {0}", Exception("Test")) _logger.info("This is an number {0}", 0) _logger.info("This is an boolean {0}", True) - _logger.verbose("{0}") - _logger.verbose("{0} {1}", 0, 1) + _logger.verb("{0}") + _logger.verb("{0} {1}", 0, 1) _logger.info("{0} {1}", 0, 1) _logger.warn("{0} {1}", 0, 1) _logger.error("{0} {1}", 0, 1) - _logger.info("this is a unicode {0}", '\u6211') - _logger.info("this is a utf-8 {0}", '\u6211'.encode('utf-8')) - _logger.info("this is a gbk {0}", 0xff ) + _logger.add_appender(logger.AppenderType.STDOUT, + logger.LogLevel.INFO, None) + _logger.info(u"啊哈this is a utf-8 {0}", u'呵呵') def test_file_appender(self): _logger = logger.Logger() @@ -60,12 +61,9 @@ class TestLogger(unittest.TestCase): self.assertTrue(tools.simple_file_grep('/tmp/testlog', msg)) msg = text(uuid.uuid4()) - _logger.verbose("Verbose should not be logged: {0}", msg) + _logger.verb("Verbose should not be logged: {0}", msg) self.assertFalse(tools.simple_file_grep('/tmp/testlog', msg)) - _logger.info("this is a unicode {0}", '\u6211') - _logger.info("this is a utf-8 {0}", '\u6211'.encode('utf-8')) - _logger.info("this is a gbk {0}", 0xff) def test_concole_appender(self): _logger = logger.Logger() @@ -78,7 +76,7 @@ class TestLogger(unittest.TestCase): self.assertTrue(tools.simple_file_grep('/tmp/testlog', msg)) msg = text(uuid.uuid4()) - _logger.verbose("Test logger: {0}", msg) + _logger.verb("Test logger: {0}", msg) self.assertFalse(tools.simple_file_grep('/tmp/testlog', msg)) diff --git a/tests/test_protocol.py b/tests/test_protocol.py index 3eff197..de74443 100644 --- a/tests/test_protocol.py +++ b/tests/test_protocol.py @@ -19,8 +19,7 @@ # http://msdn.microsoft.com/en-us/library/cc227259%28PROT.13%29.aspx import tests.env -import tests.tools as tools -from .tools import * +from tests.tools import * import uuid import unittest import os @@ -33,29 +32,15 @@ extensionDataStr = """ "vmAgent": { "agentVersion": "2.4.1198.689", "status": "Ready", - "message": "GuestAgent is running and accepting new configurations." - }, - "extensionHandlers": [{ - "handlerName": "Microsoft.Compute.CustomScript", - "handlerVersion": "1.0.0.0", + "message": "GuestAgent is running and accepting new configurations.", + "extensionHandlers": [{ + "name": "Microsoft.Compute.CustomScript", + "version": "1.0.0.0", "status": "Ready", "message": "Plugin enabled (name: Microsoft.Compute.CustomScript, version: 1.0.0.0).", - "extensionStatusList": [{ - "name": "MyDomainJoinScript", - "configurationAppliedTime": "2014-08-12T19:20:18Z", - "operation": "CommandExecutionFinished", - "status": "Success", - "sequenceNumber": "0", - "substatusList": [{ - "name": "StdOut", - "status": "Info", - "code": "0", - "message": "Joiningdomainfoo" - }] - }] - } - - ] + "extensions": [] + }] + } } """ @@ -63,25 +48,24 @@ class TestProtocolContract(unittest.TestCase): def test_get_properties(self): data = get_properties(VMInfo()) data = get_properties(Cert()) - data = get_properties(ExtensionPackageList()) - data = get_properties(InstanceMetadata()) + data = get_properties(ExtHandlerPackageList()) data = get_properties(VMStatus()) data = get_properties(TelemetryEventList()) - data = get_properties(Extension(name="hehe")) + data = get_properties(ExtHandler(name="hehe")) self.assertTrue("name" in data) self.assertTrue("properties" in data) self.assertEquals(dict, type(data["properties"])) - self.assertTrue("versionUris" not in data) + self.assertTrue("versionUris" in data) def test_set_properties(self): data = json.loads(extensionDataStr) obj = VMStatus() - set_properties(obj, data) + set_properties("vmStatus", obj, data) self.assertNotEquals(None, obj.vmAgent) self.assertEquals(VMAgentStatus, type(obj.vmAgent)) self.assertNotEquals(None, obj.vmAgent.status) - self.assertNotEquals(None, obj.extensionHandlers) - self.assertEquals(DataContractList, type(obj.extensionHandlers)) + self.assertNotEquals(None, obj.vmAgent.extensionHandlers) + self.assertEquals(DataContractList, type(obj.vmAgent.extensionHandlers)) if __name__ == '__main__': unittest.main() diff --git a/tests/test_shell_util.py b/tests/test_shell_util.py index 9862745..9f84c6d 100644 --- a/tests/test_shell_util.py +++ b/tests/test_shell_util.py @@ -1,3 +1,4 @@ +# -*- coding: utf-8 -*- # Copyright 2014 Microsoft Corporation # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -25,15 +26,19 @@ import unittest import os import azurelinuxagent.utils.shellutil as shellutil import test +from azurelinuxagent.future import text class TestrunCmd(unittest.TestCase): def test_run_get_output(self): - output = shellutil.run_get_output("ls /") + output = shellutil.run_get_output(u"ls /") self.assertNotEquals(None, output) self.assertEquals(0, output[0]) - err = shellutil.run_get_output("ls /not-exists") + err = shellutil.run_get_output(u"ls /not-exists") self.assertNotEquals(0, err[0]) + err = shellutil.run_get_output(u"ls 我") + self.assertNotEquals(0, err[0]) + if __name__ == '__main__': unittest.main() diff --git a/tests/test_text_util.py b/tests/test_text_util.py index b29beff..5c0016c 100644 --- a/tests/test_text_util.py +++ b/tests/test_text_util.py @@ -25,10 +25,13 @@ import unittest import os from azurelinuxagent.future import text import azurelinuxagent.utils.textutil as textutil +from azurelinuxagent.utils.textutil import Version class TestTextUtil(unittest.TestCase): def test_get_password_hash(self): - password_hash = textutil.gen_password_hash("asdf", True, 6, 10) + password_hash = textutil.gen_password_hash("asdf", 6, 10) + self.assertNotEquals(None, password_hash) + password_hash = textutil.gen_password_hash("asdf", 6, 0) self.assertNotEquals(None, password_hash) def test_remove_bom(self): @@ -42,6 +45,22 @@ class TestTextUtil(unittest.TestCase): data = textutil.remove_bom(data) self.assertEquals(u"h", data[0]) - + def test_version_compare(self) : + self.assertTrue(Version("1.0") < Version("1.1")) + self.assertTrue(Version("1.9") < Version("1.10")) + self.assertTrue(Version("1.9.9") < Version("1.10.0")) + self.assertTrue(Version("1.0.0.0") < Version("1.2.0.0")) + + self.assertTrue(Version("1.0") <= Version("1.1")) + self.assertTrue(Version("1.1") > Version("1.0")) + self.assertTrue(Version("1.1") >= Version("1.0")) + + self.assertTrue(Version("1.0") == Version("1.0")) + self.assertTrue(Version("1.0") >= Version("1.0")) + self.assertTrue(Version("1.0") <= Version("1.0")) + + self.assertTrue(Version("1.9") < "1.10") + self.assertTrue("1.9" < Version("1.10")) + if __name__ == '__main__': unittest.main() diff --git a/tests/test_v1.py b/tests/test_v1.py index 02225af..5a1b36b 100644 --- a/tests/test_v1.py +++ b/tests/test_v1.py @@ -35,7 +35,7 @@ from tests.test_sharedconfig import shared_config_sample from tests.test_certificates import certs_sample, transport_cert from tests.test_extensionsconfig import ext_conf_sample, manifest_sample -def mock_fetch_uri(url, headers=None, chk_proxy=False): +def mock_fetch_config(self, url, headers=None, chk_proxy=False): content = None if "versions" in url: content = VersionInfoSample @@ -55,10 +55,10 @@ def mock_fetch_uri(url, headers=None, chk_proxy=False): raise Exception("Bad url {0}".format(url)) return content -def mock_fetch_manifest(uris): +def mock_fetch_manifest(self, uris): return manifest_sample -def mock_fetch_cache(file_path): +def mock_fetch_cache(self, file_path): content = None if "Incarnation" in file_path: content = 1 @@ -90,13 +90,23 @@ class MockResp(object): def read(self): return self.data +def mock_403(): + return MockResp(status = v1.httpclient.FORBIDDEN) + +def mock_410(): + return MockResp(status = v1.httpclient.GONE) + +def mock_503(): + return MockResp(status = v1.httpclient.SERVICE_UNAVAILABLE) + class TestWireClint(unittest.TestCase): @mock(v1.restutil, 'http_get', MockFunc(retval=MockResp(data=data_with_bom))) def test_fetch_uri_with_bom(self): - v1._fetch_uri("http://foo.bar", None) + client = v1.WireClient("http://foo.bar/") + client.fetch_config("http://foo.bar", None) - @mock(v1, '_fetch_cache', mock_fetch_cache) + @mock(v1.WireClient, 'fetch_cache', mock_fetch_cache) def test_get(self): os.chdir('/tmp') client = v1.WireClient("foobar") @@ -110,14 +120,15 @@ class TestWireClint(unittest.TestCase): self.assertNotEquals(None, extensionsConfig) - @mock(v1, '_fetch_cache', mock_fetch_cache) + @mock(v1.WireClient, 'fetch_cache', mock_fetch_cache) def test_get_head_for_cert(self): client = v1.WireClient("foobar") header = client.get_header_for_cert() self.assertNotEquals(None, header) @mock(v1.WireClient, 'get_header_for_cert', MockFunc()) - @mock(v1, '_fetch_uri', mock_fetch_uri) + @mock(v1.WireClient, 'fetch_config', mock_fetch_config) + @mock(v1.WireClient, 'fetch_manifest', mock_fetch_manifest) @mock(v1.fileutil, 'write_file', MockFunc()) def test_update_goal_state(self): client = v1.WireClient("foobar") @@ -131,35 +142,54 @@ class TestWireClint(unittest.TestCase): ext_conf = client.get_ext_conf() self.assertNotEquals(None, ext_conf) + @mock(v1.time, "sleep", MockFunc()) + def test_call_wireserver(self): + client = v1.WireClient("foobar") + self.assertRaises(v1.ProtocolError, client.call_wireserver, mock_403) + self.assertRaises(v1.WireProtocolResourceGone, client.call_wireserver, + mock_410) + + @mock(v1.time, "sleep", MockFunc()) + def test_call_storage_service(self): + client = v1.WireClient("foobar") + self.assertRaises(v1.ProtocolError, client.call_storage_service, + mock_503) + + class TestStatusBlob(unittest.TestCase): def testToJson(self): vm_status = v1.VMStatus() - status_blob = v1.StatusBlob(vm_status) + status_blob = v1.StatusBlob(v1.WireClient("http://foo.bar/")) + status_blob.set_vm_status(vm_status) self.assertNotEquals(None, status_blob.to_json()) @mock(v1.restutil, 'http_put', MockFunc(retval=MockResp(httpclient.CREATED))) @mock(v1.restutil, 'http_head', MockFunc(retval=MockResp(httpclient.OK))) def test_put_page_blob(self): vm_status = v1.VMStatus() - status_blob = v1.StatusBlob(vm_status) + status_blob = v1.StatusBlob(v1.WireClient("http://foo.bar/")) + status_blob.set_vm_status(vm_status) data = 'a' * 100 status_blob.put_page_blob("http://foo.bar", data) class TestConvert(unittest.TestCase): def test_status(self): vm_status = v1.VMStatus() - handler_status = v1.ExtensionHandlerStatus() - substatus = v1.ExtensionSubStatus() - ext_status = v1.ExtensionStatus() + handler_status = v1.ExtHandlerStatus(name="foo") - vm_status.extensionHandlers.append(handler_status) - v1.vm_status_to_v1(vm_status) + ext_statuses = {} - handler_status.extensionStatusList.append(ext_status) - v1.vm_status_to_v1(vm_status) + ext_name="bar" + ext_status = v1.ExtensionStatus() + handler_status.extensions.append(ext_name) + ext_statuses[ext_name] = ext_status + substatus = v1.ExtensionSubStatus() ext_status.substatusList.append(substatus) - v1.vm_status_to_v1(vm_status) + + vm_status.vmAgent.extensionHandlers.append(handler_status) + v1_status = v1.vm_status_to_v1(vm_status, ext_statuses) + print(v1_status) def test_param(self): param = v1.TelemetryEventParam() diff --git a/tests/test_v2.py b/tests/test_v2.py new file mode 100644 index 0000000..c4a0b4d --- /dev/null +++ b/tests/test_v2.py @@ -0,0 +1,120 @@ +# Copyright 2014 Microsoft Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Requires Python 2.4+ and Openssl 1.0+ +# +# Implements parts of RFC 2131, 1541, 1497 and +# http://msdn.microsoft.com/en-us/library/cc227282%28PROT.10%29.aspx +# http://msdn.microsoft.com/en-us/library/cc227259%28PROT.13%29.aspx + +import tests.env +from tests.tools import * +import unittest +import json +import azurelinuxagent.protocol.v2 as v2 + +SAMPLE_IDENTITY=u"""{ + "vmName":"foo", + "subscriptionId":"bar" +}""" + +SAMPLE_CERTS=u"""{ + "certificates":[{ + "name":"foo", + "thumbprint":"bar", + "certificateDataUri":"baz" + }] +}""" + +SAMPLE_EXT_HANDLER=u"""[{ + "name":"foo", + "properties":{ + "version":"bar", + "upgradePolicy": "manual", + "state": "enabled", + "extensions":[{ + "name":"baz", + "sequenceNumber":0, + "publicSettings":{ + "commandToExecute": "echo 123", + "uris":[] + } + }] + }, + "versionUris":[{ + "uri":"versionUri.foo" + }] +}]""" + +SAMPLE_EXT_HANDLER_PKGS=u"""{ + "versions": [{ + "version":"foo", + "uris":[{ + "uri":"bar" + },{ + "uri":"baz" + }] + }] +}""" + +def mock_get_data(self, url, headers=None): + data = u"{}" + if url.count(u"identity") > 0: + data = SAMPLE_IDENTITY + elif url.count(u"certificates") > 0: + data = SAMPLE_CERTS + elif url.count(u"extensionHandlers") > 0: + data = SAMPLE_EXT_HANDLER + elif url.count(u"versionUri") > 0: + data = SAMPLE_EXT_HANDLER_PKGS + return json.loads(data) + +class TestMetadataProtocol(unittest.TestCase): + @mock(v2.MetadataProtocol, '_get_data', mock_get_data) + def test_getters(self): + protocol = v2.MetadataProtocol() + vminfo = protocol.get_vminfo() + self.assertNotEquals(None, vminfo) + self.assertNotEquals(None, vminfo.vmName) + self.assertNotEquals(None, vminfo.subscriptionId) + + protocol.get_certs() + + ext_handers = protocol.get_ext_handlers() + self.assertNotEquals(None, ext_handers) + self.assertNotEquals(None, ext_handers.extHandlers) + self.assertNotEquals(0, len(ext_handers.extHandlers)) + + ext_hander = ext_handers.extHandlers[0] + self.assertNotEquals(None, ext_hander) + self.assertNotEquals(0, len(ext_hander.properties.extensions)) + + ext = ext_hander.properties.extensions[0] + self.assertNotEquals(None, ext) + self.assertNotEquals(None, ext.publicSettings) + self.assertEquals("echo 123", ext.publicSettings.get('commandToExecute')) + + packages = protocol.get_ext_handler_pkgs(ext_handers.extHandlers[0]) + self.assertNotEquals(None, packages) + + @mock(v2.MetadataProtocol, '_put_data', MockFunc()) + def test_reporters(self): + protocol = v2.MetadataProtocol() + protocol.report_provision_status(v2.ProvisionStatus()) + protocol.report_vm_status(v2.VMStatus()) + protocol.report_ext_status("foo", "baz", v2.ExtensionStatus()) + protocol.report_event(v2.TelemetryEventList()) + +if __name__ == '__main__': + unittest.main() |