diff options
Diffstat (limited to 'azurelinuxagent')
32 files changed, 1279 insertions, 410 deletions
diff --git a/azurelinuxagent/agent.py b/azurelinuxagent/agent.py index bd6dd20..90b4253 100644 --- a/azurelinuxagent/agent.py +++ b/azurelinuxagent/agent.py @@ -21,6 +21,8 @@ Module agent """ +from __future__ import print_function + import os import sys import re @@ -37,17 +39,21 @@ from azurelinuxagent.common.version import AGENT_NAME, AGENT_LONG_VERSION, \ from azurelinuxagent.common.osutil import get_osutil class Agent(object): - def __init__(self, verbose): + def __init__(self, verbose, conf_file_path=None): """ Initialize agent running environment. """ + self.conf_file_path = conf_file_path self.osutil = get_osutil() + #Init stdout log level = logger.LogLevel.VERBOSE if verbose else logger.LogLevel.INFO logger.add_logger_appender(logger.AppenderType.STDOUT, level) #Init config - conf_file_path = self.osutil.get_agent_conf_file_path() + conf_file_path = self.conf_file_path \ + if self.conf_file_path is not None \ + else self.osutil.get_agent_conf_file_path() conf.load_conf_from_file(conf_file_path) #Init log @@ -67,9 +73,13 @@ class Agent(object): """ Run agent daemon """ + child_args = None \ + if self.conf_file_path is None \ + else "-configuration-path:{0}".format(self.conf_file_path) + from azurelinuxagent.daemon import get_daemon_handler daemon_handler = get_daemon_handler() - daemon_handler.run() + daemon_handler.run(child_args=child_args) def provision(self): """ @@ -113,7 +123,7 @@ def main(args=[]): """ if len(args) <= 0: args = sys.argv[1:] - command, force, verbose = parse_args(args) + command, force, verbose, conf_file_path = parse_args(args) if command == "version": version() elif command == "help": @@ -122,7 +132,7 @@ def main(args=[]): start() else: try: - agent = Agent(verbose) + agent = Agent(verbose, conf_file_path=conf_file_path) if command == "deprovision+user": agent.deprovision(force, deluser=True) elif command == "deprovision": @@ -147,8 +157,18 @@ def parse_args(sys_args): cmd = "help" force = False verbose = False + conf_file_path = None for a in sys_args: - if re.match("^([-/]*)deprovision\\+user", a): + m = re.match("^(?:[-/]*)configuration-path:([\w/\.\-_]+)", a) + if not m is None: + conf_file_path = m.group(1) + if not os.path.exists(conf_file_path): + print("Error: Configuration file {0} does not exist".format( + conf_file_path), file=sys.stderr) + usage() + sys.exit(1) + + elif re.match("^([-/]*)deprovision\\+user", a): cmd = "deprovision+user" elif re.match("^([-/]*)deprovision", a): cmd = "deprovision" @@ -171,7 +191,7 @@ def parse_args(sys_args): else: cmd = "help" break - return cmd, force, verbose + return cmd, force, verbose, conf_file_path def version(): """ @@ -191,6 +211,7 @@ def usage(): """ print("") print((("usage: {0} [-verbose] [-force] [-help] " + "-configuration-path:<path to configuration file>" "-deprovision[+user]|-register-service|-version|-daemon|-start|" "-run-exthandlers]" "").format(sys.argv[0]))) diff --git a/azurelinuxagent/common/conf.py b/azurelinuxagent/common/conf.py index 7911699..5422784 100644 --- a/azurelinuxagent/common/conf.py +++ b/azurelinuxagent/common/conf.py @@ -21,13 +21,15 @@ Module conf loads and parses configuration file """ import os +import os.path + import azurelinuxagent.common.utils.fileutil as fileutil from azurelinuxagent.common.exception import AgentConfigError class ConfigurationProvider(object): """ - Parse amd store key:values in /etc/waagent.conf. + Parse and store key:values in /etc/waagent.conf. """ def __init__(self): @@ -38,12 +40,12 @@ class ConfigurationProvider(object): raise AgentConfigError("Can't not parse empty configuration") for line in content.split('\n'): if not line.startswith("#") and "=" in line: - parts = line.split()[0].split('=') + parts = line.split('=') + if len(parts) < 2: + continue + key = parts[0].strip() value = parts[1].strip("\" ") - if value != "None": - self.values[parts[0]] = value - else: - self.values[parts[0]] = None + self.values[key] = value if value != "None" else None def get(self, key, default_val): val = self.values.get(key) @@ -113,38 +115,49 @@ def get_agent_pid_file_path(conf=__conf__): def get_ext_log_dir(conf=__conf__): return conf.get("Extension.LogDir", "/var/log/azure") +def get_fips_enabled(conf=__conf__): + return conf.get_switch("OS.EnableFIPS", False) def get_openssl_cmd(conf=__conf__): return conf.get("OS.OpensslPath", "/usr/bin/openssl") +def get_ssh_dir(conf=__conf__): + return conf.get("OS.SshDir", "/etc/ssh") def get_home_dir(conf=__conf__): return conf.get("OS.HomeDir", "/home") - def get_passwd_file_path(conf=__conf__): return conf.get("OS.PasswordPath", "/etc/shadow") - def get_sudoers_dir(conf=__conf__): return conf.get("OS.SudoersDir", "/etc/sudoers.d") - def get_sshd_conf_file_path(conf=__conf__): - return conf.get("OS.SshdConfigPath", "/etc/ssh/sshd_config") + return os.path.join(get_ssh_dir(conf), "sshd_config") +def get_ssh_key_glob(conf=__conf__): + return os.path.join(get_ssh_dir(conf), 'ssh_host_*key*') + +def get_ssh_key_private_path(conf=__conf__): + return os.path.join(get_ssh_dir(conf), + 'ssh_host_{0}_key'.format(get_ssh_host_keypair_type(conf))) + +def get_ssh_key_public_path(conf=__conf__): + return os.path.join(get_ssh_dir(conf), + 'ssh_host_{0}_key.pub'.format(get_ssh_host_keypair_type(conf))) def get_root_device_scsi_timeout(conf=__conf__): return conf.get("OS.RootDeviceScsiTimeout", None) - def get_ssh_host_keypair_type(conf=__conf__): return conf.get("Provisioning.SshHostKeyPairType", "rsa") - def get_provision_enabled(conf=__conf__): return conf.get_switch("Provisioning.Enabled", True) +def get_provision_cloudinit(conf=__conf__): + return conf.get_switch("Provisioning.UseCloudInit", False) def get_allow_reset_sys_user(conf=__conf__): return conf.get_switch("Provisioning.AllowResetSysUser", False) diff --git a/azurelinuxagent/common/dhcp.py b/azurelinuxagent/common/dhcp.py index 66346b5..6087643 100644 --- a/azurelinuxagent/common/dhcp.py +++ b/azurelinuxagent/common/dhcp.py @@ -130,7 +130,7 @@ class DhcpHandler(object): logger.info("Gateway:{0}", self.gateway) logger.info("Routes:{0}", self.routes) # Add default gateway - if self.gateway is not None: + if self.gateway is not None and self.osutil.is_missing_default_route(): self.osutil.route_add(0, 0, self.gateway) if self.routes is not None: for route in self.routes: diff --git a/azurelinuxagent/common/event.py b/azurelinuxagent/common/event.py index ce79adf..116478b 100644 --- a/azurelinuxagent/common/event.py +++ b/azurelinuxagent/common/event.py @@ -24,7 +24,11 @@ import time import datetime import threading import platform + +from datetime import datetime + import azurelinuxagent.common.logger as logger + from azurelinuxagent.common.exception import EventError, ProtocolError from azurelinuxagent.common.future import ustr from azurelinuxagent.common.protocol.restapi import TelemetryEventParam, \ @@ -45,6 +49,7 @@ class WALAEventOperation: HeartBeat = "HeartBeat" Install = "Install" InitializeHostPlugin = "InitializeHostPlugin" + ProcessGoalState = "ProcessGoalState" Provision = "Provision" ReportStatus = "ReportStatus" Restart = "Restart" @@ -111,6 +116,11 @@ class EventLogger(object): __event_logger__ = EventLogger() +def elapsed_milliseconds(utc_start): + d = datetime.utcnow() - utc_start + return int(((d.days * 24 * 60 * 60 + d.seconds) * 1000) + \ + (d.microseconds / 1000.0)) + def report_event(op, is_success=True, message=''): from azurelinuxagent.common.version import AGENT_NAME, CURRENT_VERSION add_event(AGENT_NAME, @@ -121,10 +131,11 @@ def report_event(op, is_success=True, message=''): def add_event(name, op="", is_success=True, duration=0, version=CURRENT_VERSION, - message="", evt_type="", is_internal=False, + message="", evt_type="", is_internal=False, log_event=True, reporter=__event_logger__): - log = logger.info if is_success else logger.error - log("Event: name={0}, op={1}, message={2}", name, op, message) + if log_event or not is_success: + log = logger.info if is_success else logger.error + log("Event: name={0}, op={1}, message={2}", name, op, message) if reporter.event_dir is None: logger.warn("Event reporter is not initialized.") diff --git a/azurelinuxagent/common/exception.py b/azurelinuxagent/common/exception.py index 457490c..7a0c75e 100644 --- a/azurelinuxagent/common/exception.py +++ b/azurelinuxagent/common/exception.py @@ -20,104 +20,131 @@ Defines all exceptions """ + class AgentError(Exception): """ Base class of agent error. """ + def __init__(self, errno, msg, inner=None): - msg = u"({0}){1}".format(errno, msg) + msg = u"[{0}] {1}".format(errno, msg) if inner is not None: - msg = u"{0} \n inner error: {1}".format(msg, inner) + msg = u"{0}\nInner error: {1}".format(msg, inner) super(AgentError, self).__init__(msg) + class AgentConfigError(AgentError): """ When configure file is not found or malformed. """ + def __init__(self, msg=None, inner=None): super(AgentConfigError, self).__init__('000001', msg, inner) + class AgentNetworkError(AgentError): """ When network is not avaiable. """ + def __init__(self, msg=None, inner=None): super(AgentNetworkError, self).__init__('000002', msg, inner) + class ExtensionError(AgentError): """ When failed to execute an extension """ + def __init__(self, msg=None, inner=None): super(ExtensionError, self).__init__('000003', msg, inner) + class ProvisionError(AgentError): """ When provision failed """ + def __init__(self, msg=None, inner=None): super(ProvisionError, self).__init__('000004', msg, inner) + class ResourceDiskError(AgentError): """ Mount resource disk failed """ + def __init__(self, msg=None, inner=None): super(ResourceDiskError, self).__init__('000005', msg, inner) + class DhcpError(AgentError): """ Failed to handle dhcp response """ + def __init__(self, msg=None, inner=None): super(DhcpError, self).__init__('000006', msg, inner) + class OSUtilError(AgentError): """ Failed to perform operation to OS configuration """ + def __init__(self, msg=None, inner=None): super(OSUtilError, self).__init__('000007', msg, inner) + class ProtocolError(AgentError): """ Azure protocol error """ + def __init__(self, msg=None, inner=None): super(ProtocolError, self).__init__('000008', msg, inner) + class ProtocolNotFoundError(ProtocolError): """ Azure protocol endpoint not found """ + def __init__(self, msg=None, inner=None): super(ProtocolNotFoundError, self).__init__(msg, inner) + class HttpError(AgentError): """ Http request failure """ + def __init__(self, msg=None, inner=None): super(HttpError, self).__init__('000009', msg, inner) + class EventError(AgentError): """ Event reporting error """ + def __init__(self, msg=None, inner=None): super(EventError, self).__init__('000010', msg, inner) + class CryptError(AgentError): """ Encrypt/Decrypt error """ + def __init__(self, msg=None, inner=None): super(CryptError, self).__init__('000011', msg, inner) + class UpdateError(AgentError): """ Update Guest Agent error """ + def __init__(self, msg=None, inner=None): super(UpdateError, self).__init__('000012', msg, inner) - diff --git a/azurelinuxagent/common/osutil/arch.py b/azurelinuxagent/common/osutil/arch.py new file mode 100644 index 0000000..83d3b47 --- /dev/null +++ b/azurelinuxagent/common/osutil/arch.py @@ -0,0 +1,55 @@ +# +# Copyright 2014 Microsoft Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Requires Python 2.4+ and Openssl 1.0+ +# + +import os +import azurelinuxagent.common.utils.shellutil as shellutil +from azurelinuxagent.common.osutil.default import DefaultOSUtil + +class ArchUtil(DefaultOSUtil): + def is_dhcp_enabled(self): + return True + + def start_network(self): + return shellutil.run("systemctl start systemd-networkd", chk_err=False) + + def restart_if(self, iface): + shellutil.run("systemctl restart systemd-networkd") + + def restart_ssh_service(self): + # SSH is socket activated on CoreOS. No need to restart it. + pass + + def stop_dhcp_service(self): + return shellutil.run("systemctl stop systemd-networkd", chk_err=False) + + def start_dhcp_service(self): + return shellutil.run("systemctl start systemd-networkd", chk_err=False) + + def start_agent_service(self): + return shellutil.run("systemctl start waagent", chk_err=False) + + def stop_agent_service(self): + return shellutil.run("systemctl stop waagent", chk_err=False) + + def get_dhcp_pid(self): + ret= shellutil.run_get_output("pidof systemd-networkd") + return ret[1] if ret[0] == 0 else None + + def conf_sshd(self, disable_password): + # Don't whack the system default sshd conf + pass
\ No newline at end of file diff --git a/azurelinuxagent/common/osutil/default.py b/azurelinuxagent/common/osutil/default.py index 59d5985..20dc1f3 100644 --- a/azurelinuxagent/common/osutil/default.py +++ b/azurelinuxagent/common/osutil/default.py @@ -30,13 +30,15 @@ import fcntl import base64 import glob import datetime + import azurelinuxagent.common.logger as logger import azurelinuxagent.common.conf as conf -from azurelinuxagent.common.exception import OSUtilError -from azurelinuxagent.common.future import ustr import azurelinuxagent.common.utils.fileutil as fileutil import azurelinuxagent.common.utils.shellutil as shellutil import azurelinuxagent.common.utils.textutil as textutil + +from azurelinuxagent.common.exception import OSUtilError +from azurelinuxagent.common.future import ustr from azurelinuxagent.common.utils.cryptutil import CryptUtil __RULES_FILES__ = [ "/lib/udev/rules.d/75-persistent-net-generator.rules", @@ -48,6 +50,12 @@ for all distros. Each concrete distro classes could overwrite default behavior if needed. """ +DMIDECODE_CMD = 'dmidecode --string system-uuid' +PRODUCT_ID_FILE = '/sys/class/dmi/id/product_uuid' +UUID_PATTERN = re.compile( + '^\s*[A-F0-9]{8}(?:\-[A-F0-9]{4}){3}\-[A-F0-9]{12}\s*$', + re.IGNORECASE) + class DefaultOSUtil(object): def __init__(self): @@ -58,6 +66,22 @@ class DefaultOSUtil(object): def get_agent_conf_file_path(self): return self.agent_conf_file_path + def get_instance_id(self): + ''' + Azure records a UUID as the instance ID + First check /sys/class/dmi/id/product_uuid. + If that is missing, then extracts from dmidecode + If nothing works (for old VMs), return the empty string + ''' + if os.path.isfile(PRODUCT_ID_FILE): + return fileutil.read_file(PRODUCT_ID_FILE).strip() + + rc, s = shellutil.run_get_output(DMIDECODE_CMD) + if rc != 0 or UUID_PATTERN.match(s) is None: + return "" + + return s.strip() + def get_userentry(self, username): try: return pwd.getpwnam(username) @@ -110,8 +134,8 @@ class DefaultOSUtil(object): def chpasswd(self, username, password, crypt_id=6, salt_len=10): if self.is_sys_user(username): - raise OSUtilError(("User {0} is a system user. " - "Will not set passwd.").format(username)) + raise OSUtilError(("User {0} is a system user, " + "will not set password.").format(username)) passwd_hash = textutil.gen_password_hash(password, crypt_id, salt_len) cmd = "usermod -p '{0}' {1}".format(passwd_hash, username) ret, output = shellutil.run_get_output(cmd, log_cmd=False) @@ -273,46 +297,65 @@ class DefaultOSUtil(object): .format("Disabled" if disable_password else "Enabled")) logger.info("Configured SSH client probing to keep connections alive.") - def get_dvd_device(self, dev_dir='/dev'): - pattern=r'(sr[0-9]|hd[c-z]|cdrom[0-9]|cd[0-9])' - for dvd in [re.match(pattern, dev) for dev in os.listdir(dev_dir)]: + pattern = r'(sr[0-9]|hd[c-z]|cdrom[0-9]|cd[0-9])' + device_list = os.listdir(dev_dir) + for dvd in [re.match(pattern, dev) for dev in device_list]: if dvd is not None: return "/dev/{0}".format(dvd.group(0)) - raise OSUtilError("Failed to get dvd device") - - def mount_dvd(self, max_retry=6, chk_err=True, dvd_device=None, mount_point=None): + inner_detail = "The following devices were found, but none matched " \ + "the pattern [{0}]: {1}\n".format(pattern, device_list) + raise OSUtilError(msg="Failed to get dvd device from {0}".format(dev_dir), + inner=inner_detail) + + def mount_dvd(self, + max_retry=6, + chk_err=True, + dvd_device=None, + mount_point=None, + sleep_time=5): if dvd_device is None: dvd_device = self.get_dvd_device() if mount_point is None: mount_point = conf.get_dvd_mount_point() - mountlist = shellutil.run_get_output("mount")[1] - existing = self.get_mount_point(mountlist, dvd_device) - if existing is not None: #Already mounted + mount_list = shellutil.run_get_output("mount")[1] + existing = self.get_mount_point(mount_list, dvd_device) + + if existing is not None: + # already mounted logger.info("{0} is already mounted at {1}", dvd_device, existing) return + if not os.path.isdir(mount_point): os.makedirs(mount_point) - for retry in range(0, max_retry): - retcode = self.mount(dvd_device, mount_point, option="-o ro -t udf,iso9660", - chk_err=chk_err) - if retcode == 0: + err = '' + for retry in range(1, max_retry): + return_code, err = self.mount(dvd_device, + mount_point, + option="-o ro -t udf,iso9660", + chk_err=chk_err) + if return_code == 0: logger.info("Successfully mounted dvd") return - if retry < max_retry - 1: - logger.warn("Mount dvd failed: retry={0}, ret={1}", retry, - retcode) - time.sleep(5) + else: + logger.warn( + "Mounting dvd failed [retry {0}/{1}, sleeping {2} sec]", + retry, + max_retry - 1, + sleep_time) + if retry < max_retry: + time.sleep(sleep_time) if chk_err: - raise OSUtilError("Failed to mount dvd.") + raise OSUtilError("Failed to mount dvd device", inner=err) def umount_dvd(self, chk_err=True, mount_point=None): if mount_point is None: mount_point = conf.get_dvd_mount_point() - retcode = self.umount(mount_point, chk_err=chk_err) - if chk_err and retcode != 0: - raise OSUtilError("Failed to umount dvd.") + return_code = self.umount(mount_point, chk_err=chk_err) + if chk_err and return_code != 0: + raise OSUtilError("Failed to unmount dvd device at {0}", + mount_point) def eject_dvd(self, chk_err=True): dvd = self.get_dvd_device() @@ -356,7 +399,11 @@ class DefaultOSUtil(object): def mount(self, dvd, mount_point, option="", chk_err=True): cmd = "mount {0} {1} {2}".format(option, dvd, mount_point) - return shellutil.run_get_output(cmd, chk_err)[0] + retcode, err = shellutil.run_get_output(cmd, chk_err) + if retcode != 0: + detail = "[{0}] returned {1}: {2}".format(cmd, retcode, err) + err = detail + return retcode, err def umount(self, mount_point, chk_err=True): return shellutil.run("umount {0}".format(mount_point), chk_err=chk_err) diff --git a/azurelinuxagent/common/osutil/factory.py b/azurelinuxagent/common/osutil/factory.py index eee9f97..3447651 100644 --- a/azurelinuxagent/common/osutil/factory.py +++ b/azurelinuxagent/common/osutil/factory.py @@ -19,6 +19,7 @@ import azurelinuxagent.common.logger as logger from azurelinuxagent.common.utils.textutil import Version from azurelinuxagent.common.version import * from .default import DefaultOSUtil +from .arch import ArchUtil from .clearlinux import ClearLinuxUtil from .coreos import CoreOSUtil from .debian import DebianOSUtil @@ -28,6 +29,7 @@ from .suse import SUSEOSUtil, SUSE11OSUtil from .ubuntu import UbuntuOSUtil, Ubuntu12OSUtil, Ubuntu14OSUtil, UbuntuSnappyOSUtil from .alpine import AlpineOSUtil from .bigip import BigIpOSUtil +from .gaia import GaiaOSUtil def get_osutil(distro_name=DISTRO_NAME, @@ -35,6 +37,9 @@ def get_osutil(distro_name=DISTRO_NAME, distro_version=DISTRO_VERSION, distro_full_name=DISTRO_FULL_NAME): + if distro_name == "arch": + return ArchUtil() + if distro_name == "clear linux software for intel architecture": return ClearLinuxUtil() @@ -85,6 +90,9 @@ def get_osutil(distro_name=DISTRO_NAME, elif distro_name == "bigip": return BigIpOSUtil() + elif distro_name == "gaia": + return GaiaOSUtil() + else: logger.warn("Unable to load distro implementation for {0}. Using " "default distro implementation instead.", diff --git a/azurelinuxagent/common/osutil/freebsd.py b/azurelinuxagent/common/osutil/freebsd.py index d0c40b9..0f465a9 100644 --- a/azurelinuxagent/common/osutil/freebsd.py +++ b/azurelinuxagent/common/osutil/freebsd.py @@ -67,8 +67,8 @@ class FreeBSDOSUtil(DefaultOSUtil): def chpasswd(self, username, password, crypt_id=6, salt_len=10): if self.is_sys_user(username): - raise OSUtilError(("User {0} is a system user. " - "Will not set passwd.").format(username)) + raise OSUtilError(("User {0} is a system user, " + "will not set password.").format(username)) passwd_hash = textutil.gen_password_hash(password, crypt_id, salt_len) cmd = "echo '{0}'|pw usermod {1} -H 0 ".format(passwd_hash, username) ret, output = shellutil.run_get_output(cmd, log_cmd=False) diff --git a/azurelinuxagent/common/osutil/gaia.py b/azurelinuxagent/common/osutil/gaia.py new file mode 100644 index 0000000..a1069d3 --- /dev/null +++ b/azurelinuxagent/common/osutil/gaia.py @@ -0,0 +1,151 @@ +# +# Copyright 2017 Check Point Software Technologies +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Requires Python 2.4+ and Openssl 1.0+ +# + +import socket +import struct +import time + +import azurelinuxagent.common.logger as logger +from azurelinuxagent.common.exception import OSUtilError +import azurelinuxagent.common.utils.shellutil as shellutil +import azurelinuxagent.common.utils.textutil as textutil +from azurelinuxagent.common.osutil.default import DefaultOSUtil + + +class GaiaOSUtil(DefaultOSUtil): + def __init__(self): + super(GaiaOSUtil, self).__init__() + + def _run_clish(self, cmd, log_cmd=True): + for i in xrange(10): + ret, out = shellutil.run_get_output( + "/bin/clish -s -c '" + cmd + "'", log_cmd=log_cmd) + if not ret: + break + if 'NMSHST0025' in out: # Entry for [hostname] already present + ret = 0 + break + time.sleep(2) + return ret, out + + def useradd(self, username, expiration=None): + logger.warn('useradd is not supported on GAiA') + + def chpasswd(self, username, password, crypt_id=6, salt_len=10): + logger.info('chpasswd') + passwd_hash = textutil.gen_password_hash(password, crypt_id, salt_len) + ret, out = self._run_clish( + 'set user admin password-hash ' + passwd_hash, log_cmd=False) + if ret != 0: + raise OSUtilError(("Failed to set password for {0}: {1}" + "").format('admin', out)) + + def conf_sudoer(self, username, nopasswd=False, remove=False): + logger.info('conf_sudoer is not supported on GAiA') + + def del_root_password(self): + logger.info('del_root_password') + ret, out = self._run_clish('set user admin password-hash *LOCK*') + if ret != 0: + raise OSUtilError("Failed to delete root password") + + def _replace_user(path, username): + parts = path.split('/') + for i in xrange(len(parts)): + if parts[i] == '$HOME': + parts[i + 1] = username + break + return '/'.join(parts) + + def deploy_ssh_keypair(self, username, keypair): + logger.info('deploy_ssh_keypair') + username = 'admin' + path, thumbprint = keypair + path = self._replace_user(path, username) + super(GaiaOSUtil, self).deploy_ssh_keypair( + username, (path, thumbprint)) + + def deploy_ssh_pubkey(self, username, pubkey): + logger.info('deploy_ssh_pubkey') + username = 'admin' + path, thumbprint, value = pubkey + path = self._replace_user(path, username) + super(GaiaOSUtil, self).deploy_ssh_pubkey( + 'admin', (path, thumbprint, value)) + + def eject_dvd(self, chk_err=True): + logger.warn('eject is not supported on GAiA') + + def mount(self, dvd, mount_point, option="", chk_err=True): + logger.info('mount {0} {1} {2}', dvd, mount_point, option) + if 'udf,iso9660' in option: + ret, out = super(GaiaOSUtil, self).mount( + dvd, mount_point, option=option.replace('udf,iso9660', 'udf'), + chk_err=chk_err) + if not ret: + return ret, out + return super(GaiaOSUtil, self).mount( + dvd, mount_point, option=option, chk_err=chk_err) + + def allow_dhcp_broadcast(self): + logger.info('allow_dhcp_broadcast is ignored on GAiA') + + def remove_rules_files(self, rules_files=''): + pass + + def restore_rules_files(self, rules_files=''): + logger.info('restore_rules_files is ignored on GAiA') + + def restart_ssh_service(self): + return shellutil.run('/sbin/service sshd condrestart', chk_err=False) + + def _address_to_string(addr): + return socket.inet_ntoa(struct.pack("!I", addr)) + + def _get_prefix(self, mask): + return str(sum([bin(int(x)).count('1') for x in mask.split('.')])) + + def route_add(self, net, mask, gateway): + logger.info('route_add {0} {1} {2}', net, mask, gateway) + + if net == 0 and mask == 0: + cidr = 'default' + else: + cidr = self._address_to_string(net) + '/' + self._get_prefix( + self._address_to_string(mask)) + + ret, out = self._run_clish( + 'set static-route ' + cidr + + ' nexthop gateway address ' + + self._address_to_string(gateway) + ' on') + return ret + + def set_hostname(self, hostname): + logger.warn('set_hostname is ignored on GAiA') + + def set_dhcp_hostname(self, hostname): + logger.warn('set_dhcp_hostname is ignored on GAiA') + + def publish_hostname(self, hostname): + logger.warn('publish_hostname is ignored on GAiA') + + def del_account(self, username): + logger.warn('del_account is ignored on GAiA') + + def set_admin_access_to_ip(self, dest_ip): + logger.warn('set_admin_access_to_ip is ignored on GAiA') diff --git a/azurelinuxagent/common/osutil/redhat.py b/azurelinuxagent/common/osutil/redhat.py index 5254ea5..b94610c 100644 --- a/azurelinuxagent/common/osutil/redhat.py +++ b/azurelinuxagent/common/osutil/redhat.py @@ -103,7 +103,7 @@ class RedhatOSUtil(Redhat6xOSUtil): Due to a bug in systemd in Centos-7.0, if this call fails, fallback to hostname. """ - hostnamectl_cmd = "hostnamectl set-hostname {0}".format(hostname) + hostnamectl_cmd = "hostnamectl set-hostname {0} --static".format(hostname) if shellutil.run(hostnamectl_cmd, chk_err=False) != 0: logger.warn("[{0}] failed, attempting fallback".format(hostnamectl_cmd)) DefaultOSUtil.set_hostname(self, hostname) diff --git a/azurelinuxagent/common/protocol/hostplugin.py b/azurelinuxagent/common/protocol/hostplugin.py index 70bf8b4..464fd35 100644 --- a/azurelinuxagent/common/protocol/hostplugin.py +++ b/azurelinuxagent/common/protocol/hostplugin.py @@ -19,6 +19,7 @@ import base64 import json +import traceback from azurelinuxagent.common import logger from azurelinuxagent.common.exception import ProtocolError, HttpError @@ -177,8 +178,7 @@ class HostPluginProtocol(object): logger.info("HostGAPlugin: Setting host plugin as default channel") HostPluginProtocol.set_default_channel(True) except Exception as e: - message = "HostGAPlugin: Exception Put VM status: {0}".format(e) - logger.error(message) + message = "HostGAPlugin: Exception Put VM status: {0}, {1}".format(e, traceback.format_exc()) from azurelinuxagent.common.event import WALAEventOperation, report_event report_event(op=WALAEventOperation.ReportStatus, is_success=False, diff --git a/azurelinuxagent/common/protocol/metadata.py b/azurelinuxagent/common/protocol/metadata.py index c61e373..c50b3dd 100644 --- a/azurelinuxagent/common/protocol/metadata.py +++ b/azurelinuxagent/common/protocol/metadata.py @@ -21,17 +21,19 @@ import json import os import shutil import re + import azurelinuxagent.common.conf as conf import azurelinuxagent.common.utils.fileutil as fileutil import azurelinuxagent.common.utils.shellutil as shellutil import azurelinuxagent.common.utils.textutil as textutil + from azurelinuxagent.common.future import httpclient from azurelinuxagent.common.protocol.restapi import * from azurelinuxagent.common.utils.cryptutil import CryptUtil METADATA_ENDPOINT = '169.254.169.254' APIVERSION = '2015-05-01-preview' -BASE_URI = "http://{0}/Microsoft.Compute/{1}?api-version={2}{3}" +BASE_URI = "http://{0}/Microsoft.Compute/{1}?api-version={2}" TRANSPORT_PRV_FILE_NAME = "V2TransportPrivate.pem" TRANSPORT_CERT_FILE_NAME = "V2TransportCert.pem" @@ -39,6 +41,9 @@ P7M_FILE_NAME = "Certificates.p7m" P7B_FILE_NAME = "Certificates.p7b" PEM_FILE_NAME = "Certificates.pem" +KEY_AGENT_VERSION_URIS = "versionsManifestUris" +KEY_URI = "uri" + # TODO remote workaround for azure stack MAX_PING = 30 RETRY_PING_INTERVAL = 10 @@ -56,13 +61,13 @@ class MetadataProtocol(Protocol): self.apiversion = apiversion self.endpoint = endpoint self.identity_uri = BASE_URI.format(self.endpoint, "identity", - self.apiversion, "&$expand=*") + self.apiversion) self.cert_uri = BASE_URI.format(self.endpoint, "certificates", - self.apiversion, "&$expand=*") + self.apiversion) self.ext_uri = BASE_URI.format(self.endpoint, "extensionHandlers", - self.apiversion, "&$expand=*") + self.apiversion) self.vmagent_uri = BASE_URI.format(self.endpoint, "vmAgentVersions", - self.apiversion, "&$expand=*") + self.apiversion) self.provision_status_uri = BASE_URI.format(self.endpoint, "provisioningStatus", self.apiversion, "") @@ -74,6 +79,8 @@ class MetadataProtocol(Protocol): self.event_uri = BASE_URI.format(self.endpoint, "status/telemetry", self.apiversion, "") self.certs = None + self.agent_manifests = None + self.agent_etag = None def _get_data(self, url, headers=None): try: @@ -166,32 +173,55 @@ class MetadataProtocol(Protocol): return None return self.certs - def get_vmagent_manifests(self, last_etag=None): - manifests = VMAgentManifestList() + def get_vmagent_manifests(self): self.update_goal_state() + data, etag = self._get_data(self.vmagent_uri) - if last_etag is None or last_etag < etag: - set_properties("vmAgentManifests", - manifests.vmAgentManifests, - data) - return manifests, etag + if self.agent_etag is None or self.agent_etag < etag: + self.agent_etag = etag + + # Create a list with a single manifest + # -- The protocol lacks "family," use the configured family + self.agent_manifests = VMAgentManifestList() + + manifest = VMAgentManifest() + manifest.family = family=conf.get_autoupdate_gafamily() + + if not KEY_AGENT_VERSION_URIS in data: + raise ProtocolError( + "Agent versions missing '{0}': {1}".format( + KEY_AGENT_VERSION_URIS, data)) + + for version in data[KEY_AGENT_VERSION_URIS]: + if not KEY_URI in version: + raise ProtocolError( + "Agent versions missing '{0': {1}".format( + KEY_URI, data)) + manifest_uri = VMAgentManifestUri(uri=version[KEY_URI]) + manifest.versionsManifestUris.append(manifest_uri) + + self.agent_manifests.vmAgentManifests.append(manifest) + + return self.agent_manifests, self.agent_etag def get_vmagent_pkgs(self, vmagent_manifest): - # Agent package is the same with extension handler - vmagent_pkgs = ExtHandlerPackageList() data = None + etag = None for manifest_uri in vmagent_manifest.versionsManifestUris: try: - data = self._get_data(manifest_uri.uri) + data, etag = self._get_data(manifest_uri.uri) break except ProtocolError as e: - logger.warn("Failed to get vmagent versions: {0}", e) - logger.info("Retry getting vmagent versions") + logger.verbose( + "Error retrieving agent package from {0}: {1}".format( + manifest_uri, e)) + if data is None: - raise ProtocolError(("Failed to get versions for vm agent: {0}" - "").format(vmagent_manifest.family)) + raise ProtocolError( + "Failed retrieving agent package from all URIs") + + vmagent_pkgs = ExtHandlerPackageList() set_properties("vmAgentVersions", vmagent_pkgs, data) - # TODO: What etag should this return? return vmagent_pkgs def get_ext_handlers(self, last_etag=None): @@ -251,11 +281,9 @@ class MetadataProtocol(Protocol): self._put_data(uri, data) def report_event(self, events): - # TODO disable telemetry for azure stack test - # validate_param('events', events, TelemetryEventList) - # data = get_properties(events) - # self._post_data(self.event_uri, data) - pass + validate_param('events', events, TelemetryEventList) + data = get_properties(events) + self._post_data(self.event_uri, data) def update_certs(self): certificates = self.get_certs() diff --git a/azurelinuxagent/common/protocol/ovfenv.py b/azurelinuxagent/common/protocol/ovfenv.py index 4901871..3122e3b 100644 --- a/azurelinuxagent/common/protocol/ovfenv.py +++ b/azurelinuxagent/common/protocol/ovfenv.py @@ -35,7 +35,7 @@ WA_NAME_SPACE = "http://schemas.microsoft.com/windowsazure" def _validate_ovf(val, msg): if val is None: - raise ProtocolError("Failed to parse OVF XML: {0}".format(msg)) + raise ProtocolError("Failed to validate OVF: {0}".format(msg)) class OvfEnv(object): """ diff --git a/azurelinuxagent/common/protocol/util.py b/azurelinuxagent/common/protocol/util.py index 7e7a74f..0ba03ec 100644 --- a/azurelinuxagent/common/protocol/util.py +++ b/azurelinuxagent/common/protocol/util.py @@ -33,25 +33,21 @@ from azurelinuxagent.common.protocol.ovfenv import OvfEnv from azurelinuxagent.common.protocol.wire import WireProtocol from azurelinuxagent.common.protocol.metadata import MetadataProtocol, \ METADATA_ENDPOINT -import azurelinuxagent.common.utils.shellutil as shellutil OVF_FILE_NAME = "ovf-env.xml" - -#Tag file to indicate usage of metadata protocol -TAG_FILE_NAME = "useMetadataEndpoint.tag" - +TAG_FILE_NAME = "useMetadataEndpoint.tag" PROTOCOL_FILE_NAME = "Protocol" - -#MAX retry times for protocol probing MAX_RETRY = 360 - PROBE_INTERVAL = 10 - ENDPOINT_FILE_NAME = "WireServerEndpoint" +PASSWORD_PATTERN = "<UserPassword>.*?<" +PASSWORD_REPLACEMENT = "<UserPassword>*<" + def get_protocol_util(): return ProtocolUtil() + class ProtocolUtil(object): """ ProtocolUtil handles initialization for protocol instance. 2 protocol types @@ -71,21 +67,42 @@ class ProtocolUtil(object): dvd_mount_point = conf.get_dvd_mount_point() ovf_file_path_on_dvd = os.path.join(dvd_mount_point, OVF_FILE_NAME) tag_file_path_on_dvd = os.path.join(dvd_mount_point, TAG_FILE_NAME) + ovf_file_path = os.path.join(conf.get_lib_dir(), OVF_FILE_NAME) + tag_file_path = os.path.join(conf.get_lib_dir(), TAG_FILE_NAME) + try: self.osutil.mount_dvd() + except OSUtilError as e: + raise ProtocolError("[CopyOvfEnv] Error mounting dvd: " + "{0}".format(ustr(e))) + + try: ovfxml = fileutil.read_file(ovf_file_path_on_dvd, remove_bom=True) ovfenv = OvfEnv(ovfxml) - ovfxml = re.sub("<UserPassword>.*?<", "<UserPassword>*<", ovfxml) - ovf_file_path = os.path.join(conf.get_lib_dir(), OVF_FILE_NAME) + except IOError as e: + raise ProtocolError("[CopyOvfEnv] Error reading file " + "{0}: {1}".format(ovf_file_path_on_dvd, + ustr(e))) + + try: + ovfxml = re.sub(PASSWORD_PATTERN, + PASSWORD_REPLACEMENT, + ovfxml) fileutil.write_file(ovf_file_path, ovfxml) - + except IOError as e: + raise ProtocolError("[CopyOvfEnv] Error writing file " + "{0}: {1}".format(ovf_file_path, + ustr(e))) + + try: if os.path.isfile(tag_file_path_on_dvd): logger.info("Found {0} in provisioning ISO", TAG_FILE_NAME) - tag_file_path = os.path.join(conf.get_lib_dir(), TAG_FILE_NAME) - shutil.copyfile(tag_file_path_on_dvd, tag_file_path) - - except (OSUtilError, IOError) as e: - raise ProtocolError(ustr(e)) + shutil.copyfile(tag_file_path_on_dvd, tag_file_path) + except IOError as e: + raise ProtocolError("[CopyOvfEnv] Error copying file " + "{0} to {1}: {2}".format(tag_file_path, + tag_file_path, + ustr(e))) try: self.osutil.umount_dvd() @@ -104,7 +121,7 @@ class ProtocolUtil(object): xml_text = fileutil.read_file(ovf_file_path) return OvfEnv(xml_text) else: - raise ProtocolError("ovf-env.xml is missing.") + raise ProtocolError("ovf-env.xml is missing from {0}".format(ovf_file_path)) def _get_wireserver_endpoint(self): try: @@ -146,7 +163,7 @@ class ProtocolUtil(object): protocol = MetadataProtocol() protocol.detect() - #Only allow root access METADATA_ENDPOINT + # only allow root access METADATA_ENDPOINT self.osutil.set_admin_access_to_ip(METADATA_ENDPOINT) self.save_protocol("MetadataProtocol") @@ -206,7 +223,6 @@ class ProtocolUtil(object): except IOError as e: logger.error("Failed to save protocol endpoint: {0}", e) - def clear_protocol(self): """ Cleanup previous saved endpoint. @@ -249,7 +265,6 @@ class ProtocolUtil(object): finally: self.lock.release() - def get_protocol_by_file(self): """ Detect protocol by tag file. diff --git a/azurelinuxagent/common/protocol/wire.py b/azurelinuxagent/common/protocol/wire.py index 265e2dd..936be8c 100644 --- a/azurelinuxagent/common/protocol/wire.py +++ b/azurelinuxagent/common/protocol/wire.py @@ -21,16 +21,18 @@ import os import re import time import xml.sax.saxutils as saxutils + import azurelinuxagent.common.conf as conf +import azurelinuxagent.common.utils.fileutil as fileutil +import azurelinuxagent.common.utils.textutil as textutil + from azurelinuxagent.common.exception import ProtocolNotFoundError from azurelinuxagent.common.future import httpclient, bytebuffer +from azurelinuxagent.common.protocol.hostplugin import HostPluginProtocol +from azurelinuxagent.common.protocol.restapi import * +from azurelinuxagent.common.utils.cryptutil import CryptUtil from azurelinuxagent.common.utils.textutil import parse_doc, findall, find, \ findtext, getattrib, gettext, remove_bom, get_bytes_from_pem, parse_json -import azurelinuxagent.common.utils.fileutil as fileutil -import azurelinuxagent.common.utils.textutil as textutil -from azurelinuxagent.common.utils.cryptutil import CryptUtil -from azurelinuxagent.common.protocol.restapi import * -from azurelinuxagent.common.protocol.hostplugin import HostPluginProtocol VERSION_INFO_URI = "http://{0}/?comp=versions" GOAL_STATE_URI = "http://{0}/machine/?comp=goalstate" @@ -376,48 +378,20 @@ class StatusBlob(object): self.type = blob_type def upload(self, url): - # TODO upload extension only if content has changed - upload_successful = False - self.type = self.get_blob_type(url) try: + if not self.type in ["BlockBlob", "PageBlob"]: + raise ProtocolError("Illegal blob type: {0}".format(self.type)) + if self.type == "BlockBlob": self.put_block_blob(url, self.data) - elif self.type == "PageBlob": - self.put_page_blob(url, self.data) else: - raise ProtocolError("Unknown blob type: {0}".format(self.type)) - except HttpError as e: - message = "Initial upload failed [{0}]".format(e) - logger.warn(message) - from azurelinuxagent.common.event import WALAEventOperation, report_event - report_event(op=WALAEventOperation.ReportStatus, - is_success=False, - message=message) - else: - logger.verbose("Uploading status blob succeeded") - upload_successful = True - return upload_successful - - def get_blob_type(self, url): - logger.verbose("Get blob type") - timestamp = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()) - try: - resp = self.client.call_storage_service( - restutil.http_head, - url, - { - "x-ms-date": timestamp, - "x-ms-version": self.__class__.__storage_version__ - }) - except HttpError as e: - raise ProtocolError("Failed to get status blob type: {0}", e) + self.put_page_blob(url, self.data) + return True - if resp is None or resp.status != httpclient.OK: - raise ProtocolError("Failed to get status blob type") + except Exception as e: + logger.verbose("Initial status upload failed: {0}", e) - blob_type = resp.getheader("x-ms-blob-type") - logger.verbose("Blob type: [{0}]", blob_type) - return blob_type + return False def get_block_blob_headers(self, blob_size): return { @@ -538,7 +512,6 @@ class WireClient(object): self.req_count = 0 self.host_plugin = None self.status_blob = StatusBlob(self) - self.status_blob_type_reported = False def prevent_throttling(self): """ @@ -725,7 +698,6 @@ class WireClient(object): xml_text = self.fetch_config(goal_state.ext_uri, self.get_header()) self.save_cache(local_file, xml_text) self.ext_conf = ExtensionsConfig(xml_text) - self.status_blob_type_reported = False def update_goal_state(self, forced=False, max_retry=3): uri = GOAL_STATE_URI.format(self.endpoint) @@ -815,7 +787,6 @@ class WireClient(object): local_file = os.path.join(conf.get_lib_dir(), local_file) xml_text = self.fetch_cache(local_file) self.ext_conf = ExtensionsConfig(xml_text) - self.status_blob_type_reported = False return self.ext_conf def get_ext_manifest(self, ext_handler, goal_state): @@ -852,46 +823,38 @@ class WireClient(object): def upload_status_blob(self): ext_conf = self.get_ext_conf() - if ext_conf.status_upload_blob is not None: - uploaded = False + + blob_uri = ext_conf.status_upload_blob + blob_type = ext_conf.status_upload_blob_type + + if blob_uri is not None: + + if not blob_type in ["BlockBlob", "PageBlob"]: + blob_type = "BlockBlob" + logger.info("Status Blob type is unspecified " + "-- assuming it is a BlockBlob") + try: - self.status_blob.prepare(ext_conf.status_upload_blob_type) - if not HostPluginProtocol.is_default_channel(): - uploaded = self.status_blob.upload(ext_conf.status_upload_blob) - self.report_blob_type(self.status_blob.type, - ext_conf.status_upload_blob_type) - except (HttpError, ProtocolError): - # errors have already been logged - pass + self.status_blob.prepare(blob_type) + except Exception as e: + self.report_status_event( + "Exception creating status blob: {0}", + e) + return + + uploaded = False + if not HostPluginProtocol.is_default_channel(): + try: + uploaded = self.status_blob.upload(blob_uri) + except HttpError as e: + pass + if not uploaded: host = self.get_host_plugin() host.put_vm_status(self.status_blob, ext_conf.status_upload_blob, ext_conf.status_upload_blob_type) - """ - Emit an event to determine if the type in the extension config - matches the actual type from the HTTP HEAD request. - """ - def report_blob_type(self, head_type, config_type): - if head_type and config_type: - is_match = head_type == config_type - if self.status_blob_type_reported is False: - message = \ - 'Blob type match [{0}]'.format(head_type) if is_match else \ - 'Blob type mismatch [HEAD {0}], [CONFIG {1}]'.format( - head_type, - config_type) - - from azurelinuxagent.common.event import add_event, WALAEventOperation - from azurelinuxagent.common.version import AGENT_NAME, CURRENT_VERSION - add_event(AGENT_NAME, - version=CURRENT_VERSION, - is_success=is_match, - message=message, - op=WALAEventOperation.HealthCheck) - self.status_blob_type_reported = True - def report_role_prop(self, thumbprint): goal_state = self.get_goal_state() role_prop = _build_role_properties(goal_state.container_id, @@ -980,6 +943,16 @@ class WireClient(object): if len(buf[provider_id]) > 0: self.send_event(provider_id, buf[provider_id]) + def report_status_event(self, message, *args): + from azurelinuxagent.common.event import report_event, \ + WALAEventOperation + + message = message.format(*args) + logger.warn(message) + report_event(op=WALAEventOperation.ReportStatus, + is_success=False, + message=message) + def get_header(self): return { "x-ms-agent-name": "WALinuxAgent", diff --git a/azurelinuxagent/common/utils/cryptutil.py b/azurelinuxagent/common/utils/cryptutil.py index b35bda0..6339eb3 100644 --- a/azurelinuxagent/common/utils/cryptutil.py +++ b/azurelinuxagent/common/utils/cryptutil.py @@ -31,7 +31,7 @@ class CryptUtil(object): """ Create ssl certificate for https communication with endpoint server. """ - cmd = ("{0} req -x509 -nodes -subj /CN=LinuxTransport -days 32768 " + cmd = ("{0} req -x509 -nodes -subj /CN=LinuxTransport -days 730 " "-newkey rsa:2048 -keyout {1} " "-out {2}").format(self.openssl_cmd, prv_file, crt_file) shellutil.run(cmd) diff --git a/azurelinuxagent/common/utils/fileutil.py b/azurelinuxagent/common/utils/fileutil.py index 8713d0c..bae1957 100644 --- a/azurelinuxagent/common/utils/fileutil.py +++ b/azurelinuxagent/common/utils/fileutil.py @@ -119,16 +119,20 @@ def rm_files(*args): def rm_dirs(*args): """ - Remove all the contents under the directry + Remove the contents of each directry """ - for dir_name in args: - if os.path.isdir(dir_name): - for item in os.listdir(dir_name): - path = os.path.join(dir_name, item) - if os.path.isfile(path): - os.remove(path) - elif os.path.isdir(path): - shutil.rmtree(path) + for p in args: + if not os.path.isdir(p): + continue + + for pp in os.listdir(p): + path = os.path.join(p, pp) + if os.path.isfile(path): + os.remove(path) + elif os.path.islink(path): + os.unlink(path) + elif os.path.isdir(path): + shutil.rmtree(path) def trim_ext(path, ext): if not ext.startswith("."): diff --git a/azurelinuxagent/common/utils/shellutil.py b/azurelinuxagent/common/utils/shellutil.py index 4efcbc4..fff6aa8 100644 --- a/azurelinuxagent/common/utils/shellutil.py +++ b/azurelinuxagent/common/utils/shellutil.py @@ -76,18 +76,23 @@ def run_get_output(cmd, chk_err=True, log_cmd=True): Reports exceptions to Error if chk_err parameter is True """ if log_cmd: - logger.verbose(u"run cmd '{0}'", cmd) + logger.verbose(u"Run '{0}'", cmd) try: - output = subprocess.check_output(cmd, stderr=subprocess.STDOUT, + output = subprocess.check_output(cmd, + stderr=subprocess.STDOUT, shell=True) - output = ustr(output, encoding='utf-8', errors="backslashreplace") + output = ustr(output, + encoding='utf-8', + errors="backslashreplace") except subprocess.CalledProcessError as e: - output = ustr(e.output, encoding='utf-8', errors="backslashreplace") + output = ustr(e.output, + encoding='utf-8', + errors="backslashreplace") if chk_err: if log_cmd: - logger.error(u"run cmd '{0}' failed", e.cmd) - logger.error(u"Error Code:{0}", e.returncode) - logger.error(u"Result:{0}", output) + logger.error(u"Command: '{0}'", e.cmd) + logger.error(u"Return code: {0}", e.returncode) + logger.error(u"Result: {0}", output) return e.returncode, output return 0, output diff --git a/azurelinuxagent/common/version.py b/azurelinuxagent/common/version.py index 8a81974..dc3592b 100644 --- a/azurelinuxagent/common/version.py +++ b/azurelinuxagent/common/version.py @@ -57,6 +57,22 @@ def get_f5_platform(): return result +def get_checkpoint_platform(): + take = build = release = "" + full_name = open("/etc/cp-release").read().strip() + with open("/etc/cloud-version") as f: + for line in f: + k, _, v = line.partition(": ") + v = v.strip() + if k == "release": + release = v + elif k == "take": + take = v + elif k == "build": + build = v + return ["gaia", take + "." + build, release, full_name] + + def get_distro(): if 'FreeBSD' in platform.system(): release = re.sub('\-.*\Z', '', ustr(platform.release())) @@ -84,6 +100,9 @@ def get_distro(): if os.path.exists("/shared/vadc"): osinfo = get_f5_platform() + if os.path.exists("/etc/cp-release"): + osinfo = get_checkpoint_platform() + # Remove trailing whitespace and quote in distro name osinfo[0] = osinfo[0].strip('"').strip(' ').lower() return osinfo @@ -91,9 +110,9 @@ def get_distro(): AGENT_NAME = "WALinuxAgent" AGENT_LONG_NAME = "Azure Linux Agent" -AGENT_VERSION = '2.2.9' +AGENT_VERSION = '2.2.12' AGENT_LONG_VERSION = "{0}-{1}".format(AGENT_NAME, AGENT_VERSION) -AGENT_DESCRIPTION = """\ +AGENT_DESCRIPTION = """ The Azure Linux Agent supports the provisioning and running of Linux VMs in the Azure cloud. This package should be installed on Linux disk images that are built to run in the Azure environment. @@ -104,6 +123,7 @@ AGENT_PKG_GLOB = "{0}-*.zip".format(AGENT_NAME) AGENT_PATTERN = "{0}-(.*)".format(AGENT_NAME) AGENT_NAME_PATTERN = re.compile(AGENT_PATTERN) +AGENT_PKG_PATTERN = re.compile(AGENT_PATTERN+"\.zip") AGENT_DIR_PATTERN = re.compile(".*/{0}".format(AGENT_PATTERN)) EXT_HANDLER_PATTERN = b".*/WALinuxAgent-(\w.\w.\w[.\w]*)-.*-run-exthandlers" @@ -127,6 +147,13 @@ def set_current_agent(): version = AGENT_VERSION return agent, FlexibleVersion(version) +def is_agent_package(path): + path = os.path.basename(path) + return not re.match(AGENT_PKG_PATTERN, path) is None + +def is_agent_path(path): + path = os.path.basename(path) + return not re.match(AGENT_NAME_PATTERN, path) is None CURRENT_AGENT, CURRENT_VERSION = set_current_agent() diff --git a/azurelinuxagent/daemon/main.py b/azurelinuxagent/daemon/main.py index b0da02a..5b8db2f 100644 --- a/azurelinuxagent/daemon/main.py +++ b/azurelinuxagent/daemon/main.py @@ -25,12 +25,15 @@ import traceback import azurelinuxagent.common.conf as conf import azurelinuxagent.common.logger as logger import azurelinuxagent.common.utils.fileutil as fileutil + from azurelinuxagent.common.event import add_event, WALAEventOperation from azurelinuxagent.common.future import ustr from azurelinuxagent.common.osutil import get_osutil from azurelinuxagent.common.protocol import get_protocol_util +from azurelinuxagent.common.protocol.wire import WireClient from azurelinuxagent.common.rdma import setup_rdma_device -from azurelinuxagent.common.version import AGENT_LONG_NAME, AGENT_VERSION, \ +from azurelinuxagent.common.version import AGENT_NAME, AGENT_LONG_NAME, \ + AGENT_VERSION, \ DISTRO_NAME, DISTRO_VERSION, PY_VERSION_MAJOR, PY_VERSION_MINOR, \ PY_VERSION_MICRO from azurelinuxagent.daemon.resourcedisk import get_resourcedisk_handler @@ -39,6 +42,8 @@ from azurelinuxagent.ga.update import get_update_handler from azurelinuxagent.pa.provision import get_provision_handler from azurelinuxagent.pa.rdma import get_rdma_handler +OPENSSL_FIPS_ENVIRONMENT = "OPENSSL_FIPS" + def get_daemon_handler(): return DaemonHandler() @@ -53,7 +58,7 @@ class DaemonHandler(object): self.running = True self.osutil = get_osutil() - def run(self): + def run(self, child_args=None): logger.info("{0} Version:{1}", AGENT_LONG_NAME, AGENT_VERSION) logger.info("OS: {0} {1}", DISTRO_NAME, DISTRO_VERSION) logger.info("Python: {0}.{1}.{2}", PY_VERSION_MAJOR, PY_VERSION_MINOR, @@ -61,12 +66,18 @@ class DaemonHandler(object): self.check_pid() + # If FIPS is enabled, set the OpenSSL environment variable + # Note: + # -- Subprocesses inherit the current environment + if conf.get_fips_enabled(): + os.environ[OPENSSL_FIPS_ENVIRONMENT] = '1' + while self.running: try: - self.daemon() + self.daemon(child_args) except Exception as e: err_msg = traceback.format_exc() - add_event("WALA", is_success=False, message=ustr(err_msg), + add_event(name=AGENT_NAME, is_success=False, message=ustr(err_msg), op=WALAEventOperation.UnhandledError) logger.info("Sleep 15 seconds and restart daemon") time.sleep(15) @@ -84,7 +95,7 @@ class DaemonHandler(object): fileutil.write_file(pid_file, ustr(os.getpid())) - def daemon(self): + def daemon(self, child_args=None): logger.info("Run daemon") self.protocol_util = get_protocol_util() @@ -117,6 +128,16 @@ class DaemonHandler(object): logger.info("RDMA capabilities are enabled in configuration") try: + # Ensure the most recent SharedConfig is available + # - Changes to RDMA state may not increment the goal state + # incarnation number. A forced update ensures the most + # current values. + protocol = self.protocol_util.get_protocol() + client = protocol.client + if client is None or type(client) is not WireClient: + raise Exception("Attempt to setup RDMA without Wireserver") + client.update_goal_state(forced=True) + setup_rdma_device() except Exception as e: logger.error("Error setting up rdma device: %s" % e) @@ -124,4 +145,4 @@ class DaemonHandler(object): logger.info("RDMA capabilities are not enabled, skipping") while self.running: - self.update_handler.run_latest() + self.update_handler.run_latest(child_args=child_args) diff --git a/azurelinuxagent/daemon/resourcedisk/default.py b/azurelinuxagent/daemon/resourcedisk/default.py index 2b116fb..dadd49c 100644 --- a/azurelinuxagent/daemon/resourcedisk/default.py +++ b/azurelinuxagent/daemon/resourcedisk/default.py @@ -29,6 +29,7 @@ import azurelinuxagent.common.utils.fileutil as fileutil import azurelinuxagent.common.utils.shellutil as shellutil from azurelinuxagent.common.exception import ResourceDiskError from azurelinuxagent.common.osutil import get_osutil +from azurelinuxagent.common.version import AGENT_NAME DATALOSS_WARNING_FILE_NAME = "DATALOSS_WARNING_README.txt" DATA_LOSS_WARNING = """\ @@ -74,7 +75,7 @@ class ResourceDiskHandler(object): return mount_point except ResourceDiskError as e: logger.error("Failed to mount resource disk {0}", e) - add_event(name="WALA", is_success=False, message=ustr(e), + add_event(name=AGENT_NAME, is_success=False, message=ustr(e), op=WALAEventOperation.ActivateResourceDisk) def enable_swap(self, mount_point): @@ -123,7 +124,7 @@ class ResourceDiskHandler(object): force_option = 'F' if self.fs == 'xfs': force_option = 'f' - mkfs_string = "mkfs.{0} {1} -{2}".format(self.fs, partition, force_option) + mkfs_string = "mkfs.{0} -{2} {1}".format(self.fs, partition, force_option) if "gpt" in ret[1]: logger.info("GPT detected, finding partitions") diff --git a/azurelinuxagent/ga/exthandlers.py b/azurelinuxagent/ga/exthandlers.py index e0125aa..b44ed6d 100644 --- a/azurelinuxagent/ga/exthandlers.py +++ b/azurelinuxagent/ga/exthandlers.py @@ -20,6 +20,8 @@ import glob import json import os +import os.path +import re import shutil import stat import subprocess @@ -31,6 +33,7 @@ import azurelinuxagent.common.logger as logger import azurelinuxagent.common.utils.fileutil as fileutil import azurelinuxagent.common.utils.restutil as restutil import azurelinuxagent.common.utils.shellutil as shellutil +import azurelinuxagent.common.version as version from azurelinuxagent.common.event import add_event, WALAEventOperation from azurelinuxagent.common.exception import ExtensionError, ProtocolError, HttpError @@ -55,6 +58,13 @@ VALID_EXTENSION_STATUS = ['transitioning', 'error', 'success', 'warning'] VALID_HANDLER_STATUS = ['Ready', 'NotReady', "Installing", "Unresponsive"] +HANDLER_PATTERN = "^([^-]+)-(\d+(?:\.\d+)*)" +HANDLER_NAME_PATTERN = re.compile(HANDLER_PATTERN+"$", re.IGNORECASE) +HANDLER_PKG_EXT = ".zip" +HANDLER_PKG_PATTERN = re.compile(HANDLER_PATTERN+"\\"+HANDLER_PKG_EXT+"$", + re.IGNORECASE) + + def validate_has_key(obj, key, fullname): if key not in obj: raise ExtensionError("Missing: {0}".format(fullname)) @@ -163,6 +173,7 @@ def get_exthandlers_handler(): class ExtHandlersHandler(object): def __init__(self): self.protocol_util = get_protocol_util() + self.protocol = None self.ext_handlers = None self.last_etag = None self.log_report = False @@ -188,10 +199,74 @@ class ExtHandlersHandler(object): self.last_etag = etag self.report_ext_handlers_status() + self.cleanup_outdated_handlers() def run_status(self): self.report_ext_handlers_status() return + + def cleanup_outdated_handlers(self): + handlers = [] + pkgs = [] + + # Build a collection of uninstalled handlers and orphaned packages + # Note: + # -- An orphaned package is one without a corresponding handler + # directory + for item in os.listdir(conf.get_lib_dir()): + path = os.path.join(conf.get_lib_dir(), item) + + if version.is_agent_package(path) or version.is_agent_path(path): + continue + + if os.path.isdir(path): + if re.match(HANDLER_NAME_PATTERN, item) is None: + continue + try: + eh = ExtHandler() + + separator = item.rfind('-') + + eh.name = item[0:separator] + eh.properties.version = str(FlexibleVersion(item[separator+1:])) + + handler = ExtHandlerInstance(eh, self.protocol) + except Exception as e: + continue + if handler.get_handler_state() != ExtHandlerState.NotInstalled: + continue + handlers.append(handler) + + elif os.path.isfile(path) and \ + not os.path.isdir(path[0:-len(HANDLER_PKG_EXT)]): + if not re.match(HANDLER_PKG_PATTERN, item): + continue + pkgs.append(path) + + # Then, remove the orphaned packages + for pkg in pkgs: + try: + os.remove(pkg) + logger.verbose("Removed orphaned extension package " + "{0}".format(pkg)) + except Exception as e: + logger.warn("Failed to remove orphaned package: {0}".format( + pkg)) + + # Finally, remove the directories and packages of the + # uninstalled handlers + for handler in handlers: + handler.rm_ext_handler_dir() + pkg = os.path.join(conf.get_lib_dir(), + handler.get_full_name() + HANDLER_PKG_EXT) + if os.path.isfile(pkg): + try: + os.remove(pkg) + logger.verbose("Removed extension package " + "{0}".format(pkg)) + except Exception as e: + logger.warn("Failed to remove extension package: " + "{0}".format(pkg)) def handle_ext_handlers(self, etag=None): if self.ext_handlers.extHandlers is None or \ @@ -478,6 +553,14 @@ class ExtHandlerInstance(object): separator = path.rfind('-') version = FlexibleVersion(path[separator+1:]) + state_path = os.path.join(path, 'config', 'HandlerState') + + if not os.path.exists(state_path) or \ + fileutil.read_file(state_path) == \ + ExtHandlerState.NotInstalled: + logger.verbose("Ignoring version of uninstalled extension: " + "{0}".format(path)) + continue if lastest_version is None or lastest_version < version: lastest_version = version @@ -615,6 +698,7 @@ class ExtHandlerInstance(object): except IOError as e: message = "Failed to remove extension handler directory: {0}".format(e) self.report_event(message=message, is_success=False) + self.logger.warn(message) def update(self): self.set_operation(WALAEventOperation.Update) diff --git a/azurelinuxagent/ga/monitor.py b/azurelinuxagent/ga/monitor.py index 7ef7f04..dcfd6a4 100644 --- a/azurelinuxagent/ga/monitor.py +++ b/azurelinuxagent/ga/monitor.py @@ -178,11 +178,11 @@ class MonitorHandler(object): logger.error("{0}", e) def daemon(self): - last_heartbeat = datetime.datetime.min period = datetime.timedelta(minutes=30) + last_heartbeat = datetime.datetime.utcnow() - period while True: - if (datetime.datetime.now() - last_heartbeat) > period: - last_heartbeat = datetime.datetime.now() + if datetime.datetime.utcnow() >= (last_heartbeat + period): + last_heartbeat = datetime.datetime.utcnow() add_event( op=WALAEventOperation.HeartBeat, name=CURRENT_AGENT, diff --git a/azurelinuxagent/ga/update.py b/azurelinuxagent/ga/update.py index 203bb41..67eb785 100644 --- a/azurelinuxagent/ga/update.py +++ b/azurelinuxagent/ga/update.py @@ -30,13 +30,17 @@ import time import traceback import zipfile +from datetime import datetime, timedelta + import azurelinuxagent.common.conf as conf import azurelinuxagent.common.logger as logger import azurelinuxagent.common.utils.fileutil as fileutil import azurelinuxagent.common.utils.restutil as restutil import azurelinuxagent.common.utils.textutil as textutil -from azurelinuxagent.common.event import add_event, WALAEventOperation +from azurelinuxagent.common.event import add_event, \ + elapsed_milliseconds, \ + WALAEventOperation from azurelinuxagent.common.exception import UpdateError, ProtocolError from azurelinuxagent.common.future import ustr from azurelinuxagent.common.osutil import get_osutil @@ -53,6 +57,7 @@ from azurelinuxagent.ga.exthandlers import HandlerManifest AGENT_ERROR_FILE = "error.json" # File name for agent error record AGENT_MANIFEST_FILE = "HandlerManifest.json" +AGENT_SUPPORTED_FILE = "supported.json" CHILD_HEALTH_INTERVAL = 15 * 60 CHILD_LAUNCH_INTERVAL = 5 * 60 @@ -60,16 +65,14 @@ CHILD_LAUNCH_RESTART_MAX = 3 CHILD_POLL_INTERVAL = 60 MAX_FAILURE = 3 # Max failure allowed for agent before blacklisted -RETAIN_INTERVAL = 24 * 60 * 60 # Retain interval for black list -GOAL_STATE_INTERVAL = 25 +GOAL_STATE_INTERVAL = 3 REPORT_STATUS_INTERVAL = 15 ORPHAN_WAIT_INTERVAL = 15 * 60 * 60 AGENT_SENTINAL_FILE = "current_version" - def get_update_handler(): return UpdateHandler() @@ -98,7 +101,7 @@ class UpdateHandler(object): self.signal_handler = None return - def run_latest(self): + def run_latest(self, child_args=None): """ This method is called from the daemon to find and launch the most current, downloaded agent. @@ -127,6 +130,9 @@ class UpdateHandler(object): agent_name = latest_agent.name agent_version = latest_agent.version + if child_args is not None: + agent_cmd = "{0} {1}".format(agent_cmd, child_args) + try: # Launch the correct Python version for python-based agents @@ -198,7 +204,7 @@ class UpdateHandler(object): ret) logger.warn(msg) if latest_agent is not None: - latest_agent.mark_failure() + latest_agent.mark_failure(is_fatal=True) except Exception as e: msg = u"Agent {0} launched with command '{1}' failed with exception: {2}".format( @@ -237,10 +243,11 @@ class UpdateHandler(object): migrate_handler_state() try: + send_event_time = datetime.utcnow() + self._ensure_no_orphans() self._emit_restart_event() - # TODO: Add means to stop running while self.running: if self._is_orphaned: logger.info("Goal state agent {0} was orphaned -- exiting", CURRENT_AGENT) @@ -254,8 +261,29 @@ class UpdateHandler(object): self.agents[0].name) break + utc_start = datetime.utcnow() + + last_etag = exthandlers_handler.last_etag exthandlers_handler.run() - + + log_event = last_etag != exthandlers_handler.last_etag or \ + (datetime.utcnow() >= send_event_time) + add_event( + AGENT_NAME, + version=CURRENT_VERSION, + op=WALAEventOperation.ProcessGoalState, + is_success=True, + duration=elapsed_milliseconds(utc_start), + log_event=log_event) + if log_event: + send_event_time += timedelta(minutes=REPORT_STATUS_INTERVAL) + + test_agent = self.get_test_agent() + if test_agent is not None and test_agent.in_slice: + test_agent.enable() + logger.info(u"Enabled Agent {0} as test agent", test_agent.name) + break + time.sleep(GOAL_STATE_INTERVAL) except Exception as e: @@ -305,13 +333,21 @@ class UpdateHandler(object): if not conf.get_autoupdate_enabled(): return None - self._load_agents() + self._find_agents() available_agents = [agent for agent in self.agents if agent.is_available and agent.version > FlexibleVersion(AGENT_VERSION)] return available_agents[0] if len(available_agents) >= 1 else None + def get_test_agent(self): + agent = None + agents = [agent for agent in self._load_agents() if agent.is_test] + if len(agents) > 0: + agents.sort(key=lambda agent: agent.version, reverse=True) + agent = agents[0] + return agent + def _emit_restart_event(self): if not self._is_clean_start: msg = u"{0} did not terminate cleanly".format(CURRENT_AGENT) @@ -390,6 +426,7 @@ class UpdateHandler(object): host = None if protocol and protocol.client: host = protocol.client.get_host_plugin() + self._set_agents([GuestAgent(pkg=pkg, host=host) for pkg in pkg_list.versions]) self._purge_agents() self._filter_blacklisted_agents() @@ -457,6 +494,17 @@ class UpdateHandler(object): self.agents = [agent for agent in self.agents if not agent.is_blacklisted] return + def _find_agents(self): + """ + Load all non-blacklisted agents currently on disk. + """ + try: + self._set_agents(self._load_agents()) + self._filter_blacklisted_agents() + except Exception as e: + logger.warn(u"Exception occurred loading available agents: {0}", ustr(e)) + return + def _get_pid_files(self): pid_file = conf.get_agent_pid_file_path() @@ -502,17 +550,9 @@ class UpdateHandler(object): return fileutil.read_file(conf.get_agent_pid_file_path()) != ustr(parent_pid) def _load_agents(self): - """ - Load all non-blacklisted agents currently on disk. - """ - try: - path = os.path.join(conf.get_lib_dir(), "{0}-*".format(AGENT_NAME)) - self._set_agents([GuestAgent(path=agent_dir) - for agent_dir in glob.iglob(path) if os.path.isdir(agent_dir)]) - self._filter_blacklisted_agents() - except Exception as e: - logger.warn(u"Exception occurred loading available agents: {0}", ustr(e)) - return + path = os.path.join(conf.get_lib_dir(), "{0}-*".format(AGENT_NAME)) + return [GuestAgent(path=agent_dir) + for agent_dir in glob.iglob(path) if os.path.isdir(agent_dir)] def _purge_agents(self): """ @@ -610,7 +650,11 @@ class GuestAgent(object): logger.verbose(u"Instantiating Agent {0} from {1}", self.name, location) self.error = None + self.supported = None + self._load_error() + self._load_supported() + self._ensure_downloaded() return @@ -633,10 +677,19 @@ class GuestAgent(object): def get_agent_pkg_path(self): return ".".join((os.path.join(conf.get_lib_dir(), self.name), "zip")) + def get_agent_supported_file(self): + return os.path.join(conf.get_lib_dir(), self.name, AGENT_SUPPORTED_FILE) + def clear_error(self): self.error.clear() return + def enable(self): + if self.error.is_sentinel: + self.error.clear() + self.error.save() + return + @property def is_available(self): return self.is_downloaded and not self.is_blacklisted @@ -649,6 +702,14 @@ class GuestAgent(object): def is_downloaded(self): return self.is_blacklisted or os.path.isfile(self.get_agent_manifest_path()) + @property + def is_test(self): + return self.error.is_sentinel and self.supported.is_supported + + @property + def in_slice(self): + return self.is_test and self.supported.in_slice + def mark_failure(self, is_fatal=False): try: if not os.path.isdir(self.get_agent_dir()): @@ -666,7 +727,7 @@ class GuestAgent(object): logger.verbose(u"Ensuring Agent {0} is downloaded", self.name) if self.is_blacklisted: - logger.warn(u"Agent {0} is blacklisted - skipping download", self.name) + logger.info(u"Agent {0} is blacklisted - skipping download", self.name) return if self.is_downloaded: @@ -682,6 +743,7 @@ class GuestAgent(object): self._unpack() self._load_manifest() self._load_error() + self._load_supported() msg = u"Agent {0} downloaded successfully".format(self.name) logger.verbose(msg) @@ -770,6 +832,12 @@ class GuestAgent(object): logger.warn(u"Agent {0} failed loading error state: {1}", self.name, ustr(e)) return + def _load_supported(self): + try: + self.supported = Supported(self.get_agent_supported_file()) + except Exception as e: + self.supported = Supported() + def _load_manifest(self): path = self.get_agent_manifest_path() if not os.path.isfile(path): @@ -859,18 +927,15 @@ class GuestAgentError(object): self.failure_count = 0 self.was_fatal = False return - - def clear_old_failure(self): - if self.last_failure <= 0.0: - return - if self.last_failure < (time.time() - RETAIN_INTERVAL): - self.clear() - return @property def is_blacklisted(self): return self.was_fatal or self.failure_count >= MAX_FAILURE + @property + def is_sentinel(self): + return self.was_fatal and self.last_failure == 0.0 + def load(self): if self.path is not None and os.path.isfile(self.path): with open(self.path, 'r') as f: @@ -906,3 +971,61 @@ class GuestAgentError(object): self.last_failure, self.failure_count, self.was_fatal) + +class Supported(object): + def __init__(self, path): + if path is None: + raise UpdateError(u"Supported requires a path") + self.path = path + + self._load() + return + + @property + def is_supported(self): + return self._supported_distribution is not None + + @property + def in_slice(self): + d = self._supported_distribution + return d is not None and d.in_slice + + @property + def _supported_distribution(self): + for d in self.distributions: + dd = self.distributions[d] + if dd.is_supported: + return dd + return None + + def _load(self): + self.distributions = {} + try: + if self.path is not None and os.path.isfile(self.path): + j = json.loads(fileutil.read_file(self.path)) + for d in j: + self.distributions[d] = SupportedDistribution(j[d]) + except Exception as e: + logger.warn("Failed JSON parse of {0}: {1}".format(self.path, e)) + return + +class SupportedDistribution(object): + def __init__(self, s): + if s is None or not isinstance(s, dict): + raise UpdateError(u"SupportedDisribution requires a dictionary") + + self.slice = s['slice'] + self.versions = s['versions'] + + @property + def is_supported(self): + d = ','.join(platform.linux_distribution()) + for v in self.versions: + if re.match(v, d): + return True + return False + + @property + def in_slice(self): + n = int((60 * self.slice) / 100) + return (n - datetime.utcnow().second) > 0 diff --git a/azurelinuxagent/pa/deprovision/arch.py b/azurelinuxagent/pa/deprovision/arch.py new file mode 100644 index 0000000..e661f79 --- /dev/null +++ b/azurelinuxagent/pa/deprovision/arch.py @@ -0,0 +1,33 @@ +# Microsoft Azure Linux Agent +# +# Copyright 2014 Microsoft Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Requires Python 2.4+ and Openssl 1.0+ +# + +import azurelinuxagent.common.utils.fileutil as fileutil +from azurelinuxagent.pa.deprovision.default import DeprovisionHandler, \ + DeprovisionAction + +class ArchDeprovisionHandler(DeprovisionHandler): + def __init__(self): + super(ArchDeprovisionHandler, self).__init__() + + def setup(self, deluser): + warnings, actions = super(ArchDeprovisionHandler, self).setup(deluser) + warnings.append("WARNING! /etc/machine-id will be removed.") + files_to_del = ['/etc/machine-id'] + actions.append(DeprovisionAction(fileutil.rm_files, files_to_del)) + return warnings, actions diff --git a/azurelinuxagent/pa/deprovision/default.py b/azurelinuxagent/pa/deprovision/default.py index ced87ee..90d16c7 100644 --- a/azurelinuxagent/pa/deprovision/default.py +++ b/azurelinuxagent/pa/deprovision/default.py @@ -17,13 +17,17 @@ # Requires Python 2.4+ and Openssl 1.0+ # +import glob +import os.path import signal import sys + import azurelinuxagent.common.conf as conf -from azurelinuxagent.common.exception import ProtocolError -from azurelinuxagent.common.future import read_input import azurelinuxagent.common.utils.fileutil as fileutil import azurelinuxagent.common.utils.shellutil as shellutil + +from azurelinuxagent.common.exception import ProtocolError +from azurelinuxagent.common.future import read_input from azurelinuxagent.common.osutil import get_osutil from azurelinuxagent.common.protocol import get_protocol_util @@ -68,15 +72,19 @@ class DeprovisionHandler(object): def regen_ssh_host_key(self, warnings, actions): warnings.append("WARNING! All SSH host key pairs will be deleted.") actions.append(DeprovisionAction(fileutil.rm_files, - ['/etc/ssh/ssh_host_*key*'])) + [conf.get_ssh_key_glob()])) def stop_agent_service(self, warnings, actions): warnings.append("WARNING! The waagent service will be stopped.") actions.append(DeprovisionAction(self.osutil.stop_agent_service)) + def del_dirs(self, warnings, actions): + dirs = [conf.get_lib_dir(), conf.get_ext_log_dir()] + actions.append(DeprovisionAction(fileutil.rm_dirs, dirs)) + def del_files(self, warnings, actions): - files_to_del = ['/root/.bash_history', '/var/log/waagent.log'] - actions.append(DeprovisionAction(fileutil.rm_files, files_to_del)) + files = ['/root/.bash_history', '/var/log/waagent.log'] + actions.append(DeprovisionAction(fileutil.rm_files, files)) def del_resolv(self, warnings, actions): warnings.append("WARNING! /etc/resolv.conf will be deleted.") @@ -92,9 +100,63 @@ class DeprovisionHandler(object): actions.append(DeprovisionAction(fileutil.rm_files, ["/var/db/dhclient.leases.hn0", "/var/lib/NetworkManager/dhclient-*.lease"])) - def del_lib_dir(self, warnings, actions): - dirs_to_del = [conf.get_lib_dir()] - actions.append(DeprovisionAction(fileutil.rm_dirs, dirs_to_del)) + + def del_lib_dir_files(self, warnings, actions): + known_files = [ + 'HostingEnvironmentConfig.xml', + 'Incarnation', + 'Protocol', + 'SharedConfig.xml', + 'WireServerEndpoint' + ] + known_files_glob = [ + 'Extensions.*.xml', + 'ExtensionsConfig.*.xml', + 'GoalState.*.xml' + ] + + lib_dir = conf.get_lib_dir() + files = [f for f in \ + [os.path.join(lib_dir, kf) for kf in known_files] \ + if os.path.isfile(f)] + for p in known_files_glob: + files += glob.glob(os.path.join(lib_dir, p)) + + if len(files) > 0: + actions.append(DeprovisionAction(fileutil.rm_files, files)) + + def cloud_init_dirs(self, include_once=True): + dirs = [ + "/var/lib/cloud/instance", + "/var/lib/cloud/instances/", + "/var/lib/cloud/data" + ] + if include_once: + dirs += [ + "/var/lib/cloud/scripts/per-once" + ] + return dirs + + def cloud_init_files(self, include_once=True): + files = [ + "/etc/sudoers.d/90-cloud-init-users" + ] + if include_once: + files += [ + "/var/lib/cloud/sem/config_scripts_per_once.once" + ] + return files + + def del_cloud_init(self, warnings, actions, include_once=True): + dirs = [d for d in self.cloud_init_dirs(include_once=include_once) \ + if os.path.isdir(d)] + if len(dirs) > 0: + actions.append(DeprovisionAction(fileutil.rm_dirs, dirs)) + + files = [f for f in self.cloud_init_files(include_once=include_once) \ + if os.path.isfile(f)] + if len(files) > 0: + actions.append(DeprovisionAction(fileutil.rm_files, files)) def reset_hostname(self, warnings, actions): localhost = ["localhost.localdomain"] @@ -117,7 +179,8 @@ class DeprovisionHandler(object): if conf.get_delete_root_password(): self.del_root_password(warnings, actions) - self.del_lib_dir(warnings, actions) + self.del_cloud_init(warnings, actions) + self.del_dirs(warnings, actions) self.del_files(warnings, actions) self.del_resolv(warnings, actions) @@ -126,19 +189,55 @@ class DeprovisionHandler(object): return warnings, actions + def setup_changed_unique_id(self): + warnings = [] + actions = [] + + self.del_cloud_init(warnings, actions, include_once=False) + self.del_dhcp_lease(warnings, actions) + self.del_lib_dir_files(warnings, actions) + self.del_resolv(warnings, actions) + + return warnings, actions + def run(self, force=False, deluser=False): warnings, actions = self.setup(deluser) - for warning in warnings: - print(warning) - if not force: - confirm = read_input("Do you want to proceed (y/n)") - if not confirm.lower().startswith('y'): - return + self.do_warnings(warnings) + self.do_confirmation(force=force) + self.do_actions(actions) + + def run_changed_unique_id(self): + ''' + Clean-up files and directories that may interfere when the VM unique + identifier has changed. + While users *should* manually deprovision a VM, the files removed by + this routine will help keep the agent from getting confused + (since incarnation and extension settings, among other items, will + no longer be monotonically increasing). + ''' + warnings, actions = self.setup_changed_unique_id() + + self.do_warnings(warnings) + self.do_actions(actions) + + def do_actions(self, actions): self.actions_running = True for action in actions: action.invoke() + self.actions_running = False + + def do_confirmation(self, force=False): + if force: + return True + + confirm = read_input("Do you want to proceed (y/n)") + return True if confirm.lower().startswith('y') else False + + def do_warnings(self, warnings): + for warning in warnings: + print(warning) def handle_interrupt_signal(self, signum, frame): if not self.actions_running: diff --git a/azurelinuxagent/pa/deprovision/factory.py b/azurelinuxagent/pa/deprovision/factory.py index 72a5be1..5ac35a7 100644 --- a/azurelinuxagent/pa/deprovision/factory.py +++ b/azurelinuxagent/pa/deprovision/factory.py @@ -21,6 +21,7 @@ from azurelinuxagent.common.version import DISTRO_NAME, DISTRO_VERSION, \ DISTRO_FULL_NAME from .default import DeprovisionHandler +from .arch import ArchDeprovisionHandler from .clearlinux import ClearLinuxDeprovisionHandler from .coreos import CoreOSDeprovisionHandler from .ubuntu import UbuntuDeprovisionHandler @@ -28,6 +29,8 @@ from .ubuntu import UbuntuDeprovisionHandler def get_deprovision_handler(distro_name=DISTRO_NAME, distro_version=DISTRO_VERSION, distro_full_name=DISTRO_FULL_NAME): + if distro_name == "arch": + return ArchDeprovisionHandler() if distro_name == "ubuntu": return UbuntuDeprovisionHandler() if distro_name == "coreos": diff --git a/azurelinuxagent/pa/provision/cloudinit.py b/azurelinuxagent/pa/provision/cloudinit.py new file mode 100644 index 0000000..5789e9a --- /dev/null +++ b/azurelinuxagent/pa/provision/cloudinit.py @@ -0,0 +1,132 @@ +# Microsoft Azure Linux Agent +# +# Copyright 2014 Microsoft Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Requires Python 2.4+ and Openssl 1.0+ +# + +import os +import os.path +import time + +from datetime import datetime + +import azurelinuxagent.common.conf as conf +import azurelinuxagent.common.logger as logger +import azurelinuxagent.common.utils.fileutil as fileutil +import azurelinuxagent.common.utils.shellutil as shellutil + +from azurelinuxagent.common.event import elapsed_milliseconds +from azurelinuxagent.common.exception import ProvisionError, ProtocolError +from azurelinuxagent.common.future import ustr +from azurelinuxagent.common.protocol import OVF_FILE_NAME +from azurelinuxagent.common.protocol.ovfenv import OvfEnv +from azurelinuxagent.pa.provision.default import ProvisionHandler + + +class CloudInitProvisionHandler(ProvisionHandler): + def __init__(self): + super(CloudInitProvisionHandler, self).__init__() + + def run(self): + # If provision is enabled, run default provision handler + if conf.get_provision_enabled(): + logger.warn("Provisioning flag is enabled, which overrides using " + "cloud-init; running the default provisioning code") + super(CloudInitProvisionHandler, self).run() + return + + try: + if super(CloudInitProvisionHandler, self).is_provisioned(): + logger.info("Provisioning already completed, skipping.") + return + + utc_start = datetime.utcnow() + logger.info("Running CloudInit provisioning handler") + self.wait_for_ovfenv() + self.protocol_util.get_protocol() + self.report_not_ready("Provisioning", "Starting") + + thumbprint = self.wait_for_ssh_host_key() + self.write_provisioned() + logger.info("Finished provisioning") + + self.report_ready(thumbprint) + self.report_event("Provision succeed", + is_success=True, + duration=elapsed_milliseconds(utc_start)) + + except ProvisionError as e: + logger.error("Provisioning failed: {0}", ustr(e)) + self.report_not_ready("ProvisioningFailed", ustr(e)) + self.report_event(ustr(e)) + return + + def wait_for_ovfenv(self, max_retry=360, sleep_time=5): + """ + Wait for cloud-init to copy ovf-env.xml file from provision ISO + """ + ovf_file_path = os.path.join(conf.get_lib_dir(), OVF_FILE_NAME) + for retry in range(0, max_retry): + if os.path.isfile(ovf_file_path): + try: + OvfEnv(fileutil.read_file(ovf_file_path)) + return + except ProtocolError as pe: + raise ProvisionError("OVF xml could not be parsed " + "[{0}]: {1}".format(ovf_file_path, + ustr(pe))) + else: + if retry < max_retry - 1: + logger.info( + "Waiting for cloud-init to copy ovf-env.xml to {0} " + "[{1} retries remaining, " + "sleeping {2}s]".format(ovf_file_path, + max_retry - retry, + sleep_time)) + if not self.validate_cloud_init(): + logger.warn("cloud-init does not appear to be running") + time.sleep(sleep_time) + raise ProvisionError("Giving up, ovf-env.xml was not copied to {0} " + "after {1}s".format(ovf_file_path, + max_retry * sleep_time)) + + def wait_for_ssh_host_key(self, max_retry=360, sleep_time=5): + """ + Wait for cloud-init to generate ssh host key + """ + keypair_type = conf.get_ssh_host_keypair_type() + path = conf.get_ssh_key_public_path() + for retry in range(0, max_retry): + if os.path.isfile(path): + logger.info("ssh host key found at: {0}".format(path)) + try: + thumbprint = self.get_ssh_host_key_thumbprint(chk_err=False) + logger.info("Thumbprint obtained from : {0}".format(path)) + return thumbprint + except ProvisionError: + logger.warn("Could not get thumbprint from {0}".format(path)) + if retry < max_retry - 1: + logger.info("Waiting for ssh host key be generated at {0} " + "[{1} attempts remaining, " + "sleeping {2}s]".format(path, + max_retry - retry, + sleep_time)) + if not self.validate_cloud_init(): + logger.warn("cloud-init does not appear to be running") + time.sleep(sleep_time) + raise ProvisionError("Giving up, ssh host key was not found at {0} " + "after {1}s".format(path, + max_retry * sleep_time)) diff --git a/azurelinuxagent/pa/provision/default.py b/azurelinuxagent/pa/provision/default.py index 3a3f36f..d4870f1 100644 --- a/azurelinuxagent/pa/provision/default.py +++ b/azurelinuxagent/pa/provision/default.py @@ -20,19 +20,31 @@ Provision handler """ import os +import os.path +import re + +from datetime import datetime + +import azurelinuxagent.common.conf as conf import azurelinuxagent.common.logger as logger +import azurelinuxagent.common.utils.shellutil as shellutil +import azurelinuxagent.common.utils.fileutil as fileutil + from azurelinuxagent.common.future import ustr -import azurelinuxagent.common.conf as conf -from azurelinuxagent.common.event import add_event, WALAEventOperation +from azurelinuxagent.common.event import add_event, WALAEventOperation, \ + elapsed_milliseconds from azurelinuxagent.common.exception import ProvisionError, ProtocolError, \ OSUtilError -from azurelinuxagent.common.protocol.restapi import ProvisionStatus -import azurelinuxagent.common.utils.shellutil as shellutil -import azurelinuxagent.common.utils.fileutil as fileutil from azurelinuxagent.common.osutil import get_osutil +from azurelinuxagent.common.protocol.restapi import ProvisionStatus from azurelinuxagent.common.protocol import get_protocol_util +from azurelinuxagent.common.version import AGENT_NAME CUSTOM_DATA_FILE = "CustomData" +CLOUD_INIT_PATTERN = b".*/bin/cloud-init.*" +CLOUD_INIT_REGEX = re.compile(CLOUD_INIT_PATTERN) + +PROVISIONED_FILE = 'provisioned' class ProvisionHandler(object): @@ -41,54 +53,85 @@ class ProvisionHandler(object): self.protocol_util = get_protocol_util() def run(self): - # if provisioning is already done, return - provisioned = os.path.join(conf.get_lib_dir(), "provisioned") - if os.path.isfile(provisioned): - logger.info("Provisioning already completed, skipping.") - return - - thumbprint = None # If provision is not enabled, report ready and then return if not conf.get_provision_enabled(): logger.info("Provisioning is disabled, skipping.") - else: - logger.info("Running provisioning handler") - try: - logger.info("Copying ovf-env.xml") - ovf_env = self.protocol_util.copy_ovf_env() - self.protocol_util.get_protocol_by_file() - self.report_not_ready("Provisioning", "Starting") - logger.info("Starting provisioning") - self.provision(ovf_env) - thumbprint = self.reg_ssh_host_key() - self.osutil.restart_ssh_service() - self.report_event("Provision succeed", is_success=True) - except ProtocolError as e: - logger.error("[ProtocolError] Provisioning failed: {0}", e) - self.report_not_ready("ProvisioningFailed", ustr(e)) - self.report_event("Failed to copy ovf-env.xml: {0}".format(e)) - return - except ProvisionError as e: - logger.error("[ProvisionError] Provisioning failed: {0}", e) - self.report_not_ready("ProvisioningFailed", ustr(e)) - self.report_event(ustr(e)) + return + + try: + utc_start = datetime.utcnow() + thumbprint = None + + # if provisioning is already done, return + if self.is_provisioned(): + logger.info("Provisioning already completed, skipping.") return - # write out provisioned file and report Ready - fileutil.write_file(provisioned, "") - self.report_ready(thumbprint) - logger.info("Provisioning complete") + + logger.info("Running default provisioning handler") + + if not self.validate_cloud_init(is_expected=False): + raise ProvisionError("cloud-init appears to be running, " + "this is not expected, cannot continue") + + logger.info("Copying ovf-env.xml") + ovf_env = self.protocol_util.copy_ovf_env() + + self.protocol_util.get_protocol_by_file() + self.report_not_ready("Provisioning", "Starting") + logger.info("Starting provisioning") + + self.provision(ovf_env) + + thumbprint = self.reg_ssh_host_key() + self.osutil.restart_ssh_service() + + # write out provisioned file and report Ready + self.write_provisioned() + + self.report_event("Provision succeed", + is_success=True, + duration=elapsed_milliseconds(utc_start)) + + self.report_ready(thumbprint) + logger.info("Provisioning complete") + + except (ProtocolError, ProvisionError) as e: + self.report_not_ready("ProvisioningFailed", ustr(e)) + self.report_event(ustr(e)) + logger.error("Provisioning failed: {0}", ustr(e)) + return + + @staticmethod + def validate_cloud_init(is_expected=True): + pids = [pid for pid in os.listdir('/proc') if pid.isdigit()] + is_running = False + for pid in pids: + try: + pname = open(os.path.join('/proc', pid, 'cmdline'), 'rb').read() + if CLOUD_INIT_REGEX.match(pname): + is_running = True + msg = "cloud-init is running [PID {0}, {1}]".format(pid, + pname) + if is_expected: + logger.verbose(msg) + else: + logger.error(msg) + break + except IOError: + continue + return is_running == is_expected def reg_ssh_host_key(self): keypair_type = conf.get_ssh_host_keypair_type() if conf.get_regenerate_ssh_host_key(): - fileutil.rm_files("/etc/ssh/ssh_host_*key*") - keygen_cmd = "ssh-keygen -N '' -t {0} -f /etc/ssh/ssh_host_{1}_key" - shellutil.run(keygen_cmd.format(keypair_type, keypair_type)) - thumbprint = self.get_ssh_host_key_thumbprint(keypair_type) - return thumbprint - - def get_ssh_host_key_thumbprint(self, keypair_type, chk_err=True): - cmd = "ssh-keygen -lf /etc/ssh/ssh_host_{0}_key.pub".format(keypair_type) + fileutil.rm_files(conf.get_ssh_key_glob()) + keygen_cmd = "ssh-keygen -N '' -t {0} -f {1}" + shellutil.run(keygen_cmd.format(keypair_type, + conf.get_ssh_key_private_path())) + return self.get_ssh_host_key_thumbprint() + + def get_ssh_host_key_thumbprint(self, chk_err=True): + cmd = "ssh-keygen -lf {0}".format(conf.get_ssh_key_public_path()) ret = shellutil.run_get_output(cmd, chk_err=chk_err) if ret[0] == 0: return ret[1].rstrip().split()[1].replace(':', '') @@ -96,6 +139,45 @@ class ProvisionHandler(object): raise ProvisionError(("Failed to generate ssh host key: " "ret={0}, out= {1}").format(ret[0], ret[1])) + def provisioned_file_path(self): + return os.path.join(conf.get_lib_dir(), PROVISIONED_FILE) + + def is_provisioned(self): + ''' + A VM is considered provisionend *anytime* the provisioning + sentinel file exists and not provisioned *anytime* the file + is absent. + + If the VM was provisioned using an agent that did not record + the VM unique identifier, the provisioning file will be re-written + to include the identifier. + + A warning is logged *if* the VM unique identifier has changed + since VM was provisioned. + ''' + if not os.path.isfile(self.provisioned_file_path()): + return False + + s = fileutil.read_file(self.provisioned_file_path()).strip() + if s != self.osutil.get_instance_id(): + if len(s) > 0: + logger.warn("VM is provisioned, " + "but the VM unique identifier has changed -- " + "clearing cached state") + from azurelinuxagent.pa.deprovision \ + import get_deprovision_handler + deprovision_handler = get_deprovision_handler() + deprovision_handler.run_changed_unique_id() + + self.write_provisioned() + + return True + + def write_provisioned(self): + fileutil.write_file( + self.provisioned_file_path(), + get_osutil().get_instance_id()) + def provision(self, ovfenv): logger.info("Handle ovf-env.xml.") try: @@ -113,7 +195,7 @@ class ProvisionHandler(object): self.osutil.del_root_password() except OSUtilError as e: - raise ProvisionError("Failed to handle ovf-env.xml: {0}".format(e)) + raise ProvisionError("Failed to provision: {0}".format(ustr(e))) def config_user_account(self, ovfenv): logger.info("Create user account if not exists") @@ -141,11 +223,12 @@ class ProvisionHandler(object): if customdata is None: return - logger.info("Save custom data") lib_dir = conf.get_lib_dir() - if conf.get_decode_customdata(): + if conf.get_decode_customdata() or conf.get_execute_customdata(): + logger.info("Decode custom data") customdata = self.osutil.decode_customdata(customdata) + logger.info("Save custom data") customdata_file = os.path.join(lib_dir, CUSTOM_DATA_FILE) fileutil.write_file(customdata_file, customdata) @@ -164,9 +247,12 @@ class ProvisionHandler(object): logger.info("Deploy ssh key pairs.") self.osutil.deploy_ssh_keypair(ovfenv.username, keypair) - def report_event(self, message, is_success=False): - add_event(name="WALA", message=message, is_success=is_success, - op=WALAEventOperation.Provision) + def report_event(self, message, is_success=False, duration=0): + add_event(name=AGENT_NAME, + message=message, + duration=duration, + is_success=is_success, + op=WALAEventOperation.Provision) def report_not_ready(self, sub_status, description): status = ProvisionStatus(status="NotReady", subStatus=sub_status, diff --git a/azurelinuxagent/pa/provision/factory.py b/azurelinuxagent/pa/provision/factory.py index 9bbe35c..d87765f 100644 --- a/azurelinuxagent/pa/provision/factory.py +++ b/azurelinuxagent/pa/provision/factory.py @@ -15,18 +15,22 @@ # Requires Python 2.4+ and Openssl 1.0+ # +import azurelinuxagent.common.conf as conf import azurelinuxagent.common.logger as logger + from azurelinuxagent.common.utils.textutil import Version from azurelinuxagent.common.version import DISTRO_NAME, DISTRO_VERSION, \ DISTRO_FULL_NAME + from .default import ProvisionHandler -from .ubuntu import UbuntuProvisionHandler +from .cloudinit import CloudInitProvisionHandler def get_provision_handler(distro_name=DISTRO_NAME, distro_version=DISTRO_VERSION, distro_full_name=DISTRO_FULL_NAME): - if distro_name == "ubuntu": - return UbuntuProvisionHandler() + + if conf.get_provision_cloudinit(): + return CloudInitProvisionHandler() return ProvisionHandler() diff --git a/azurelinuxagent/pa/provision/ubuntu.py b/azurelinuxagent/pa/provision/ubuntu.py deleted file mode 100644 index 66866b2..0000000 --- a/azurelinuxagent/pa/provision/ubuntu.py +++ /dev/null @@ -1,102 +0,0 @@ -# Microsoft Azure Linux Agent -# -# Copyright 2014 Microsoft Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# Requires Python 2.4+ and Openssl 1.0+ -# - -import os -import time - -import azurelinuxagent.common.conf as conf -import azurelinuxagent.common.logger as logger -import azurelinuxagent.common.utils.fileutil as fileutil -from azurelinuxagent.common.exception import ProvisionError, ProtocolError -from azurelinuxagent.common.future import ustr -from azurelinuxagent.pa.provision.default import ProvisionHandler - -""" -On ubuntu image, provision could be disabled. -""" - - -class UbuntuProvisionHandler(ProvisionHandler): - def __init__(self): - super(UbuntuProvisionHandler, self).__init__() - - def run(self): - # If provision is enabled, run default provision handler - if conf.get_provision_enabled(): - super(UbuntuProvisionHandler, self).run() - return - - logger.info("run Ubuntu provision handler") - provisioned = os.path.join(conf.get_lib_dir(), "provisioned") - if os.path.isfile(provisioned): - return - - logger.info("Waiting cloud-init to copy ovf-env.xml.") - self.wait_for_ovfenv() - self.protocol_util.get_protocol() - self.report_not_ready("Provisioning", "Starting") - logger.info("Sleeping 1 second to avoid throttling.") - time.sleep(1) - try: - logger.info("Wait for ssh host key to be generated.") - thumbprint = self.wait_for_ssh_host_key() - fileutil.write_file(provisioned, "") - logger.info("Finished provisioning") - except ProvisionError as e: - logger.error("Provision failed: {0}", e) - self.report_not_ready("ProvisioningFailed", ustr(e)) - self.report_event(ustr(e)) - return - - self.report_ready(thumbprint) - self.report_event("Provision succeed", is_success=True) - - def wait_for_ovfenv(self, max_retry=60): - """ - Wait for cloud-init to copy ovf-env.xml file from provision ISO - """ - for retry in range(0, max_retry): - try: - self.protocol_util.get_ovf_env() - return - except ProtocolError: - if retry < max_retry - 1: - logger.info("Wait for cloud-init to copy ovf-env.xml") - time.sleep(5) - raise ProvisionError("ovf-env.xml is not copied") - - def wait_for_ssh_host_key(self, max_retry=60): - """ - Wait for cloud-init to generate ssh host key - """ - keypair_type = conf.get_ssh_host_keypair_type() - path = '/etc/ssh/ssh_host_{0}_key.pub'.format(keypair_type) - for retry in range(0, max_retry): - if os.path.isfile(path): - logger.info("ssh host key found at: {0}".format(path)) - try: - thumbprint = self.get_ssh_host_key_thumbprint(keypair_type, chk_err=False) - logger.info("Thumbprint obtained from : {0}".format(path)) - return thumbprint - except ProvisionError: - logger.warn("Could not get thumbprint from {0}".format(path)) - if retry < max_retry - 1: - logger.info("Wait for ssh host key be generated: {0}".format(path)) - time.sleep(5) - raise ProvisionError("ssh host key is not generated.") |