diff options
Diffstat (limited to 'azurelinuxagent/common/utils')
| -rw-r--r-- | azurelinuxagent/common/utils/cryptutil.py | 11 | ||||
| -rw-r--r-- | azurelinuxagent/common/utils/fileutil.py | 35 | ||||
| -rw-r--r-- | azurelinuxagent/common/utils/flexible_version.py | 2 | ||||
| -rw-r--r-- | azurelinuxagent/common/utils/restutil.py | 105 | ||||
| -rw-r--r-- | azurelinuxagent/common/utils/shellutil.py | 26 |
5 files changed, 136 insertions, 43 deletions
diff --git a/azurelinuxagent/common/utils/cryptutil.py b/azurelinuxagent/common/utils/cryptutil.py index 6339eb3..b34c1a5 100644 --- a/azurelinuxagent/common/utils/cryptutil.py +++ b/azurelinuxagent/common/utils/cryptutil.py @@ -19,8 +19,11 @@ import base64 import struct + from azurelinuxagent.common.future import ustr, bytebuffer from azurelinuxagent.common.exception import CryptError + +import azurelinuxagent.common.logger as logger import azurelinuxagent.common.utils.shellutil as shellutil class CryptUtil(object): @@ -34,7 +37,10 @@ class CryptUtil(object): cmd = ("{0} req -x509 -nodes -subj /CN=LinuxTransport -days 730 " "-newkey rsa:2048 -keyout {1} " "-out {2}").format(self.openssl_cmd, prv_file, crt_file) - shellutil.run(cmd) + rc = shellutil.run(cmd) + if rc != 0: + logger.error("Failed to create {0} and {1} certificates".format( + prv_file, crt_file)) def get_pubkey_from_prv(self, file_name): cmd = "{0} rsa -in {1} -pubout 2>/dev/null".format(self.openssl_cmd, @@ -61,6 +67,9 @@ class CryptUtil(object): "").format(self.openssl_cmd, p7m_file, trans_prv_file, trans_cert_file, self.openssl_cmd, pem_file) shellutil.run(cmd) + rc = shellutil.run(cmd) + if rc != 0: + logger.error("Failed to decrypt {0}".format(p7m_file)) def crt_to_ssh(self, input_file, output_file): shellutil.run("ssh-keygen -i -m PKCS8 -f {0} >> {1}".format(input_file, diff --git a/azurelinuxagent/common/utils/fileutil.py b/azurelinuxagent/common/utils/fileutil.py index 96b5b82..1f0c7ac 100644 --- a/azurelinuxagent/common/utils/fileutil.py +++ b/azurelinuxagent/common/utils/fileutil.py @@ -27,7 +27,6 @@ import os import pwd import re import shutil -import string import azurelinuxagent.common.logger as logger import azurelinuxagent.common.utils.textutil as textutil @@ -42,9 +41,10 @@ KNOWN_IOERRORS = [ errno.ENOSPC, # Out of space errno.ENAMETOOLONG, # Name too long errno.ELOOP, # Too many symbolic links encountered - errno.EREMOTEIO # Remote I/O error + 121 # Remote I/O error (errno.EREMOTEIO -- not present in all Python 2.7+) ] + def copy_file(from_path, to_path=None, to_dir=None): if to_path is None: to_path = os.path.join(to_dir, os.path.basename(from_path)) @@ -66,11 +66,12 @@ def read_file(filepath, asbin=False, remove_bom=False, encoding='utf-8'): return data if remove_bom: - #Remove bom on bytes data before it is converted into string. + # remove bom on bytes data before it is converted into string. data = textutil.remove_bom(data) data = ustr(data, encoding=encoding) return data + def write_file(filepath, contents, asbin=False, encoding='utf-8', append=False): """ Write 'contents' to 'filepath'. @@ -82,6 +83,7 @@ def write_file(filepath, contents, asbin=False, encoding='utf-8', append=False): with open(filepath, mode) as out_file: out_file.write(data) + def append_file(filepath, contents, asbin=False, encoding='utf-8'): """ Append 'contents' to 'filepath'. @@ -93,6 +95,7 @@ def base_name(path): head, tail = os.path.split(path) return tail + def get_line_startingwith(prefix, filepath): """ Return line from 'filepath' if the line startswith 'prefix' @@ -102,7 +105,6 @@ def get_line_startingwith(prefix, filepath): return line return None -#End File operation util functions def mkdir(dirpath, mode=None, owner=None): if not os.path.isdir(dirpath): @@ -112,6 +114,7 @@ def mkdir(dirpath, mode=None, owner=None): if owner is not None: chowner(dirpath, owner) + def chowner(path, owner): if not os.path.exists(path): logger.error("Path does not exist: {0}".format(path)) @@ -119,19 +122,22 @@ def chowner(path, owner): owner_info = pwd.getpwnam(owner) os.chown(path, owner_info[2], owner_info[3]) + def chmod(path, mode): if not os.path.exists(path): logger.error("Path does not exist: {0}".format(path)) else: os.chmod(path, mode) + def rm_files(*args): for paths in args: - #Find all possible file paths + # find all possible file paths for path in glob.glob(paths): if os.path.isfile(path): os.remove(path) + def rm_dirs(*args): """ Remove the contents of each directry @@ -149,20 +155,24 @@ def rm_dirs(*args): elif os.path.isdir(path): shutil.rmtree(path) + def trim_ext(path, ext): if not ext.startswith("."): ext = "." + ext return path.split(ext)[0] if path.endswith(ext) else path + def update_conf_file(path, line_start, val, chk_err=False): conf = [] if not os.path.isfile(path) and chk_err: raise IOError("Can't find config file:{0}".format(path)) conf = read_file(path).split('\n') - conf = [x for x in conf if x is not None and len(x) > 0 and not x.startswith(line_start)] + conf = [x for x in conf + if x is not None and len(x) > 0 and not x.startswith(line_start)] conf.append(val) write_file(path, '\n'.join(conf) + '\n') + def search_file(target_dir_name, target_file_name): for root, dirs, files in os.walk(target_dir_name): for file_name in files: @@ -170,24 +180,28 @@ def search_file(target_dir_name, target_file_name): return os.path.join(root, file_name) return None + def chmod_tree(path, mode): for root, dirs, files in os.walk(path): for file_name in files: os.chmod(os.path.join(root, file_name), mode) + def findstr_in_file(file_path, line_str): """ Return True if the line is in the file; False otherwise. - (Trailing whitespace is ignore.) + (Trailing whitespace is ignored.) """ try: for line in (open(file_path, 'r')).readlines(): if line_str == line.rstrip(): return True - except Exception as e: + except Exception: + # swallow exception pass return False + def findre_in_file(file_path, line_re): """ Return match object if found in file. @@ -203,6 +217,7 @@ def findre_in_file(file_path, line_re): return None + def get_all_files(root_path): """ Find all files under the given root path @@ -213,6 +228,7 @@ def get_all_files(root_path): return result + def clean_ioerror(e, paths=[]): """ Clean-up possibly bad files and directories after an IO error. @@ -228,5 +244,6 @@ def clean_ioerror(e, paths=[]): shutil.rmtree(path, ignore_errors=True) else: os.remove(path) - except Exception as e: + except Exception: + # swallow exception pass diff --git a/azurelinuxagent/common/utils/flexible_version.py b/azurelinuxagent/common/utils/flexible_version.py index 2fce88d..14c2a73 100644 --- a/azurelinuxagent/common/utils/flexible_version.py +++ b/azurelinuxagent/common/utils/flexible_version.py @@ -37,7 +37,7 @@ class FlexibleVersion(version.Version): self.prerelease = None self.version = () if vstring: - self._parse(vstring) + self._parse(str(vstring)) return _nn_version = 'version' diff --git a/azurelinuxagent/common/utils/restutil.py b/azurelinuxagent/common/utils/restutil.py index ddd930b..807be29 100644 --- a/azurelinuxagent/common/utils/restutil.py +++ b/azurelinuxagent/common/utils/restutil.py @@ -18,6 +18,7 @@ # import os +import threading import time import traceback @@ -32,10 +33,11 @@ from azurelinuxagent.common.version import PY_VERSION_MAJOR SECURE_WARNING_EMITTED = False -DEFAULT_RETRIES = 3 +DEFAULT_RETRIES = 6 +DELAY_IN_SECONDS = 1 -SHORT_DELAY_IN_SECONDS = 5 -LONG_DELAY_IN_SECONDS = 15 +THROTTLE_RETRIES = 25 +THROTTLE_DELAY_IN_SECONDS = 1 RETRY_CODES = [ httpclient.RESET_CONTENT, @@ -63,7 +65,8 @@ OK_CODES = [ THROTTLE_CODES = [ httpclient.FORBIDDEN, - httpclient.SERVICE_UNAVAILABLE + httpclient.SERVICE_UNAVAILABLE, + 429, # Request Rate Limit Exceeded ] RETRY_EXCEPTIONS = [ @@ -76,6 +79,48 @@ RETRY_EXCEPTIONS = [ HTTP_PROXY_ENV = "http_proxy" HTTPS_PROXY_ENV = "https_proxy" +DEFAULT_PROTOCOL_ENDPOINT='168.63.129.16' +HOST_PLUGIN_PORT = 32526 + + +class IOErrorCounter(object): + _lock = threading.RLock() + _protocol_endpoint = DEFAULT_PROTOCOL_ENDPOINT + _counts = {"hostplugin":0, "protocol":0, "other":0} + + @staticmethod + def increment(host=None, port=None): + with IOErrorCounter._lock: + if host == IOErrorCounter._protocol_endpoint: + if port == HOST_PLUGIN_PORT: + IOErrorCounter._counts["hostplugin"] += 1 + else: + IOErrorCounter._counts["protocol"] += 1 + else: + IOErrorCounter._counts["other"] += 1 + + @staticmethod + def get_and_reset(): + with IOErrorCounter._lock: + counts = IOErrorCounter._counts.copy() + IOErrorCounter.reset() + return counts + + @staticmethod + def reset(): + with IOErrorCounter._lock: + IOErrorCounter._counts = {"hostplugin":0, "protocol":0, "other":0} + + @staticmethod + def set_protocol_endpoint(endpoint=DEFAULT_PROTOCOL_ENDPOINT): + IOErrorCounter._protocol_endpoint = endpoint + + +def _compute_delay(retry_attempt=1, delay=DELAY_IN_SECONDS): + fib = (1, 1) + for n in range(retry_attempt): + fib = (fib[1], fib[0]+fib[1]) + return delay*fib[1] def _is_retry_status(status, retry_codes=RETRY_CODES): return status in retry_codes @@ -166,7 +211,7 @@ def http_request(method, use_proxy=False, max_retry=DEFAULT_RETRIES, retry_codes=RETRY_CODES, - retry_delay=SHORT_DELAY_IN_SECONDS): + retry_delay=DELAY_IN_SECONDS): global SECURE_WARNING_EMITTED @@ -208,18 +253,31 @@ def http_request(method, msg = '' attempt = 0 - delay = retry_delay + delay = 0 + was_throttled = False while attempt < max_retry: if attempt > 0: - logger.info("[HTTP Retry] Attempt {0} of {1}: {2}", + # Compute the request delay + # -- Use a fixed delay if the server ever rate-throttles the request + # (with a safe, minimum number of retry attempts) + # -- Otherwise, compute a delay that is the product of the next + # item in the Fibonacci series and the initial delay value + delay = THROTTLE_DELAY_IN_SECONDS \ + if was_throttled \ + else _compute_delay(retry_attempt=attempt, + delay=retry_delay) + + logger.verbose("[HTTP Retry] " + "Attempt {0} of {1} will delay {2} seconds: {3}", attempt+1, max_retry, + delay, msg) + time.sleep(delay) attempt += 1 - delay = retry_delay try: resp = _http_request(method, @@ -235,13 +293,13 @@ def http_request(method, if request_failed(resp): if _is_retry_status(resp.status, retry_codes=retry_codes): - msg = '[HTTP Retry] HTTP {0} Status Code {1}'.format( - method, resp.status) + msg = '[HTTP Retry] {0} {1} -- Status Code {2}'.format( + method, url, resp.status) + # Note if throttled and ensure a safe, minimum number of + # retry attempts if _is_throttle_status(resp.status): - delay = LONG_DELAY_IN_SECONDS - logger.info("[HTTP Delay] Delay {0} seconds for " \ - "Status Code {1}".format( - delay, resp.status)) + was_throttled = True + max_retry = max(max_retry, THROTTLE_RETRIES) continue if resp.status in RESOURCE_GONE_CODES: @@ -250,22 +308,25 @@ def http_request(method, return resp except httpclient.HTTPException as e: - msg = '[HTTP Failed] HTTP {0} HttpException {1}'.format(method, e) + msg = '[HTTP Failed] {0} {1} -- HttpException {2}'.format( + method, url, e) if _is_retry_exception(e): continue break except IOError as e: - msg = '[HTTP Failed] HTTP {0} IOError {1}'.format(method, e) + IOErrorCounter.increment(host=host, port=port) + msg = '[HTTP Failed] {0} {1} -- IOError {2}'.format( + method, url, e) continue - raise HttpError(msg) + raise HttpError("{0} -- {1} attempts made".format(msg,attempt)) def http_get(url, headers=None, use_proxy=False, max_retry=DEFAULT_RETRIES, retry_codes=RETRY_CODES, - retry_delay=SHORT_DELAY_IN_SECONDS): + retry_delay=DELAY_IN_SECONDS): return http_request("GET", url, None, headers=headers, use_proxy=use_proxy, @@ -277,7 +338,7 @@ def http_get(url, headers=None, use_proxy=False, def http_head(url, headers=None, use_proxy=False, max_retry=DEFAULT_RETRIES, retry_codes=RETRY_CODES, - retry_delay=SHORT_DELAY_IN_SECONDS): + retry_delay=DELAY_IN_SECONDS): return http_request("HEAD", url, None, headers=headers, use_proxy=use_proxy, @@ -289,7 +350,7 @@ def http_head(url, headers=None, use_proxy=False, def http_post(url, data, headers=None, use_proxy=False, max_retry=DEFAULT_RETRIES, retry_codes=RETRY_CODES, - retry_delay=SHORT_DELAY_IN_SECONDS): + retry_delay=DELAY_IN_SECONDS): return http_request("POST", url, data, headers=headers, use_proxy=use_proxy, @@ -301,7 +362,7 @@ def http_post(url, data, headers=None, use_proxy=False, def http_put(url, data, headers=None, use_proxy=False, max_retry=DEFAULT_RETRIES, retry_codes=RETRY_CODES, - retry_delay=SHORT_DELAY_IN_SECONDS): + retry_delay=DELAY_IN_SECONDS): return http_request("PUT", url, data, headers=headers, use_proxy=use_proxy, @@ -313,7 +374,7 @@ def http_put(url, data, headers=None, use_proxy=False, def http_delete(url, headers=None, use_proxy=False, max_retry=DEFAULT_RETRIES, retry_codes=RETRY_CODES, - retry_delay=SHORT_DELAY_IN_SECONDS): + retry_delay=DELAY_IN_SECONDS): return http_request("DELETE", url, None, headers=headers, use_proxy=use_proxy, diff --git a/azurelinuxagent/common/utils/shellutil.py b/azurelinuxagent/common/utils/shellutil.py index fff6aa8..7b8e0c9 100644 --- a/azurelinuxagent/common/utils/shellutil.py +++ b/azurelinuxagent/common/utils/shellutil.py @@ -84,16 +84,22 @@ def run_get_output(cmd, chk_err=True, log_cmd=True): output = ustr(output, encoding='utf-8', errors="backslashreplace") - except subprocess.CalledProcessError as e: - output = ustr(e.output, - encoding='utf-8', - errors="backslashreplace") - if chk_err: - if log_cmd: - logger.error(u"Command: '{0}'", e.cmd) - logger.error(u"Return code: {0}", e.returncode) - logger.error(u"Result: {0}", output) - return e.returncode, output + except Exception as e: + if type(e) is subprocess.CalledProcessError: + output = ustr(e.output, + encoding='utf-8', + errors="backslashreplace") + if chk_err: + if log_cmd: + logger.error(u"Command: '{0}'", e.cmd) + logger.error(u"Return code: {0}", e.returncode) + logger.error(u"Result: {0}", output) + return e.returncode, output + else: + logger.error( + u"'{0}' raised unexpected exception: '{1}'".format( + cmd, ustr(e))) + return -1, ustr(e) return 0, output |