From a456f4fdc26eba06ee98ffe2eeeafc6e15a39e88 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Fri, 15 Jun 2012 18:26:34 -0700 Subject: Inclusion of more utility functions included: 1. Adjustments to using more selinux guards around directory creation, chmod... 2. Adding util functions to check if values are false or true (used internally and externally) 3. Move find_devs_with to util.py and allow it to serve multiple use cases 4. Add fork_cb which will fork a process and then call a certain callback (used right now by the resize nonblocking mode) 5. Move functions that performed time_rfc2822 time fetching and uptime fetching to here. 6. Allow the subp util function to act in shell mode 7. Increase logging usefulness in shellify function --- cloudinit/util.py | 280 ++++++++++++++++++++++++++++++++++++++++++------------ 1 file changed, 220 insertions(+), 60 deletions(-) diff --git a/cloudinit/util.py b/cloudinit/util.py index 7d5932c1..7259d933 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -28,14 +28,18 @@ import errno import glob import grp import gzip +import hashlib import os import platform import pwd +import random import shutil import socket +import string import subprocess import sys import tempfile +import time import traceback import types import urlparse @@ -68,8 +72,11 @@ CONTAINER_TESTS = ['running-in-container', 'lxc-is-container'] class ProcessExecutionError(IOError): - MESSAGE_TMPL = ('%(description)s\nCommand: %(cmd)s\n' - 'Exit code: %(exit_code)s\nStdout: %(stdout)r\n' + MESSAGE_TMPL = ('%(description)s\n' + 'Command: %(cmd)s\n' + 'Exit code: %(exit_code)s\n' + 'Reason: %(reason)s\n' + 'Stdout: %(stdout)r\n' 'Stderr: %(stderr)r') def __init__(self, stdout=None, stderr=None, @@ -100,31 +107,37 @@ class ProcessExecutionError(IOError): else: self.stdout = stdout + if reason: + self.reason = reason + else: + self.reason = '-' + message = self.MESSAGE_TMPL % { 'description': self.description, 'cmd': self.cmd, 'exit_code': self.exit_code, 'stdout': self.stdout, 'stderr': self.stderr, + 'reason': self.reason, } IOError.__init__(self, message) - self.reason = reason class SeLinuxGuard(object): def __init__(self, path, recursive=False): self.path = path self.recursive = recursive - self.engaged = False + self.enabled = False if HAVE_LIBSELINUX and selinux.is_selinux_enabled(): - self.engaged = True + self.enabled = True def __enter__(self): - return self.engaged + # TODO: Should we try to engage selinux here?? + return None def __exit__(self, excp_type, excp_value, excp_traceback): - if self.engaged: - LOG.debug("Disengaging selinux mode for %s: %s", + if self.enabled: + LOG.debug("Restoring selinux mode for %s (recursive=%s)", self.path, self.recursive) selinux.restorecon(self.path, recursive=self.recursive) @@ -133,14 +146,72 @@ class MountFailedError(Exception): pass -def translate_bool(val): +def SilentTemporaryFile(**kwargs): + fh = tempfile.NamedTemporaryFile(**kwargs) + # Replace its unlink with a quiet version + # that does not raise errors when the + # file to unlink has been unlinked elsewhere.. + LOG.debug("Created temporary file %s", fh.name) + fh.unlink = del_file + # Add a new method that will unlink + # right 'now' but still lets the exit + # method attempt to remove it (which will + # not throw due to our del file being quiet + # about files that are not there) + def unlink_now(): + fh.unlink(fh.name) + setattr(fh, 'unlink_now', unlink_now) + return fh + + +def fork_cb(child_cb, *args): + fid = os.fork() + if fid == 0: + try: + child_cb(*args) + os._exit(0) # pylint: disable=W0212 + except: + logexc(LOG, "Failed forking and calling callback %s", obj_name(child_cb)) + os._exit(1) # pylint: disable=W0212 + else: + LOG.debug("Forked child %s who will run callback %s", + fid, obj_name(child_cb)) + + +def is_true_str(val, addons=None): + check_set = ['true', '1', 'on', 'yes'] + if addons: + check_set = check_set + addons + if str(val).lower().strip() in check_set: + return True + return False + + +def is_false_str(val, addons=None): + check_set = ['off', '0', 'no', 'false'] + if addons: + check_set = check_set + addons + if str(val).lower().strip() in check_set: + return True + return False + + +def translate_bool(val, addons=None): if not val: + # This handles empty lists and false and + # other things that python believes are false return False - if val is isinstance(val, bool): + # If its already a boolean skip + if isinstance(val, (bool)): return val - if str(val).lower().strip() in ['true', '1', 'on', 'yes']: - return True - return False + return is_true_str(val, addons) + + +def rand_str(strlen=32, select_from=None): + if not select_from: + select_from = string.letters + string.digits + return "".join([random.choice(select_from) for _x in range(0, strlen)]) + def read_conf(fname): @@ -221,7 +292,10 @@ def get_cfg_option_bool(yobj, key, default=False): def get_cfg_option_str(yobj, key, default=None): if key not in yobj: return default - return yobj[key] + val = yobj[key] + if not isinstance(val, (str, basestring)): + val = str(val) + return val def system_info(): @@ -233,7 +307,7 @@ def system_info(): } -def get_cfg_option_list_or_str(yobj, key, default=None): +def get_cfg_option_list(yobj, key, default=None): """ Gets the C{key} config option from C{yobj} as a list of strings. If the key is present as a single string it will be returned as a list with one @@ -249,9 +323,14 @@ def get_cfg_option_list_or_str(yobj, key, default=None): return default if yobj[key] is None: return [] - if isinstance(yobj[key], (list)): - return yobj[key] - return [yobj[key]] + val = yobj[key] + if isinstance(val, (list)): + # Should we ensure they are all strings?? + cval = [str(v) for v in val] + return cval + if not isinstance(val, (str, basestring)): + val = str(val) + return [val] # get a cfg entry by its path array @@ -419,21 +498,21 @@ def runparts(dirp, skip_no_exist=True): if skip_no_exist and not os.path.isdir(dirp): return - failed = 0 - attempted = 0 + failed = [] + attempted = [] for exe_name in sorted(os.listdir(dirp)): exe_path = os.path.join(dirp, exe_name) if os.path.isfile(exe_path) and os.access(exe_path, os.X_OK): - attempted += 1 + attempted.append(exe_path) try: subp([exe_path]) except ProcessExecutionError as e: logexc(LOG, "Failed running %s [%s]", exe_path, e.exit_code) - failed += 1 + failed.append(e) if failed and attempted: raise RuntimeError('Runparts: %s failures in %s attempted commands' - % (failed, attempted)) + % (len(failed), len(attempted))) # read_optional_seed @@ -470,7 +549,7 @@ def load_yaml(blob, default=None, allowed=(dict,)): converted = yaml.load(blob) if not isinstance(converted, allowed): # Yes this will just be caught, but thats ok for now... - raise TypeError("Yaml load allows %s types, but got %s instead" % + raise TypeError("Yaml load allows %s root types, but got %s instead" % (allowed, obj_name(converted))) loaded = converted except (yaml.YAMLError, TypeError, ValueError) as exc: @@ -718,7 +797,8 @@ def close_stdin(): os.dup2(fp.fileno(), sys.stdin.fileno()) -def find_devs_with(criteria=None): +def find_devs_with(criteria=None, oformat='device', + tag=None, no_cache=False, path=None): """ find devices matching given criteria (via blkid) criteria can be *one* of: @@ -726,38 +806,58 @@ def find_devs_with(criteria=None): LABEL=