diff options
Diffstat (limited to 'cloudinit/util.py')
| -rw-r--r-- | cloudinit/util.py | 208 |
1 files changed, 129 insertions, 79 deletions
diff --git a/cloudinit/util.py b/cloudinit/util.py index ba9becda..e4337e3a 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -38,20 +38,22 @@ try: except ImportError: HAVE_LIBSELINUX = False + def read_conf(fname): try: - stream = open(fname,"r") + stream = open(fname, "r") conf = yaml.load(stream) stream.close() return conf except IOError as e: if e.errno == errno.ENOENT: - return { } + return {} raise -def get_base_cfg(cfgfile,cfg_builtin="", parsed_cfgs=None): - kerncfg = { } - syscfg = { } + +def get_base_cfg(cfgfile, cfg_builtin="", parsed_cfgs=None): + kerncfg = {} + syscfg = {} if parsed_cfgs and cfgfile in parsed_cfgs: return(parsed_cfgs[cfgfile]) @@ -66,7 +68,7 @@ def get_base_cfg(cfgfile,cfg_builtin="", parsed_cfgs=None): if cfg_builtin: builtin = yaml.load(cfg_builtin) - fin = mergedict(combined,builtin) + fin = mergedict(combined, builtin) else: fin = combined @@ -74,18 +76,24 @@ def get_base_cfg(cfgfile,cfg_builtin="", parsed_cfgs=None): parsed_cfgs[cfgfile] = fin return(fin) + def get_cfg_option_bool(yobj, key, default=False): - if not yobj.has_key(key): return default + if key not in yobj: + return default val = yobj[key] - if val is True: return True - if str(val).lower() in [ 'true', '1', 'on', 'yes']: + if val is True: + return True + if str(val).lower() in ['true', '1', 'on', 'yes']: return True return False + def get_cfg_option_str(yobj, key, default=None): - if not yobj.has_key(key): return default + if key not in yobj: + return default return yobj[key] + def get_cfg_option_list_or_str(yobj, key, default=None): """ Gets the C{key} config option from C{yobj} as a list of strings. If the @@ -106,15 +114,18 @@ def get_cfg_option_list_or_str(yobj, key, default=None): return yobj[key] return [yobj[key]] + # get a cfg entry by its path array # for f['a']['b']: get_cfg_by_path(mycfg,('a','b')) -def get_cfg_by_path(yobj,keyp,default=None): +def get_cfg_by_path(yobj, keyp, default=None): cur = yobj for tok in keyp: - if tok not in cur: return(default) + if tok not in cur: + return(default) cur = cur[tok] return(cur) + def mergedict(src, cand): """ Merge values from C{cand} into C{src}. If C{src} has a key C{cand} will @@ -128,6 +139,7 @@ def mergedict(src, cand): src[k] = mergedict(src[k], v) return src + def delete_dir_contents(dirname): """ Deletes all contents of a directory without deleting the directory itself. @@ -141,6 +153,7 @@ def delete_dir_contents(dirname): else: os.unlink(node_fullpath) + def write_file(filename, content, mode=0644, omode="wb"): """ Writes a file with the given content and sets the file mode as specified. @@ -164,13 +177,15 @@ def write_file(filename, content, mode=0644, omode="wb"): f.close() restorecon_if_possible(filename) + def restorecon_if_possible(path, recursive=False): if HAVE_LIBSELINUX and selinux.is_selinux_enabled(): selinux.restorecon(path, recursive=recursive) + # get keyid from keyserver -def getkeybyid(keyid,keyserver): - shcmd=""" +def getkeybyid(keyid, keyserver): + shcmd = """ k=${1} ks=${2}; exec 2>/dev/null [ -n "$k" ] || exit 1; @@ -182,36 +197,43 @@ def getkeybyid(keyid,keyserver): fi [ -n "${armour}" ] && echo "${armour}" """ - args=['sh', '-c', shcmd, "export-gpg-keyid", keyid, keyserver] + args = ['sh', '-c', shcmd, "export-gpg-keyid", keyid, keyserver] return(subp(args)[0]) + def runparts(dirp, skip_no_exist=True): - if skip_no_exist and not os.path.isdir(dirp): return - + if skip_no_exist and not os.path.isdir(dirp): + return + # per bug 857926, Fedora's run-parts will exit failure on empty dir - if os.path.isdir(dirp) and os.listdir(dirp) == []: return + if os.path.isdir(dirp) and os.listdir(dirp) == []: + return - cmd = [ 'run-parts', '--regex', '.*', dirp ] + cmd = ['run-parts', '--regex', '.*', dirp] sp = subprocess.Popen(cmd) sp.communicate() if sp.returncode is not 0: - raise subprocess.CalledProcessError(sp.returncode,cmd) + raise subprocess.CalledProcessError(sp.returncode, cmd) return + def subp(args, input_=None): sp = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE) - out,err = sp.communicate(input_) + out, err = sp.communicate(input_) if sp.returncode is not 0: - raise subprocess.CalledProcessError(sp.returncode,args, (out,err)) - return(out,err) + raise subprocess.CalledProcessError(sp.returncode, args, (out, err)) + return(out, err) + def render_to_file(template, outfile, searchList): - t = Template(file='/etc/cloud/templates/%s.tmpl' % template, searchList=[searchList]) + t = Template(file='/etc/cloud/templates/%s.tmpl' % template, + searchList=[searchList]) f = open(outfile, 'w') f.write(t.respond()) f.close() + def render_string(template, searchList): return(Template(template, searchList=[searchList]).respond()) @@ -220,22 +242,22 @@ def render_string(template, searchList): # returns boolean indicating success or failure (presense of files) # if files are present, populates 'fill' dictionary with 'user-data' and # 'meta-data' entries -def read_optional_seed(fill,base="",ext="", timeout=5): +def read_optional_seed(fill, base="", ext="", timeout=5): try: - (md,ud) = read_seeded(base,ext,timeout) - fill['user-data']= ud - fill['meta-data']= md + (md, ud) = read_seeded(base, ext, timeout) + fill['user-data'] = ud + fill['meta-data'] = md return True except OSError, e: if e.errno == errno.ENOENT: return False raise - + # raise OSError with enoent if not found def read_seeded(base="", ext="", timeout=5, retries=10, file_retries=0): if base.startswith("/"): - base="file://%s" % base + base = "file://%s" % base # default retries for file is 0. for network is 10 if base.startswith("file://"): @@ -248,20 +270,22 @@ def read_seeded(base="", ext="", timeout=5, retries=10, file_retries=0): ud_url = "%s%s%s" % (base, "user-data", ext) md_url = "%s%s%s" % (base, "meta-data", ext) - raise_err = None - for attempt in range(0,retries+1): + no_exc = object() + raise_err = no_exc + for attempt in range(0, retries + 1): try: md_str = readurl(md_url, timeout=timeout) ud = readurl(ud_url, timeout=timeout) md = yaml.load(md_str) - - return(md,ud) + + return(md, ud) except urllib2.HTTPError as e: raise_err = e except urllib2.URLError as e: raise_err = e - if isinstance(e.reason,OSError) and e.reason.errno == errno.ENOENT: - raise_err = e.reason + if (isinstance(e.reason, OSError) and + e.reason.errno == errno.ENOENT): + raise_err = e.reason if attempt == retries: break @@ -271,13 +295,16 @@ def read_seeded(base="", ext="", timeout=5, retries=10, file_retries=0): raise(raise_err) -def logexc(log,lvl=logging.DEBUG): - log.log(lvl,traceback.format_exc()) + +def logexc(log, lvl=logging.DEBUG): + log.log(lvl, traceback.format_exc()) + class RecursiveInclude(Exception): pass -def read_file_with_includes(fname, rel = ".", stack=None, patt = None): + +def read_file_with_includes(fname, rel=".", stack=None, patt=None): if stack is None: stack = [] if not fname.startswith("/"): @@ -292,7 +319,7 @@ def read_file_with_includes(fname, rel = ".", stack=None, patt = None): (fname, len(stack)))) if patt == None: - patt = re.compile("^#(opt_include|include)[ \t].*$",re.MULTILINE) + patt = re.compile("^#(opt_include|include)[ \t].*$", re.MULTILINE) try: fp = open(fname) @@ -307,11 +334,12 @@ def read_file_with_includes(fname, rel = ".", stack=None, patt = None): cur = 0 while True: match = patt.search(contents[cur:]) - if not match: break + if not match: + break loc = match.start() + cur endl = match.end() + cur - (key, cur_fname) = contents[loc:endl].split(None,2) + (key, cur_fname) = contents[loc:endl].split(None, 2) cur_fname = cur_fname.strip() try: @@ -321,43 +349,47 @@ def read_file_with_includes(fname, rel = ".", stack=None, patt = None): inc_contents = "" else: raise - contents = contents[0:loc] + inc_contents + contents[endl+1:] + contents = contents[0:loc] + inc_contents + contents[endl + 1:] cur = loc + len(inc_contents) stack.pop() return(contents) + def read_conf_d(confd): # get reverse sorted list (later trumps newer) - confs = sorted(os.listdir(confd),reverse=True) - + confs = sorted(os.listdir(confd), reverse=True) + # remove anything not ending in '.cfg' confs = [f for f in confs if f.endswith(".cfg")] # remove anything not a file - confs = [f for f in confs if os.path.isfile("%s/%s" % (confd,f))] + confs = [f for f in confs if os.path.isfile("%s/%s" % (confd, f))] - cfg = { } + cfg = {} for conf in confs: - cfg = mergedict(cfg,read_conf("%s/%s" % (confd,conf))) + cfg = mergedict(cfg, read_conf("%s/%s" % (confd, conf))) return(cfg) + def read_conf_with_confd(cfgfile): cfg = read_conf(cfgfile) confd = False if "conf_d" in cfg: if cfg['conf_d'] is not None: confd = cfg['conf_d'] - if not isinstance(confd,str): - raise Exception("cfgfile %s contains 'conf_d' with non-string" % cfgfile) + if not isinstance(confd, str): + raise Exception("cfgfile %s contains 'conf_d' " + "with non-string" % cfgfile) elif os.path.isdir("%s.d" % cfgfile): confd = "%s.d" % cfgfile - if not confd: return(cfg) + if not confd: + return(cfg) confd_cfg = read_conf_d(confd) - return(mergedict(confd_cfg,cfg)) + return(mergedict(confd_cfg, cfg)) def get_cmdline(): @@ -371,7 +403,8 @@ def get_cmdline(): except: cmdline = "" return(cmdline) - + + def read_cc_from_cmdline(cmdline=None): # this should support reading cloud-config information from # the kernel command line. It is intended to support content of the @@ -384,23 +417,25 @@ def read_cc_from_cmdline(cmdline=None): if cmdline is None: cmdline = get_cmdline() - tag_begin="cc:" - tag_end="end_cc" + tag_begin = "cc:" + tag_end = "end_cc" begin_l = len(tag_begin) end_l = len(tag_end) clen = len(cmdline) - tokens = [ ] + tokens = [] begin = cmdline.find(tag_begin) while begin >= 0: end = cmdline.find(tag_end, begin + begin_l) if end < 0: end = clen - tokens.append(cmdline[begin+begin_l:end].lstrip().replace("\\n","\n")) - + tokens.append(cmdline[begin + begin_l:end].lstrip().replace("\\n", + "\n")) + begin = cmdline.find(tag_begin, end + end_l) return('\n'.join(tokens)) + def ensure_dirs(dirlist, mode=0755): fixmodes = [] for d in dirlist: @@ -410,16 +445,20 @@ def ensure_dirs(dirlist, mode=0755): else: os.makedirs(d, mode) except OSError as e: - if e.errno != errno.EEXIST: raise - if mode != None: fixmodes.append(d) + if e.errno != errno.EEXIST: + raise + if mode != None: + fixmodes.append(d) for d in fixmodes: os.chmod(d, mode) -def chownbyname(fname,user=None,group=None): + +def chownbyname(fname, user=None, group=None): uid = -1 gid = -1 - if user == None and group == None: return + if user == None and group == None: + return if user: import pwd uid = pwd.getpwnam(user).pw_uid @@ -427,10 +466,11 @@ def chownbyname(fname,user=None,group=None): import grp gid = grp.getgrnam(group).gr_gid - os.chown(fname,uid,gid) + os.chown(fname, uid, gid) + def readurl(url, data=None, timeout=None): - openargs = { } + openargs = {} if timeout != None: openargs['timeout'] = timeout @@ -443,44 +483,48 @@ def readurl(url, data=None, timeout=None): response = urllib2.urlopen(req, **openargs) return(response.read()) + # shellify, takes a list of commands # for each entry in the list # if it is an array, shell protect it (with single ticks) # if it is a string, do nothing def shellify(cmdlist): - content="#!/bin/sh\n" - escaped="%s%s%s%s" % ( "'", '\\', "'", "'" ) + content = "#!/bin/sh\n" + escaped = "%s%s%s%s" % ("'", '\\', "'", "'") for args in cmdlist: # if the item is a list, wrap all items in single tick # if its not, then just write it directly - if isinstance(args,list): - fixed = [ ] + if isinstance(args, list): + fixed = [] for f in args: - fixed.append("'%s'" % str(f).replace("'",escaped)) - content="%s%s\n" % ( content, ' '.join(fixed) ) + fixed.append("'%s'" % str(f).replace("'", escaped)) + content = "%s%s\n" % (content, ' '.join(fixed)) else: - content="%s%s\n" % ( content, str(args) ) + content = "%s%s\n" % (content, str(args)) return content + def dos2unix(string): # find first end of line pos = string.find('\n') - if pos <= 0 or string[pos-1] != '\r': return(string) - return(string.replace('\r\n','\n')) + if pos <= 0 or string[pos - 1] != '\r': + return(string) + return(string.replace('\r\n', '\n')) + def islxc(): # is this host running lxc? try: with open("/proc/1/cgroup") as f: - if f.read() == "/": + if f.read() == "/": return True except IOError as e: if e.errno != errno.ENOENT: raise try: - # try to run a program named 'lxc-is-container'. if it returns true, then - # we're inside a container. otherwise, no + # try to run a program named 'lxc-is-container'. if it returns true, + # then we're inside a container. otherwise, no sp = subprocess.Popen(['lxc-is-container'], stdout=subprocess.PIPE, stderr=subprocess.PIPE) sp.communicate(None) @@ -491,13 +535,14 @@ def islxc(): return False + def get_hostname_fqdn(cfg, cloud): # return the hostname and fqdn from 'cfg'. If not found in cfg, # then fall back to data from cloud if "fqdn" in cfg: # user specified a fqdn. Default hostname then is based off that fqdn = cfg['fqdn'] - hostname = get_cfg_option_str(cfg,"hostname",fqdn.split('.')[0]) + hostname = get_cfg_option_str(cfg, "hostname", fqdn.split('.')[0]) else: if "hostname" in cfg and cfg['hostname'].find('.') > 0: # user specified hostname, and it had '.' in it @@ -505,7 +550,7 @@ def get_hostname_fqdn(cfg, cloud): fqdn = cfg['hostname'] hostname = cfg['hostname'][:fqdn.find('.')] else: - # no fqdn set, get fqdn from cloud. + # no fqdn set, get fqdn from cloud. # get hostname from cfg if available otherwise cloud fqdn = cloud.get_hostname(fqdn=True) if "hostname" in cfg: @@ -514,9 +559,10 @@ def get_hostname_fqdn(cfg, cloud): hostname = cloud.get_hostname() return(hostname, fqdn) + def get_fqdn_from_hosts(hostname, filename="/etc/hosts"): # this parses /etc/hosts to get a fqdn. It should return the same - # result as 'hostname -f <hostname>' if /etc/hosts.conf + # result as 'hostname -f <hostname>' if /etc/hosts.conf # did not have did not have 'bind' in the order attribute fqdn = None try: @@ -542,6 +588,7 @@ def get_fqdn_from_hosts(hostname, filename="/etc/hosts"): return fqdn + def is_resolvable(name): """ determine if a url is resolvable, return a boolean """ try: @@ -550,10 +597,12 @@ def is_resolvable(name): except socket.gaierror: return False + def is_resolvable_url(url): """ determine if this url is resolvable (existing or ip) """ return(is_resolvable(urlparse.urlparse(url).hostname)) + def search_for_mirror(candidates): """ Search through a list of mirror urls for one that works """ for cand in candidates: @@ -565,6 +614,7 @@ def search_for_mirror(candidates): return None + def close_stdin(): """ reopen stdin as /dev/null so even subprocesses or other os level things get |