From 869402301c9793cece24a9357ee3c13dcdafb6e2 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Thu, 7 Jun 2012 12:45:28 -0700 Subject: Darn it. Those shouldn't be there! --- cloudinit/sources/__init__.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 cloudinit/sources/__init__.py (limited to 'cloudinit/sources/__init__.py') diff --git a/cloudinit/sources/__init__.py b/cloudinit/sources/__init__.py new file mode 100644 index 00000000..e69de29b -- cgit v1.2.3 From 32362c0ac63f10d7a33e3b95dd91a544a1cbdf54 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Thu, 7 Jun 2012 13:46:54 -0700 Subject: 1. Move cloud init and cloud config objects to a cloud file. 2. Cleanup main __init__ file with shell additions, constants usage, os.path usage. --- cloudinit/__init__.py | 608 ++++-------------------------------------- cloudinit/sources/__init__.py | 214 +++++++++++++++ 2 files changed, 260 insertions(+), 562 deletions(-) (limited to 'cloudinit/sources/__init__.py') diff --git a/cloudinit/__init__.py b/cloudinit/__init__.py index 85c6fd1b..f223fbe8 100644 --- a/cloudinit/__init__.py +++ b/cloudinit/__init__.py @@ -20,627 +20,111 @@ # along with this program. If not, see . # -varlibdir = '/var/lib/cloud' -cur_instance_link = varlibdir + "/instance" -boot_finished = cur_instance_link + "/boot-finished" -system_config = '/etc/cloud/cloud.cfg' -seeddir = varlibdir + "/seed" -cfg_env_name = "CLOUD_CFG" - -cfg_builtin = """ -log_cfgs: [] -datasource_list: ["NoCloud", "ConfigDrive", "OVF", "MAAS", "Ec2", "CloudStack"] -def_log_file: /var/log/cloud-init.log -syslog_fix_perms: syslog:adm -""" -logger_name = "cloudinit" - -pathmap = { - "handlers": "/handlers", - "scripts": "/scripts", - "sem": "/sem", - "boothooks": "/boothooks", - "userdata_raw": "/user-data.txt", - "userdata": "/user-data.txt.i", - "obj_pkl": "/obj.pkl", - "cloud_config": "/cloud-config.txt", - "data": "/data", - None: "", -} - -per_instance = "once-per-instance" -per_always = "always" -per_once = "once" - -parsed_cfgs = {} - import os -import cPickle import sys import os.path import errno import subprocess import yaml -import logging -import logging.config -import StringIO import glob import traceback +import cloudinit.log as logging +import cloudinit.shell as sh import cloudinit.util as util +from cloudinit.constants import (VAR_LIB_DIR, CFG_BUILTIN, CLOUD_CONFIG, + BOOT_FINISHED, CUR_INSTANCE_LINK, PATH_MAP) -class NullHandler(logging.Handler): - def emit(self, record): - pass - - -log = logging.getLogger(logger_name) -log.addHandler(NullHandler()) - - -def logging_set_from_cfg_file(cfg_file=system_config): - logging_set_from_cfg(util.get_base_cfg(cfg_file, cfg_builtin, parsed_cfgs)) - - -def logging_set_from_cfg(cfg): - log_cfgs = [] - logcfg = util.get_cfg_option_str(cfg, "log_cfg", False) - if logcfg: - # if there is a 'logcfg' entry in the config, respect - # it, it is the old keyname - log_cfgs = [logcfg] - elif "log_cfgs" in cfg: - for cfg in cfg['log_cfgs']: - if isinstance(cfg, list): - log_cfgs.append('\n'.join(cfg)) - else: - log_cfgs.append() - - if not len(log_cfgs): - sys.stderr.write("Warning, no logging configured\n") - return - - for logcfg in log_cfgs: - try: - logging.config.fileConfig(StringIO.StringIO(logcfg)) - return - except: - pass - - raise Exception("no valid logging found\n") - - -import cloudinit.DataSource as DataSource -import cloudinit.UserDataHandler as UserDataHandler - - -class CloudInit: - cfg = None - part_handlers = {} - old_conffile = '/etc/ec2-init/ec2-config.cfg' - ds_deps = [DataSource.DEP_FILESYSTEM, DataSource.DEP_NETWORK] - datasource = None - cloud_config_str = '' - datasource_name = '' - - builtin_handlers = [] - - def __init__(self, ds_deps=None, sysconfig=system_config): - self.builtin_handlers = [ - ['text/x-shellscript', self.handle_user_script, per_always], - ['text/cloud-config', self.handle_cloud_config, per_always], - ['text/upstart-job', self.handle_upstart_job, per_instance], - ['text/cloud-boothook', self.handle_cloud_boothook, per_always], - ] - - if ds_deps != None: - self.ds_deps = ds_deps - - self.sysconfig = sysconfig - - self.cfg = self.read_cfg() - - def read_cfg(self): - if self.cfg: - return(self.cfg) - - try: - conf = util.get_base_cfg(self.sysconfig, cfg_builtin, parsed_cfgs) - except Exception: - conf = get_builtin_cfg() - - # support reading the old ConfigObj format file and merging - # it into the yaml dictionary - try: - from configobj import ConfigObj - oldcfg = ConfigObj(self.old_conffile) - if oldcfg is None: - oldcfg = {} - conf = util.mergedict(conf, oldcfg) - except: - pass - - return(conf) - - def restore_from_cache(self): - try: - # we try to restore from a current link and static path - # by using the instance link, if purge_cache was called - # the file wont exist - cache = get_ipath_cur('obj_pkl') - f = open(cache, "rb") - data = cPickle.load(f) - f.close() - self.datasource = data - return True - except: - return False - - def write_to_cache(self): - cache = self.get_ipath("obj_pkl") - try: - os.makedirs(os.path.dirname(cache)) - except OSError as e: - if e.errno != errno.EEXIST: - return False - - try: - f = open(cache, "wb") - cPickle.dump(self.datasource, f) - f.close() - os.chmod(cache, 0400) - except: - raise - - def get_data_source(self): - if self.datasource is not None: - return True - - if self.restore_from_cache(): - log.debug("restored from cache type %s" % self.datasource) - return True - - cfglist = self.cfg['datasource_list'] - dslist = list_sources(cfglist, self.ds_deps) - dsnames = [f.__name__ for f in dslist] - - log.debug("searching for data source in %s" % dsnames) - for cls in dslist: - ds = cls.__name__ - try: - s = cls(sys_cfg=self.cfg) - if s.get_data(): - self.datasource = s - self.datasource_name = ds - log.debug("found data source %s" % ds) - return True - except Exception as e: - log.warn("get_data of %s raised %s" % (ds, e)) - util.logexc(log) - msg = "Did not find data source. searched classes: %s" % dsnames - log.debug(msg) - raise DataSourceNotFoundException(msg) - - def set_cur_instance(self): - try: - os.unlink(cur_instance_link) - except OSError as e: - if e.errno != errno.ENOENT: - raise - - iid = self.get_instance_id() - os.symlink("./instances/%s" % iid, cur_instance_link) - idir = self.get_ipath() - dlist = [] - for d in ["handlers", "scripts", "sem"]: - dlist.append("%s/%s" % (idir, d)) +LOG = logging.getLogger(__name__) - util.ensure_dirs(dlist) +INIT_SUBDIRS = [ + 'scripts', + os.path.join('scripts', 'per-instance'), + os.path.join('scripts', 'per-once'), + os.path.join('scripts', 'per-boot'), + 'seed', + 'instances', + 'handlers', + 'sem', + 'data' +] - ds = "%s: %s\n" % (self.datasource.__class__, str(self.datasource)) - dp = self.get_cpath('data') - util.write_file("%s/%s" % (idir, 'datasource'), ds) - util.write_file("%s/%s" % (dp, 'previous-datasource'), ds) - util.write_file("%s/%s" % (dp, 'previous-instance-id'), "%s\n" % iid) - - def get_userdata(self): - return(self.datasource.get_userdata()) - - def get_userdata_raw(self): - return(self.datasource.get_userdata_raw()) - - def get_instance_id(self): - return(self.datasource.get_instance_id()) - - def update_cache(self): - self.write_to_cache() - self.store_userdata() - - def store_userdata(self): - util.write_file(self.get_ipath('userdata_raw'), - self.datasource.get_userdata_raw(), 0600) - util.write_file(self.get_ipath('userdata'), - self.datasource.get_userdata(), 0600) - - def sem_getpath(self, name, freq): - if freq == 'once-per-instance': - return("%s/%s" % (self.get_ipath("sem"), name)) - return("%s/%s.%s" % (get_cpath("sem"), name, freq)) - - def sem_has_run(self, name, freq): - if freq == per_always: - return False - semfile = self.sem_getpath(name, freq) - if os.path.exists(semfile): - return True - return False - - def sem_acquire(self, name, freq): - from time import time - semfile = self.sem_getpath(name, freq) - - try: - os.makedirs(os.path.dirname(semfile)) - except OSError as e: - if e.errno != errno.EEXIST: - raise e - - if os.path.exists(semfile) and freq != per_always: - return False - - # race condition - try: - f = open(semfile, "w") - f.write("%s\n" % str(time())) - f.close() - except: - return(False) - return(True) - def sem_clear(self, name, freq): - semfile = self.sem_getpath(name, freq) - try: - os.unlink(semfile) - except OSError as e: - if e.errno != errno.ENOENT: - return False - - return True - - # acquire lock on 'name' for given 'freq' - # if that does not exist, then call 'func' with given 'args' - # if 'clear_on_fail' is True and func throws an exception - # then remove the lock (so it would run again) - def sem_and_run(self, semname, freq, func, args=None, clear_on_fail=False): - if args is None: - args = [] - if self.sem_has_run(semname, freq): - log.debug("%s already ran %s", semname, freq) - return False - try: - if not self.sem_acquire(semname, freq): - raise Exception("Failed to acquire lock on %s" % semname) - - func(*args) - except: - if clear_on_fail: - self.sem_clear(semname, freq) - raise - - return True - - # get_ipath : get the instance path for a name in pathmap - # (/var/lib/cloud/instances//name)) - def get_ipath(self, name=None): - return("%s/instances/%s%s" - % (varlibdir, self.get_instance_id(), pathmap[name])) - - def consume_userdata(self, frequency=per_instance): - self.get_userdata() - data = self - - cdir = get_cpath("handlers") - idir = self.get_ipath("handlers") - - # add the path to the plugins dir to the top of our list for import - # instance dir should be read before cloud-dir - sys.path.insert(0, cdir) - sys.path.insert(0, idir) - - part_handlers = {} - # add handlers in cdir - for fname in glob.glob("%s/*.py" % cdir): - if not os.path.isfile(fname): - continue - modname = os.path.basename(fname)[0:-3] - try: - mod = __import__(modname) - handler_register(mod, part_handlers, data, frequency) - log.debug("added handler for [%s] from %s" % (mod.list_types(), - fname)) - except: - log.warn("failed to initialize handler in %s" % fname) - util.logexc(log) - - # add the internal handers if their type hasn't been already claimed - for (btype, bhand, bfreq) in self.builtin_handlers: - if btype in part_handlers: - continue - handler_register(InternalPartHandler(bhand, [btype], bfreq), - part_handlers, data, frequency) - - # walk the data - pdata = {'handlers': part_handlers, 'handlerdir': idir, - 'data': data, 'frequency': frequency} - UserDataHandler.walk_userdata(self.get_userdata(), - partwalker_callback, data=pdata) - - # give callbacks opportunity to finalize - called = [] - for (_mtype, mod) in part_handlers.iteritems(): - if mod in called: - continue - handler_call_end(mod, data, frequency) - - def handle_user_script(self, _data, ctype, filename, payload, _frequency): - if ctype == "__end__": - return - if ctype == "__begin__": - # maybe delete existing things here - return - - filename = filename.replace(os.sep, '_') - scriptsdir = get_ipath_cur('scripts') - util.write_file("%s/%s" % - (scriptsdir, filename), util.dos2unix(payload), 0700) - - def handle_upstart_job(self, _data, ctype, filename, payload, frequency): - # upstart jobs are only written on the first boot - if frequency != per_instance: - return - - if ctype == "__end__" or ctype == "__begin__": - return - if not filename.endswith(".conf"): - filename = filename + ".conf" - - util.write_file("%s/%s" % ("/etc/init", filename), - util.dos2unix(payload), 0644) - - def handle_cloud_config(self, _data, ctype, filename, payload, _frequency): - if ctype == "__begin__": - self.cloud_config_str = "" - return - if ctype == "__end__": - cloud_config = self.get_ipath("cloud_config") - util.write_file(cloud_config, self.cloud_config_str, 0600) - - ## this could merge the cloud config with the system config - ## for now, not doing this as it seems somewhat circular - ## as CloudConfig does that also, merging it with this cfg - ## - # ccfg = yaml.load(self.cloud_config_str) - # if ccfg is None: ccfg = {} - # self.cfg = util.mergedict(ccfg, self.cfg) - - return - - self.cloud_config_str += "\n#%s\n%s" % (filename, payload) - - def handle_cloud_boothook(self, _data, ctype, filename, payload, - _frequency): - if ctype == "__end__": - return - if ctype == "__begin__": - return - - filename = filename.replace(os.sep, '_') - payload = util.dos2unix(payload) - prefix = "#cloud-boothook" - start = 0 - if payload.startswith(prefix): - start = len(prefix) + 1 - - boothooks_dir = self.get_ipath("boothooks") - filepath = "%s/%s" % (boothooks_dir, filename) - util.write_file(filepath, payload[start:], 0700) - try: - env = os.environ.copy() - env['INSTANCE_ID'] = self.datasource.get_instance_id() - subprocess.check_call([filepath], env=env) - except subprocess.CalledProcessError as e: - log.error("boothooks script %s returned %i" % - (filepath, e.returncode)) - except Exception as e: - log.error("boothooks unknown exception %s when running %s" % - (e, filepath)) - - def get_public_ssh_keys(self): - return(self.datasource.get_public_ssh_keys()) - - def get_locale(self): - return(self.datasource.get_locale()) - - def get_mirror(self): - return(self.datasource.get_local_mirror()) - - def get_hostname(self, fqdn=False): - return(self.datasource.get_hostname(fqdn=fqdn)) - - def device_name_to_device(self, name): - return(self.datasource.device_name_to_device(name)) - - # I really don't know if this should be here or not, but - # I needed it in cc_update_hostname, where that code had a valid 'cloud' - # reference, but did not have a cloudinit handle - # (ie, no cloudinit.get_cpath()) - def get_cpath(self, name=None): - return(get_cpath(name)) +# TODO: get rid of this global +parsed_cfgs = {} def initfs(): - subds = ['scripts/per-instance', 'scripts/per-once', 'scripts/per-boot', - 'seed', 'instances', 'handlers', 'sem', 'data'] + + # TODO don't do this every time this function is called? dlist = [] - for subd in subds: - dlist.append("%s/%s" % (varlibdir, subd)) - util.ensure_dirs(dlist) + for subd in INIT_SUBDIRS: + dlist.append(os.path.join(VAR_LIB_DIR, subd)) + sh.ensure_dirs(dlist) - cfg = util.get_base_cfg(system_config, cfg_builtin, parsed_cfgs) + cfg = util.get_base_cfg(CLOUD_CONFIG, get_builtin_cfg(), parsed_cfgs) log_file = util.get_cfg_option_str(cfg, 'def_log_file', None) perms = util.get_cfg_option_str(cfg, 'syslog_fix_perms', None) if log_file: - fp = open(log_file, "ab") - fp.close() + sh.ensure_file(log_file) if log_file and perms: (u, g) = perms.split(':', 1) if u == "-1" or u == "None": u = None if g == "-1" or g == "None": g = None - util.chownbyname(log_file, u, g) + sh.chownbyname(log_file, u, g) def purge_cache(rmcur=True): - rmlist = [boot_finished] + rmlist = [BOOT_FINISHED] if rmcur: - rmlist.append(cur_instance_link) + rmlist.append(CUR_INSTANCE_LINK) for f in rmlist: try: - os.unlink(f) + sh.unlink(f) except OSError as e: if e.errno == errno.ENOENT: continue - return(False) + return False except: - return(False) - return(True) + return False + return True # get_ipath_cur: get the current instance path for an item def get_ipath_cur(name=None): - return("%s/%s%s" % (varlibdir, "instance", pathmap[name])) + add_on = PATH_MAP.get(name) + ipath = os.path.join(VAR_LIB_DIR, 'instance') + if add_on: + ipath = os.path.join(ipath, add_on) + return ipath # get_cpath : get the "clouddir" (/var/lib/cloud/) # for a name in dirmap def get_cpath(name=None): - return("%s%s" % (varlibdir, pathmap[name])) + cpath = VAR_LIB_DIR + add_on = PATH_MAP.get(name) + if add_on: + cpath = os.path.join(cpath, add_on) + return cpath def get_base_cfg(cfg_path=None): if cfg_path is None: - cfg_path = system_config - return(util.get_base_cfg(cfg_path, cfg_builtin, parsed_cfgs)) + cfg_path = CLOUD_CONFIG + return util.get_base_cfg(cfg_path, get_builtin_cfg(), parsed_cfgs) def get_builtin_cfg(): - return(yaml.load(cfg_builtin)) - - -class DataSourceNotFoundException(Exception): - pass + return dict(CFG_BUILTIN) def list_sources(cfg_list, depends): - return(DataSource.list_sources(cfg_list, depends, ["cloudinit", ""])) - - -def handler_register(mod, part_handlers, data, frequency=per_instance): - if not hasattr(mod, "handler_version"): - setattr(mod, "handler_version", 1) - - for mtype in mod.list_types(): - part_handlers[mtype] = mod - - handler_call_begin(mod, data, frequency) - return(mod) - - -def handler_call_begin(mod, data, frequency): - handler_handle_part(mod, data, "__begin__", None, None, frequency) - - -def handler_call_end(mod, data, frequency): - handler_handle_part(mod, data, "__end__", None, None, frequency) - - -def handler_handle_part(mod, data, ctype, filename, payload, frequency): - # only add the handler if the module should run - modfreq = getattr(mod, "frequency", per_instance) - if not (modfreq == per_always or - (frequency == per_instance and modfreq == per_instance)): - return - try: - if mod.handler_version == 1: - mod.handle_part(data, ctype, filename, payload) - else: - mod.handle_part(data, ctype, filename, payload, frequency) - except: - util.logexc(log) - traceback.print_exc(file=sys.stderr) - - -def partwalker_handle_handler(pdata, _ctype, _filename, payload): - curcount = pdata['handlercount'] - modname = 'part-handler-%03d' % curcount - frequency = pdata['frequency'] - - modfname = modname + ".py" - util.write_file("%s/%s" % (pdata['handlerdir'], modfname), payload, 0600) - - try: - mod = __import__(modname) - handler_register(mod, pdata['handlers'], pdata['data'], frequency) - pdata['handlercount'] = curcount + 1 - except: - util.logexc(log) - traceback.print_exc(file=sys.stderr) - - -def partwalker_callback(pdata, ctype, filename, payload): - # data here is the part_handlers array and then the data to pass through - if ctype == "text/part-handler": - if 'handlercount' not in pdata: - pdata['handlercount'] = 0 - partwalker_handle_handler(pdata, ctype, filename, payload) - return - if ctype not in pdata['handlers']: - if ctype == "text/x-not-multipart": - # Extract the first line or 24 bytes for displaying in the log - start = payload.split("\n", 1)[0][:24] - if start < payload: - details = "starting '%s...'" % start.encode("string-escape") - else: - details = repr(payload) - log.warning("Unhandled non-multipart userdata %s", details) - return - handler_handle_part(pdata['handlers'][ctype], pdata['data'], - ctype, filename, payload, pdata['frequency']) - - -class InternalPartHandler: - freq = per_instance - mtypes = [] - handler_version = 1 - handler = None - - def __init__(self, handler, mtypes, frequency, version=2): - self.handler = handler - self.mtypes = mtypes - self.frequency = frequency - self.handler_version = version - - def __repr__(self): - return("InternalPartHandler: [%s]" % self.mtypes) - - def list_types(self): - return(self.mtypes) - - def handle_part(self, data, ctype, filename, payload, frequency): - return(self.handler(data, ctype, filename, payload, frequency)) + return (DataSource.list_sources(cfg_list, depends, ["cloudinit", ""])) def get_cmdline_url(names=('cloud-config-url', 'url'), diff --git a/cloudinit/sources/__init__.py b/cloudinit/sources/__init__.py index e69de29b..e2a9150d 100644 --- a/cloudinit/sources/__init__.py +++ b/cloudinit/sources/__init__.py @@ -0,0 +1,214 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2009-2010 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# +# Author: Scott Moser +# Author: Juerg Hafliger +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + + +DEP_FILESYSTEM = "FILESYSTEM" +DEP_NETWORK = "NETWORK" + +import cloudinit.UserDataHandler as ud +import cloudinit.util as util +import socket + + +class DataSource: + userdata = None + metadata = None + userdata_raw = None + cfgname = "" + # system config (passed in from cloudinit, + # cloud-config before input from the DataSource) + sys_cfg = {} + # datasource config, the cloud-config['datasource']['__name__'] + ds_cfg = {} # datasource config + + def __init__(self, sys_cfg=None): + if not self.cfgname: + name = str(self.__class__).split(".")[-1] + if name.startswith("DataSource"): + name = name[len("DataSource"):] + self.cfgname = name + if sys_cfg: + self.sys_cfg = sys_cfg + + self.ds_cfg = util.get_cfg_by_path(self.sys_cfg, + ("datasource", self.cfgname), self.ds_cfg) + + def get_userdata(self): + if self.userdata == None: + self.userdata = ud.preprocess_userdata(self.userdata_raw) + return self.userdata + + def get_userdata_raw(self): + return(self.userdata_raw) + + # the data sources' config_obj is a cloud-config formated + # object that came to it from ways other than cloud-config + # because cloud-config content would be handled elsewhere + def get_config_obj(self): + return({}) + + def get_public_ssh_keys(self): + keys = [] + if 'public-keys' not in self.metadata: + return([]) + + if isinstance(self.metadata['public-keys'], str): + return(str(self.metadata['public-keys']).splitlines()) + + if isinstance(self.metadata['public-keys'], list): + return(self.metadata['public-keys']) + + for _keyname, klist in self.metadata['public-keys'].items(): + # lp:506332 uec metadata service responds with + # data that makes boto populate a string for 'klist' rather + # than a list. + if isinstance(klist, str): + klist = [klist] + for pkey in klist: + # there is an empty string at the end of the keylist, trim it + if pkey: + keys.append(pkey) + + return(keys) + + def device_name_to_device(self, _name): + # translate a 'name' to a device + # the primary function at this point is on ec2 + # to consult metadata service, that has + # ephemeral0: sdb + # and return 'sdb' for input 'ephemeral0' + return(None) + + def get_locale(self): + return('en_US.UTF-8') + + def get_local_mirror(self): + return None + + def get_instance_id(self): + if 'instance-id' not in self.metadata: + return "iid-datasource" + return(self.metadata['instance-id']) + + def get_hostname(self, fqdn=False): + defdomain = "localdomain" + defhost = "localhost" + + domain = defdomain + if not 'local-hostname' in self.metadata: + + # this is somewhat questionable really. + # the cloud datasource was asked for a hostname + # and didn't have one. raising error might be more appropriate + # but instead, basically look up the existing hostname + toks = [] + + hostname = socket.gethostname() + + fqdn = util.get_fqdn_from_hosts(hostname) + + if fqdn and fqdn.find(".") > 0: + toks = str(fqdn).split(".") + elif hostname: + toks = [hostname, defdomain] + else: + toks = [defhost, defdomain] + + else: + # if there is an ipv4 address in 'local-hostname', then + # make up a hostname (LP: #475354) in format ip-xx.xx.xx.xx + lhost = self.metadata['local-hostname'] + if is_ipv4(lhost): + toks = "ip-%s" % lhost.replace(".", "-") + else: + toks = lhost.split(".") + + if len(toks) > 1: + hostname = toks[0] + domain = '.'.join(toks[1:]) + else: + hostname = toks[0] + + if fqdn: + return "%s.%s" % (hostname, domain) + else: + return hostname + + +# return a list of classes that have the same depends as 'depends' +# iterate through cfg_list, loading "DataSourceCollections" modules +# and calling their "get_datasource_list". +# return an ordered list of classes that match +# +# - modules must be named "DataSource", where 'item' is an entry +# in cfg_list +# - if pkglist is given, it will iterate try loading from that package +# ie, pkglist=[ "foo", "" ] +# will first try to load foo.DataSource +# then DataSource +def list_sources(cfg_list, depends, pkglist=None): + if pkglist is None: + pkglist = [] + retlist = [] + for ds_coll in cfg_list: + for pkg in pkglist: + if pkg: + pkg = "%s." % pkg + try: + mod = __import__("%sDataSource%s" % (pkg, ds_coll)) + if pkg: + mod = getattr(mod, "DataSource%s" % ds_coll) + lister = getattr(mod, "get_datasource_list") + retlist.extend(lister(depends)) + break + except: + raise + return(retlist) + + +# depends is a list of dependencies (DEP_FILESYSTEM) +# dslist is a list of 2 item lists +# dslist = [ +# ( class, ( depends-that-this-class-needs ) ) +# } +# it returns a list of 'class' that matched these deps exactly +# it is a helper function for DataSourceCollections +def list_from_depends(depends, dslist): + retlist = [] + depset = set(depends) + for elem in dslist: + (cls, deps) = elem + if depset == set(deps): + retlist.append(cls) + return(retlist) + + +def is_ipv4(instr): + """ determine if input string is a ipv4 address. return boolean""" + toks = instr.split('.') + if len(toks) != 4: + return False + + try: + toks = [x for x in toks if (int(x) < 256 and int(x) > 0)] + except: + return False + + return (len(toks) == 4) -- cgit v1.2.3 From d78bc08df59051bc0f118e990bf1a1660584c38e Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Fri, 8 Jun 2012 18:05:27 -0700 Subject: Remove is ipv4 function from here and move to utils + move exceptions to here as well as other find datasource function. --- cloudinit/sources/__init__.py | 42 +++++++++++++++++++++++++----------------- 1 file changed, 25 insertions(+), 17 deletions(-) (limited to 'cloudinit/sources/__init__.py') diff --git a/cloudinit/sources/__init__.py b/cloudinit/sources/__init__.py index e2a9150d..9a9c1316 100644 --- a/cloudinit/sources/__init__.py +++ b/cloudinit/sources/__init__.py @@ -18,13 +18,16 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . +from cloudinit import user_data as ud +from cloudinit import util + +import socket DEP_FILESYSTEM = "FILESYSTEM" DEP_NETWORK = "NETWORK" -import cloudinit.UserDataHandler as ud -import cloudinit.util as util -import socket +class DataSourceNotFoundException(Exception): + pass class DataSource: @@ -152,6 +155,25 @@ class DataSource: return hostname +def find_source(cfg, ds_deps): + cfglist = cfg.get('datasource_list') or [] + dslist = list_sources(cfglist, ds_deps) + dsnames = [f.__name__ for f in dslist] + + LOG.debug("Searching for data source in %s", dsnames) + for cls in dslist: + ds = cls.__name__ + try: + s = cls(sys_cfg=cfg) + if s.get_data(): + return (s, ds) + except Exception as e: + LOG.exception("Getting data from %s raised %s", ds, e) + + msg = "Did not find any data source, searched classes: %s" % dsnames + raise DataSourceNotFoundException(msg) + + # return a list of classes that have the same depends as 'depends' # iterate through cfg_list, loading "DataSourceCollections" modules # and calling their "get_datasource_list". @@ -198,17 +220,3 @@ def list_from_depends(depends, dslist): if depset == set(deps): retlist.append(cls) return(retlist) - - -def is_ipv4(instr): - """ determine if input string is a ipv4 address. return boolean""" - toks = instr.split('.') - if len(toks) != 4: - return False - - try: - toks = [x for x in toks if (int(x) < 256 and int(x) > 0)] - except: - return False - - return (len(toks) == 4) -- cgit v1.2.3 From ad9122034ff59fb6113adfac2713c44af09bc91b Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Sat, 9 Jun 2012 12:38:19 -0700 Subject: Initial cleanups --- cloudinit/sources/__init__.py | 188 +++++++++++++++++++++--------------------- 1 file changed, 93 insertions(+), 95 deletions(-) (limited to 'cloudinit/sources/__init__.py') diff --git a/cloudinit/sources/__init__.py b/cloudinit/sources/__init__.py index 9a9c1316..05c8bfad 100644 --- a/cloudinit/sources/__init__.py +++ b/cloudinit/sources/__init__.py @@ -1,10 +1,12 @@ # vi: ts=4 expandtab # -# Copyright (C) 2009-2010 Canonical Ltd. +# Copyright (C) 2012 Canonical Ltd. # Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# Copyright (C) 2012 Yahoo! Inc. # # Author: Scott Moser -# Author: Juerg Hafliger +# Author: Juerg Haefliger +# Author: Joshua Harlow # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License version 3, as @@ -18,78 +20,78 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -from cloudinit import user_data as ud +from cloudinit import importer +from cloudinit import log as logging from cloudinit import util -import socket - DEP_FILESYSTEM = "FILESYSTEM" DEP_NETWORK = "NETWORK" +DS_PREFIX = 'DataSource' +LOG = logging.getLogger(__name__) + class DataSourceNotFoundException(Exception): pass -class DataSource: - userdata = None - metadata = None - userdata_raw = None - cfgname = "" - # system config (passed in from cloudinit, - # cloud-config before input from the DataSource) - sys_cfg = {} - # datasource config, the cloud-config['datasource']['__name__'] - ds_cfg = {} # datasource config - - def __init__(self, sys_cfg=None): - if not self.cfgname: - name = str(self.__class__).split(".")[-1] - if name.startswith("DataSource"): - name = name[len("DataSource"):] - self.cfgname = name +class DataSource(object): + def __init__(self, ud_proc, cfg): + name = util.obj_name(self) + if name.startswith(DS_PREFIX): + name = name[DS_PREFIX:] + self.cfgname = name if sys_cfg: self.sys_cfg = sys_cfg - + else: + self.sys_cfg = {} + self.ud_proc = ud_proc + self.userdata = None + self.metadata = None + self.userdata_raw = None self.ds_cfg = util.get_cfg_by_path(self.sys_cfg, ("datasource", self.cfgname), self.ds_cfg) def get_userdata(self): - if self.userdata == None: - self.userdata = ud.preprocess_userdata(self.userdata_raw) + if self.userdata is None: + raw_data = self.get_userdata_raw() + self.userdata = self.ud_proc.process(raw_data) return self.userdata def get_userdata_raw(self): - return(self.userdata_raw) + return self.userdata_raw # the data sources' config_obj is a cloud-config formated # object that came to it from ways other than cloud-config # because cloud-config content would be handled elsewhere def get_config_obj(self): - return({}) + return {} def get_public_ssh_keys(self): keys = [] - if 'public-keys' not in self.metadata: - return([]) - if isinstance(self.metadata['public-keys'], str): - return(str(self.metadata['public-keys']).splitlines()) + if not self.metadata or 'public-keys' not in self.metadata: + return keys - if isinstance(self.metadata['public-keys'], list): - return(self.metadata['public-keys']) + if isinstance(self.metadata['public-keys'], (str)): + return str(self.metadata['public-keys']).splitlines() - for _keyname, klist in self.metadata['public-keys'].items(): - # lp:506332 uec metadata service responds with - # data that makes boto populate a string for 'klist' rather - # than a list. - if isinstance(klist, str): - klist = [klist] - for pkey in klist: - # there is an empty string at the end of the keylist, trim it - if pkey: - keys.append(pkey) + if isinstance(self.metadata['public-keys'], (list, set)): + return list(self.metadata['public-keys']) - return(keys) + if isinstance(self.metadata['public-keys'], (dict)): + for _keyname, klist in self.metadata['public-keys'].items(): + # lp:506332 uec metadata service responds with + # data that makes boto populate a string for 'klist' rather + # than a list. + if isinstance(klist, (str)): + klist = [klist] + if isinstance(klist, (list)): + for pkey in klist: + # there is an empty string at the end of the keylist, trim it + if pkey: + keys.append(pkey) + + return keys def device_name_to_device(self, _name): # translate a 'name' to a device @@ -97,48 +99,43 @@ class DataSource: # to consult metadata service, that has # ephemeral0: sdb # and return 'sdb' for input 'ephemeral0' - return(None) + return None def get_locale(self): - return('en_US.UTF-8') + return 'en_US.UTF-8' def get_local_mirror(self): return None def get_instance_id(self): - if 'instance-id' not in self.metadata: + if not self.metadata or 'instance-id' not in self.metadata: return "iid-datasource" - return(self.metadata['instance-id']) + return str(self.metadata['instance-id']) def get_hostname(self, fqdn=False): defdomain = "localdomain" defhost = "localhost" - domain = defdomain - if not 'local-hostname' in self.metadata: + if not self.metadata or not 'local-hostname' in self.metadata: # this is somewhat questionable really. # the cloud datasource was asked for a hostname # and didn't have one. raising error might be more appropriate # but instead, basically look up the existing hostname toks = [] - - hostname = socket.gethostname() - + hostname = util.get_hostname() fqdn = util.get_fqdn_from_hosts(hostname) - if fqdn and fqdn.find(".") > 0: toks = str(fqdn).split(".") elif hostname: toks = [hostname, defdomain] else: toks = [defhost, defdomain] - else: # if there is an ipv4 address in 'local-hostname', then # make up a hostname (LP: #475354) in format ip-xx.xx.xx.xx lhost = self.metadata['local-hostname'] - if is_ipv4(lhost): + if util.is_ipv4(lhost): toks = "ip-%s" % lhost.replace(".", "-") else: toks = lhost.split(".") @@ -155,22 +152,22 @@ class DataSource: return hostname -def find_source(cfg, ds_deps): - cfglist = cfg.get('datasource_list') or [] - dslist = list_sources(cfglist, ds_deps) - dsnames = [f.__name__ for f in dslist] - - LOG.debug("Searching for data source in %s", dsnames) - for cls in dslist: - ds = cls.__name__ +def find_source(cfg, ds_deps, cfg_list, pkg_list, **kwargs): + ds_list = list_sources(cfg_list, ds_deps, pkg_list) + ds_names = [util.obj_name(f) for f in ds_list] + ds_args = dict(kwargs) + ds_args['cfg'] = cfg + LOG.info("Searching for data source in: %s", ds_names) + for cls in ds_list: + ds = util.obj_name(cls) try: - s = cls(sys_cfg=cfg) + s = cls(**ds_args) if s.get_data(): return (s, ds) except Exception as e: - LOG.exception("Getting data from %s raised %s", ds, e) + LOG.exception("Getting data from %s failed", ds) - msg = "Did not find any data source, searched classes: %s" % dsnames + msg = "Did not find any data source, searched classes: %s" % (ds_names) raise DataSourceNotFoundException(msg) @@ -178,31 +175,33 @@ def find_source(cfg, ds_deps): # iterate through cfg_list, loading "DataSourceCollections" modules # and calling their "get_datasource_list". # return an ordered list of classes that match -# -# - modules must be named "DataSource", where 'item' is an entry -# in cfg_list -# - if pkglist is given, it will iterate try loading from that package -# ie, pkglist=[ "foo", "" ] -# will first try to load foo.DataSource -# then DataSource -def list_sources(cfg_list, depends, pkglist=None): - if pkglist is None: - pkglist = [] - retlist = [] +def list_sources(cfg_list, depends, pkg_list): + src_list = [] + LOG.info("Looking for for data source in: %s, %s that match %s", cfg_list, pkg_list, depends) for ds_coll in cfg_list: - for pkg in pkglist: + ds_name = str(ds_coll) + if not ds_name.startswith(DS_PREFIX): + ds_name = '%s%s' % (DS_PREFIX, ds_name) + for pkg in pkg_list: + pkg_name = [] + if pkg: + pkg_name.append(str(pkg)) + pkg_name.append(ds_name) + mod_name = ".".join(pkg_name) + mod = importer.import_module(mod_name) if pkg: - pkg = "%s." % pkg - try: - mod = __import__("%sDataSource%s" % (pkg, ds_coll)) - if pkg: - mod = getattr(mod, "DataSource%s" % ds_coll) - lister = getattr(mod, "get_datasource_list") - retlist.extend(lister(depends)) - break - except: - raise - return(retlist) + mod = getattr(mod, ds_name, None) + if not mod: + continue + lister = getattr(mod, "get_datasource_list", None) + if not lister: + continue + cls_matches = lister(depends) + if not cls_matches: + continue + src_list.extend(cls_matches) + break + return src_list # depends is a list of dependencies (DEP_FILESYSTEM) @@ -213,10 +212,9 @@ def list_sources(cfg_list, depends, pkglist=None): # it returns a list of 'class' that matched these deps exactly # it is a helper function for DataSourceCollections def list_from_depends(depends, dslist): - retlist = [] + ret_list = [] depset = set(depends) - for elem in dslist: - (cls, deps) = elem + for (cls, deps) in dslist: if depset == set(deps): - retlist.append(cls) - return(retlist) + ret_list.append(cls) + return ret_list -- cgit v1.2.3 From 08775384a2d9b538e9eeeede0f67369fd17a6d26 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Mon, 11 Jun 2012 17:17:05 -0700 Subject: Add more logging in onto what is being searched for. --- cloudinit/sources/__init__.py | 35 +++++++++++++++++++---------------- 1 file changed, 19 insertions(+), 16 deletions(-) (limited to 'cloudinit/sources/__init__.py') diff --git a/cloudinit/sources/__init__.py b/cloudinit/sources/__init__.py index 05c8bfad..dfd1fff3 100644 --- a/cloudinit/sources/__init__.py +++ b/cloudinit/sources/__init__.py @@ -24,9 +24,12 @@ from cloudinit import importer from cloudinit import log as logging from cloudinit import util +from cloudinit.user_data import processor as ud_proc + DEP_FILESYSTEM = "FILESYSTEM" DEP_NETWORK = "NETWORK" DS_PREFIX = 'DataSource' + LOG = logging.getLogger(__name__) @@ -35,26 +38,25 @@ class DataSourceNotFoundException(Exception): class DataSource(object): - def __init__(self, ud_proc, cfg): + def __init__(self, sys_cfg, distro, paths): name = util.obj_name(self) if name.startswith(DS_PREFIX): name = name[DS_PREFIX:] self.cfgname = name - if sys_cfg: - self.sys_cfg = sys_cfg - else: - self.sys_cfg = {} - self.ud_proc = ud_proc + self.sys_cfg = sys_cfg + self.distro = distro + self.paths = paths + self.userdata_proc = ud_proc.UserDataProcessor(paths) self.userdata = None self.metadata = None self.userdata_raw = None self.ds_cfg = util.get_cfg_by_path(self.sys_cfg, - ("datasource", self.cfgname), self.ds_cfg) + ("datasource", self.cfgname), {}) def get_userdata(self): if self.userdata is None: raw_data = self.get_userdata_raw() - self.userdata = self.ud_proc.process(raw_data) + self.userdata = self.userdata_proc.process(raw_data) return self.userdata def get_userdata_raw(self): @@ -85,7 +87,7 @@ class DataSource(object): # than a list. if isinstance(klist, (str)): klist = [klist] - if isinstance(klist, (list)): + if isinstance(klist, (list, set)): for pkey in klist: # there is an empty string at the end of the keylist, trim it if pkey: @@ -105,6 +107,7 @@ class DataSource(object): return 'en_US.UTF-8' def get_local_mirror(self): + # ?? return None def get_instance_id(self): @@ -152,20 +155,18 @@ class DataSource(object): return hostname -def find_source(cfg, ds_deps, cfg_list, pkg_list, **kwargs): +def find_source(sys_cfg, distro, paths, ds_deps, cfg_list, pkg_list): ds_list = list_sources(cfg_list, ds_deps, pkg_list) ds_names = [util.obj_name(f) for f in ds_list] - ds_args = dict(kwargs) - ds_args['cfg'] = cfg LOG.info("Searching for data source in: %s", ds_names) for cls in ds_list: ds = util.obj_name(cls) try: - s = cls(**ds_args) + s = cls(distro, sys_cfg, paths) if s.get_data(): return (s, ds) except Exception as e: - LOG.exception("Getting data from %s failed", ds) + LOG.exception("Getting data from %s failed due to %s", ds, e) msg = "Did not find any data source, searched classes: %s" % (ds_names) raise DataSourceNotFoundException(msg) @@ -187,8 +188,7 @@ def list_sources(cfg_list, depends, pkg_list): if pkg: pkg_name.append(str(pkg)) pkg_name.append(ds_name) - mod_name = ".".join(pkg_name) - mod = importer.import_module(mod_name) + mod = importer.import_module(".".join(pkg_name)) if pkg: mod = getattr(mod, ds_name, None) if not mod: @@ -196,10 +196,13 @@ def list_sources(cfg_list, depends, pkg_list): lister = getattr(mod, "get_datasource_list", None) if not lister: continue + LOG.debug("Seeing if %s matches using function %s", mod, lister) cls_matches = lister(depends) if not cls_matches: continue src_list.extend(cls_matches) + LOG.debug("Found a match for data source %s in %s with matches %s", + ds_name, mod, cls_matches) break return src_list -- cgit v1.2.3 From 508168acb95aee070d493b45656f781a42bdd262 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Fri, 15 Jun 2012 18:01:03 -0700 Subject: Complete initial cleanup for refactoring/rework. Some of the cleanups were the following 1. Using standard (logged) utility functions for sub process work, writing, reading files, and other file system/operating system options 2. Having distrobutions impelement there own subclasses to handle system specifics (if applicable) 3. Having a cloud wrapper that provides just the functionality we want to expose (cloud.py) 4. Using a path class instead of globals for all cloud init paths (it is configured via config) 5. Removal of as much shared global state as possible (there should be none, minus a set of constants) 6. Other various cleanups that remove transforms/handlers/modules from reading/writing/chmoding there own files. a. They should be using util functions to take advantage of the logging that is now enabled in those util functions (very useful for debugging) 7. Urls being read and checked from a single module that serves this and only this purpose (+1 for code organization) 8. Updates to log whenever a transform decides not to run 9. Ensure whenever a exception is thrown (and possibly captured) that the util.logexc function is called a. For debugging, tracing this is important to not just drop them on the floor. 10. Code shuffling into utils.py where it makes sense (and where it could serve a benefit for other code now or in the future) --- cloudinit/sources/DataSourceMAAS.py | 226 +++++++---------------- cloudinit/sources/DataSourceNoCloud.py | 143 +++++++-------- cloudinit/sources/DataSourceOVF.py | 227 ++++++++++-------------- cloudinit/sources/__init__.py | 12 +- cloudinit/transforms/__init__.py | 190 ++------------------ cloudinit/transforms/cc_apt_pipelining.py | 9 +- cloudinit/transforms/cc_apt_update_upgrade.py | 116 ++++++------ cloudinit/transforms/cc_bootcmd.py | 50 +++--- cloudinit/transforms/cc_byobu.py | 22 +-- cloudinit/transforms/cc_ca_certs.py | 25 +-- cloudinit/transforms/cc_chef.py | 101 +++++------ cloudinit/transforms/cc_disable_ec2_metadata.py | 14 +- cloudinit/transforms/cc_final_message.py | 63 ++++--- cloudinit/transforms/cc_foo.py | 35 +++- cloudinit/transforms/cc_grub_dpkg.py | 19 +- cloudinit/transforms/cc_keys_to_console.py | 14 +- cloudinit/transforms/cc_landscape.py | 43 +++-- cloudinit/transforms/cc_locale.py | 36 ++-- cloudinit/transforms/cc_mcollective.py | 80 ++++----- cloudinit/transforms/cc_mounts.py | 84 +++++---- cloudinit/transforms/cc_phone_home.py | 53 +++--- cloudinit/transforms/cc_puppet.py | 94 +++++----- cloudinit/transforms/cc_resizefs.py | 142 +++++++++------ cloudinit/transforms/cc_rightscale_userdata.py | 62 ++++--- cloudinit/transforms/cc_rsyslog.py | 52 +++--- cloudinit/transforms/cc_runcmd.py | 15 +- cloudinit/transforms/cc_salt_minion.py | 49 ++--- cloudinit/transforms/cc_scripts_per_boot.py | 20 ++- cloudinit/transforms/cc_scripts_per_instance.py | 20 ++- cloudinit/transforms/cc_scripts_per_once.py | 20 ++- cloudinit/transforms/cc_scripts_user.py | 19 +- cloudinit/transforms/cc_set_hostname.py | 23 +-- cloudinit/transforms/cc_set_passwords.py | 108 ++++++----- cloudinit/transforms/cc_ssh.py | 93 ++++++---- cloudinit/transforms/cc_ssh_import_id.py | 25 ++- cloudinit/transforms/cc_timezone.py | 41 +---- cloudinit/transforms/cc_update_etc_hosts.py | 82 +++------ cloudinit/transforms/cc_update_hostname.py | 80 +-------- 38 files changed, 1113 insertions(+), 1394 deletions(-) (limited to 'cloudinit/sources/__init__.py') diff --git a/cloudinit/sources/DataSourceMAAS.py b/cloudinit/sources/DataSourceMAAS.py index 61a0038f..27196265 100644 --- a/cloudinit/sources/DataSourceMAAS.py +++ b/cloudinit/sources/DataSourceMAAS.py @@ -1,8 +1,10 @@ # vi: ts=4 expandtab # # Copyright (C) 2012 Canonical Ltd. +# Copyright (C) 2012 Yahoo! Inc. # # Author: Scott Moser +# Author: Joshua Harlow # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License version 3, as @@ -16,22 +18,22 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -import cloudinit.DataSource as DataSource - -from cloudinit import seeddir as base_seeddir -from cloudinit import log -import cloudinit.util as util +import os import errno import oauth.oauth as oauth -import os.path -import urllib2 import time +import urllib2 +from cloudinit import log as logging +from cloudinit import sources +from cloudinit import url_helper as uhelp +from cloudinit import util +LOG = logging.getLogger(__name__) MD_VERSION = "2012-03-01" -class DataSourceMAAS(DataSource.DataSource): +class DataSourceMAAS(sources.DataSource): """ DataSourceMAAS reads instance information from MAAS. Given a config metadata_url, and oauth tokens, it expects to find @@ -40,61 +42,64 @@ class DataSourceMAAS(DataSource.DataSource): user-data hostname """ - seeddir = base_seeddir + '/maas' - baseurl = None + def __init__(self, sys_cfg, distro, paths): + sources.DataSource.__init__(self, sys_cfg, distro, paths) + self.base_url = None + self.seed_dir = os.path.join(paths.seed_dir, 'maas') def __str__(self): - return("DataSourceMAAS[%s]" % self.baseurl) + return "%s[%s]" % (util.obj_name(self), self.base_url) def get_data(self): mcfg = self.ds_cfg try: - (userdata, metadata) = read_maas_seed_dir(self.seeddir) + (userdata, metadata) = read_maas_seed_dir(self.seed_dir) self.userdata_raw = userdata self.metadata = metadata - self.baseurl = self.seeddir + self.base_url = self.seed_dir return True except MAASSeedDirNone: pass except MAASSeedDirMalformed as exc: - log.warn("%s was malformed: %s\n" % (self.seeddir, exc)) + LOG.warn("%s was malformed: %s" % (self.seed_dir, exc)) raise - try: - # if there is no metadata_url, then we're not configured - url = mcfg.get('metadata_url', None) - if url == None: - return False + # If there is no metadata_url, then we're not configured + url = mcfg.get('metadata_url', None) + if not url: + return False + try: if not self.wait_for_metadata_service(url): return False - self.baseurl = url + self.base_url = url - (userdata, metadata) = read_maas_seed_url(self.baseurl, - self.md_headers) + (userdata, metadata) = read_maas_seed_url(self.base_url, + self.md_headers) self.userdata_raw = userdata self.metadata = metadata return True except Exception: - util.logexc(log) + util.logexc(LOG, "Failed fetching metadata from url %s", url) return False def md_headers(self, url): mcfg = self.ds_cfg - # if we are missing token_key, token_secret or consumer_key + # If we are missing token_key, token_secret or consumer_key # then just do non-authed requests for required in ('token_key', 'token_secret', 'consumer_key'): if required not in mcfg: - return({}) + return {} consumer_secret = mcfg.get('consumer_secret', "") - - return(oauth_headers(url=url, consumer_key=mcfg['consumer_key'], - token_key=mcfg['token_key'], token_secret=mcfg['token_secret'], - consumer_secret=consumer_secret)) + return oauth_headers(url=url, + consumer_key=mcfg['consumer_key'], + token_key=mcfg['token_key'], + token_secret=mcfg['token_secret'], + consumer_secret=consumer_secret) def wait_for_metadata_service(self, url): mcfg = self.ds_cfg @@ -103,32 +108,31 @@ class DataSourceMAAS(DataSource.DataSource): try: max_wait = int(mcfg.get("max_wait", max_wait)) except Exception: - util.logexc(log) - log.warn("Failed to get max wait. using %s" % max_wait) + util.logexc(LOG, "Failed to get max wait. using %s", max_wait) if max_wait == 0: return False timeout = 50 try: - timeout = int(mcfg.get("timeout", timeout)) + if timeout in mcfg: + timeout = int(mcfg.get("timeout", timeout)) except Exception: - util.logexc(log) - log.warn("Failed to get timeout, using %s" % timeout) + LOG.warn("Failed to get timeout, using %s" % timeout) starttime = time.time() check_url = "%s/%s/meta-data/instance-id" % (url, MD_VERSION) url = util.wait_for_url(urls=[check_url], max_wait=max_wait, - timeout=timeout, status_cb=log.warn, - headers_cb=self.md_headers) + timeout=timeout, status_cb=LOG.warn, + headers_cb=self.md_headers) if url: - log.debug("Using metadata source: '%s'" % url) + LOG.info("Using metadata source: '%s'", url) else: - log.critical("giving up on md after %i seconds\n" % - int(time.time() - starttime)) + LOG.critical("Giving up on md from %s after %i seconds", + urls, int(time.time() - starttime)) - return (bool(url)) + return bool(url) def read_maas_seed_dir(seed_d): @@ -139,22 +143,19 @@ def read_maas_seed_dir(seed_d): * local-hostname * user-data """ - files = ('local-hostname', 'instance-id', 'user-data', 'public-keys') - md = {} - if not os.path.isdir(seed_d): raise MAASSeedDirNone("%s: not a directory") + files = ('local-hostname', 'instance-id', 'user-data', 'public-keys') + md = {} for fname in files: try: - with open(os.path.join(seed_d, fname)) as fp: - md[fname] = fp.read() - fp.close() + md[fname] = util.load_file(os.path.join(seed_d, fname)) except IOError as e: if e.errno != errno.ENOENT: raise - return(check_seed_contents(md, seed_d)) + return check_seed_contents(md, seed_d) def read_maas_seed_url(seed_url, header_cb=None, timeout=None, @@ -169,29 +170,26 @@ def read_maas_seed_url(seed_url, header_cb=None, timeout=None, * //meta-data/local-hostname * //user-data """ - files = ('meta-data/local-hostname', - 'meta-data/instance-id', - 'meta-data/public-keys', - 'user-data') - base_url = "%s/%s" % (seed_url, version) + files = { + 'local-hostname': "%s/%s" % (base_url, 'meta-data/local-hostname'), + 'instance-id': "%s/%s" % (base_url, 'meta-data/instance-id'), + 'public-keys': "%s/%s" % (base_url, 'meta-data/public-keys'), + 'user-data': "%s/%s" % (base_url, 'user-data'), + } md = {} - for fname in files: - url = "%s/%s" % (base_url, fname) + for (name, url) in files: if header_cb: headers = header_cb(url) else: headers = {} - try: - req = urllib2.Request(url, data=None, headers=headers) - resp = urllib2.urlopen(req, timeout=timeout) - md[os.path.basename(fname)] = resp.read() + (resp, sc) = uhelp.readurl(url, headers=headers, timeout=timeout) + md[name] = resp except urllib2.HTTPError as e: if e.code != 404: raise - - return(check_seed_contents(md, seed_url)) + return check_seed_contents(md, seed_url) def check_seed_contents(content, seed): @@ -201,11 +199,10 @@ def check_seed_contents(content, seed): Raise MAASSeedDirMalformed or MAASSeedDirNone """ md_required = ('instance-id', 'local-hostname') - found = content.keys() - if len(content) == 0: raise MAASSeedDirNone("%s: no data files found" % seed) + found = content.keys() missing = [k for k in md_required if k not in found] if len(missing): raise MAASSeedDirMalformed("%s: missing files %s" % (seed, missing)) @@ -217,7 +214,7 @@ def check_seed_contents(content, seed): continue md[key] = val - return(userdata, md) + return (userdata, md) def oauth_headers(url, consumer_key, token_key, token_secret, consumer_secret): @@ -232,8 +229,8 @@ def oauth_headers(url, consumer_key, token_key, token_secret, consumer_secret): } req = oauth.OAuthRequest(http_url=url, parameters=params) req.sign_request(oauth.OAuthSignatureMethod_PLAINTEXT(), - consumer, token) - return(req.to_header()) + consumer, token) + return req.to_header() class MAASSeedDirNone(Exception): @@ -244,102 +241,11 @@ class MAASSeedDirMalformed(Exception): pass +# Used to match classes to dependencies datasources = [ - (DataSourceMAAS, (DataSource.DEP_FILESYSTEM, DataSource.DEP_NETWORK)), + (DataSourceMAAS, (sources.DEP_FILESYSTEM, sources.DEP_NETWORK)), ] - -# return a list of data sources that match this set of dependencies +# Return a list of data sources that match this set of dependencies def get_datasource_list(depends): - return(DataSource.list_from_depends(depends, datasources)) - - -if __name__ == "__main__": - def main(): - """ - Call with single argument of directory or http or https url. - If url is given additional arguments are allowed, which will be - interpreted as consumer_key, token_key, token_secret, consumer_secret - """ - import argparse - import pprint - - parser = argparse.ArgumentParser(description='Interact with MAAS DS') - parser.add_argument("--config", metavar="file", - help="specify DS config file", default=None) - parser.add_argument("--ckey", metavar="key", - help="the consumer key to auth with", default=None) - parser.add_argument("--tkey", metavar="key", - help="the token key to auth with", default=None) - parser.add_argument("--csec", metavar="secret", - help="the consumer secret (likely '')", default="") - parser.add_argument("--tsec", metavar="secret", - help="the token secret to auth with", default=None) - parser.add_argument("--apiver", metavar="version", - help="the apiver to use ("" can be used)", default=MD_VERSION) - - subcmds = parser.add_subparsers(title="subcommands", dest="subcmd") - subcmds.add_parser('crawl', help="crawl the datasource") - subcmds.add_parser('get', help="do a single GET of provided url") - subcmds.add_parser('check-seed', help="read andn verify seed at url") - - parser.add_argument("url", help="the data source to query") - - args = parser.parse_args() - - creds = {'consumer_key': args.ckey, 'token_key': args.tkey, - 'token_secret': args.tsec, 'consumer_secret': args.csec} - - if args.config: - import yaml - with open(args.config) as fp: - cfg = yaml.load(fp) - if 'datasource' in cfg: - cfg = cfg['datasource']['MAAS'] - for key in creds.keys(): - if key in cfg and creds[key] == None: - creds[key] = cfg[key] - - def geturl(url, headers_cb): - req = urllib2.Request(url, data=None, headers=headers_cb(url)) - return(urllib2.urlopen(req).read()) - - def printurl(url, headers_cb): - print "== %s ==\n%s\n" % (url, geturl(url, headers_cb)) - - def crawl(url, headers_cb=None): - if url.endswith("/"): - for line in geturl(url, headers_cb).splitlines(): - if line.endswith("/"): - crawl("%s%s" % (url, line), headers_cb) - else: - printurl("%s%s" % (url, line), headers_cb) - else: - printurl(url, headers_cb) - - def my_headers(url): - headers = {} - if creds.get('consumer_key', None) != None: - headers = oauth_headers(url, **creds) - return headers - - if args.subcmd == "check-seed": - if args.url.startswith("http"): - (userdata, metadata) = read_maas_seed_url(args.url, - header_cb=my_headers, version=args.apiver) - else: - (userdata, metadata) = read_maas_seed_url(args.url) - print "=== userdata ===" - print userdata - print "=== metadata ===" - pprint.pprint(metadata) - - elif args.subcmd == "get": - printurl(args.url, my_headers) - - elif args.subcmd == "crawl": - if not args.url.endswith("/"): - args.url = "%s/" % args.url - crawl(args.url, my_headers) - - main() + return sources.list_from_depends(depends, datasources) diff --git a/cloudinit/sources/DataSourceNoCloud.py b/cloudinit/sources/DataSourceNoCloud.py index e8c56b8f..84d0f99d 100644 --- a/cloudinit/sources/DataSourceNoCloud.py +++ b/cloudinit/sources/DataSourceNoCloud.py @@ -2,9 +2,11 @@ # # Copyright (C) 2009-2010 Canonical Ltd. # Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# Copyright (C) 2012 Yahoo! Inc. # # Author: Scott Moser # Author: Juerg Hafliger +# Author: Joshua Harlow # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License version 3, as @@ -18,33 +20,34 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -import cloudinit.DataSource as DataSource - -from cloudinit import seeddir as base_seeddir -from cloudinit import log -import cloudinit.util as util import errno -import subprocess +import os + +from cloudinit import log as logging +from cloudinit import sources +from cloudinit import util + +LOG = logging.getLogger(__name__) -class DataSourceNoCloud(DataSource.DataSource): - metadata = None - userdata = None - userdata_raw = None - supported_seed_starts = ("/", "file://") - dsmode = "local" - seed = None - cmdline_id = "ds=nocloud" - seeddir = base_seeddir + '/nocloud' +class DataSourceNoCloud(sources.DataSource): + def __init__(self, sys_cfg, distro, paths): + sources.DataSource.__init__(self, sys_cfg, distro, paths) + self.dsmode = 'local' + self.seed = None + self.cmdline_id = "ds=nocloud" + self.seed_dir = os.path.join(paths.seed_dir, 'nocloud') + self.supported_seed_starts = ("/", "file://") def __str__(self): - mstr = "DataSourceNoCloud" - mstr = mstr + " [seed=%s]" % self.seed - return(mstr) + mstr = "%s [seed=%s][dsmode=%s]" % (util.obj_name(self), + self.seed, self.dsmode) + return mstr def get_data(self): defaults = { - "instance-id": "nocloud", "dsmode": self.dsmode + "instance-id": "nocloud", + "dsmode": self.dsmode, } found = [] @@ -52,24 +55,24 @@ class DataSourceNoCloud(DataSource.DataSource): ud = "" try: - # parse the kernel command line, getting data passed in + # Parse the kernel command line, getting data passed in if parse_cmdline_data(self.cmdline_id, md): found.append("cmdline") except: - util.logexc(log) + util.logexc(LOG, "Unable to parse command line data") return False - # check to see if the seeddir has data. + # Check to see if the seed dir has data. seedret = {} - if util.read_optional_seed(seedret, base=self.seeddir + "/"): + if util.read_optional_seed(seedret, base=self.seed_dir + "/"): md = util.mergedict(md, seedret['meta-data']) ud = seedret['user-data'] - found.append(self.seeddir) - log.debug("using seeded cache data in %s" % self.seeddir) + found.append(self.seed_dir) + LOG.debug("Using seeded cache data from %s", self.seed_dir) - # if the datasource config had a 'seedfrom' entry, then that takes + # If the datasource config had a 'seedfrom' entry, then that takes # precedence over a 'seedfrom' that was found in a filesystem - # but not over external medi + # but not over external media if 'seedfrom' in self.ds_cfg and self.ds_cfg['seedfrom']: found.append("ds_config") md["seedfrom"] = self.ds_cfg['seedfrom'] @@ -83,35 +86,36 @@ class DataSourceNoCloud(DataSource.DataSource): for dev in devlist: try: - (newmd, newud) = util.mount_callback_umount(dev, - util.read_seeded) + LOG.debug("Attempting to use data from %s", dev) + + (newmd, newud) = util.mount_cb(dev, util.read_seeded) md = util.mergedict(newmd, md) ud = newud - # for seed from a device, the default mode is 'net'. + # For seed from a device, the default mode is 'net'. # that is more likely to be what is desired. # If they want dsmode of local, then they must # specify that. if 'dsmode' not in md: md['dsmode'] = "net" - log.debug("using data from %s" % dev) + LOG.debug("Using data from %s", dev) found.append(dev) break - except OSError, e: + except OSError as e: if e.errno != errno.ENOENT: raise - except util.mountFailedError: - log.warn("Failed to mount %s when looking for seed" % dev) + except util.MountFailedError: + util.logexc(LOG, "Failed to mount %s when looking for seed", dev) - # there was no indication on kernel cmdline or data + # There was no indication on kernel cmdline or data # in the seeddir suggesting this handler should be used. if len(found) == 0: return False seeded_interfaces = None - # the special argument "seedfrom" indicates we should + # The special argument "seedfrom" indicates we should # attempt to seed the userdata / metadata from its value # its primarily value is in allowing the user to type less # on the command line, ie: ds=nocloud;s=http://bit.ly/abcdefg @@ -123,57 +127,46 @@ class DataSourceNoCloud(DataSource.DataSource): seedfound = proto break if not seedfound: - log.debug("seed from %s not supported by %s" % - (seedfrom, self.__class__)) + LOG.debug("Seed from %s not supported by %s", seedfrom, self) return False if 'network-interfaces' in md: seeded_interfaces = self.dsmode - # this could throw errors, but the user told us to do it + # This could throw errors, but the user told us to do it # so if errors are raised, let them raise (md_seed, ud) = util.read_seeded(seedfrom, timeout=None) - log.debug("using seeded cache data from %s" % seedfrom) + LOG.debug("Using seeded cache data from %s", seedfrom) - # values in the command line override those from the seed + # Values in the command line override those from the seed md = util.mergedict(md, md_seed) found.append(seedfrom) + # Now that we have exhausted any other places merge in the defaults md = util.mergedict(md, defaults) - # update the network-interfaces if metadata had 'network-interfaces' + # Update the network-interfaces if metadata had 'network-interfaces' # entry and this is the local datasource, or 'seedfrom' was used # and the source of the seed was self.dsmode # ('local' for NoCloud, 'net' for NoCloudNet') if ('network-interfaces' in md and (self.dsmode in ("local", seeded_interfaces))): - log.info("updating network interfaces from nocloud") - - util.write_file("/etc/network/interfaces", - md['network-interfaces']) - try: - (out, err) = util.subp(['ifup', '--all']) - if len(out) or len(err): - log.warn("ifup --all had stderr: %s" % err) - - except subprocess.CalledProcessError as exc: - log.warn("ifup --all failed: %s" % (exc.output[1])) - - self.seed = ",".join(found) - self.metadata = md - self.userdata_raw = ud - + LOG.info("Updating network interfaces from %s", self) + self.distro.apply_network(md['network-interfaces']) + if md['dsmode'] == self.dsmode: + self.seed = ",".join(found) + self.metadata = md + self.userdata_raw = ud return True - log.debug("%s: not claiming datasource, dsmode=%s" % - (self, md['dsmode'])) + LOG.debug("%s: not claiming datasource, dsmode=%s", self, md['dsmode']) return False -# returns true or false indicating if cmdline indicated +# Returns true or false indicating if cmdline indicated # that this module should be used -# example cmdline: +# Example cmdline: # root=LABEL=uec-rootfs ro ds=nocloud def parse_cmdline_data(ds_id, fill, cmdline=None): if cmdline is None: @@ -210,23 +203,25 @@ def parse_cmdline_data(ds_id, fill, cmdline=None): k = s2l[k] fill[k] = v - return(True) + return True class DataSourceNoCloudNet(DataSourceNoCloud): - cmdline_id = "ds=nocloud-net" - supported_seed_starts = ("http://", "https://", "ftp://") - seeddir = base_seeddir + '/nocloud-net' - dsmode = "net" + def __init__(self, sys_cfg, distro, paths): + DataSourceNoCloud.__init__(self, sys_cfg, distro, paths) + self.cmdline_id = "ds=nocloud-net" + self.supported_seed_starts = ("http://", "https://", "ftp://") + self.seed_dir = os.path.join(paths.seed_dir, 'nocloud-net') + self.dsmode = "net" -datasources = ( - (DataSourceNoCloud, (DataSource.DEP_FILESYSTEM, )), - (DataSourceNoCloudNet, - (DataSource.DEP_FILESYSTEM, DataSource.DEP_NETWORK)), -) +# Used to match classes to dependencies +datasources = [ + (DataSourceNoCloud, (sources.DEP_FILESYSTEM, )), + (DataSourceNoCloudNet, (sources.DEP_FILESYSTEM, sources.DEP_NETWORK)), +] -# return a list of data sources that match this set of dependencies +# Return a list of data sources that match this set of dependencies def get_datasource_list(depends): - return(DataSource.list_from_depends(depends, datasources)) + return sources.list_from_depends(depends, datasources) diff --git a/cloudinit/sources/DataSourceOVF.py b/cloudinit/sources/DataSourceOVF.py index a0b1b518..bb0f46c2 100644 --- a/cloudinit/sources/DataSourceOVF.py +++ b/cloudinit/sources/DataSourceOVF.py @@ -2,9 +2,11 @@ # # Copyright (C) 2011 Canonical Ltd. # Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# Copyright (C) 2012 Yahoo! Inc. # # Author: Scott Moser # Author: Juerg Hafliger +# Author: Joshua Harlow # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License version 3, as @@ -18,33 +20,30 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -import cloudinit.DataSource as DataSource - -from cloudinit import seeddir as base_seeddir -from cloudinit import log -import cloudinit.util as util -import os.path -import os from xml.dom import minidom import base64 +import os import re import tempfile -import subprocess +from cloudinit import log as logging +from cloudinit import sources +from cloudinit import util -class DataSourceOVF(DataSource.DataSource): - seed = None - seeddir = base_seeddir + '/ovf' - environment = None - cfg = {} - userdata_raw = None - metadata = None - supported_seed_starts = ("/", "file://") +LOG = logging.getLogger(__name__) + + +class DataSourceOVF(sources.DataSource): + def __init__(self, sys_cfg, distro, paths): + sources.DataSource.__init__(self, sys_cfg, distro, paths) + self.seed = None + self.seed_dir = os.path.join(paths.seed_dir, 'ovf') + self.environment = None + self.cfg = {} + self.supported_seed_starts = ("/", "file://") def __str__(self): - mstr = "DataSourceOVF" - mstr = mstr + " [seed=%s]" % self.seed - return(mstr) + return "%s [seed=%s]" % (util.obj_name(self), self.seed) def get_data(self): found = [] @@ -55,13 +54,12 @@ class DataSourceOVF(DataSource.DataSource): "instance-id": "iid-dsovf" } - (seedfile, contents) = get_ovf_env(base_seeddir) + (seedfile, contents) = get_ovf_env(self.paths.seed_dir) if seedfile: - # found a seed dir - seed = "%s/%s" % (base_seeddir, seedfile) + # Found a seed dir + seed = os.path.join(self.paths.seed_dir, seedfile) (md, ud, cfg) = read_ovf_environment(contents) self.environment = contents - found.append(seed) else: np = {'iso': transport_iso9660, @@ -71,7 +69,6 @@ class DataSourceOVF(DataSource.DataSource): (contents, _dev, _fname) = transfunc() if contents: break - if contents: (md, ud, cfg) = read_ovf_environment(contents) self.environment = contents @@ -89,17 +86,19 @@ class DataSourceOVF(DataSource.DataSource): seedfound = proto break if not seedfound: - log.debug("seed from %s not supported by %s" % - (seedfrom, self.__class__)) + LOG.debug("Seed from %s not supported by %s", + seedfrom, self) return False (md_seed, ud) = util.read_seeded(seedfrom, timeout=None) - log.debug("using seeded cache data from %s" % seedfrom) + LOG.debug("Using seeded cache data from %s", seedfrom) md = util.mergedict(md, md_seed) found.append(seedfrom) + # Now that we have exhausted any other places merge in the defaults md = util.mergedict(md, defaults) + self.seed = ",".join(found) self.metadata = md self.userdata_raw = ud @@ -108,31 +107,37 @@ class DataSourceOVF(DataSource.DataSource): def get_public_ssh_keys(self): if not 'public-keys' in self.metadata: - return([]) - return([self.metadata['public-keys'], ]) + return [] + pks = self.metadata['public-keys'] + if isinstance(pks, (list)): + return pks + else: + return [pks] - # the data sources' config_obj is a cloud-config formated + # The data sources' config_obj is a cloud-config formatted # object that came to it from ways other than cloud-config # because cloud-config content would be handled elsewhere def get_config_obj(self): - return(self.cfg) + return self.cfg class DataSourceOVFNet(DataSourceOVF): - seeddir = base_seeddir + '/ovf-net' - supported_seed_starts = ("http://", "https://", "ftp://") + def __init__(self, sys_cfg, distro, paths): + DataSourceOVF.__init__(self, sys_cfg, distro, paths) + self.seed_dir = os.path.join(paths.seed_dir, 'ovf-net') + self.supported_seed_starts = ("http://", "https://", "ftp://") -# this will return a dict with some content -# meta-data, user-data +# This will return a dict with some content +# meta-data, user-data, some config def read_ovf_environment(contents): - props = getProperties(contents) + props = get_properties(contents) md = {} cfg = {} ud = "" - cfg_props = ['password', ] + cfg_props = ['password'] md_props = ['seedfrom', 'local-hostname', 'public-keys', 'instance-id'] - for prop, val in props.iteritems(): + for (prop, val) in props.iteritems(): if prop == 'hostname': prop = "local-hostname" if prop in md_props: @@ -144,23 +149,25 @@ def read_ovf_environment(contents): ud = base64.decodestring(val) except: ud = val - return(md, ud, cfg) + return (md, ud, cfg) -# returns tuple of filename (in 'dirname', and the contents of the file) +# Returns tuple of filename (in 'dirname', and the contents of the file) # on "not found", returns 'None' for filename and False for contents def get_ovf_env(dirname): env_names = ("ovf-env.xml", "ovf_env.xml", "OVF_ENV.XML", "OVF-ENV.XML") for fname in env_names: - if os.path.isfile("%s/%s" % (dirname, fname)): - fp = open("%s/%s" % (dirname, fname)) - contents = fp.read() - fp.close() - return(fname, contents) - return(None, False) + full_fn = os.path.join(dirname, fname) + if os.path.isfile(full_fn): + try: + contents = util.load_file(full_fn) + return (fname, contents) + except: + util.logexc(LOG, "Failed loading ovf file %s", full_fn) + return (None, False) -# transport functions take no input and return +# Transport functions take no input and return # a 3 tuple of content, path, filename def transport_iso9660(require_iso=True): @@ -173,79 +180,45 @@ def transport_iso9660(require_iso=True): devname_regex = os.environ.get(envname, default_regex) cdmatch = re.compile(devname_regex) - # go through mounts to see if it was already mounted - fp = open("/proc/mounts") - mounts = fp.readlines() - fp.close() - - mounted = {} - for mpline in mounts: - (dev, mp, fstype, _opts, _freq, _passno) = mpline.split() - mounted[dev] = (dev, fstype, mp, False) - mp = mp.replace("\\040", " ") + # Go through mounts to see if it was already mounted + mounts = util.mounts() + for (dev, info) in mounts.iteritems(): + fstype = info['fstype'] if fstype != "iso9660" and require_iso: continue - if cdmatch.match(dev[5:]) == None: # take off '/dev/' continue - + mp = info['mountpoint'] (fname, contents) = get_ovf_env(mp) if contents is not False: - return(contents, dev, fname) - - tmpd = None - dvnull = None + return (contents, dev, fname) devs = os.listdir("/dev/") devs.sort() - for dev in devs: - fullp = "/dev/%s" % dev + fullp = os.path.join("/dev/", dev) - if fullp in mounted or not cdmatch.match(dev) or os.path.isdir(fullp): + if (fullp in mounted or + not cdmatch.match(dev) or os.path.isdir(fullp)): continue - fp = None try: - fp = open(fullp, "rb") - fp.read(512) - fp.close() + # See if we can read anything at all...?? + with open(fullp, 'rb') as fp: + fp.read(512) except: - if fp: - fp.close() continue - if tmpd is None: - tmpd = tempfile.mkdtemp() - if dvnull is None: - try: - dvnull = open("/dev/null") - except: - pass - - cmd = ["mount", "-o", "ro", fullp, tmpd] - if require_iso: - cmd.extend(('-t', 'iso9660')) - - rc = subprocess.call(cmd, stderr=dvnull, stdout=dvnull, stdin=dvnull) - if rc: + try: + (fname, contents) = utils.mount_cb(fullp, get_ovf_env, mtype="iso9660") + except util.MountFailedError: + util.logexc(LOG, "Failed mounting %s", fullp) continue - (fname, contents) = get_ovf_env(tmpd) - - subprocess.call(["umount", tmpd]) - if contents is not False: - os.rmdir(tmpd) - return(contents, fullp, fname) - - if tmpd: - os.rmdir(tmpd) - - if dvnull: - dvnull.close() + return (contents, fullp, fname) - return(False, None, None) + return (False, None, None) def transport_vmware_guestd(): @@ -259,74 +232,60 @@ def transport_vmware_guestd(): # # would need to error check here and see why this failed # # to know if log/error should be raised # return(False, None, None) - return(False, None, None) + return (False, None, None) -def findChild(node, filter_func): +def find_child(node, filter_func): ret = [] if not node.hasChildNodes(): return ret for child in node.childNodes: if filter_func(child): ret.append(child) - return(ret) + return ret -def getProperties(environString): - dom = minidom.parseString(environString) +def get_properties(contents): + + dom = minidom.parseString(contents) if dom.documentElement.localName != "Environment": - raise Exception("No Environment Node") + raise XmlError("No Environment Node") if not dom.documentElement.hasChildNodes(): - raise Exception("No Child Nodes") + raise XmlError("No Child Nodes") envNsURI = "http://schemas.dmtf.org/ovf/environment/1" # could also check here that elem.namespaceURI == # "http://schemas.dmtf.org/ovf/environment/1" - propSections = findChild(dom.documentElement, + propSections = find_child(dom.documentElement, lambda n: n.localName == "PropertySection") if len(propSections) == 0: - raise Exception("No 'PropertySection's") + raise XmlError("No 'PropertySection's") props = {} - propElems = findChild(propSections[0], lambda n: n.localName == "Property") + propElems = find_child(propSections[0], lambda n: n.localName == "Property") for elem in propElems: key = elem.attributes.getNamedItemNS(envNsURI, "key").value val = elem.attributes.getNamedItemNS(envNsURI, "value").value props[key] = val - return(props) + return props + + +class XmlError(Exception): + pass +# Used to match classes to dependencies datasources = ( - (DataSourceOVF, (DataSource.DEP_FILESYSTEM, )), - (DataSourceOVFNet, - (DataSource.DEP_FILESYSTEM, DataSource.DEP_NETWORK)), + (DataSourceOVF, (sources.DEP_FILESYSTEM, )), + (DataSourceOVFNet, (sources.DEP_FILESYSTEM, sources.DEP_NETWORK)), ) -# return a list of data sources that match this set of dependencies +# Return a list of data sources that match this set of dependencies def get_datasource_list(depends): - return(DataSource.list_from_depends(depends, datasources)) - - -if __name__ == "__main__": - def main(): - import sys - envStr = open(sys.argv[1]).read() - props = getProperties(envStr) - import pprint - pprint.pprint(props) - - md, ud, cfg = read_ovf_environment(envStr) - print "=== md ===" - pprint.pprint(md) - print "=== ud ===" - pprint.pprint(ud) - print "=== cfg ===" - pprint.pprint(cfg) - - main() + return sources.list_from_depends(depends, datasources) diff --git a/cloudinit/sources/__init__.py b/cloudinit/sources/__init__.py index dfd1fff3..08669f5d 100644 --- a/cloudinit/sources/__init__.py +++ b/cloudinit/sources/__init__.py @@ -39,10 +39,6 @@ class DataSourceNotFoundException(Exception): class DataSource(object): def __init__(self, sys_cfg, distro, paths): - name = util.obj_name(self) - if name.startswith(DS_PREFIX): - name = name[DS_PREFIX:] - self.cfgname = name self.sys_cfg = sys_cfg self.distro = distro self.paths = paths @@ -50,8 +46,11 @@ class DataSource(object): self.userdata = None self.metadata = None self.userdata_raw = None + name = util.obj_name(self) + if name.startswith(DS_PREFIX): + name = name[DS_PREFIX:] self.ds_cfg = util.get_cfg_by_path(self.sys_cfg, - ("datasource", self.cfgname), {}) + ("datasource", name), {}) def get_userdata(self): if self.userdata is None: @@ -112,6 +111,7 @@ class DataSource(object): def get_instance_id(self): if not self.metadata or 'instance-id' not in self.metadata: + # Return a magic not really instance id string return "iid-datasource" return str(self.metadata['instance-id']) @@ -166,7 +166,7 @@ def find_source(sys_cfg, distro, paths, ds_deps, cfg_list, pkg_list): if s.get_data(): return (s, ds) except Exception as e: - LOG.exception("Getting data from %s failed due to %s", ds, e) + util.logexc(LOG, "Getting data from %s failed", ds) msg = "Did not find any data source, searched classes: %s" % (ds_names) raise DataSourceNotFoundException(msg) diff --git a/cloudinit/transforms/__init__.py b/cloudinit/transforms/__init__.py index 5d70ac43..8275b375 100644 --- a/cloudinit/transforms/__init__.py +++ b/cloudinit/transforms/__init__.py @@ -19,183 +19,12 @@ # along with this program. If not, see . # -import os -import subprocess -import sys -import time -import traceback - -import yaml - -from cloudinit.settings import (PER_INSTANCE, PER_ALWAYS, PER_ONCE) +from cloudinit.settings import (PER_INSTANCE, FREQUENCIES) from cloudinit import log as logging -from cloudinit import util LOG = logging.getLogger(__name__) -DEF_HANDLER_VERSION = 1 -DEF_FREQ = PER_INSTANCE - - -# reads a cloudconfig module list, returns -# a 2 dimensional array suitable to pass to run_cc_modules -def read_cc_modules(cfg, name): - if name not in cfg: - return([]) - module_list = [] - # create 'module_list', an array of arrays - # where array[0] = config - # array[1] = freq - # array[2:] = arguemnts - for item in cfg[name]: - if isinstance(item, str): - module_list.append((item,)) - elif isinstance(item, list): - module_list.append(item) - else: - raise TypeError("failed to read '%s' item in config") - return(module_list) - - -def run_cc_modules(cc, module_list, log): - failures = [] - for cfg_mod in module_list: - name = cfg_mod[0] - freq = None - run_args = [] - if len(cfg_mod) > 1: - freq = cfg_mod[1] - if len(cfg_mod) > 2: - run_args = cfg_mod[2:] - - try: - log.debug("handling %s with freq=%s and args=%s" % - (name, freq, run_args)) - cc.handle(name, run_args, freq=freq) - except: - log.warn(traceback.format_exc()) - log.error("config handling of %s, %s, %s failed\n" % - (name, freq, run_args)) - failures.append(name) - - return(failures) - - -# always returns well formated values -# cfg is expected to have an entry 'output' in it, which is a dictionary -# that includes entries for 'init', 'config', 'final' or 'all' -# init: /var/log/cloud.out -# config: [ ">> /var/log/cloud-config.out", /var/log/cloud-config.err ] -# final: -# output: "| logger -p" -# error: "> /dev/null" -# this returns the specific 'mode' entry, cleanly formatted, with value -# None if if none is given -def get_output_cfg(cfg, mode="init"): - ret = [None, None] - if not 'output' in cfg: - return ret - - outcfg = cfg['output'] - if mode in outcfg: - modecfg = outcfg[mode] - else: - if 'all' not in outcfg: - return ret - # if there is a 'all' item in the output list - # then it applies to all users of this (init, config, final) - modecfg = outcfg['all'] - - # if value is a string, it specifies stdout and stderr - if isinstance(modecfg, str): - ret = [modecfg, modecfg] - - # if its a list, then we expect (stdout, stderr) - if isinstance(modecfg, list): - if len(modecfg) > 0: - ret[0] = modecfg[0] - if len(modecfg) > 1: - ret[1] = modecfg[1] - - # if it is a dictionary, expect 'out' and 'error' - # items, which indicate out and error - if isinstance(modecfg, dict): - if 'output' in modecfg: - ret[0] = modecfg['output'] - if 'error' in modecfg: - ret[1] = modecfg['error'] - - # if err's entry == "&1", then make it same as stdout - # as in shell syntax of "echo foo >/dev/null 2>&1" - if ret[1] == "&1": - ret[1] = ret[0] - - swlist = [">>", ">", "|"] - for i in range(len(ret)): - if not ret[i]: - continue - val = ret[i].lstrip() - found = False - for s in swlist: - if val.startswith(s): - val = "%s %s" % (s, val[len(s):].strip()) - found = True - break - if not found: - # default behavior is append - val = "%s %s" % (">>", val.strip()) - ret[i] = val - - return(ret) - - -# redirect_output(outfmt, errfmt, orig_out, orig_err) -# replace orig_out and orig_err with filehandles specified in outfmt or errfmt -# fmt can be: -# > FILEPATH -# >> FILEPATH -# | program [ arg1 [ arg2 [ ... ] ] ] -# -# with a '|', arguments are passed to shell, so one level of -# shell escape is required. -def redirect_output(outfmt, errfmt, o_out=sys.stdout, o_err=sys.stderr): - if outfmt: - (mode, arg) = outfmt.split(" ", 1) - if mode == ">" or mode == ">>": - owith = "ab" - if mode == ">": - owith = "wb" - new_fp = open(arg, owith) - elif mode == "|": - proc = subprocess.Popen(arg, shell=True, stdin=subprocess.PIPE) - new_fp = proc.stdin - else: - raise TypeError("invalid type for outfmt: %s" % outfmt) - - if o_out: - os.dup2(new_fp.fileno(), o_out.fileno()) - if errfmt == outfmt: - os.dup2(new_fp.fileno(), o_err.fileno()) - return - - if errfmt: - (mode, arg) = errfmt.split(" ", 1) - if mode == ">" or mode == ">>": - owith = "ab" - if mode == ">": - owith = "wb" - new_fp = open(arg, owith) - elif mode == "|": - proc = subprocess.Popen(arg, shell=True, stdin=subprocess.PIPE) - new_fp = proc.stdin - else: - raise TypeError("invalid type for outfmt: %s" % outfmt) - - if o_err: - os.dup2(new_fp.fileno(), o_err.fileno()) - return - def form_module_name(name): canon_name = name.replace("-", "_") @@ -209,13 +38,18 @@ def form_module_name(name): return canon_name -def fixup_module(mod): - freq = getattr(mod, "frequency", None) - if not freq: - setattr(mod, 'frequency', PER_INSTANCE) - handler = getattr(mod, "handle", None) - if not handler: +def fixup_module(mod, def_freq=PER_INSTANCE): + if not hasattr(mod, 'frequency'): + setattr(mod, 'frequency', def_freq) + else: + freq = mod.frequency + if freq and freq not in FREQUENCIES: + LOG.warn("Module %s has an unknown frequency %s", mod, freq) + if not hasattr(mod, 'handle'): def empty_handle(_name, _cfg, _cloud, _log, _args): pass setattr(mod, 'handle', empty_handle) + # Used only for warning if possibly running on a not checked distro... + if not hasattr(mod, 'distros'): + setattr(mod, 'distros', None) return mod diff --git a/cloudinit/transforms/cc_apt_pipelining.py b/cloudinit/transforms/cc_apt_pipelining.py index 0286a9ae..69027b0c 100644 --- a/cloudinit/transforms/cc_apt_pipelining.py +++ b/cloudinit/transforms/cc_apt_pipelining.py @@ -16,10 +16,13 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -import cloudinit.util as util -from cloudinit.CloudConfig import per_instance +from cloudinit import util +from cloudinit.settings import PER_INSTANCE + +frequency = PER_INSTANCE + +distros = ['ubuntu', 'debian'] -frequency = per_instance default_file = "/etc/apt/apt.conf.d/90cloud-init-pipelining" diff --git a/cloudinit/transforms/cc_apt_update_upgrade.py b/cloudinit/transforms/cc_apt_update_upgrade.py index a7049bce..c4a543ed 100644 --- a/cloudinit/transforms/cc_apt_update_upgrade.py +++ b/cloudinit/transforms/cc_apt_update_upgrade.py @@ -18,12 +18,13 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -import cloudinit.util as util -import subprocess -import traceback -import os import glob -import cloudinit.CloudConfig as cc +import os + +from cloudinit import templater +from cloudinit import util + +distros = ['ubuntu', 'debian'] def handle(_name, cfg, cloud, log, _args): @@ -34,13 +35,13 @@ def handle(_name, cfg, cloud, log, _args): mirror = find_apt_mirror(cloud, cfg) - log.debug("selected mirror at: %s" % mirror) + log.debug("Selected mirror at: %s" % mirror) - if not util.get_cfg_option_bool(cfg, \ - 'apt_preserve_sources_list', False): - generate_sources_list(release, mirror) - old_mir = util.get_cfg_option_str(cfg, 'apt_old_mirror', \ - "archive.ubuntu.com/ubuntu") + if not util.get_cfg_option_bool(cfg, + 'apt_preserve_sources_list', False): + generate_sources_list(release, mirror, cloud, log) + old_mir = util.get_cfg_option_str(cfg, 'apt_old_mirror', + "archive.ubuntu.com/ubuntu") rename_apt_lists(old_mir, mirror) # set up proxy @@ -49,19 +50,18 @@ def handle(_name, cfg, cloud, log, _args): if proxy: try: contents = "Acquire::HTTP::Proxy \"%s\";\n" - with open(proxy_filename, "w") as fp: - fp.write(contents % proxy) + util.write_file(proxy_filename, contents % (proxy)) except Exception as e: - log.warn("Failed to write proxy to %s" % proxy_filename) + util.logexc(log, "Failed to write proxy to %s", proxy_filename) elif os.path.isfile(proxy_filename): - os.unlink(proxy_filename) + util.del_file(proxy_filename) # process 'apt_sources' if 'apt_sources' in cfg: errors = add_sources(cfg['apt_sources'], {'MIRROR': mirror, 'RELEASE': release}) for e in errors: - log.warn("Source Error: %s\n" % ':'.join(e)) + log.warn("Source Error: %s", ':'.join(e)) dconf_sel = util.get_cfg_option_str(cfg, 'debconf_selections', False) if dconf_sel: @@ -69,41 +69,35 @@ def handle(_name, cfg, cloud, log, _args): try: util.subp(('debconf-set-selections', '-'), dconf_sel) except: - log.error("Failed to run debconf-set-selections") - log.debug(traceback.format_exc()) + util.logexc(log, "Failed to run debconf-set-selections") pkglist = util.get_cfg_option_list_or_str(cfg, 'packages', []) errors = [] if update or len(pkglist) or upgrade: try: - cc.update_package_sources() - except subprocess.CalledProcessError as e: - log.warn("apt-get update failed") - log.debug(traceback.format_exc()) + cloud.distro.update_package_sources() + except Exception as e: + util.logexc(log, "Package update failed") errors.append(e) if upgrade: try: - cc.apt_get("upgrade") - except subprocess.CalledProcessError as e: - log.warn("apt upgrade failed") - log.debug(traceback.format_exc()) + cloud.distro.package_command("upgrade") + except Exception as e: + util.logexc(log, "Package upgrade failed") errors.append(e) if len(pkglist): try: - cc.install_packages(pkglist) - except subprocess.CalledProcessError as e: - log.warn("Failed to install packages: %s " % pkglist) - log.debug(traceback.format_exc()) + cloud.distro.install_packages(pkglist) + except Exception as e: + util.logexc(log, "Failed to install packages: %s ", pkglist) errors.append(e) if len(errors): raise errors[0] - return(True) - def mirror2lists_fileprefix(mirror): string = mirror @@ -120,37 +114,40 @@ def mirror2lists_fileprefix(mirror): def rename_apt_lists(omirror, new_mirror, lists_d="/var/lib/apt/lists"): oprefix = "%s/%s" % (lists_d, mirror2lists_fileprefix(omirror)) nprefix = "%s/%s" % (lists_d, mirror2lists_fileprefix(new_mirror)) - if(oprefix == nprefix): + if oprefix == nprefix: return olen = len(oprefix) for filename in glob.glob("%s_*" % oprefix): - os.rename(filename, "%s%s" % (nprefix, filename[olen:])) + util.rename(filename, "%s%s" % (nprefix, filename[olen:])) def get_release(): - stdout, _stderr = subprocess.Popen(['lsb_release', '-cs'], - stdout=subprocess.PIPE).communicate() - return(str(stdout).strip()) + (stdout, _stderr) = util.subp(['lsb_release', '-cs']) + return stdout.strip() -def generate_sources_list(codename, mirror): - util.render_to_file('sources.list', '/etc/apt/sources.list', \ - {'mirror': mirror, 'codename': codename}) +def generate_sources_list(codename, mirror, cloud, log): + template_fn = cloud.get_template_filename('sources.list') + if template_fn: + params = {'mirror': mirror, 'codename': codename} + templater.render_to_file(template_fn, '/etc/apt/sources.list', params) + else: + log.warn("No template found, not rendering /etc/apt/sources.list") -def add_sources(srclist, searchList=None): +def add_sources(srclist, template_params=None): """ add entries in /etc/apt/sources.list.d for each abbreviated sources.list entry in 'srclist'. When rendering template, also include the values in dictionary searchList """ - if searchList is None: - searchList = {} - elst = [] + if template_params is None: + template_params = {} + errorlist = [] for ent in srclist: if 'source' not in ent: - elst.append(["", "missing source"]) + errorlist.append(["", "missing source"]) continue source = ent['source'] @@ -158,17 +155,17 @@ def add_sources(srclist, searchList=None): try: util.subp(["add-apt-repository", source]) except: - elst.append([source, "add-apt-repository failed"]) + errorlist.append([source, "add-apt-repository failed"]) continue - source = util.render_string(source, searchList) + source = templater.render_string(source, template_params) if 'filename' not in ent: ent['filename'] = 'cloud_config_sources.list' if not ent['filename'].startswith("/"): - ent['filename'] = "%s/%s" % \ - ("/etc/apt/sources.list.d/", ent['filename']) + ent['filename'] = os.path.join("/etc/apt/sources.list.d/", + ent['filename']) if ('keyid' in ent and 'key' not in ent): ks = "keyserver.ubuntu.com" @@ -177,32 +174,26 @@ def add_sources(srclist, searchList=None): try: ent['key'] = util.getkeybyid(ent['keyid'], ks) except: - elst.append([source, "failed to get key from %s" % ks]) + errorlist.append([source, "failed to get key from %s" % ks]) continue if 'key' in ent: try: util.subp(('apt-key', 'add', '-'), ent['key']) except: - elst.append([source, "failed add key"]) + errorlist.append([source, "failed add key"]) try: util.write_file(ent['filename'], source + "\n", omode="ab") except: - elst.append([source, "failed write to file %s" % ent['filename']]) + errorlist.append([source, "failed write to file %s" % ent['filename']]) - return(elst) + return errorlist def find_apt_mirror(cloud, cfg): """ find an apt_mirror given the cloud and cfg provided """ - # TODO: distro and defaults should be configurable - distro = "ubuntu" - defaults = { - 'ubuntu': "http://archive.ubuntu.com/ubuntu", - 'debian': "http://archive.debian.org/debian", - } mirror = None cfg_mirror = cfg.get("apt_mirror", None) @@ -211,14 +202,13 @@ def find_apt_mirror(cloud, cfg): elif "apt_mirror_search" in cfg: mirror = util.search_for_mirror(cfg['apt_mirror_search']) else: - if cloud: - mirror = cloud.get_mirror() + mirror = cloud.get_local_mirror() mydom = "" doms = [] - if not mirror and cloud: + if not mirror: # if we have a fqdn, then search its domain portion first (_hostname, fqdn) = util.get_hostname_fqdn(cfg, cloud) mydom = ".".join(fqdn.split(".")[1:]) @@ -236,6 +226,6 @@ def find_apt_mirror(cloud, cfg): mirror = util.search_for_mirror(mirror_list) if not mirror: - mirror = defaults[distro] + mirror = cloud.distro.get_package_mirror() return mirror diff --git a/cloudinit/transforms/cc_bootcmd.py b/cloudinit/transforms/cc_bootcmd.py index f584da02..a2efad32 100644 --- a/cloudinit/transforms/cc_bootcmd.py +++ b/cloudinit/transforms/cc_bootcmd.py @@ -17,32 +17,36 @@ # # You should have received a copy of the GNU General Public License # along with this program. If not, see . -import cloudinit.util as util -import subprocess -import tempfile + import os -from cloudinit.CloudConfig import per_always -frequency = per_always +import tempfile + +from cloudinit import util +from cloudinit.settings import PER_ALWAYS +frequency = PER_ALWAYS + + +def handle(name, cfg, cloud, log, _args): -def handle(_name, cfg, cloud, log, _args): if "bootcmd" not in cfg: + log.debug("Skipping module named %s, no 'bootcomd' key in configuration", name) return - try: - content = util.shellify(cfg["bootcmd"]) - tmpf = tempfile.TemporaryFile() - tmpf.write(content) - tmpf.seek(0) - except: - log.warn("failed to shellify bootcmd") - raise - - try: - env = os.environ.copy() - env['INSTANCE_ID'] = cloud.get_instance_id() - subprocess.check_call(['/bin/sh'], env=env, stdin=tmpf) - tmpf.close() - except: - log.warn("failed to run commands from bootcmd") - raise + with tempfile.NamedTemporaryFile(suffix=".sh") as tmpf: + try: + content = util.shellify(cfg["bootcmd"]) + tmpf.write(content) + tmpf.flush() + except: + log.warn("Failed to shellify bootcmd") + raise + + try: + env = os.environ.copy() + env['INSTANCE_ID'] = cloud.get_instance_id() + cmd = ['/bin/sh', tmpf.name] + util.subp(cmd, env=env, capture=False) + except: + log.warn("Failed to run commands from bootcmd") + raise diff --git a/cloudinit/transforms/cc_byobu.py b/cloudinit/transforms/cc_byobu.py index e821b261..38586174 100644 --- a/cloudinit/transforms/cc_byobu.py +++ b/cloudinit/transforms/cc_byobu.py @@ -18,18 +18,19 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -import cloudinit.util as util -import subprocess -import traceback +from cloudinit import util +distros = ['ubuntu', 'debian'] -def handle(_name, cfg, _cloud, log, args): + +def handle(name, cfg, _cloud, log, args): if len(args) != 0: value = args[0] else: value = util.get_cfg_option_str(cfg, "byobu_by_default", "") if not value: + log.debug("Skipping module named %s, no 'byobu' values found", name) return if value == "user" or value == "system": @@ -38,7 +39,7 @@ def handle(_name, cfg, _cloud, log, args): valid = ("enable-user", "enable-system", "enable", "disable-user", "disable-system", "disable") if not value in valid: - log.warn("Unknown value %s for byobu_by_default" % value) + log.warn("Unknown value %s for byobu_by_default", value) mod_user = value.endswith("-user") mod_sys = value.endswith("-system") @@ -65,13 +66,6 @@ def handle(_name, cfg, _cloud, log, args): cmd = ["/bin/sh", "-c", "%s %s %s" % ("X=0;", shcmd, "exit $X")] - log.debug("setting byobu to %s" % value) + log.debug("Setting byobu to %s", value) - try: - subprocess.check_call(cmd) - except subprocess.CalledProcessError as e: - log.debug(traceback.format_exc(e)) - raise Exception("Cmd returned %s: %s" % (e.returncode, cmd)) - except OSError as e: - log.debug(traceback.format_exc(e)) - raise Exception("Cmd failed to execute: %s" % (cmd)) + util.subp(cmd) diff --git a/cloudinit/transforms/cc_ca_certs.py b/cloudinit/transforms/cc_ca_certs.py index 3af6238a..8ca9a200 100644 --- a/cloudinit/transforms/cc_ca_certs.py +++ b/cloudinit/transforms/cc_ca_certs.py @@ -13,10 +13,10 @@ # # You should have received a copy of the GNU General Public License # along with this program. If not, see . + import os -from subprocess import check_call -from cloudinit.util import (write_file, get_cfg_option_list_or_str, - delete_dir_contents, subp) + +from cloudinit import util CA_CERT_PATH = "/usr/share/ca-certificates/" CA_CERT_FILENAME = "cloud-init-ca-certs.crt" @@ -28,7 +28,7 @@ def update_ca_certs(): """ Updates the CA certificate cache on the current machine. """ - check_call(["update-ca-certificates"]) + util.subp(["update-ca-certificates"]) def add_ca_certs(certs): @@ -41,9 +41,9 @@ def add_ca_certs(certs): if certs: cert_file_contents = "\n".join(certs) cert_file_fullpath = os.path.join(CA_CERT_PATH, CA_CERT_FILENAME) - write_file(cert_file_fullpath, cert_file_contents, mode=0644) + util.write_file(cert_file_fullpath, cert_file_contents, mode=0644) # Append cert filename to CA_CERT_CONFIG file. - write_file(CA_CERT_CONFIG, "\n%s" % CA_CERT_FILENAME, omode="a") + util.write_file(CA_CERT_CONFIG, "\n%s" % CA_CERT_FILENAME, omode="ab") def remove_default_ca_certs(): @@ -51,14 +51,14 @@ def remove_default_ca_certs(): Removes all default trusted CA certificates from the system. To actually apply the change you must also call L{update_ca_certs}. """ - delete_dir_contents(CA_CERT_PATH) - delete_dir_contents(CA_CERT_SYSTEM_PATH) - write_file(CA_CERT_CONFIG, "", mode=0644) + util.delete_dir_contents(CA_CERT_PATH) + util.delete_dir_contents(CA_CERT_SYSTEM_PATH) + util.write_file(CA_CERT_CONFIG, "", mode=0644) debconf_sel = "ca-certificates ca-certificates/trust_new_crts select no" - subp(('debconf-set-selections', '-'), debconf_sel) + util.subp(('debconf-set-selections', '-'), debconf_sel) -def handle(_name, cfg, _cloud, log, _args): +def handle(name, cfg, _cloud, log, _args): """ Call to handle ca-cert sections in cloud-config file. @@ -70,6 +70,7 @@ def handle(_name, cfg, _cloud, log, _args): """ # If there isn't a ca-certs section in the configuration don't do anything if "ca-certs" not in cfg: + log.debug("Skipping module named %s, no 'ca-certs' key in configuration", name) return ca_cert_cfg = cfg['ca-certs'] @@ -81,7 +82,7 @@ def handle(_name, cfg, _cloud, log, _args): # If we are given any new trusted CA certs to add, add them. if "trusted" in ca_cert_cfg: - trusted_certs = get_cfg_option_list_or_str(ca_cert_cfg, "trusted") + trusted_certs = util.get_cfg_option_list_or_str(ca_cert_cfg, "trusted") if trusted_certs: log.debug("adding %d certificates" % len(trusted_certs)) add_ca_certs(trusted_certs) diff --git a/cloudinit/transforms/cc_chef.py b/cloudinit/transforms/cc_chef.py index 941e04fe..12c2f539 100644 --- a/cloudinit/transforms/cc_chef.py +++ b/cloudinit/transforms/cc_chef.py @@ -18,53 +18,59 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -import os -import subprocess import json -import cloudinit.CloudConfig as cc -import cloudinit.util as util +import os + +from cloudinit import templater +from cloudinit import util ruby_version_default = "1.8" -def handle(_name, cfg, cloud, log, _args): +def handle(name, cfg, cloud, log, _args): + # If there isn't a chef key in the configuration don't do anything if 'chef' not in cfg: + log.debug("Skipping module named %s, no 'chef' key in configuration", name) return chef_cfg = cfg['chef'] # ensure the chef directories we use exist - mkdirs(['/etc/chef', '/var/log/chef', '/var/lib/chef', - '/var/cache/chef', '/var/backups/chef', '/var/run/chef']) + util.ensure_dirs(['/etc/chef', '/var/log/chef', '/var/lib/chef', + '/var/cache/chef', '/var/backups/chef', '/var/run/chef']) # set the validation key based on the presence of either 'validation_key' # or 'validation_cert'. In the case where both exist, 'validation_key' # takes precedence for key in ('validation_key', 'validation_cert'): if key in chef_cfg and chef_cfg[key]: - with open('/etc/chef/validation.pem', 'w') as validation_key_fh: - validation_key_fh.write(chef_cfg[key]) + util.write_file('/etc/chef/validation.pem', chef_cfg[key]) break # create the chef config from template - util.render_to_file('chef_client.rb', '/etc/chef/client.rb', - {'server_url': chef_cfg['server_url'], - 'node_name': util.get_cfg_option_str(chef_cfg, 'node_name', - cloud.datasource.get_instance_id()), - 'environment': util.get_cfg_option_str(chef_cfg, 'environment', - '_default'), - 'validation_name': chef_cfg['validation_name']}) + template_fn = cloud.get_template_filename('chef_client.rb') + if template_fn: + params = { + 'server_url': chef_cfg['server_url'], + 'node_name': util.get_cfg_option_str(chef_cfg, 'node_name', + cloud.datasource.get_instance_id()), + 'environment': util.get_cfg_option_str(chef_cfg, 'environment', + '_default'), + 'validation_name': chef_cfg['validation_name'] + } + templater.render_to_file(template_fn, '/etc/chef/client.rb', params) + else: + log.warn("No template found, not rendering to /etc/chef/client.rb") # set the firstboot json - with open('/etc/chef/firstboot.json', 'w') as firstboot_json_fh: - initial_json = {} - if 'run_list' in chef_cfg: - initial_json['run_list'] = chef_cfg['run_list'] - if 'initial_attributes' in chef_cfg: - initial_attributes = chef_cfg['initial_attributes'] - for k in initial_attributes.keys(): - initial_json[k] = initial_attributes[k] - firstboot_json_fh.write(json.dumps(initial_json)) + initial_json = {} + if 'run_list' in chef_cfg: + initial_json['run_list'] = chef_cfg['run_list'] + if 'initial_attributes' in chef_cfg: + initial_attributes = chef_cfg['initial_attributes'] + for k in initial_attributes.keys(): + initial_json[k] = initial_attributes[k] + util.write_file('/etc/chef/firstboot.json', json.dumps(initial_json)) # If chef is not installed, we install chef based on 'install_type' if not os.path.isfile('/usr/bin/chef-client'): @@ -75,14 +81,15 @@ def handle(_name, cfg, cloud, log, _args): chef_version = util.get_cfg_option_str(chef_cfg, 'version', None) ruby_version = util.get_cfg_option_str(chef_cfg, 'ruby_version', ruby_version_default) - install_chef_from_gems(ruby_version, chef_version) + install_chef_from_gems(cloud.distro, ruby_version, chef_version) # and finally, run chef-client - log.debug('running chef-client') - subprocess.check_call(['/usr/bin/chef-client', '-d', '-i', '1800', - '-s', '20']) - else: + log.debug('Running chef-client') + util.subp(['/usr/bin/chef-client', '-d', '-i', '1800', '-s', '20']) + elif install_type == 'packages': # this will install and run the chef-client from packages - cc.install_packages(('chef',)) + cloud.distro.install_packages(('chef',)) + else: + log.warn("Unknown chef install type %s", install_type) def get_ruby_packages(version): @@ -90,30 +97,20 @@ def get_ruby_packages(version): pkgs = ['ruby%s' % version, 'ruby%s-dev' % version] if version == "1.8": pkgs.extend(('libopenssl-ruby1.8', 'rubygems1.8')) - return(pkgs) + return pkgs -def install_chef_from_gems(ruby_version, chef_version=None): - cc.install_packages(get_ruby_packages(ruby_version)) +def install_chef_from_gems(ruby_version, chef_version, distro): + distro.install_packages(get_ruby_packages(ruby_version)) if not os.path.exists('/usr/bin/gem'): - os.symlink('/usr/bin/gem%s' % ruby_version, '/usr/bin/gem') + util.sym_link('/usr/bin/gem%s' % ruby_version, '/usr/bin/gem') if not os.path.exists('/usr/bin/ruby'): - os.symlink('/usr/bin/ruby%s' % ruby_version, '/usr/bin/ruby') + util.sym_link('/usr/bin/ruby%s' % ruby_version, '/usr/bin/ruby') if chef_version: - subprocess.check_call(['/usr/bin/gem', 'install', 'chef', - '-v %s' % chef_version, '--no-ri', - '--no-rdoc', '--bindir', '/usr/bin', '-q']) + util.subp(['/usr/bin/gem', 'install', 'chef', + '-v %s' % chef_version, '--no-ri', + '--no-rdoc', '--bindir', '/usr/bin', '-q']) else: - subprocess.check_call(['/usr/bin/gem', 'install', 'chef', - '--no-ri', '--no-rdoc', '--bindir', - '/usr/bin', '-q']) - - -def ensure_dir(d): - if not os.path.exists(d): - os.makedirs(d) - - -def mkdirs(dirs): - for d in dirs: - ensure_dir(d) + util.subp(['/usr/bin/gem', 'install', 'chef', + '--no-ri', '--no-rdoc', '--bindir', + '/usr/bin', '-q']) diff --git a/cloudinit/transforms/cc_disable_ec2_metadata.py b/cloudinit/transforms/cc_disable_ec2_metadata.py index 6b31ea8e..4d2a7f55 100644 --- a/cloudinit/transforms/cc_disable_ec2_metadata.py +++ b/cloudinit/transforms/cc_disable_ec2_metadata.py @@ -17,14 +17,16 @@ # # You should have received a copy of the GNU General Public License # along with this program. If not, see . -import cloudinit.util as util -import subprocess -from cloudinit.CloudConfig import per_always -frequency = per_always +from cloudinit import util + +from cloudinit.settings import PER_ALWAYS + +frequency = PER_ALWAYS + +reject_cmd = ['route', 'add', '-host', '169.254.169.254', 'reject'] def handle(_name, cfg, _cloud, _log, _args): if util.get_cfg_option_bool(cfg, "disable_ec2_metadata", False): - fwall = "route add -host 169.254.169.254 reject" - subprocess.call(fwall.split(' ')) + util.subp(reject_cmd) diff --git a/cloudinit/transforms/cc_final_message.py b/cloudinit/transforms/cc_final_message.py index abb4ca32..dc4ae34c 100644 --- a/cloudinit/transforms/cc_final_message.py +++ b/cloudinit/transforms/cc_final_message.py @@ -18,41 +18,54 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -from cloudinit.CloudConfig import per_always import sys -from cloudinit import util, boot_finished -import time -frequency = per_always +from cloudinit import templater +from cloudinit import util +from cloudinit import version -final_message = "cloud-init boot finished at $TIMESTAMP. Up $UPTIME seconds" +from cloudinit.settings import PER_ALWAYS +frequency = PER_ALWAYS -def handle(_name, cfg, _cloud, log, args): +final_message_def = ("Cloud-init v. {{version}} finished at {{timestamp}}." + " Up {{uptime}} seconds.") + + +def handle(name, cfg, cloud, log, args): + + msg_in = None if len(args) != 0: msg_in = args[0] else: - msg_in = util.get_cfg_option_str(cfg, "final_message", final_message) + msg_in = util.get_cfg_option_str(cfg, "final_message") + + if not msg_in: + template_fn = cloud.get_template_filename('final_message') + if template_fn: + msg_in = util.load_file(template_fn) - try: - uptimef = open("/proc/uptime") - uptime = uptimef.read().split(" ")[0] - uptimef.close() - except IOError as e: - log.warn("unable to open /proc/uptime\n") - uptime = "na" + if not msg_in: + msg_in = final_message_def + uptime = util.uptime() + ts = util.time_rfc2822() + cver = version.version_string() try: - ts = time.strftime("%a, %d %b %Y %H:%M:%S %z", time.gmtime()) - except: - ts = "na" - - try: - subs = {'UPTIME': uptime, 'TIMESTAMP': ts} - sys.stdout.write("%s\n" % util.render_string(msg_in, subs)) + subs = { + 'uptime': uptime, + 'timestamp': ts, + 'version': cver, + } + # Use stdout, stderr or the logger?? + content = templater.render_string(msg_in, subs) + sys.stderr.write("%s\n" % (content)) except Exception as e: - log.warn("failed to render string to stdout: %s" % e) + util.logexc(log, "Failed to render final message template") - fp = open(boot_finished, "wb") - fp.write(uptime + "\n") - fp.close() + boot_fin_fn = cloud.paths.boot_finished + try: + contents = "%s - %s - v. %s\n" % (uptime, ts, cver) + util.write_file(boot_fin_fn, contents) + except: + util.logexc(log, "Failed to write boot finished file %s", boot_fin_fn) diff --git a/cloudinit/transforms/cc_foo.py b/cloudinit/transforms/cc_foo.py index 35ec3fa7..8007f981 100644 --- a/cloudinit/transforms/cc_foo.py +++ b/cloudinit/transforms/cc_foo.py @@ -18,12 +18,35 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -#import cloudinit -#import cloudinit.util as util -from cloudinit.CloudConfig import per_instance +from cloudinit.settings import PER_INSTANCE -frequency = per_instance +# Modules are expected to have the following attributes. +# 1. A required 'handle' method which takes the following params. +# a) The name will not be this files name, but instead +# the name specified in configuration (which is the name +# which will be used to find this module). +# b) A configuration object that is the result of the merging +# of cloud configs configuration with legacy configuration +# as well as any datasource provided configuration +# c) A cloud object that can be used to access various +# datasource and paths for the given distro and data provided +# by the various datasource instance types. +# d) A argument list that may or may not be empty to this module. +# Typically those are from module configuration where the module +# is defined with some extra configuration that will eventually +# be translated from yaml into arguments to this module. +# 2. A optional 'frequency' that defines how often this module should be ran. +# Typically one of PER_INSTANCE, PER_ALWAYS, PER_ONCE. If not +# provided PER_INSTANCE will be assumed. +# See settings.py for these constants. +# 3. A optional 'distros' array/set/tuple that defines the known distros +# this module will work with (if not all of them). This is used to write +# a warning out if a module is being ran on a untested distribution for +# informational purposes. If non existent all distros are assumed and +# no warning occurs. +frequency = settings.PER_INSTANCE -def handle(_name, _cfg, _cloud, _log, _args): - print "hi" + +def handle(name, _cfg, _cloud, _log, _args): + print("Hi from %s" % (name)) diff --git a/cloudinit/transforms/cc_grub_dpkg.py b/cloudinit/transforms/cc_grub_dpkg.py index 9f3a7eaf..c048d5cc 100644 --- a/cloudinit/transforms/cc_grub_dpkg.py +++ b/cloudinit/transforms/cc_grub_dpkg.py @@ -18,10 +18,12 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -import cloudinit.util as util -import traceback import os +from cloudinit import util + +distros = ['ubuntu', 'debian'] + def handle(_name, cfg, _cloud, log, _args): idevs = None @@ -52,13 +54,14 @@ def handle(_name, cfg, _cloud, log, _args): # now idevs and idevs_empty are set to determined values # or, those set by user - dconf_sel = "grub-pc grub-pc/install_devices string %s\n" % idevs + \ - "grub-pc grub-pc/install_devices_empty boolean %s\n" % idevs_empty - log.debug("setting grub debconf-set-selections with '%s','%s'" % + dconf_sel = ("grub-pc grub-pc/install_devices string %s\n" + "grub-pc grub-pc/install_devices_empty boolean %s\n") % + (idevs, idevs_empty) + + log.debug("Setting grub debconf-set-selections with '%s','%s'" % (idevs, idevs_empty)) try: - util.subp(('debconf-set-selections'), dconf_sel) + util.subp(['debconf-set-selections'], dconf_sel) except: - log.error("Failed to run debconf-set-selections for grub-dpkg") - log.debug(traceback.format_exc()) + util.logexc(log, "Failed to run debconf-set-selections for grub-dpkg") diff --git a/cloudinit/transforms/cc_keys_to_console.py b/cloudinit/transforms/cc_keys_to_console.py index 73a477c0..2f2a5297 100644 --- a/cloudinit/transforms/cc_keys_to_console.py +++ b/cloudinit/transforms/cc_keys_to_console.py @@ -18,11 +18,10 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -from cloudinit.CloudConfig import per_instance -import cloudinit.util as util -import subprocess +from cloudinit.settings import PER_INSTANCE +from cloudinit import util -frequency = per_instance +frequency = PER_INSTANCE def handle(_name, cfg, _cloud, log, _args): @@ -32,11 +31,10 @@ def handle(_name, cfg, _cloud, log, _args): key_blacklist = util.get_cfg_option_list_or_str(cfg, "ssh_key_console_blacklist", ["ssh-dss"]) try: - confp = open('/dev/console', "wb") cmd.append(','.join(fp_blacklist)) cmd.append(','.join(key_blacklist)) - subprocess.call(cmd, stdout=confp) - confp.close() + (stdout, stderr) = util.subp(cmd) + util.write_file('/dev/console', stdout) except: - log.warn("writing keys to console value") + log.warn("Writing keys to console failed!") raise diff --git a/cloudinit/transforms/cc_landscape.py b/cloudinit/transforms/cc_landscape.py index a4113cbe..48491992 100644 --- a/cloudinit/transforms/cc_landscape.py +++ b/cloudinit/transforms/cc_landscape.py @@ -19,14 +19,24 @@ # along with this program. If not, see . import os -import os.path -from cloudinit.CloudConfig import per_instance -from configobj import ConfigObj -frequency = per_instance +from StringIO import StringIO + +try: + from configobj import ConfigObj +except ImportError: + ConfigObj = None + +from cloudinit import util + +from cloudinit.settings import PER_INSTANCE + +frequency = PER_INSTANCE lsc_client_cfg_file = "/etc/landscape/client.conf" +distros = ['ubuntu'] + # defaults taken from stock client.conf in landscape-client 11.07.1.1-0ubuntu2 lsc_builtincfg = { 'client': { @@ -38,36 +48,43 @@ lsc_builtincfg = { } -def handle(_name, cfg, _cloud, log, _args): +def handle(name, cfg, _cloud, log, _args): """ Basically turn a top level 'landscape' entry with a 'client' dict and render it to ConfigObj format under '[client]' section in /etc/landscape/client.conf """ + if not ConfigObj: + log.warn("'ConfigObj' support not enabled, running %s disabled", name) + return ls_cloudcfg = cfg.get("landscape", {}) if not isinstance(ls_cloudcfg, dict): - raise(Exception("'landscape' existed in config, but not a dict")) + raise Exception(("'landscape' key existed in config," + " but not a dictionary type," + " is a %s instead"), util.obj_name(ls_cloudcfg)) - merged = mergeTogether([lsc_builtincfg, lsc_client_cfg_file, ls_cloudcfg]) + merged = merge_together([lsc_builtincfg, lsc_client_cfg_file, ls_cloudcfg]) if not os.path.isdir(os.path.dirname(lsc_client_cfg_file)): - os.makedirs(os.path.dirname(lsc_client_cfg_file)) - - with open(lsc_client_cfg_file, "w") as fp: - merged.write(fp) + util.ensure_dir(os.path.dirname(lsc_client_cfg_file)) - log.debug("updated %s" % lsc_client_cfg_file) + contents = StringIO() + merged.write(contents) + util.write_file(lsc_client_cfg_file, contents.getvalue()) + log.debug("Wrote landscape config file to %s", lsc_client_cfg_file) -def mergeTogether(objs): +def merge_together(objs): """ merge together ConfigObj objects or things that ConfigObj() will take in later entries override earlier """ cfg = ConfigObj({}) for obj in objs: + if not obj: + continue if isinstance(obj, ConfigObj): cfg.merge(obj) else: diff --git a/cloudinit/transforms/cc_locale.py b/cloudinit/transforms/cc_locale.py index 2bb22fdb..3fb4c5d9 100644 --- a/cloudinit/transforms/cc_locale.py +++ b/cloudinit/transforms/cc_locale.py @@ -18,22 +18,28 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -import cloudinit.util as util -import os.path -import subprocess -import traceback +import os +from cloudinit import templater +from cloudinit import util -def apply_locale(locale, cfgfile): + +def apply_locale(locale, cfgfile, cloud, log): + # TODO this command might not work on RH... if os.path.exists('/usr/sbin/locale-gen'): - subprocess.Popen(['locale-gen', locale]).communicate() + util.subp(['locale-gen', locale], capture=False) if os.path.exists('/usr/sbin/update-locale'): - subprocess.Popen(['update-locale', locale]).communicate() - - util.render_to_file('default-locale', cfgfile, {'locale': locale}) + util.subp(['update-locale', locale], capture=False) + if not cfgfile: + return + template_fn = cloud.get_template_filename('default-locale') + if not template_fn: + log.warn("No template filename found to write to %s", cfgfile) + else: + templater.render_to_file(template_fn, cfgfile, {'locale': locale}) -def handle(_name, cfg, cloud, log, args): +def handle(name, cfg, cloud, log, args): if len(args) != 0: locale = args[0] else: @@ -43,12 +49,10 @@ def handle(_name, cfg, cloud, log, args): "/etc/default/locale") if not locale: + log.debug(("Skipping module named %s, " + "no 'locale' configuration found"), name) return - log.debug("setting locale to %s" % locale) + log.debug("Setting locale to %s", locale) - try: - apply_locale(locale, locale_cfgfile) - except Exception as e: - log.debug(traceback.format_exc(e)) - raise Exception("failed to apply locale %s" % locale) + apply_locale(locale, locale_cfgfile, cloud, log) diff --git a/cloudinit/transforms/cc_mcollective.py b/cloudinit/transforms/cc_mcollective.py index a2a6230c..aeeda9d2 100644 --- a/cloudinit/transforms/cc_mcollective.py +++ b/cloudinit/transforms/cc_mcollective.py @@ -19,50 +19,53 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . +from ConfigParser import ConfigParser +from StringIO import StringIO + import os -import subprocess -import StringIO -import ConfigParser -import cloudinit.CloudConfig as cc -import cloudinit.util as util + +from cloudinit import util +from cloudinit import cfg pubcert_file = "/etc/mcollective/ssl/server-public.pem" pricert_file = "/etc/mcollective/ssl/server-private.pem" -# Our fake header section -class FakeSecHead(object): - def __init__(self, fp): - self.fp = fp - self.sechead = '[nullsection]\n' - - def readline(self): - if self.sechead: - try: - return self.sechead - finally: - self.sechead = None - else: - return self.fp.readline() +def handle(name, cfg, cloud, log, _args): - -def handle(_name, cfg, _cloud, _log, _args): # If there isn't a mcollective key in the configuration don't do anything if 'mcollective' not in cfg: + log.debug(("Skipping module named %s, " + "no 'mcollective' key in configuration"), name) return + mcollective_cfg = cfg['mcollective'] + # Start by installing the mcollective package ... - cc.install_packages(("mcollective",)) + cloud.distro.install_packages(("mcollective",)) # ... and then update the mcollective configuration if 'conf' in mcollective_cfg: # Create object for reading server.cfg values - mcollective_config = ConfigParser.ConfigParser() + mcollective_config = cfg.DefaultingConfigParser() # Read server.cfg values from original file in order to be able to mix # the rest up - mcollective_config.readfp(FakeSecHead(open('/etc/mcollective/' - 'server.cfg'))) - for cfg_name, cfg in mcollective_cfg['conf'].iteritems(): + old_contents = util.load_file('/etc/mcollective/server.cfg') + # It doesn't contain any sections so just add one temporarily + # Use a hash id based off the contents, + # just incase of conflicts... (try to not have any...) + # This is so that an error won't occur when reading (and no + # sections exist in the file) + section_tpl = "[nullsection_%s]" + attempts = 0 + section_head = section_tpl % (attempts) + while old_contents.find(section_head) != -1: + attempts += 1 + section_head = section_tpl % (attempts) + sectioned_contents = "%s\n%s" % (section_head, old_contents) + mcollective_config.readfp(StringIO(sectioned_contents), + filename='/etc/mcollective/server.cfg') + for (cfg_name, cfg) in mcollective_cfg['conf'].iteritems(): if cfg_name == 'public-cert': util.write_file(pubcert_file, cfg, mode=0644) mcollective_config.set(cfg_name, @@ -76,24 +79,19 @@ def handle(_name, cfg, _cloud, _log, _args): else: # Iterate throug the config items, we'll use ConfigParser.set # to overwrite or create new items as needed - for o, v in cfg.iteritems(): + for (o, v) in cfg.iteritems(): mcollective_config.set(cfg_name, o, v) # We got all our config as wanted we'll rename # the previous server.cfg and create our new one - os.rename('/etc/mcollective/server.cfg', - '/etc/mcollective/server.cfg.old') - outputfile = StringIO.StringIO() - mcollective_config.write(outputfile) - # Now we got the whole file, write to disk except first line + util.rename('/etc/mcollective/server.cfg', + '/etc/mcollective/server.cfg.old') + # Now we got the whole file, write to disk except the section + # we added so that config parser won't error out when trying to read. # Note below, that we've just used ConfigParser because it generally - # works. Below, we remove the initial 'nullsection' header - # and then change 'key = value' to 'key: value'. The global - # search and replace of '=' with ':' could be problematic though. - # this most likely needs fixing. - util.write_file('/etc/mcollective/server.cfg', - outputfile.getvalue().replace('[nullsection]\n', '').replace(' =', - ':'), - mode=0644) + # works. Below, we remove the initial 'nullsection' header. + contents = mcollective_config.stringify() + contents = contents.replace("%s\n" % (section_head), "") + util.write_file('/etc/mcollective/server.cfg', contents, mode=0644) # Start mcollective - subprocess.check_call(['service', 'mcollective', 'start']) + util.subp(['service', 'mcollective', 'start'], capture=False) diff --git a/cloudinit/transforms/cc_mounts.py b/cloudinit/transforms/cc_mounts.py index 6cdd74e8..babcbda1 100644 --- a/cloudinit/transforms/cc_mounts.py +++ b/cloudinit/transforms/cc_mounts.py @@ -18,10 +18,17 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -import cloudinit.util as util +from string import whitespace # pylint: disable=W0402 + import os import re -from string import whitespace # pylint: disable=W0402 + +from cloudinit import util + +# shortname matches 'sda', 'sda1', 'xvda', 'hda', 'sdb', xvdb, vda, vdd1 +shortname_filter = r"^[x]{0,1}[shv]d[a-z][0-9]*$" +shortname = re.compile(shortname_filter) +ws = re.compile("[%s]+" % whitespace) def is_mdname(name): @@ -49,38 +56,46 @@ def handle(_name, cfg, cloud, log, _args): if "mounts" in cfg: cfgmnt = cfg["mounts"] - # shortname matches 'sda', 'sda1', 'xvda', 'hda', 'sdb', xvdb, vda, vdd1 - shortname_filter = r"^[x]{0,1}[shv]d[a-z][0-9]*$" - shortname = re.compile(shortname_filter) - + for i in range(len(cfgmnt)): # skip something that wasn't a list if not isinstance(cfgmnt[i], list): + log.warn("Mount option %s not a list, got a %s instead", + (i + 1), util.obj_name(cfgmnt[i])) continue + startname = str(cfgmnt[i][0]) + LOG.debug("Attempting to determine the real name of %s", startname) + # workaround, allow user to specify 'ephemeral' # rather than more ec2 correct 'ephemeral0' - if cfgmnt[i][0] == "ephemeral": + if startname == "ephemeral": cfgmnt[i][0] = "ephemeral0" + log.debug("Adjusted mount option %s name from ephemeral to ephemeral0", (i + 1)) - if is_mdname(cfgmnt[i][0]): - newname = cloud.device_name_to_device(cfgmnt[i][0]) + if is_mdname(startname): + newname = cloud.device_name_to_device(startname) if not newname: - log.debug("ignoring nonexistant named mount %s" % cfgmnt[i][0]) + log.debug("Ignoring nonexistant named mount %s", startname) cfgmnt[i][1] = None else: - if newname.startswith("/"): - cfgmnt[i][0] = newname - else: - cfgmnt[i][0] = "/dev/%s" % newname + renamed = newname + if not newname.startswith("/"): + renamed = "/dev/%s" % newname + cfgmnt[i][0] = renamed + log.debug("Mapped metadata name %s to %s", startname, renamed) else: - if shortname.match(cfgmnt[i][0]): - cfgmnt[i][0] = "/dev/%s" % cfgmnt[i][0] + if shortname.match(startname): + renamed = "/dev/%s" % startname + log.debug("Mapped shortname name %s to %s", startname, renamed) + cfgmnt[i][0] = renamed # in case the user did not quote a field (likely fs-freq, fs_passno) # but do not convert None to 'None' (LP: #898365) for j in range(len(cfgmnt[i])): - if isinstance(cfgmnt[i][j], int): + if j is None: + continue + else: cfgmnt[i][j] = str(cfgmnt[i][j]) for i in range(len(cfgmnt)): @@ -102,14 +117,18 @@ def handle(_name, cfg, cloud, log, _args): # for each of the "default" mounts, add them only if no other # entry has the same device name for defmnt in defmnts: - devname = cloud.device_name_to_device(defmnt[0]) + startname = defmnt[0] + devname = cloud.device_name_to_device(startname) if devname is None: + log.debug("Ignoring nonexistant named default mount %s", startname) continue if devname.startswith("/"): defmnt[0] = devname else: defmnt[0] = "/dev/%s" % devname + log.debug("Mapped default device %s to %s", startname, defmnt[0]) + cfgmnt_has = False for cfgm in cfgmnt: if cfgm[0] == defmnt[0]: @@ -117,14 +136,21 @@ def handle(_name, cfg, cloud, log, _args): break if cfgmnt_has: + log.debug("Not including %s, already previously included", startname) continue cfgmnt.append(defmnt) # now, each entry in the cfgmnt list has all fstab values # if the second field is None (not the string, the value) we skip it - actlist = [x for x in cfgmnt if x[1] is not None] + actlist = [] + for x in cfgmnt: + if x[1] is None: + log.debug("Skipping non-existent device named %s", x[0]) + else: + actlist.append(x) if len(actlist) == 0: + log.debug("No modifications to fstab needed.") return comment = "comment=cloudconfig" @@ -141,8 +167,7 @@ def handle(_name, cfg, cloud, log, _args): cc_lines.append('\t'.join(line)) fstab_lines = [] - fstab = open("/etc/fstab", "r+") - ws = re.compile("[%s]+" % whitespace) + fstab = util.load_file("/etc/fstab") for line in fstab.read().splitlines(): try: toks = ws.split(line) @@ -153,27 +178,22 @@ def handle(_name, cfg, cloud, log, _args): fstab_lines.append(line) fstab_lines.extend(cc_lines) - - fstab.seek(0) - fstab.write("%s\n" % '\n'.join(fstab_lines)) - fstab.truncate() - fstab.close() + contents = "%s\n" % ('\n'.join(fstab_lines)) + util.write_file("/etc/fstab", contents) if needswap: try: util.subp(("swapon", "-a")) except: - log.warn("Failed to enable swap") + util.logexc(log, "Activating swap via 'swapon -a' failed") for d in dirs: - if os.path.exists(d): - continue try: - os.makedirs(d) + util.ensure_dir(d) except: - log.warn("Failed to make '%s' config-mount\n", d) + util.logexc(log, "Failed to make '%s' config-mount", d) try: util.subp(("mount", "-a")) except: - log.warn("'mount -a' failed") + util.logexc(log, "Activating mounts via 'mount -a' failed") diff --git a/cloudinit/transforms/cc_phone_home.py b/cloudinit/transforms/cc_phone_home.py index a7ff74e1..36af6dfa 100644 --- a/cloudinit/transforms/cc_phone_home.py +++ b/cloudinit/transforms/cc_phone_home.py @@ -17,13 +17,18 @@ # # You should have received a copy of the GNU General Public License # along with this program. If not, see . -from cloudinit.CloudConfig import per_instance -import cloudinit.util as util + +from cloudinit import templater +from cloudinit import url_helper as uhelp +from cloudinit import util + +from cloudinit.settings import PER_INSTANCE + from time import sleep -frequency = per_instance -post_list_all = ['pub_key_dsa', 'pub_key_rsa', 'pub_key_ecdsa', 'instance_id', - 'hostname'] +frequency = PER_INSTANCE +post_list_all = ['pub_key_dsa', 'pub_key_rsa', 'pub_key_ecdsa', + 'instance_id', 'hostname'] # phone_home: @@ -35,7 +40,7 @@ post_list_all = ['pub_key_dsa', 'pub_key_rsa', 'pub_key_ecdsa', 'instance_id', # url: http://my.foo.bar/$INSTANCE_ID/ # post: [ pub_key_dsa, pub_key_rsa, pub_key_ecdsa, instance_id # -def handle(_name, cfg, cloud, log, args): +def handle(name, cfg, cloud, log, args): if len(args) != 0: ph_cfg = util.read_conf(args[0]) else: @@ -44,7 +49,8 @@ def handle(_name, cfg, cloud, log, args): ph_cfg = cfg['phone_home'] if 'url' not in ph_cfg: - log.warn("no 'url' token in phone_home") + log.warn(("Skipping module named %s, " + "no 'url' found in 'phone_home' configuration"), name) return url = ph_cfg['url'] @@ -53,8 +59,8 @@ def handle(_name, cfg, cloud, log, args): try: tries = int(tries) except: - log.warn("tries is not an integer. using 10") tries = 10 + util.logexc(log, "Configuration entry 'tries' is not an integer, using %s", tries) if post_list == "all": post_list = post_list_all @@ -71,11 +77,9 @@ def handle(_name, cfg, cloud, log, args): for n, path in pubkeys.iteritems(): try: - fp = open(path, "rb") - all_keys[n] = fp.read() - fp.close() + all_keys[n] = util.load_file(path) except: - log.warn("%s: failed to open in phone_home" % path) + util.logexc(log, "%s: failed to open, can not phone home that data", path) submit_keys = {} for k in post_list: @@ -83,24 +87,11 @@ def handle(_name, cfg, cloud, log, args): submit_keys[k] = all_keys[k] else: submit_keys[k] = "N/A" - log.warn("requested key %s from 'post' list not available") + log.warn("Requested key %s from 'post' configuration list not available", k) - url = util.render_string(url, {'INSTANCE_ID': all_keys['instance_id']}) + url = templater.render_string(url, {'INSTANCE_ID': all_keys['instance_id']}) - null_exc = object() - last_e = null_exc - for i in range(0, tries): - try: - util.readurl(url, submit_keys) - log.debug("succeeded submit to %s on try %i" % (url, i + 1)) - return - except Exception as e: - log.debug("failed to post to %s on try %i" % (url, i + 1)) - last_e = e - sleep(3) - - log.warn("failed to post to %s in %i tries" % (url, tries)) - if last_e is not null_exc: - raise(last_e) - - return + try: + uhelp.readurl(url, data=submit_keys, retries=tries, sec_between=3) + except: + util.logexc(log, "Failed to post phone home data to %s in %s tries", url, tries) diff --git a/cloudinit/transforms/cc_puppet.py b/cloudinit/transforms/cc_puppet.py index 6fc475f6..0a21a929 100644 --- a/cloudinit/transforms/cc_puppet.py +++ b/cloudinit/transforms/cc_puppet.py @@ -18,91 +18,85 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . +from StringIO import StringIO + import os -import os.path import pwd import socket -import subprocess -import StringIO -import ConfigParser -import cloudinit.CloudConfig as cc -import cloudinit.util as util + +from cloudinit import util +from cloudinit import cfg -def handle(_name, cfg, cloud, log, _args): +def handle(name, cfg, cloud, log, _args): # If there isn't a puppet key in the configuration don't do anything if 'puppet' not in cfg: + log.debug(("Skipping module named %s," + " no 'puppet' configuration found"), name) return + puppet_cfg = cfg['puppet'] + # Start by installing the puppet package ... - cc.install_packages(("puppet",)) + cloud.distro.install_packages(("puppet",)) # ... and then update the puppet configuration if 'conf' in puppet_cfg: # Add all sections from the conf object to puppet.conf - puppet_conf_fh = open('/etc/puppet/puppet.conf', 'r') + contents = util.load_file('/etc/puppet/puppet.conf') # Create object for reading puppet.conf values - puppet_config = ConfigParser.ConfigParser() + puppet_config = cfg.DefaultingConfigParser() # Read puppet.conf values from original file in order to be able to - # mix the rest up - puppet_config.readfp(StringIO.StringIO(''.join(i.lstrip() for i in - puppet_conf_fh.readlines()))) - # Close original file, no longer needed - puppet_conf_fh.close() - for cfg_name, cfg in puppet_cfg['conf'].iteritems(): + # mix the rest up. First clean them up (TODO is this really needed??) + cleaned_contents = '\n'.join([i.lstrip() for i in contents.splitlines()]) + puppet_config.readfp(StringIO(cleaned_contents), + filename='/etc/puppet/puppet.conf') + for (cfg_name, cfg) in puppet_cfg['conf'].iteritems(): # ca_cert configuration is a special case # Dump the puppetmaster ca certificate in the correct place if cfg_name == 'ca_cert': # Puppet ssl sub-directory isn't created yet # Create it with the proper permissions and ownership - os.makedirs('/var/lib/puppet/ssl') - os.chmod('/var/lib/puppet/ssl', 0771) - os.chown('/var/lib/puppet/ssl', - pwd.getpwnam('puppet').pw_uid, 0) - os.makedirs('/var/lib/puppet/ssl/certs/') - os.chown('/var/lib/puppet/ssl/certs/', - pwd.getpwnam('puppet').pw_uid, 0) - ca_fh = open('/var/lib/puppet/ssl/certs/ca.pem', 'w') - ca_fh.write(cfg) - ca_fh.close() - os.chown('/var/lib/puppet/ssl/certs/ca.pem', - pwd.getpwnam('puppet').pw_uid, 0) - util.restorecon_if_possible('/var/lib/puppet', recursive=True) + util.ensure_dir('/var/lib/puppet/ssl', 0771) + util.chownbyid('/var/lib/puppet/ssl', + pwd.getpwnam('puppet').pw_uid, 0) + util.ensure_dir('/var/lib/puppet/ssl/certs/') + util.chownbyid('/var/lib/puppet/ssl/certs/', + pwd.getpwnam('puppet').pw_uid, 0) + util.write_file('/var/lib/puppet/ssl/certs/ca.pem', cfg) + util.chownbyid('/var/lib/puppet/ssl/certs/ca.pem', + pwd.getpwnam('puppet').pw_uid, 0) else: - #puppet_conf_fh.write("\n[%s]\n" % (cfg_name)) - # If puppet.conf already has this section we don't want to - # write it again - if puppet_config.has_section(cfg_name) == False: - puppet_config.add_section(cfg_name) # Iterate throug the config items, we'll use ConfigParser.set # to overwrite or create new items as needed - for o, v in cfg.iteritems(): + for (o, v) in cfg.iteritems(): if o == 'certname': # Expand %f as the fqdn + # TODO should this use the cloud fqdn?? v = v.replace("%f", socket.getfqdn()) # Expand %i as the instance id - v = v.replace("%i", - cloud.datasource.get_instance_id()) - # certname needs to be downcase + v = v.replace("%i", cloud.get_instance_id()) + # certname needs to be downcased v = v.lower() puppet_config.set(cfg_name, o, v) - #puppet_conf_fh.write("%s=%s\n" % (o, v)) # We got all our config as wanted we'll rename # the previous puppet.conf and create our new one - os.rename('/etc/puppet/puppet.conf', '/etc/puppet/puppet.conf.old') - with open('/etc/puppet/puppet.conf', 'wb') as configfile: - puppet_config.write(configfile) - util.restorecon_if_possible('/etc/puppet/puppet.conf') + util.rename('/etc/puppet/puppet.conf', '/etc/puppet/puppet.conf.old') + contents = puppet_config.stringify() + util.write_file('/etc/puppet/puppet.conf', contents) + # Set puppet to automatically start if os.path.exists('/etc/default/puppet'): - subprocess.check_call(['sed', '-i', - '-e', 's/^START=.*/START=yes/', - '/etc/default/puppet']) + util.subp(['sed', '-i', + '-e', 's/^START=.*/START=yes/', + '/etc/default/puppet'], capture=False) elif os.path.exists('/bin/systemctl'): - subprocess.check_call(['/bin/systemctl', 'enable', 'puppet.service']) + util.subp(['/bin/systemctl', 'enable', 'puppet.service'], capture=False) elif os.path.exists('/sbin/chkconfig'): - subprocess.check_call(['/sbin/chkconfig', 'puppet', 'on']) + util.subp(['/sbin/chkconfig', 'puppet', 'on'], capture=False) else: - log.warn("Do not know how to enable puppet service on this system") + log.warn(("Sorry we do not know how to enable" + " puppet services on this system")) + # Start puppetd - subprocess.check_call(['service', 'puppet', 'start']) + util.subp(['service', 'puppet', 'start'], capture=False) diff --git a/cloudinit/transforms/cc_resizefs.py b/cloudinit/transforms/cc_resizefs.py index 2dc66def..daaf4da9 100644 --- a/cloudinit/transforms/cc_resizefs.py +++ b/cloudinit/transforms/cc_resizefs.py @@ -18,91 +18,117 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -import cloudinit.util as util -import subprocess +import errno import os import stat import sys -import time import tempfile -from cloudinit.CloudConfig import per_always - -frequency = per_always +import time +from cloudinit import util +from cloudinit.settings import PER_ALWAYS -def handle(_name, cfg, _cloud, log, args): - if len(args) != 0: - resize_root = False - if str(args[0]).lower() in ['true', '1', 'on', 'yes']: - resize_root = True - else: - resize_root = util.get_cfg_option_str(cfg, "resize_rootfs", True) +frequency = PER_ALWAYS - if str(resize_root).lower() in ['false', '0']: - return +resize_fs_prefixes_cmds = [ + ('ext', 'resize2fs'), + ('xfs', 'xfs_growfs'), +] - # we use mktemp rather than mkstemp because early in boot nothing - # else should be able to race us for this, and we need to mknod. - devpth = tempfile.mktemp(prefix="cloudinit.resizefs.", dir="/run") +def nodeify_path(devpth, where, log): try: - st_dev = os.stat("/").st_dev + st_dev = os.stat(where).st_dev dev = os.makedev(os.major(st_dev), os.minor(st_dev)) os.mknod(devpth, 0400 | stat.S_IFBLK, dev) + return st_dev except: if util.is_container(): - log.debug("inside container, ignoring mknod failure in resizefs") + log.debug("Inside container, ignoring mknod failure in resizefs") return - log.warn("Failed to make device node to resize /") + log.warn("Failed to make device node to resize %s at %s", where, devpth) raise - cmd = ['blkid', '-c', '/dev/null', '-sTYPE', '-ovalue', devpth] + +def get_fs_type(st_dev, path, log): try: - (fstype, _err) = util.subp(cmd) - except subprocess.CalledProcessError as e: - log.warn("Failed to get filesystem type of maj=%s, min=%s via: %s" % - (os.major(st_dev), os.minor(st_dev), cmd)) - log.warn("output=%s\nerror=%s\n", e.output[0], e.output[1]) - os.unlink(devpth) + fs_type = util.find_devs_with(tag='TYPE', oformat='value', + no_cache=True, path=path) + return fs_type + except util.ProcessExecutionError: + util.logexc(log, ("Failed to get filesystem type" + " of maj=%s, min=%s for path %s"), + os.major(st_dev), os.minor(st_dev), path) raise - if str(fstype).startswith("ext"): - resize_cmd = ['resize2fs', devpth] - elif fstype == "xfs": - resize_cmd = ['xfs_growfs', devpth] + +def handle(name, cfg, _cloud, log, args): + if len(args) != 0: + resize_root = args[0] else: - os.unlink(devpth) - log.debug("not resizing unknown filesystem %s" % fstype) + resize_root = util.get_cfg_option_str(cfg, "resize_rootfs", True) + + if not util.translate_bool(resize_root): + log.debug("Skipping module named %s, resizing disabled", name) return - if resize_root == "noblock": - fid = os.fork() - if fid == 0: - try: - do_resize(resize_cmd, devpth, log) - os._exit(0) # pylint: disable=W0212 - except Exception as exc: - sys.stderr.write("Failed: %s" % exc) - os._exit(1) # pylint: disable=W0212 - else: - do_resize(resize_cmd, devpth, log) + # TODO is the directory ok to be used?? + resize_root_d = util.get_cfg_option_str(cfg, "resize_rootfs_tmp", "/run") + util.ensure_dir(resize_root_d) + with util.SilentTemporaryFile(prefix="cloudinit.resizefs.", + dir=resize_root_d, delete=True) as tfh: + devpth = tfh.name + + # Delete the file so that mknod will work + # but don't change the file handle to know that its + # removed so that when a later call that recreates + # occurs this temporary file will still benefit from + # auto deletion + tfh.unlink_now() + + # TODO: allow what is to be resized to + # be configurable?? + st_dev = nodeify_path(devpth, "/", log) + fs_type = get_fs_type(st_dev, devpath, log) + + resizer = None + fstype_lc = fstype.lower() + for (pfix, root_cmd) in resize_fs_prefixes_cmds: + if fstype_lc.startswith(pfix): + resizer = root_cmd + break - log.debug("resizing root filesystem (type=%s, maj=%i, min=%i, val=%s)" % - (str(fstype).rstrip("\n"), os.major(st_dev), os.minor(st_dev), - resize_root)) + if not resizer: + log.warn("Not resizing unknown filesystem type %s", fs_type) + return + + log.debug("Resizing using %s", resizer) + resize_cmd = [resizer, devpth] - return + if resize_root == "noblock": + # Fork to a child that will run + # the resize command + util.fork_cb(do_resize, resize_cmd, log) + # Don't delete the file now in the parent + tfh.delete = False + else: + do_resize(resize_cmd, log) + action = 'Resized' + if resize_root == "noblock": + action = 'Resizing (via forking)' + log.debug("%s root filesystem (type=%s, maj=%i, min=%i, val=%s)", + action, fs_type, os.major(st_dev), os.minor(st_dev), resize_root) -def do_resize(resize_cmd, devpth, log): + +def do_resize(resize_cmd, log): + start = time.time() try: - start = time.time() util.subp(resize_cmd) - except subprocess.CalledProcessError as e: - log.warn("Failed to resize filesystem (%s)" % resize_cmd) - log.warn("output=%s\nerror=%s\n", e.output[0], e.output[1]) - os.unlink(devpth) + except util.ProcessExecutionError as e: + util.logexc(log, "Failed to resize filesystem (using %s)", resize_cmd) raise - - os.unlink(devpth) - log.debug("resize took %s seconds" % (time.time() - start)) + tot_time = int(time.time() - start) + log.debug("Resizing took %s seconds", tot_time) + # TODO: Should we add a fsck check after this to make + # sure we didn't corrupt anything? diff --git a/cloudinit/transforms/cc_rightscale_userdata.py b/cloudinit/transforms/cc_rightscale_userdata.py index 5ed0848f..cde11b54 100644 --- a/cloudinit/transforms/cc_rightscale_userdata.py +++ b/cloudinit/transforms/cc_rightscale_userdata.py @@ -35,44 +35,64 @@ ## ## -import cloudinit.util as util -from cloudinit.CloudConfig import per_instance -from cloudinit import get_ipath_cur +import os + +from cloudinit import url_helper as uhelp +from cloudinit import util +from cloudinit.settings import PER_INSTANCE + from urlparse import parse_qs -frequency = per_instance +frequency = PER_INSTANCE + my_name = "cc_rightscale_userdata" my_hookname = 'CLOUD_INIT_REMOTE_HOOK' -def handle(_name, _cfg, cloud, log, _args): +def handle(name, _cfg, cloud, log, _args): try: ud = cloud.get_userdata_raw() except: - log.warn("failed to get raw userdata in %s" % my_name) + log.warn("Failed to get raw userdata in module %s", name) return try: mdict = parse_qs(ud) - if not my_hookname in mdict: + if not mdict or not my_hookname in mdict: + log.debug("Skipping module %s, did not find %s in parsed raw userdata", name, my_hookname) return except: - log.warn("failed to urlparse.parse_qa(userdata_raw())") + log.warn("Failed to parse query string %s into a dictionary", ud) raise - scripts_d = get_ipath_cur('scripts') - i = 0 - first_e = None - for url in mdict[my_hookname]: - fname = "%s/rightscale-%02i" % (scripts_d, i) - i = i + 1 + wrote_fns = [] + captured_excps = [] + + # These will eventually be then ran by the cc_scripts_user + # TODO: maybe this should just be a new user data handler?? + # Instead of a late transform that acts like a user data handler? + scripts_d = cloud.get_ipath_cur('scripts') + urls = mdict[my_hookname] + for (i, url) in enumerate(urls): + fname = os.path.join(scripts_d, "rightscale-%02i" % (i)) try: - content = util.readurl(url) - util.write_file(fname, content, mode=0700) + (content, st) = uhelp.readurl(url) + # Ensure its a valid http response (and something gotten) + if uhelp.ok_http_code(st) and content: + util.write_file(fname, content, mode=0700) + wrote_fns.append(fname) except Exception as e: - if not first_e: - first_e = None - log.warn("%s failed to read %s: %s" % (my_name, url, e)) + captured_excps.append(e) + util.logexc(log, "%s failed to read %s and write %s", my_name, url, fname) + + if wrote_fns: + log.debug("Wrote out rightscale userdata to %s files", len(wrote_fns)) + + if len(wrote_fns) != len(urls): + skipped = len(urls) - len(wrote_fns) + log.debug("%s urls were skipped or failed", skipped) + + if captured_excps: + log.warn("%s failed with exceptions, re-raising the last one", len(captured_excps)) + raise captured_excps[-1] - if first_e: - raise(e) diff --git a/cloudinit/transforms/cc_rsyslog.py b/cloudinit/transforms/cc_rsyslog.py index ac7f2c74..ccbe68ff 100644 --- a/cloudinit/transforms/cc_rsyslog.py +++ b/cloudinit/transforms/cc_rsyslog.py @@ -18,16 +18,15 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -import cloudinit -import logging -import cloudinit.util as util -import traceback +import os + +from cloudinit import util DEF_FILENAME = "20-cloud-config.conf" DEF_DIR = "/etc/rsyslog.d" -def handle(_name, cfg, _cloud, log, _args): +def handle(name, cfg, cloud, log, _args): # rsyslog: # - "*.* @@192.158.1.1" # - content: "*.* @@192.0.2.1:10514" @@ -37,17 +36,17 @@ def handle(_name, cfg, _cloud, log, _args): # process 'rsyslog' if not 'rsyslog' in cfg: + log.debug("Skipping module named %s, no 'rsyslog' key in configuration", name) return def_dir = cfg.get('rsyslog_dir', DEF_DIR) def_fname = cfg.get('rsyslog_filename', DEF_FILENAME) files = [] - elst = [] - for ent in cfg['rsyslog']: + for i, ent in enumerate(cfg['rsyslog']): if isinstance(ent, dict): if not "content" in ent: - elst.append((ent, "no 'content' entry")) + log.warn("No 'content' entry in config entry %s", i + 1) continue content = ent['content'] filename = ent.get("filename", def_fname) @@ -55,8 +54,13 @@ def handle(_name, cfg, _cloud, log, _args): content = ent filename = def_fname + filename = filename.strip() + if not filename: + log.warn("Entry %s has an empty filename", i + 1) + continue + if not filename.startswith("/"): - filename = "%s/%s" % (def_dir, filename) + filename = os.path.join(def_dir, filename) omode = "ab" # truncate filename first time you see it @@ -67,35 +71,29 @@ def handle(_name, cfg, _cloud, log, _args): try: util.write_file(filename, content + "\n", omode=omode) except Exception as e: - log.debug(traceback.format_exc(e)) - elst.append((content, "failed to write to %s" % filename)) + util.logexc(log, "Failed to write to %s", filename) - # need to restart syslogd + # Attempt to restart syslogd restarted = False try: - # if this config module is running at cloud-init time + # If this config module is running at cloud-init time # (before rsyslog is running) we don't actually have to # restart syslog. # - # upstart actually does what we want here, in that it doesn't + # Upstart actually does what we want here, in that it doesn't # start a service that wasn't running already on 'restart' # it will also return failure on the attempt, so 'restarted' - # won't get set - log.debug("restarting rsyslog") + # won't get set. + log.debug("Restarting rsyslog") util.subp(['service', 'rsyslog', 'restart']) restarted = True - except Exception as e: - elst.append(("restart", str(e))) + util.logexc("Failed restarting rsyslog") if restarted: - # this only needs to run if we *actually* restarted + # This only needs to run if we *actually* restarted # syslog above. - cloudinit.logging_set_from_cfg_file() - log = logging.getLogger() - log.debug("rsyslog configured %s" % files) - - for e in elst: - log.warn("rsyslog error: %s\n" % ':'.join(e)) - - return + cloud.cycle_logging() + # This should now use rsyslog if + # the logging was setup to use it... + log.debug("%s configured %s files", name, files) diff --git a/cloudinit/transforms/cc_runcmd.py b/cloudinit/transforms/cc_runcmd.py index f7e8c671..19c0e721 100644 --- a/cloudinit/transforms/cc_runcmd.py +++ b/cloudinit/transforms/cc_runcmd.py @@ -18,15 +18,20 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -import cloudinit.util as util +import os +from cloudinit import util -def handle(_name, cfg, cloud, log, _args): + +def handle(name, cfg, cloud, log, _args): if "runcmd" not in cfg: + log.debug("Skipping module named %s, no 'runcmd' key in configuration", name) return - outfile = "%s/runcmd" % cloud.get_ipath('scripts') + + outfile = os.path.join(cloud.get_ipath('scripts'), "runcmd") + cmd = cfg["runcmd"] try: - content = util.shellify(cfg["runcmd"]) + content = util.shellify(cmd) util.write_file(outfile, content, 0700) except: - log.warn("failed to open %s for runcmd" % outfile) + util.logexc(log, "Failed to shellify %s into file %s", cmd, outfile) diff --git a/cloudinit/transforms/cc_salt_minion.py b/cloudinit/transforms/cc_salt_minion.py index 1a3b5039..47cbc194 100644 --- a/cloudinit/transforms/cc_salt_minion.py +++ b/cloudinit/transforms/cc_salt_minion.py @@ -15,42 +15,43 @@ # along with this program. If not, see . import os -import os.path -import subprocess -import cloudinit.CloudConfig as cc -import yaml +from cloudinit import util -def handle(_name, cfg, _cloud, _log, _args): +# Note: see http://saltstack.org/topics/installation/ + + +def handle(name, cfg, cloud, _log, _args): # If there isn't a salt key in the configuration don't do anything if 'salt_minion' not in cfg: + log.debug("Skipping module named %s, no 'salt_minion' key in configuration", name) return + salt_cfg = cfg['salt_minion'] + # Start by installing the salt package ... - cc.install_packages(("salt",)) - config_dir = '/etc/salt' - if not os.path.isdir(config_dir): - os.makedirs(config_dir) + cloud.distro.install_packages(("salt",)) + + # Ensure we can configure files at the right dir + config_dir = salt_cfg.get("config_dir", '/etc/salt') + util.ensure_dir(config_dir) + # ... and then update the salt configuration if 'conf' in salt_cfg: # Add all sections from the conf object to /etc/salt/minion minion_config = os.path.join(config_dir, 'minion') - yaml.dump(salt_cfg['conf'], - file(minion_config, 'w'), - default_flow_style=False) + minion_data = util.yaml_dumps(salt_cfg.get('conf')) + util.write_file(minion_config, minion_data) + # ... copy the key pair if specified if 'public_key' in salt_cfg and 'private_key' in salt_cfg: - pki_dir = '/etc/salt/pki' - cumask = os.umask(077) - if not os.path.isdir(pki_dir): - os.makedirs(pki_dir) - pub_name = os.path.join(pki_dir, 'minion.pub') - pem_name = os.path.join(pki_dir, 'minion.pem') - with open(pub_name, 'w') as f: - f.write(salt_cfg['public_key']) - with open(pem_name, 'w') as f: - f.write(salt_cfg['private_key']) - os.umask(cumask) + pki_dir = salt_cfg.get('pki_dir', '/etc/salt/pki') + with util.umask(077): + util.ensure_dir(pki_dir) + pub_name = os.path.join(pki_dir, 'minion.pub') + pem_name = os.path.join(pki_dir, 'minion.pem') + util.write_file(pub_name, salt_cfg['public_key']) + util.write_file(pem_name, salt_cfg['private_key']) # Start salt-minion - subprocess.check_call(['service', 'salt-minion', 'start']) + util.subp(['service', 'salt-minion', 'start'], capture=False) diff --git a/cloudinit/transforms/cc_scripts_per_boot.py b/cloudinit/transforms/cc_scripts_per_boot.py index 41a74754..bcdf4400 100644 --- a/cloudinit/transforms/cc_scripts_per_boot.py +++ b/cloudinit/transforms/cc_scripts_per_boot.py @@ -18,17 +18,23 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -import cloudinit.util as util -from cloudinit.CloudConfig import per_always -from cloudinit import get_cpath +import os -frequency = per_always -runparts_path = "%s/%s" % (get_cpath(), "scripts/per-boot") +from cloudinit import util +from cloudinit.settings import PER_ALWAYS -def handle(_name, _cfg, _cloud, log, _args): +frequency = PER_ALWAYS + +script_subdir = 'per-boot' + + +def handle(_name, _cfg, cloud, log, _args): + # Comes from the following: + # https://forums.aws.amazon.com/thread.jspa?threadID=96918 + runparts_path = os.path.join(cloud.get_cpath(), 'scripts', script_subdir) try: util.runparts(runparts_path) except: - log.warn("failed to run-parts in %s" % runparts_path) + log.warn("Failed to run-parts(%s) in %s", script_subdir, runparts_path) raise diff --git a/cloudinit/transforms/cc_scripts_per_instance.py b/cloudinit/transforms/cc_scripts_per_instance.py index a2981eab..8d6609a1 100644 --- a/cloudinit/transforms/cc_scripts_per_instance.py +++ b/cloudinit/transforms/cc_scripts_per_instance.py @@ -18,17 +18,23 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -import cloudinit.util as util -from cloudinit.CloudConfig import per_instance -from cloudinit import get_cpath +import os -frequency = per_instance -runparts_path = "%s/%s" % (get_cpath(), "scripts/per-instance") +from cloudinit import util +from cloudinit.settings import PER_INSTANCE -def handle(_name, _cfg, _cloud, log, _args): +frequency = PER_INSTANCE + +script_subdir = 'per-instance' + + +def handle(_name, _cfg, cloud, log, _args): + # Comes from the following: + # https://forums.aws.amazon.com/thread.jspa?threadID=96918 + runparts_path = os.path.join(cloud.get_cpath(), 'scripts', script_subdir) try: util.runparts(runparts_path) except: - log.warn("failed to run-parts in %s" % runparts_path) + log.warn("Failed to run-parts(%s) in %s", script_subdir, runparts_path) raise diff --git a/cloudinit/transforms/cc_scripts_per_once.py b/cloudinit/transforms/cc_scripts_per_once.py index a69151da..dbcec05d 100644 --- a/cloudinit/transforms/cc_scripts_per_once.py +++ b/cloudinit/transforms/cc_scripts_per_once.py @@ -18,17 +18,23 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -import cloudinit.util as util -from cloudinit.CloudConfig import per_once -from cloudinit import get_cpath +import os -frequency = per_once -runparts_path = "%s/%s" % (get_cpath(), "scripts/per-once") +from cloudinit import util +from cloudinit.settings import PER_ONCE -def handle(_name, _cfg, _cloud, log, _args): +frequency = PER_ONCE + +script_subdir = 'per-once' + + +def handle(_name, _cfg, cloud, log, _args): + # Comes from the following: + # https://forums.aws.amazon.com/thread.jspa?threadID=96918 + runparts_path = os.path.join(cloud.get_cpath(), 'scripts', script_subdir) try: util.runparts(runparts_path) except: - log.warn("failed to run-parts in %s" % runparts_path) + log.warn("Failed to run-parts(%s) in %s", script_subdir, runparts_path) raise diff --git a/cloudinit/transforms/cc_scripts_user.py b/cloudinit/transforms/cc_scripts_user.py index 933aa4e0..1e438ee6 100644 --- a/cloudinit/transforms/cc_scripts_user.py +++ b/cloudinit/transforms/cc_scripts_user.py @@ -18,17 +18,22 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -import cloudinit.util as util -from cloudinit.CloudConfig import per_instance -from cloudinit import get_ipath_cur +import os -frequency = per_instance -runparts_path = "%s/%s" % (get_ipath_cur(), "scripts") +from cloudinit import util +from cloudinit.settings import PER_INSTANCE -def handle(_name, _cfg, _cloud, log, _args): +frequency = PER_INSTANCE + + +def handle(_name, _cfg, cloud, log, _args): + # This is written to by the user data handlers + # Ie, any custom shell scripts that come down + # go here... + runparts_path = os.path.join(cloud.get_ipath_cur(), "scripts") try: util.runparts(runparts_path) except: - log.warn("failed to run-parts in %s" % runparts_path) + log.warn("Failed to run-parts(%s) in %s", "user-data", runparts_path) raise diff --git a/cloudinit/transforms/cc_set_hostname.py b/cloudinit/transforms/cc_set_hostname.py index acea74d9..fa2b59c2 100644 --- a/cloudinit/transforms/cc_set_hostname.py +++ b/cloudinit/transforms/cc_set_hostname.py @@ -18,25 +18,18 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -import cloudinit.util as util +from cloudinit import util -def handle(_name, cfg, cloud, log, _args): +def handle(name, cfg, cloud, log, _args): if util.get_cfg_option_bool(cfg, "preserve_hostname", False): - log.debug("preserve_hostname is set. not setting hostname") - return(True) + log.debug(("Configuration option 'preserve_hostname' is set," + " not setting the hostname in %s"), name) + return (hostname, _fqdn) = util.get_hostname_fqdn(cfg, cloud) try: - set_hostname(hostname, log) + log.debug("Setting hostname to %s", hostname) + cloud.distro.set_hostname(hostname) except Exception: - util.logexc(log) - log.warn("failed to set hostname to %s\n", hostname) - - return(True) - - -def set_hostname(hostname, log): - util.subp(['hostname', hostname]) - util.write_file("/etc/hostname", "%s\n" % hostname, 0644) - log.debug("populated /etc/hostname with %s on first boot", hostname) + util.logexc(log, "Failed to set hostname to %s", hostname) diff --git a/cloudinit/transforms/cc_set_passwords.py b/cloudinit/transforms/cc_set_passwords.py index 9d0bbdb8..4f2cdb97 100644 --- a/cloudinit/transforms/cc_set_passwords.py +++ b/cloudinit/transforms/cc_set_passwords.py @@ -18,13 +18,18 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -import cloudinit.util as util import sys -import random -from string import letters, digits # pylint: disable=W0402 +from cloudinit import util -def handle(_name, cfg, _cloud, log, args): +from string import letters, digits + +# We are removing certain 'painful' letters/numbers +pw_set = (letters.translate(None, 'loLOI') + + digits.translate(None, '01')) + + +def handle(_name, cfg, cloud, log, args): if len(args) != 0: # if run from command line, and give args, wipe the chpasswd['list'] password = args[0] @@ -62,68 +67,83 @@ def handle(_name, cfg, _cloud, log, args): ch_in = '\n'.join(plist_in) try: + log.debug("Changing password for %s:", users) util.subp(['chpasswd'], ch_in) - log.debug("changed password for %s:" % users) except Exception as e: errors.append(e) - log.warn("failed to set passwords with chpasswd: %s" % e) + util.logexc(log, "Failed to set passwords with chpasswd for %s", users) if len(randlist): - sys.stdout.write("%s\n%s\n" % ("Set the following passwords\n", + sys.stderr.write("%s\n%s\n" % ("Set the following 'random' passwords\n", '\n'.join(randlist))) if expire: - enum = len(errors) + expired_users = [] for u in users: try: util.subp(['passwd', '--expire', u]) + expired_users.append(u) except Exception as e: errors.append(e) - log.warn("failed to expire account for %s" % u) - if enum == len(errors): - log.debug("expired passwords for: %s" % u) + util.logexc(log, "Failed to set 'expire' for %s", u) + if expired_users: + log.debug("Expired passwords for: %s users", expired_users) + change_pwauth = False + pw_auth = None if 'ssh_pwauth' in cfg: - val = str(cfg['ssh_pwauth']).lower() - if val in ("true", "1", "yes"): - pw_auth = "yes" - change_pwauth = True - elif val in ("false", "0", "no"): - pw_auth = "no" - change_pwauth = True - else: - change_pwauth = False + change_pwauth = True + if util.is_true_str(cfg['ssh_pwauth']): + pw_auth = 'yes' + if util.is_false_str(cfg['ssh_pwauth']): + pw_auth = 'no' if change_pwauth: - pa_s = "\(#*\)\(PasswordAuthentication[[:space:]]\+\)\(yes\|no\)" - msg = "set PasswordAuthentication to '%s'" % pw_auth - try: - cmd = ['sed', '-i', 's,%s,\\2%s,' % (pa_s, pw_auth), - '/etc/ssh/sshd_config'] - util.subp(cmd) - log.debug(msg) - except Exception as e: - log.warn("failed %s" % msg) - errors.append(e) + new_lines = [] + replaced_auth = False + replacement = "PasswordAuthentication %s" % (pw_auth) + + # See http://linux.die.net/man/5/sshd_config + old_lines = util.load_file('/etc/ssh/sshd_config').splitlines() + for i, line in enumerate(old_lines): + if not line.strip() or line.startswith("#"): + new_lines.append(line) + continue + splitup = line.split(None, 1) + if len(splitup) <= 1: + new_lines.append(line) + continue + (cmd, args) = splitup + # Keywords are case-insensitive and arguments are case-sensitive + cmd = cmd.lower().strip() + if cmd == 'passwordauthentication': + log.debug("Replacing auth line %s with %s", i + 1, replacement) + replaced_auth = True + new_lines.append(replacement) + else: + new_lines.append(line) + + if not replaced_auth: + log.debug("Adding new auth line %s", replacement) + replaced_auth = True + new_lines.append(replacement) + + new_contents = "\n".join(new_lines) + util.write_file('/etc/ssh/sshd_config', new_contents) try: - p = util.subp(['service', cfg.get('ssh_svcname', 'ssh'), - 'restart']) - log.debug("restarted sshd") + cmd = ['service'] + cmd.append(cloud.distro.get_option('ssh_svcname', 'ssh')) + cmd.append('restart') + util.subp(cmd) + log.debug("Restarted the ssh daemon") except: - log.warn("restart of ssh failed") + util.logexc(log, "Restarting of the ssh daemon failed") if len(errors): - raise(errors[0]) - - return - - -def rand_str(strlen=32, select_from=letters + digits): - return("".join([random.choice(select_from) for _x in range(0, strlen)])) + log.debug("%s errors occured, re-raising the last one", len(errors)) + raise errors[-1] def rand_user_password(pwlen=9): - selfrom = (letters.translate(None, 'loLOI') + - digits.translate(None, '01')) - return(rand_str(pwlen, select_from=selfrom)) + return util.rand_str(pwlen, select_from=pw_set) diff --git a/cloudinit/transforms/cc_ssh.py b/cloudinit/transforms/cc_ssh.py index 48eb58bc..db6848d9 100644 --- a/cloudinit/transforms/cc_ssh.py +++ b/cloudinit/transforms/cc_ssh.py @@ -18,15 +18,34 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -import cloudinit.util as util -import cloudinit.SshUtil as sshutil import os import glob -import subprocess -DISABLE_ROOT_OPTS = "no-port-forwarding,no-agent-forwarding," \ -"no-X11-forwarding,command=\"echo \'Please login as the user \\\"$USER\\\" " \ -"rather than the user \\\"root\\\".\';echo;sleep 10\"" +from cloudinit import util +from cloudinit import ssh_util + +DISABLE_ROOT_OPTS = ( "no-port-forwarding,no-agent-forwarding," +"no-X11-forwarding,command=\"echo \'Please login as the user \\\"$USER\\\" " +"rather than the user \\\"root\\\".\';echo;sleep 10\"") + +key2file = { + "rsa_private": ("/etc/ssh/ssh_host_rsa_key", 0600), + "rsa_public": ("/etc/ssh/ssh_host_rsa_key.pub", 0644), + "dsa_private": ("/etc/ssh/ssh_host_dsa_key", 0600), + "dsa_public": ("/etc/ssh/ssh_host_dsa_key.pub", 0644), + "ecdsa_private": ("/etc/ssh/ssh_host_ecdsa_key", 0600), + "ecdsa_public": ("/etc/ssh/ssh_host_ecdsa_key.pub", 0644), +} + +priv2pub = { + 'rsa_private': 'rsa_public', + 'dsa_private': 'dsa_public', + 'ecdsa_private': 'ecdsa_public', +} + +key_gen_tpl = 'o=$(ssh-keygen -yf "%s") && echo "$o" root@localhost > "%s"' + +generate_keys = ['rsa', 'dsa', 'ecdsa'] def handle(_name, cfg, cloud, log, _args): @@ -35,72 +54,70 @@ def handle(_name, cfg, cloud, log, _args): if cfg.get("ssh_deletekeys", True): for f in glob.glob("/etc/ssh/ssh_host_*key*"): try: - os.unlink(f) + util.del_file(f) except: - pass - + util.logexc(log, "Failed deleting key file %s", f) + if "ssh_keys" in cfg: # if there are keys in cloud-config, use them - key2file = { - "rsa_private": ("/etc/ssh/ssh_host_rsa_key", 0600), - "rsa_public": ("/etc/ssh/ssh_host_rsa_key.pub", 0644), - "dsa_private": ("/etc/ssh/ssh_host_dsa_key", 0600), - "dsa_public": ("/etc/ssh/ssh_host_dsa_key.pub", 0644), - "ecdsa_private": ("/etc/ssh/ssh_host_ecdsa_key", 0600), - "ecdsa_public": ("/etc/ssh/ssh_host_ecdsa_key.pub", 0644), - } - - for key, val in cfg["ssh_keys"].items(): + for (key, val) in cfg["ssh_keys"].iteritems(): if key in key2file: - util.write_file(key2file[key][0], val, key2file[key][1]) - - priv2pub = {'rsa_private': 'rsa_public', 'dsa_private': 'dsa_public', - 'ecdsa_private': 'ecdsa_public', } - + tgt_fn = key2file[key][0] + tgt_perms = key2file[key][1] + util.write_file(tgt_fn, val, tgt_perms) + cmd = 'o=$(ssh-keygen -yf "%s") && echo "$o" root@localhost > "%s"' for priv, pub in priv2pub.iteritems(): if pub in cfg['ssh_keys'] or not priv in cfg['ssh_keys']: continue pair = (key2file[priv][0], key2file[pub][0]) - subprocess.call(('sh', '-xc', cmd % pair)) - log.debug("generated %s from %s" % pair) + cmd = ['sh', '-xc', key_gen_tpl % pair] + try: + # TODO: Is this guard needed? + with util.SeLinuxGuard("/etc/ssh", recursive=True): + util.subp(cmd, capture=False) + log.debug("Generated a key for %s from %s", pair[0], pair[1]) + except: + util.logexc(log, "Failed generated a key for %s from %s", pair[0], pair[1]) else: # if not, generate them - for keytype in util.get_cfg_option_list_or_str(cfg, 'ssh_genkeytypes', - ['rsa', 'dsa', 'ecdsa']): + for keytype in util.get_cfg_option_list_or_str(cfg, 'ssh_genkeytypes', generate_keys): keyfile = '/etc/ssh/ssh_host_%s_key' % keytype if not os.path.exists(keyfile): - subprocess.call(['ssh-keygen', '-t', keytype, '-N', '', - '-f', keyfile]) - - util.restorecon_if_possible('/etc/ssh', recursive=True) + cmd = ['ssh-keygen', '-t', keytype, '-N', '', '-f', keyfile] + try: + # TODO: Is this guard needed? + with util.SeLinuxGuard("/etc/ssh", recursive=True): + util.subp(cmd, capture=False) + except: + util.logexc(log, "Failed generating key type %s to file %s", keytype, keyfile) try: user = util.get_cfg_option_str(cfg, 'user') disable_root = util.get_cfg_option_bool(cfg, "disable_root", True) disable_root_opts = util.get_cfg_option_str(cfg, "disable_root_opts", DISABLE_ROOT_OPTS) - keys = cloud.get_public_ssh_keys() + keys = cloud.get_public_ssh_keys() or [] if "ssh_authorized_keys" in cfg: cfgkeys = cfg["ssh_authorized_keys"] keys.extend(cfgkeys) apply_credentials(keys, user, disable_root, disable_root_opts, log) except: - util.logexc(log) - log.warn("applying credentials failed!\n") + util.logexc(log, "Applying ssh credentials failed!") def apply_credentials(keys, user, disable_root, disable_root_opts=DISABLE_ROOT_OPTS, log=None): + keys = set(keys) if user: - sshutil.setup_user_keys(keys, user, '', log) + ssh_util.setup_user_keys(keys, user, '') - if disable_root: + if disable_root and user: key_prefix = disable_root_opts.replace('$USER', user) else: key_prefix = '' - sshutil.setup_user_keys(keys, 'root', key_prefix, log) + ssh_util.setup_user_keys(keys, 'root', key_prefix) diff --git a/cloudinit/transforms/cc_ssh_import_id.py b/cloudinit/transforms/cc_ssh_import_id.py index bbf5bd83..019413d4 100644 --- a/cloudinit/transforms/cc_ssh_import_id.py +++ b/cloudinit/transforms/cc_ssh_import_id.py @@ -18,12 +18,14 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -import cloudinit.util as util -import subprocess -import traceback +from cloudinit import util +# The ssh-import-id only seems to exist on ubuntu (for now) +# https://launchpad.net/ssh-import-id +distros = ['ubuntu'] -def handle(_name, cfg, _cloud, log, args): + +def handle(name, cfg, _cloud, log, args): if len(args) != 0: user = args[0] ids = [] @@ -34,17 +36,14 @@ def handle(_name, cfg, _cloud, log, args): ids = util.get_cfg_option_list_or_str(cfg, "ssh_import_id", []) if len(ids) == 0: + log.debug("Skipping module named %s, no ids found to import", name) return cmd = ["sudo", "-Hu", user, "ssh-import-id"] + ids - - log.debug("importing ssh ids. cmd = %s" % cmd) + log.debug("Importing ssh ids for user %s.", user) try: - subprocess.check_call(cmd) - except subprocess.CalledProcessError as e: - log.debug(traceback.format_exc(e)) - raise Exception("Cmd returned %s: %s" % (e.returncode, cmd)) - except OSError as e: - log.debug(traceback.format_exc(e)) - raise Exception("Cmd failed to execute: %s" % (cmd)) + util.subp(cmd, capture=False) + except util.ProcessExecutionError as e: + util.logexc(log, "Failed to run command to import %s ssh ids", user) + raise e diff --git a/cloudinit/transforms/cc_timezone.py b/cloudinit/transforms/cc_timezone.py index e5c9901b..6fb5edc0 100644 --- a/cloudinit/transforms/cc_timezone.py +++ b/cloudinit/transforms/cc_timezone.py @@ -18,50 +18,19 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -from cloudinit.CloudConfig import per_instance -from cloudinit import util -import os.path -import shutil +from cloudinit.settings import PER_INSTANCE -frequency = per_instance -tz_base = "/usr/share/zoneinfo" +frequency = PER_INSTANCE -def handle(_name, cfg, _cloud, log, args): +def handle(_name, cfg, cloud, log, args): if len(args) != 0: timezone = args[0] else: timezone = util.get_cfg_option_str(cfg, "timezone", False) if not timezone: + log.debug("Skipping module named %s, no 'timezone' specified", name) return - tz_file = "%s/%s" % (tz_base, timezone) - - if not os.path.isfile(tz_file): - log.debug("Invalid timezone %s" % tz_file) - raise Exception("Invalid timezone %s" % tz_file) - - try: - fp = open("/etc/timezone", "wb") - fp.write("%s\n" % timezone) - fp.close() - except: - log.debug("failed to write to /etc/timezone") - raise - if os.path.exists("/etc/sysconfig/clock"): - try: - with open("/etc/sysconfig/clock", "w") as fp: - fp.write('ZONE="%s"\n' % timezone) - except: - log.debug("failed to write to /etc/sysconfig/clock") - raise - - try: - shutil.copy(tz_file, "/etc/localtime") - except: - log.debug("failed to copy %s to /etc/localtime" % tz_file) - raise - - log.debug("set timezone to %s" % timezone) - return + cloud.distro.set_timezone(timezone) diff --git a/cloudinit/transforms/cc_update_etc_hosts.py b/cloudinit/transforms/cc_update_etc_hosts.py index 6ad2fca8..361097a6 100644 --- a/cloudinit/transforms/cc_update_etc_hosts.py +++ b/cloudinit/transforms/cc_update_etc_hosts.py @@ -18,70 +18,34 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -import cloudinit.util as util -from cloudinit.CloudConfig import per_always -import StringIO +from cloudinit import util +from cloudinit import templater -frequency = per_always +from cloudinit.settings import PER_ALWAYS +frequency = PER_ALWAYS -def handle(_name, cfg, cloud, log, _args): - (hostname, fqdn) = util.get_hostname_fqdn(cfg, cloud) +def handle(name, cfg, cloud, log, _args): manage_hosts = util.get_cfg_option_str(cfg, "manage_etc_hosts", False) - if manage_hosts in ("True", "true", True, "template"): - # render from template file - try: - if not hostname: - log.info("manage_etc_hosts was set, but no hostname found") - return - - util.render_to_file('hosts', '/etc/hosts', + if util.translate_bool(manage_hosts, addons=['template']): + (hostname, fqdn) = util.get_hostname_fqdn(cfg, cloud) + # Render from template file + if not hostname: + log.warn("Option 'manage_etc_hosts' was set, but no hostname was found") + return + tpl_fn_name = cloud.get_template_filename("hosts.%s" % (cloud.distro.name())) + if not tpl_fn_name: + raise Exception("No hosts template could be found for distro %s" % (cloud.distro.name())) + templater.render_to_file(tpl_fn_name, '/etc/hosts', {'hostname': hostname, 'fqdn': fqdn}) - except Exception: - log.warn("failed to update /etc/hosts") - raise elif manage_hosts == "localhost": - log.debug("managing 127.0.1.1 in /etc/hosts") - update_etc_hosts(hostname, fqdn, log) - return + log.debug("Managing localhost in /etc/hosts") + (hostname, fqdn) = util.get_hostname_fqdn(cfg, cloud) + if not hostname: + log.warn("Option 'manage_etc_hosts' was set, but no hostname was found") + return + cloud.distro.update_etc_hosts(hostname, fqdn) else: - if manage_hosts not in ("False", False): - log.warn("Unknown value for manage_etc_hosts. Assuming False") - else: - log.debug("not managing /etc/hosts") - - -def update_etc_hosts(hostname, fqdn, _log): - with open('/etc/hosts', 'r') as etchosts: - header = "# Added by cloud-init\n" - hosts_line = "127.0.1.1\t%s %s\n" % (fqdn, hostname) - need_write = False - need_change = True - new_etchosts = StringIO.StringIO() - for line in etchosts: - split_line = [s.strip() for s in line.split()] - if len(split_line) < 2: - new_etchosts.write(line) - continue - if line == header: - continue - ip, hosts = split_line[0], split_line[1:] - if ip == "127.0.1.1": - if sorted([hostname, fqdn]) == sorted(hosts): - need_change = False - if need_change == True: - line = "%s%s" % (header, hosts_line) - need_change = False - need_write = True - new_etchosts.write(line) - etchosts.close() - if need_change == True: - new_etchosts.write("%s%s" % (header, hosts_line)) - need_write = True - if need_write == True: - new_etcfile = open('/etc/hosts', 'wb') - new_etcfile.write(new_etchosts.getvalue()) - new_etcfile.close() - new_etchosts.close() - return + log.debug(("Configuration option 'manage_etc_hosts' is not set," + " not managing /etc/hosts in %s"), name) diff --git a/cloudinit/transforms/cc_update_hostname.py b/cloudinit/transforms/cc_update_hostname.py index b9d1919a..439bdcb3 100644 --- a/cloudinit/transforms/cc_update_hostname.py +++ b/cloudinit/transforms/cc_update_hostname.py @@ -18,84 +18,22 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -import cloudinit.util as util -import subprocess -import errno -from cloudinit.CloudConfig import per_always +from cloudinit import util +from cloudinit.settings import PER_ALWAYS -frequency = per_always +frequency = PER_ALWAYS -def handle(_name, cfg, cloud, log, _args): +def handle(name, cfg, cloud, log, _args): if util.get_cfg_option_bool(cfg, "preserve_hostname", False): - log.debug("preserve_hostname is set. not updating hostname") + log.debug(("Configuration option 'preserve_hostname' is set," + " not updating the hostname in %s"), name) return (hostname, _fqdn) = util.get_hostname_fqdn(cfg, cloud) try: - prev = "%s/%s" % (cloud.get_cpath('data'), "previous-hostname") - update_hostname(hostname, prev, log) + prev_fn = os.path.join(cloud.get_cpath('data'), "previous-hostname") + cloud.distro.update_hostname(hostname, prev_fn) except Exception: - log.warn("failed to set hostname\n") + util.logexc(log, "Failed to set the hostname to %s", hostname) raise - - -# read hostname from a 'hostname' file -# allow for comments and stripping line endings. -# if file doesn't exist, or no contents, return default -def read_hostname(filename, default=None): - try: - fp = open(filename, "r") - lines = fp.readlines() - fp.close() - for line in lines: - hpos = line.find("#") - if hpos != -1: - line = line[0:hpos] - line = line.rstrip() - if line: - return line - except IOError as e: - if e.errno != errno.ENOENT: - raise - return default - - -def update_hostname(hostname, prev_file, log): - etc_file = "/etc/hostname" - - hostname_prev = None - hostname_in_etc = None - - try: - hostname_prev = read_hostname(prev_file) - except Exception as e: - log.warn("Failed to open %s: %s" % (prev_file, e)) - - try: - hostname_in_etc = read_hostname(etc_file) - except: - log.warn("Failed to open %s" % etc_file) - - update_files = [] - if not hostname_prev or hostname_prev != hostname: - update_files.append(prev_file) - - if (not hostname_in_etc or - (hostname_in_etc == hostname_prev and hostname_in_etc != hostname)): - update_files.append(etc_file) - - try: - for fname in update_files: - util.write_file(fname, "%s\n" % hostname, 0644) - log.debug("wrote %s to %s" % (hostname, fname)) - except: - log.warn("failed to write hostname to %s" % fname) - - if hostname_in_etc and hostname_prev and hostname_in_etc != hostname_prev: - log.debug("%s differs from %s. assuming user maintained" % - (prev_file, etc_file)) - - if etc_file in update_files: - log.debug("setting hostname to %s" % hostname) - subprocess.Popen(['hostname', hostname]).communicate() -- cgit v1.2.3 From 450261a1fcf1f8929a2f7a25c2c278ba40689289 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Fri, 15 Jun 2012 21:33:55 -0700 Subject: Fixups to ensure that pylint does not find anything major wrong. --- cloudinit/cloud.py | 2 - cloudinit/distros/__init__.py | 21 +++--- cloudinit/distros/ubuntu.py | 23 +++--- cloudinit/handlers/boot_hook.py | 14 ++-- cloudinit/handlers/cloud_config.py | 4 +- cloudinit/handlers/shell_script.py | 5 +- cloudinit/handlers/upstart_job.py | 7 +- cloudinit/helpers.py | 41 ++--------- cloudinit/log.py | 2 +- cloudinit/sources/DataSourceCloudStack.py | 2 +- cloudinit/sources/DataSourceConfigDrive.py | 4 +- cloudinit/sources/DataSourceMAAS.py | 14 ++-- cloudinit/sources/DataSourceNoCloud.py | 3 +- cloudinit/sources/DataSourceOVF.py | 12 +-- cloudinit/sources/__init__.py | 23 +++--- cloudinit/stages.py | 98 ++++++++++++++++--------- cloudinit/transforms/__init__.py | 2 +- cloudinit/transforms/cc_apt_update_upgrade.py | 12 ++- cloudinit/transforms/cc_bootcmd.py | 8 +- cloudinit/transforms/cc_byobu.py | 2 +- cloudinit/transforms/cc_ca_certs.py | 13 +++- cloudinit/transforms/cc_chef.py | 3 +- cloudinit/transforms/cc_disable_ec2_metadata.py | 3 +- cloudinit/transforms/cc_final_message.py | 4 +- cloudinit/transforms/cc_foo.py | 6 +- cloudinit/transforms/cc_grub_dpkg.py | 4 +- cloudinit/transforms/cc_keys_to_console.py | 23 ++++-- cloudinit/transforms/cc_landscape.py | 3 +- cloudinit/transforms/cc_locale.py | 2 +- cloudinit/transforms/cc_mcollective.py | 9 +-- cloudinit/transforms/cc_mounts.py | 15 ++-- cloudinit/transforms/cc_phone_home.py | 36 ++++++--- cloudinit/transforms/cc_puppet.py | 17 +++-- cloudinit/transforms/cc_resizefs.py | 38 ++++++---- cloudinit/transforms/cc_rightscale_userdata.py | 16 ++-- cloudinit/transforms/cc_rsyslog.py | 14 ++-- cloudinit/transforms/cc_runcmd.py | 9 ++- cloudinit/transforms/cc_salt_minion.py | 7 +- cloudinit/transforms/cc_scripts_per_boot.py | 5 +- cloudinit/transforms/cc_scripts_per_instance.py | 5 +- cloudinit/transforms/cc_scripts_per_once.py | 5 +- cloudinit/transforms/cc_scripts_user.py | 9 ++- cloudinit/transforms/cc_set_hostname.py | 2 +- cloudinit/transforms/cc_set_passwords.py | 10 ++- cloudinit/transforms/cc_ssh.py | 22 +++--- cloudinit/transforms/cc_ssh_import_id.py | 8 +- cloudinit/transforms/cc_timezone.py | 8 +- cloudinit/transforms/cc_update_etc_hosts.py | 22 ++++-- cloudinit/transforms/cc_update_hostname.py | 4 +- cloudinit/transforms/cc_welcome.py | 6 +- cloudinit/user_data.py | 13 ++-- cloudinit/util.py | 48 ++++++------ 52 files changed, 388 insertions(+), 300 deletions(-) (limited to 'cloudinit/sources/__init__.py') diff --git a/cloudinit/cloud.py b/cloudinit/cloud.py index b2dfc749..8372d123 100644 --- a/cloudinit/cloud.py +++ b/cloudinit/cloud.py @@ -23,8 +23,6 @@ import copy import os -from cloudinit import distros -from cloudinit import helpers from cloudinit import log as logging LOG = logging.getLogger(__name__) diff --git a/cloudinit/distros/__init__.py b/cloudinit/distros/__init__.py index 90607668..fd4c70c1 100644 --- a/cloudinit/distros/__init__.py +++ b/cloudinit/distros/__init__.py @@ -20,29 +20,32 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . +from StringIO import StringIO + import abc import copy from cloudinit import importer +from cloudinit import log as logging from cloudinit import util -from StringIO import StringIO - # TODO: Make this via config?? IFACE_ACTIONS = { 'up': ['ifup', '--all'], 'down': ['ifdown', '--all'], } +LOG = logging.getLogger(__name__) + class Distro(object): __metaclass__ = abc.ABCMeta - def __init__(self, cfg, runner): + def __init__(self, name, cfg, runner): self._runner = runner - self._cfg = util.get_cfg_by_path(cfg, ('system_info', ), {}) - self.name = self._cfg.pop("distro", 'generic') + self._cfg = cfg + self.name = name @abc.abstractmethod def install_packages(self, pkglist): @@ -135,10 +138,9 @@ class Distro(object): action, cmd) (_out, err) = util.subp(cmd) if len(err): - LOG.warn("Running %s resulted in stderr output: %s", - IF_UP_CMD, err) + LOG.warn("Running %s resulted in stderr output: %s", cmd, err) return True - except util.ProcessExecutionError as exc: + except util.ProcessExecutionError: util.logexc(LOG, "Running %s failed", cmd) return False @@ -152,7 +154,8 @@ def fetch(distro_name, mods=(__name__, )): except RuntimeError: pass if not mod: - raise RuntimeError("No distribution found for distro %s" % (distro_name)) + raise RuntimeError("No distribution found for distro %s" + % (distro_name)) distro_cls = getattr(mod, 'Distro') return distro_cls diff --git a/cloudinit/distros/ubuntu.py b/cloudinit/distros/ubuntu.py index 6b0aff47..9252a1c4 100644 --- a/cloudinit/distros/ubuntu.py +++ b/cloudinit/distros/ubuntu.py @@ -20,17 +20,13 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -from StringIO import StringIO - import os -import socket from cloudinit import distros from cloudinit import log as logging -from cloudinit import templater from cloudinit import util -from cloudinit.settings import (PER_INSTANCE) +from cloudinit.settings import PER_INSTANCE LOG = logging.getLogger(__name__) @@ -65,9 +61,11 @@ class Distro(distros.Distro): try: util.write_file(fn, "%s\n" % hostname, 0644) except: - util.logexc(LOG, "Failed to write hostname %s to %s", hostname, fn) - if hostname_in_etc and hostname_prev and hostname_in_etc != hostname_prev: - LOG.debug(("%s differs from /etc/hostname." + util.logexc(LOG, "Failed to write hostname %s to %s", + hostname, fn) + if (hostname_in_etc and hostname_prev and + hostname_in_etc != hostname_prev): + LOG.debug(("%s differs from /etc/hostname." " Assuming user maintained hostname."), prev_file) if "/etc/hostname" in update_files: LOG.debug("Setting hostname to %s", hostname) @@ -91,7 +89,8 @@ class Distro(distros.Distro): def set_timezone(self, tz): tz_file = os.path.join("/usr/share/zoneinfo", tz) if not os.path.isfile(tz_file): - raise Exception("Invalid timezone %s, no file found at %s" % (tz, tz_file)) + raise Exception(("Invalid timezone %s," + " no file found at %s") % (tz, tz_file)) tz_contents = "%s\n" % tz util.write_file("/etc/timezone", tz_contents) # TODO, this should be in a rhel distro subclass?? @@ -101,9 +100,6 @@ class Distro(distros.Distro): # This ensures that the correct tz will be used for the system util.copy(tz_file, "/etc/localtime") - def name(self): - return "ubuntu" - # apt_get top level command (install, update...), and args to pass it def _apt_get(self, tlc, args=None): e = os.environ.copy() @@ -116,4 +112,5 @@ class Distro(distros.Distro): util.subp(cmd, env=e, capture=False) def _update_package_sources(self): - self.runner.run("update-sources", self._apt_get, ["update"], freq=PER_INSTANCE) \ No newline at end of file + self._runner.run("update-sources", self._apt_get, + ["update"], freq=PER_INSTANCE) \ No newline at end of file diff --git a/cloudinit/handlers/boot_hook.py b/cloudinit/handlers/boot_hook.py index c75aeb72..b3aab366 100644 --- a/cloudinit/handlers/boot_hook.py +++ b/cloudinit/handlers/boot_hook.py @@ -32,9 +32,9 @@ LOG = logging.getLogger(__name__) class BootHookPartHandler(ud.PartHandler): - def __init__(self, boothook_dir, instance_id): + def __init__(self, paths, instance_id, **_kwargs): ud.PartHandler.__init__(self, PER_ALWAYS) - self.boothook_dir = boothook_dir + self.boothook_dir = paths.get_ipath("boothooks") self.instance_id = instance_id def list_types(self): @@ -54,13 +54,15 @@ class BootHookPartHandler(ud.PartHandler): start = len(prefix) + 1 filepath = os.path.join(self.boothook_dir, filename) - util.write_file(filepath, payload[start:], 0700) + contents = payload[start:] + util.write_file(filepath, contents, 0700) try: env = os.environ.copy() - env['INSTANCE_ID'] = str(self.instance_id) + if self.instance_id: + env['INSTANCE_ID'] = str(self.instance_id) util.subp([filepath], env=env) - except util.ProcessExecutionError as e: + except util.ProcessExecutionError: util.logexc(LOG, "Boothooks script %s execution error", filepath) - except Exception as e: + except Exception: util.logexc(LOG, ("Boothooks unknown " "error when running %s"), filepath) diff --git a/cloudinit/handlers/cloud_config.py b/cloudinit/handlers/cloud_config.py index f0e88eeb..12d1bd96 100644 --- a/cloudinit/handlers/cloud_config.py +++ b/cloudinit/handlers/cloud_config.py @@ -30,10 +30,10 @@ LOG = logging.getLogger(__name__) class CloudConfigPartHandler(ud.PartHandler): - def __init__(self, cloud_fn): + def __init__(self, paths, **_kwargs): ud.PartHandler.__init__(self, PER_ALWAYS) self.cloud_buf = [] - self.cloud_fn = cloud_fn + self.cloud_fn = paths.get_ipath("cloud_config") def list_types(self): return [ diff --git a/cloudinit/handlers/shell_script.py b/cloudinit/handlers/shell_script.py index 564e4623..f6e2ef16 100644 --- a/cloudinit/handlers/shell_script.py +++ b/cloudinit/handlers/shell_script.py @@ -32,10 +32,9 @@ LOG = logging.getLogger(__name__) class ShellScriptPartHandler(ud.PartHandler): - - def __init__(self, script_dir): + def __init__(self, paths, **_kwargs): ud.PartHandler.__init__(self, PER_ALWAYS) - self.script_dir = script_dir + self.script_dir = paths.get_ipath_cur('scripts') def list_types(self): return [ diff --git a/cloudinit/handlers/upstart_job.py b/cloudinit/handlers/upstart_job.py index 568a644a..059a4851 100644 --- a/cloudinit/handlers/upstart_job.py +++ b/cloudinit/handlers/upstart_job.py @@ -33,9 +33,9 @@ LOG = logging.getLogger(__name__) class UpstartJobPartHandler(ud.PartHandler): - def __init__(self, upstart_dir): + def __init__(self, paths, **_kwargs): ud.PartHandler.__init__(self, PER_INSTANCE) - self.upstart_dir = upstart_dir + self.upstart_dir = paths.upstart_conf_d def list_types(self): return [ @@ -46,6 +46,9 @@ class UpstartJobPartHandler(ud.PartHandler): if ctype in ud.CONTENT_SIGNALS: return + if not self.upstart_dir: + return + filename = util.clean_filename(filename) (_name, ext) = os.path.splitext(filename) if not ext: diff --git a/cloudinit/helpers.py b/cloudinit/helpers.py index 2ecda3e9..c276a54c 100644 --- a/cloudinit/helpers.py +++ b/cloudinit/helpers.py @@ -30,11 +30,6 @@ from cloudinit.settings import (PER_INSTANCE, PER_ALWAYS, PER_ONCE) from cloudinit import log as logging from cloudinit import util -from cloudinit.user_data import boot_hook as bh_part -from cloudinit.user_data import cloud_config as cc_part -from cloudinit.user_data import shell_script as ss_part -from cloudinit.user_data import upstart_job as up_part - LOG = logging.getLogger(__name__) @@ -77,7 +72,7 @@ class FileSemaphores(object): sem_file = self._get_path(name, freq) try: util.del_file(sem_file) - except (IOError, OSError) as e: + except (IOError, OSError): util.logexc(LOG, "Failed deleting semaphore %s", sem_file) return False return True @@ -99,7 +94,7 @@ class FileSemaphores(object): contents = "%s: %s\n" % (os.getpid(), time()) try: util.write_file(sem_file, contents) - except (IOError, OSError) as e: + except (IOError, OSError): util.logexc(LOG, "Failed writing semaphore file %s", sem_file) return None return sem_file @@ -162,9 +157,10 @@ class Runners(object): class ContentHandlers(object): - def __init__(self, paths): + def __init__(self, paths, iid=None): self.paths = paths self.registered = {} + self.iid = iid def __contains__(self, item): return self.is_registered(item) @@ -191,34 +187,9 @@ class ContentHandlers(object): def iteritems(self): return self.registered.iteritems() - def _get_default_handlers(self): - def_handlers = [] - - cc_path = self.paths.get_ipath("cloud_config") - if cc_path: - cc_h = cc_part.CloudConfigPartHandler(cc_path) - def_handlers.append(cc_h) - - sc_path = self.paths.get_ipath_cur('scripts') - if sc_path: - ss_h = ss_part.ShellScriptPartHandler(sc_path) - def_handlers.append(ss_h) - - bh_path = self.paths.get_ipath("boothooks") - if bh_path: - bh_h = bh_part.BootHookPartHandler(bh_path) - def_handlers.append(bh_h) - - upstart_pth = self.paths.upstart_conf_d - if upstart_pth: - up_h = up_part.UpstartJobPartHandler(upstart_pth) - def_handlers.append(up_h) - - return def_handlers - - def register_defaults(self): + def register_defaults(self, defs): registered = set() - for mod in self._get_default_handlers(): + for mod in defs: for t in mod.list_types(): if not self.is_registered(t): self.registered[t] = mod diff --git a/cloudinit/log.py b/cloudinit/log.py index c247eb9e..5fcb77ef 100644 --- a/cloudinit/log.py +++ b/cloudinit/log.py @@ -56,7 +56,7 @@ def setupBasicLogging(): cfile = logging.FileHandler('/var/log/cloud-init.log') cfile.setFormatter(logging.Formatter(DEF_CON_FORMAT)) cfile.setLevel(DEBUG) - root.addHandle(cfile) + root.addHandler(cfile) except (IOError, OSError): # Likely that u can't write to that file... # Make console now have DEBUG?? diff --git a/cloudinit/sources/DataSourceCloudStack.py b/cloudinit/sources/DataSourceCloudStack.py index 791df68f..27217e65 100644 --- a/cloudinit/sources/DataSourceCloudStack.py +++ b/cloudinit/sources/DataSourceCloudStack.py @@ -79,7 +79,7 @@ class DataSourceCloudStack(sources.DataSource): tot_time = (time.time() - start) LOG.debug("Crawl of metadata service took %s", int(tot_time)) return True - except Exception as e: + except Exception: util.logexc(LOG, ('Failed fetching from metadata ' 'service %s'), self.metadata_address) return False diff --git a/cloudinit/sources/DataSourceConfigDrive.py b/cloudinit/sources/DataSourceConfigDrive.py index 176b62b0..7450572f 100644 --- a/cloudinit/sources/DataSourceConfigDrive.py +++ b/cloudinit/sources/DataSourceConfigDrive.py @@ -51,8 +51,8 @@ class DataSourceConfigDrive(sources.DataSource): self.seed_dir = os.path.join(paths.seed_dir, 'config_drive') def __str__(self): - mstr = "%s[%s]" % (util.obj_name(self), self.dsmode) - mstr += " [seed=%s]" % (self.seed) + mstr = "%s [%s]" % (util.obj_name(self), self.dsmode) + mstr += "[seed=%s]" % (self.seed) return mstr def get_data(self): diff --git a/cloudinit/sources/DataSourceMAAS.py b/cloudinit/sources/DataSourceMAAS.py index 27196265..9e639649 100644 --- a/cloudinit/sources/DataSourceMAAS.py +++ b/cloudinit/sources/DataSourceMAAS.py @@ -18,9 +18,9 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -import os import errno import oauth.oauth as oauth +import os import time import urllib2 @@ -48,7 +48,7 @@ class DataSourceMAAS(sources.DataSource): self.seed_dir = os.path.join(paths.seed_dir, 'maas') def __str__(self): - return "%s[%s]" % (util.obj_name(self), self.base_url) + return "%s [%s]" % (util.obj_name(self), self.base_url) def get_data(self): mcfg = self.ds_cfg @@ -122,9 +122,10 @@ class DataSourceMAAS(sources.DataSource): starttime = time.time() check_url = "%s/%s/meta-data/instance-id" % (url, MD_VERSION) - url = util.wait_for_url(urls=[check_url], max_wait=max_wait, - timeout=timeout, status_cb=LOG.warn, - headers_cb=self.md_headers) + urls = [check_url] + url = uhelp.wait_for_url(urls=urls, max_wait=max_wait, + timeout=timeout, status_cb=LOG.warn, + headers_cb=self.md_headers) if url: LOG.info("Using metadata source: '%s'", url) @@ -185,7 +186,8 @@ def read_maas_seed_url(seed_url, header_cb=None, timeout=None, headers = {} try: (resp, sc) = uhelp.readurl(url, headers=headers, timeout=timeout) - md[name] = resp + if uhelp.ok_http_code(sc): + md[name] = resp except urllib2.HTTPError as e: if e.code != 404: raise diff --git a/cloudinit/sources/DataSourceNoCloud.py b/cloudinit/sources/DataSourceNoCloud.py index 84d0f99d..2b016d1c 100644 --- a/cloudinit/sources/DataSourceNoCloud.py +++ b/cloudinit/sources/DataSourceNoCloud.py @@ -106,7 +106,8 @@ class DataSourceNoCloud(sources.DataSource): if e.errno != errno.ENOENT: raise except util.MountFailedError: - util.logexc(LOG, "Failed to mount %s when looking for seed", dev) + util.logexc(LOG, ("Failed to mount %s" + " when looking for data"), dev) # There was no indication on kernel cmdline or data # in the seeddir suggesting this handler should be used. diff --git a/cloudinit/sources/DataSourceOVF.py b/cloudinit/sources/DataSourceOVF.py index bb0f46c2..258d8d03 100644 --- a/cloudinit/sources/DataSourceOVF.py +++ b/cloudinit/sources/DataSourceOVF.py @@ -21,10 +21,10 @@ # along with this program. If not, see . from xml.dom import minidom + import base64 import os import re -import tempfile from cloudinit import log as logging from cloudinit import sources @@ -51,7 +51,7 @@ class DataSourceOVF(sources.DataSource): ud = "" defaults = { - "instance-id": "iid-dsovf" + "instance-id": "iid-dsovf", } (seedfile, contents) = get_ovf_env(self.paths.seed_dir) @@ -198,7 +198,7 @@ def transport_iso9660(require_iso=True): for dev in devs: fullp = os.path.join("/dev/", dev) - if (fullp in mounted or + if (fullp in mounts or not cdmatch.match(dev) or os.path.isdir(fullp)): continue @@ -210,7 +210,8 @@ def transport_iso9660(require_iso=True): continue try: - (fname, contents) = utils.mount_cb(fullp, get_ovf_env, mtype="iso9660") + (fname, contents) = util.mount_cb(fullp, + get_ovf_env, mtype="iso9660") except util.MountFailedError: util.logexc(LOG, "Failed mounting %s", fullp) continue @@ -265,7 +266,8 @@ def get_properties(contents): raise XmlError("No 'PropertySection's") props = {} - propElems = find_child(propSections[0], lambda n: n.localName == "Property") + propElems = find_child(propSections[0], + (lambda n: n.localName == "Property")) for elem in propElems: key = elem.attributes.getNamedItemNS(envNsURI, "key").value diff --git a/cloudinit/sources/__init__.py b/cloudinit/sources/__init__.py index 08669f5d..beb0f3d7 100644 --- a/cloudinit/sources/__init__.py +++ b/cloudinit/sources/__init__.py @@ -22,10 +22,9 @@ from cloudinit import importer from cloudinit import log as logging +from cloudinit import user_data as ud from cloudinit import util -from cloudinit.user_data import processor as ud_proc - DEP_FILESYSTEM = "FILESYSTEM" DEP_NETWORK = "NETWORK" DS_PREFIX = 'DataSource' @@ -42,7 +41,6 @@ class DataSource(object): self.sys_cfg = sys_cfg self.distro = distro self.paths = paths - self.userdata_proc = ud_proc.UserDataProcessor(paths) self.userdata = None self.metadata = None self.userdata_raw = None @@ -55,7 +53,7 @@ class DataSource(object): def get_userdata(self): if self.userdata is None: raw_data = self.get_userdata_raw() - self.userdata = self.userdata_proc.process(raw_data) + self.userdata = ud.UserDataProcessor(self.paths).process(raw_data) return self.userdata def get_userdata_raw(self): @@ -73,7 +71,7 @@ class DataSource(object): if not self.metadata or 'public-keys' not in self.metadata: return keys - if isinstance(self.metadata['public-keys'], (str)): + if isinstance(self.metadata['public-keys'], (basestring, str)): return str(self.metadata['public-keys']).splitlines() if isinstance(self.metadata['public-keys'], (list, set)): @@ -84,11 +82,12 @@ class DataSource(object): # lp:506332 uec metadata service responds with # data that makes boto populate a string for 'klist' rather # than a list. - if isinstance(klist, (str)): + if isinstance(klist, (str, basestring)): klist = [klist] if isinstance(klist, (list, set)): for pkey in klist: - # there is an empty string at the end of the keylist, trim it + # There is an empty string at + # the end of the keylist, trim it if pkey: keys.append(pkey) @@ -159,13 +158,14 @@ def find_source(sys_cfg, distro, paths, ds_deps, cfg_list, pkg_list): ds_list = list_sources(cfg_list, ds_deps, pkg_list) ds_names = [util.obj_name(f) for f in ds_list] LOG.info("Searching for data source in: %s", ds_names) + for cls in ds_list: ds = util.obj_name(cls) try: s = cls(distro, sys_cfg, paths) if s.get_data(): return (s, ds) - except Exception as e: + except Exception: util.logexc(LOG, "Getting data from %s failed", ds) msg = "Did not find any data source, searched classes: %s" % (ds_names) @@ -178,7 +178,8 @@ def find_source(sys_cfg, distro, paths, ds_deps, cfg_list, pkg_list): # return an ordered list of classes that match def list_sources(cfg_list, depends, pkg_list): src_list = [] - LOG.info("Looking for for data source in: %s, %s that match %s", cfg_list, pkg_list, depends) + LOG.info(("Looking for for data source in: %s," + " %s that matches %s"), cfg_list, pkg_list, depends) for ds_coll in cfg_list: ds_name = str(ds_coll) if not ds_name.startswith(DS_PREFIX): @@ -201,8 +202,8 @@ def list_sources(cfg_list, depends, pkg_list): if not cls_matches: continue src_list.extend(cls_matches) - LOG.debug("Found a match for data source %s in %s with matches %s", - ds_name, mod, cls_matches) + LOG.debug(("Found a match for data source %s" + " in %s with matches %s"), ds_name, mod, cls_matches) break return src_list diff --git a/cloudinit/stages.py b/cloudinit/stages.py index 2615d59f..b9076881 100644 --- a/cloudinit/stages.py +++ b/cloudinit/stages.py @@ -31,19 +31,23 @@ try: except ImportError: ConfigObj = None -from cloudinit.settings import (PER_INSTANCE, FREQUENCIES) from cloudinit.settings import (OLD_CLOUD_CONFIG) +from cloudinit.settings import (PER_INSTANCE, FREQUENCIES) + +from cloudinit.handlers import boot_hook as bh_part +from cloudinit.handlers import cloud_config as cc_part +from cloudinit.handlers import shell_script as ss_part +from cloudinit.handlers import upstart_job as up_part from cloudinit import cloud from cloudinit import distros -from cloudinit import modules from cloudinit import helpers from cloudinit import importer from cloudinit import log as logging from cloudinit import sources -from cloudinit import util - +from cloudinit import transforms from cloudinit import user_data as ud +from cloudinit import util LOG = logging.getLogger(__name__) @@ -73,12 +77,19 @@ class Init(object): def distro(self): if not self._distro: d_cfg = util.get_cfg_by_path(self.cfg, ('system_info'), {}) + # Ensure its a dictionary + if not isinstance(d_cfg, (dict)): + d_cfg = {} # Ensure not modified indirectly d_cfg = copy.deepcopy(d_cfg) + # Remove this since its path config, not distro config d_cfg.pop('paths', None) - distro_cls = distros.fetch(sys_cfg.pop('distro', 'ubuntu')) + # Try to find the right class to use + distro_name = d_cfg.pop('distro', 'ubuntu') + distro_cls = distros.fetch(distro_name) LOG.debug("Using distro class %s", distro_cls) - distro = distro_cls(d_cfg, helpers.Runners(self.paths)) + distro = distro_cls(distro_name, d_cfg, + helpers.Runners(self.paths)) self._distro = distro return self._distro @@ -93,7 +104,8 @@ class Init(object): @property def paths(self): if not self._paths: - path_info = util.get_cfg_by_path(self.cfg, ('system_info', 'paths'), {}) + path_info = util.get_cfg_by_path(self.cfg, + ('system_info', 'paths'), {}) # Ensure not modified indirectly path_info = copy.deepcopy(path_info) self._paths = helpers.Paths(path_info, self.datasource) @@ -156,7 +168,7 @@ class Init(object): # by using the instance link, if purge_cache was called # the file wont exist return pickle.loads(util.load_file(pickled_fn)) - except Exception as e: + except Exception: util.logexc(LOG, "Failed loading pickled datasource from %s", pickled_fn) return None @@ -166,7 +178,7 @@ class Init(object): try: contents = pickle.dumps(self.datasource) util.write_file(pickled_fn, contents, mode=0400) - except Exception as e: + except Exception: util.logexc(LOG, "Failed pickling datasource to %s", pickled_fn) return False @@ -192,7 +204,8 @@ class Init(object): # (which will affect user-data handlers down the line...) sys_cfg = copy.deepcopy(self.cfg) ds_deps = copy.deepcopy(self.ds_deps) - (ds, dsname) = sources.find_source(sys_cfg, self.distro, self.paths, + (ds, dsname) = sources.find_source(sys_cfg, self.distro, + self.paths, ds_deps, cfg_list, pkg_list) LOG.debug("Loaded datasource %s - %s", dsname, ds) self.datasource = ds @@ -270,6 +283,20 @@ class Init(object): processed_ud = "%s" % (self.datasource.get_userdata()) util.write_file(self.paths.get_ipath('userdata'), processed_ud, 0600) + def _default_userdata_handlers(self): + opts = { + 'paths': self.paths, + 'instance_id': self.datasource.get_instance_id(), + } + # TODO Hmmm, should we dynamically import these?? + def_handlers = [ + cc_part.CloudConfigPartHandler(**opts), + ss_part.ShellScriptPartHandler(**opts), + bh_part.BootHookPartHandler(**opts), + up_part.UpstartJobPartHandler(**opts), + ] + return def_handlers + def consume(self, frequency=PER_INSTANCE): cdir = self.paths.get_cpath("handlers") idir = self.paths.get_ipath("handlers") @@ -279,8 +306,11 @@ class Init(object): sys.path.insert(0, cdir) sys.path.insert(0, idir) + # Ensure datasource fetched before activation (just incase) + ud_obj = self.datasource.get_userdata() + # This keeps track of all the active handlers - c_handlers = helpers.ContentHandlers(self.paths) + c_handlers = helpers.ContentHandlers(paths=self.paths) # Add handlers in cdir potential_handlers = util.find_modules(cdir) @@ -292,13 +322,10 @@ class Init(object): except: util.logexc(LOG, "Failed to register handler from %s", fname) - def_handlers = c_handlers.register_defaults() - if def_handlers: - LOG.debug("Registered default handlers for %s", def_handlers) - - - # Ensure userdata fetched before activation (just incase) - ud_obj = self.datasource.get_userdata() + def_handlers = self._default_userdata_handlers() + applied_def_handlers = c_handlers.register_defaults(def_handlers) + if applied_def_handlers: + LOG.debug("Registered default handlers: %s", applied_def_handlers) # Form our cloud interface data = self.cloudify() @@ -334,11 +361,11 @@ class Init(object): class Transforms(object): - def __init__(self, cloudobj, cfgfile=None): - self.datasource = cloudobj.datasource + def __init__(self, init, cfgfile=None): + self.datasource = init.fetch() self.cfgfile = cfgfile - self.basecfg = copy.deepcopy(cloudobj.cfg) - self.cloudobj = cloudobj + self.basecfg = copy.deepcopy(init.cfg) + self.init = init # Created on first use self._cachedcfg = None @@ -409,25 +436,28 @@ class Transforms(object): (item, util.obj_name(item))) return module_list - def _transforms_modules(self, raw_mods): + def _fixup_transforms(self, raw_mods): mostly_mods = [] for raw_mod in raw_mods: raw_name = raw_mod['mod'] freq = raw_mod.get('freq') run_args = raw_mod.get('args') or [] - mod_name = modules.form_module_name(raw_name) + mod_name = transforms.form_module_name(raw_name) if not mod_name: continue if freq and freq not in FREQUENCIES: - LOG.warn("Config specified module %s has an unknown frequency %s", raw_name, freq) + LOG.warn(("Config specified transform %s" + " has an unknown frequency %s"), raw_name, freq) # Reset it so when ran it will get set to a known value freq = None - mod = modules.fixup_module(importer.import_module(mod_name)) + mod = transforms.fixup_module(importer.import_module(mod_name)) mostly_mods.append([mod, raw_name, freq, run_args]) return mostly_mods def _run_transforms(self, mostly_mods): failures = [] + d_name = self.init.distro.name + c_cloud = self.init.cloudify() for (mod, name, freq, args) in mostly_mods: try: # Try the modules frequency, otherwise fallback to a known one @@ -436,17 +466,17 @@ class Transforms(object): if not freq in FREQUENCIES: freq = PER_INSTANCE worked_distros = mod.distros - if worked_distros and self.cloud.distro.name() not in worked_distros: - LOG.warn(("Module %s is verified on %s distros" + if (worked_distros and d_name not in worked_distros): + LOG.warn(("Transform %s is verified on %s distros" " but not on %s distro. It may or may not work" - " correctly."), name, worked_distros, - self.cloud.distro.name()) + " correctly."), name, worked_distros, d_name) # Deep copy the config so that modules can't alter it + # Use the transforms logger and not our own func_args = [name, copy.deepcopy(self.cfg), - self.cloudobj, LOG, args] - # This name will affect the semphapore name created + c_cloud, transforms.LOG, args] + # This name will affect the semaphore name created run_name = "config-%s" % (name) - self.cloudobj.run(run_name, mod.handle, func_args, freq=freq) + c_cloud.run(run_name, mod.handle, func_args, freq=freq) except Exception as e: util.logexc(LOG, "Running %s failed", mod) failures.append((name, e)) @@ -454,5 +484,5 @@ class Transforms(object): def run(self, name): raw_mods = self._read_transforms(name) - mostly_mods = self._transforms_modules(raw_mods) + mostly_mods = self._fixup_transforms(raw_mods) return self._run_transforms(mostly_mods) diff --git a/cloudinit/transforms/__init__.py b/cloudinit/transforms/__init__.py index 8275b375..40affc4b 100644 --- a/cloudinit/transforms/__init__.py +++ b/cloudinit/transforms/__init__.py @@ -44,7 +44,7 @@ def fixup_module(mod, def_freq=PER_INSTANCE): else: freq = mod.frequency if freq and freq not in FREQUENCIES: - LOG.warn("Module %s has an unknown frequency %s", mod, freq) + LOG.warn("Transform %s has an unknown frequency %s", mod, freq) if not hasattr(mod, 'handle'): def empty_handle(_name, _cfg, _cloud, _log, _args): pass diff --git a/cloudinit/transforms/cc_apt_update_upgrade.py b/cloudinit/transforms/cc_apt_update_upgrade.py index c4a543ed..a4e058c6 100644 --- a/cloudinit/transforms/cc_apt_update_upgrade.py +++ b/cloudinit/transforms/cc_apt_update_upgrade.py @@ -71,7 +71,7 @@ def handle(_name, cfg, cloud, log, _args): except: util.logexc(log, "Failed to run debconf-set-selections") - pkglist = util.get_cfg_option_list_or_str(cfg, 'packages', []) + pkglist = util.get_cfg_option_list(cfg, 'packages', []) errors = [] if update or len(pkglist) or upgrade: @@ -96,7 +96,9 @@ def handle(_name, cfg, cloud, log, _args): errors.append(e) if len(errors): - raise errors[0] + log.warn("%s failed with exceptions, re-raising the last one", + len(errors)) + raise errors[-1] def mirror2lists_fileprefix(mirror): @@ -186,7 +188,8 @@ def add_sources(srclist, template_params=None): try: util.write_file(ent['filename'], source + "\n", omode="ab") except: - errorlist.append([source, "failed write to file %s" % ent['filename']]) + errorlist.append([source, + "failed write to file %s" % ent['filename']]) return errorlist @@ -219,9 +222,10 @@ def find_apt_mirror(cloud, cfg): doms.extend((".localdomain", "",)) mirror_list = [] + distro = cloud.distro.name mirrorfmt = "http://%s-mirror%s/%s" % (distro, "%s", distro) for post in doms: - mirror_list.append(mirrorfmt % post) + mirror_list.append(mirrorfmt % (post)) mirror = util.search_for_mirror(mirror_list) diff --git a/cloudinit/transforms/cc_bootcmd.py b/cloudinit/transforms/cc_bootcmd.py index a2efad32..80afb5e7 100644 --- a/cloudinit/transforms/cc_bootcmd.py +++ b/cloudinit/transforms/cc_bootcmd.py @@ -30,7 +30,8 @@ frequency = PER_ALWAYS def handle(name, cfg, cloud, log, _args): if "bootcmd" not in cfg: - log.debug("Skipping module named %s, no 'bootcomd' key in configuration", name) + log.debug(("Skipping transform named %s," + " no 'bootcomd' key in configuration"), name) return with tempfile.NamedTemporaryFile(suffix=".sh") as tmpf: @@ -39,7 +40,7 @@ def handle(name, cfg, cloud, log, _args): tmpf.write(content) tmpf.flush() except: - log.warn("Failed to shellify bootcmd") + util.logexc(log, "Failed to shellify bootcmd") raise try: @@ -48,5 +49,6 @@ def handle(name, cfg, cloud, log, _args): cmd = ['/bin/sh', tmpf.name] util.subp(cmd, env=env, capture=False) except: - log.warn("Failed to run commands from bootcmd") + util.logexc(log, + ("Failed to run bootcmd transform %s"), name) raise diff --git a/cloudinit/transforms/cc_byobu.py b/cloudinit/transforms/cc_byobu.py index 38586174..741aa934 100644 --- a/cloudinit/transforms/cc_byobu.py +++ b/cloudinit/transforms/cc_byobu.py @@ -30,7 +30,7 @@ def handle(name, cfg, _cloud, log, args): value = util.get_cfg_option_str(cfg, "byobu_by_default", "") if not value: - log.debug("Skipping module named %s, no 'byobu' values found", name) + log.debug("Skipping transform named %s, no 'byobu' values found", name) return if value == "user" or value == "system": diff --git a/cloudinit/transforms/cc_ca_certs.py b/cloudinit/transforms/cc_ca_certs.py index 8ca9a200..e0802bfe 100644 --- a/cloudinit/transforms/cc_ca_certs.py +++ b/cloudinit/transforms/cc_ca_certs.py @@ -23,6 +23,8 @@ CA_CERT_FILENAME = "cloud-init-ca-certs.crt" CA_CERT_CONFIG = "/etc/ca-certificates.conf" CA_CERT_SYSTEM_PATH = "/etc/ssl/certs/" +distros = ['ubuntu'] + def update_ca_certs(): """ @@ -70,22 +72,25 @@ def handle(name, cfg, _cloud, log, _args): """ # If there isn't a ca-certs section in the configuration don't do anything if "ca-certs" not in cfg: - log.debug("Skipping module named %s, no 'ca-certs' key in configuration", name) + log.debug(("Skipping transform named %s," + " no 'ca-certs' key in configuration"), name) return + ca_cert_cfg = cfg['ca-certs'] # If there is a remove-defaults option set to true, remove the system # default trusted CA certs first. if ca_cert_cfg.get("remove-defaults", False): - log.debug("removing default certificates") + log.debug("Removing default certificates") remove_default_ca_certs() # If we are given any new trusted CA certs to add, add them. if "trusted" in ca_cert_cfg: - trusted_certs = util.get_cfg_option_list_or_str(ca_cert_cfg, "trusted") + trusted_certs = util.get_cfg_option_list(ca_cert_cfg, "trusted") if trusted_certs: - log.debug("adding %d certificates" % len(trusted_certs)) + log.debug("Adding %d certificates" % len(trusted_certs)) add_ca_certs(trusted_certs) # Update the system with the new cert configuration. + log.debug("Updating certificates") update_ca_certs() diff --git a/cloudinit/transforms/cc_chef.py b/cloudinit/transforms/cc_chef.py index 12c2f539..473e5f8b 100644 --- a/cloudinit/transforms/cc_chef.py +++ b/cloudinit/transforms/cc_chef.py @@ -31,7 +31,8 @@ def handle(name, cfg, cloud, log, _args): # If there isn't a chef key in the configuration don't do anything if 'chef' not in cfg: - log.debug("Skipping module named %s, no 'chef' key in configuration", name) + log.debug(("Skipping transform named %s," + " no 'chef' key in configuration"), name) return chef_cfg = cfg['chef'] diff --git a/cloudinit/transforms/cc_disable_ec2_metadata.py b/cloudinit/transforms/cc_disable_ec2_metadata.py index 4d2a7f55..3c0dd57b 100644 --- a/cloudinit/transforms/cc_disable_ec2_metadata.py +++ b/cloudinit/transforms/cc_disable_ec2_metadata.py @@ -28,5 +28,6 @@ reject_cmd = ['route', 'add', '-host', '169.254.169.254', 'reject'] def handle(_name, cfg, _cloud, _log, _args): - if util.get_cfg_option_bool(cfg, "disable_ec2_metadata", False): + disabled = util.get_cfg_option_bool(cfg, "disable_ec2_metadata", False) + if disabled: util.subp(reject_cmd) diff --git a/cloudinit/transforms/cc_final_message.py b/cloudinit/transforms/cc_final_message.py index dc4ae34c..c257b6d0 100644 --- a/cloudinit/transforms/cc_final_message.py +++ b/cloudinit/transforms/cc_final_message.py @@ -32,7 +32,7 @@ final_message_def = ("Cloud-init v. {{version}} finished at {{timestamp}}." " Up {{uptime}} seconds.") -def handle(name, cfg, cloud, log, args): +def handle(_name, cfg, cloud, log, args): msg_in = None if len(args) != 0: @@ -60,7 +60,7 @@ def handle(name, cfg, cloud, log, args): # Use stdout, stderr or the logger?? content = templater.render_string(msg_in, subs) sys.stderr.write("%s\n" % (content)) - except Exception as e: + except Exception: util.logexc(log, "Failed to render final message template") boot_fin_fn = cloud.paths.boot_finished diff --git a/cloudinit/transforms/cc_foo.py b/cloudinit/transforms/cc_foo.py index 8007f981..99135704 100644 --- a/cloudinit/transforms/cc_foo.py +++ b/cloudinit/transforms/cc_foo.py @@ -45,8 +45,8 @@ from cloudinit.settings import PER_INSTANCE # informational purposes. If non existent all distros are assumed and # no warning occurs. -frequency = settings.PER_INSTANCE +frequency = PER_INSTANCE -def handle(name, _cfg, _cloud, _log, _args): - print("Hi from %s" % (name)) +def handle(name, _cfg, _cloud, log, _args): + log.debug("Hi from transform %s", name) diff --git a/cloudinit/transforms/cc_grub_dpkg.py b/cloudinit/transforms/cc_grub_dpkg.py index c048d5cc..02f05ce3 100644 --- a/cloudinit/transforms/cc_grub_dpkg.py +++ b/cloudinit/transforms/cc_grub_dpkg.py @@ -54,9 +54,9 @@ def handle(_name, cfg, _cloud, log, _args): # now idevs and idevs_empty are set to determined values # or, those set by user - dconf_sel = ("grub-pc grub-pc/install_devices string %s\n" + dconf_sel = (("grub-pc grub-pc/install_devices string %s\n" "grub-pc grub-pc/install_devices_empty boolean %s\n") % - (idevs, idevs_empty) + (idevs, idevs_empty)) log.debug("Setting grub debconf-set-selections with '%s','%s'" % (idevs, idevs_empty)) diff --git a/cloudinit/transforms/cc_keys_to_console.py b/cloudinit/transforms/cc_keys_to_console.py index 2f2a5297..e974375f 100644 --- a/cloudinit/transforms/cc_keys_to_console.py +++ b/cloudinit/transforms/cc_keys_to_console.py @@ -18,23 +18,34 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . +import os + from cloudinit.settings import PER_INSTANCE from cloudinit import util frequency = PER_INSTANCE +# This is a tool that cloud init provides +helper_tool = '/usr/lib/cloud-init/write-ssh-key-fingerprints' + -def handle(_name, cfg, _cloud, log, _args): - cmd = ['/usr/lib/cloud-init/write-ssh-key-fingerprints'] - fp_blacklist = util.get_cfg_option_list_or_str(cfg, +def handle(name, cfg, _cloud, log, _args): + if not os.path.exists(helper_tool): + log.warn(("Unable to activate transform %s," + " helper tool not found at %s"), name, helper_tool) + return + + fp_blacklist = util.get_cfg_option_list(cfg, "ssh_fp_console_blacklist", []) - key_blacklist = util.get_cfg_option_list_or_str(cfg, + key_blacklist = util.get_cfg_option_list(cfg, "ssh_key_console_blacklist", ["ssh-dss"]) + try: + cmd = [helper_tool] cmd.append(','.join(fp_blacklist)) cmd.append(','.join(key_blacklist)) - (stdout, stderr) = util.subp(cmd) + (stdout, _stderr) = util.subp(cmd) util.write_file('/dev/console', stdout) except: - log.warn("Writing keys to console failed!") + log.warn("Writing keys to /dev/console failed!") raise diff --git a/cloudinit/transforms/cc_landscape.py b/cloudinit/transforms/cc_landscape.py index 48491992..19948d0e 100644 --- a/cloudinit/transforms/cc_landscape.py +++ b/cloudinit/transforms/cc_landscape.py @@ -55,7 +55,8 @@ def handle(name, cfg, _cloud, log, _args): /etc/landscape/client.conf """ if not ConfigObj: - log.warn("'ConfigObj' support not enabled, running %s disabled", name) + log.warn(("'ConfigObj' support not available," + " running transform %s disabled"), name) return ls_cloudcfg = cfg.get("landscape", {}) diff --git a/cloudinit/transforms/cc_locale.py b/cloudinit/transforms/cc_locale.py index 3fb4c5d9..7f273123 100644 --- a/cloudinit/transforms/cc_locale.py +++ b/cloudinit/transforms/cc_locale.py @@ -49,7 +49,7 @@ def handle(name, cfg, cloud, log, args): "/etc/default/locale") if not locale: - log.debug(("Skipping module named %s, " + log.debug(("Skipping transform named %s, " "no 'locale' configuration found"), name) return diff --git a/cloudinit/transforms/cc_mcollective.py b/cloudinit/transforms/cc_mcollective.py index aeeda9d2..5464fe8c 100644 --- a/cloudinit/transforms/cc_mcollective.py +++ b/cloudinit/transforms/cc_mcollective.py @@ -19,13 +19,10 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -from ConfigParser import ConfigParser from StringIO import StringIO -import os - +from cloudinit import cfg as config from cloudinit import util -from cloudinit import cfg pubcert_file = "/etc/mcollective/ssl/server-public.pem" pricert_file = "/etc/mcollective/ssl/server-private.pem" @@ -35,7 +32,7 @@ def handle(name, cfg, cloud, log, _args): # If there isn't a mcollective key in the configuration don't do anything if 'mcollective' not in cfg: - log.debug(("Skipping module named %s, " + log.debug(("Skipping transform named %s, " "no 'mcollective' key in configuration"), name) return @@ -47,7 +44,7 @@ def handle(name, cfg, cloud, log, _args): # ... and then update the mcollective configuration if 'conf' in mcollective_cfg: # Create object for reading server.cfg values - mcollective_config = cfg.DefaultingConfigParser() + mcollective_config = config.DefaultingConfigParser() # Read server.cfg values from original file in order to be able to mix # the rest up old_contents = util.load_file('/etc/mcollective/server.cfg') diff --git a/cloudinit/transforms/cc_mounts.py b/cloudinit/transforms/cc_mounts.py index babcbda1..44182b87 100644 --- a/cloudinit/transforms/cc_mounts.py +++ b/cloudinit/transforms/cc_mounts.py @@ -20,7 +20,6 @@ from string import whitespace # pylint: disable=W0402 -import os import re from cloudinit import util @@ -28,7 +27,7 @@ from cloudinit import util # shortname matches 'sda', 'sda1', 'xvda', 'hda', 'sdb', xvdb, vda, vdd1 shortname_filter = r"^[x]{0,1}[shv]d[a-z][0-9]*$" shortname = re.compile(shortname_filter) -ws = re.compile("[%s]+" % whitespace) +ws = re.compile("[%s]+" % (whitespace)) def is_mdname(name): @@ -65,13 +64,14 @@ def handle(_name, cfg, cloud, log, _args): continue startname = str(cfgmnt[i][0]) - LOG.debug("Attempting to determine the real name of %s", startname) + log.debug("Attempting to determine the real name of %s", startname) # workaround, allow user to specify 'ephemeral' # rather than more ec2 correct 'ephemeral0' if startname == "ephemeral": cfgmnt[i][0] = "ephemeral0" - log.debug("Adjusted mount option %s name from ephemeral to ephemeral0", (i + 1)) + log.debug(("Adjusted mount option %s " + "name from ephemeral to ephemeral0"), (i + 1)) if is_mdname(startname): newname = cloud.device_name_to_device(startname) @@ -136,7 +136,8 @@ def handle(_name, cfg, cloud, log, _args): break if cfgmnt_has: - log.debug("Not including %s, already previously included", startname) + log.debug(("Not including %s, already" + " previously included"), startname) continue cfgmnt.append(defmnt) @@ -159,7 +160,7 @@ def handle(_name, cfg, cloud, log, _args): dirs = [] for line in actlist: # write 'comment' in the fs_mntops, entry, claiming this - line[3] = "%s,comment=cloudconfig" % line[3] + line[3] = "%s,%s" % (line[3], comment) if line[2] == "swap": needswap = True if line[1].startswith("/"): @@ -168,7 +169,7 @@ def handle(_name, cfg, cloud, log, _args): fstab_lines = [] fstab = util.load_file("/etc/fstab") - for line in fstab.read().splitlines(): + for line in fstab.splitlines(): try: toks = ws.split(line) if toks[3].find(comment) != -1: diff --git a/cloudinit/transforms/cc_phone_home.py b/cloudinit/transforms/cc_phone_home.py index 36af6dfa..98ff2b85 100644 --- a/cloudinit/transforms/cc_phone_home.py +++ b/cloudinit/transforms/cc_phone_home.py @@ -24,9 +24,8 @@ from cloudinit import util from cloudinit.settings import PER_INSTANCE -from time import sleep - frequency = PER_INSTANCE + post_list_all = ['pub_key_dsa', 'pub_key_rsa', 'pub_key_ecdsa', 'instance_id', 'hostname'] @@ -49,7 +48,7 @@ def handle(name, cfg, cloud, log, args): ph_cfg = cfg['phone_home'] if 'url' not in ph_cfg: - log.warn(("Skipping module named %s, " + log.warn(("Skipping transform named %s, " "no 'url' found in 'phone_home' configuration"), name) return @@ -60,7 +59,8 @@ def handle(name, cfg, cloud, log, args): tries = int(tries) except: tries = 10 - util.logexc(log, "Configuration entry 'tries' is not an integer, using %s", tries) + util.logexc(log, ("Configuration entry 'tries'" + " is not an integer, using %s instead"), tries) if post_list == "all": post_list = post_list_all @@ -75,23 +75,37 @@ def handle(name, cfg, cloud, log, args): 'pub_key_ecdsa': '/etc/ssh/ssh_host_ecdsa_key.pub', } - for n, path in pubkeys.iteritems(): + for (n, path) in pubkeys.iteritems(): try: all_keys[n] = util.load_file(path) except: - util.logexc(log, "%s: failed to open, can not phone home that data", path) + util.logexc(log, ("%s: failed to open, can not" + " phone home that data"), path) submit_keys = {} for k in post_list: if k in all_keys: submit_keys[k] = all_keys[k] else: - submit_keys[k] = "N/A" - log.warn("Requested key %s from 'post' configuration list not available", k) + submit_keys[k] = None + log.warn(("Requested key %s from 'post'" + " configuration list not available"), k) - url = templater.render_string(url, {'INSTANCE_ID': all_keys['instance_id']}) + # Get them read to be posted + real_submit_keys = {} + for (k, v) in submit_keys.iteritems(): + if v is None: + real_submit_keys[k] = 'N/A' + else: + real_submit_keys[k] = str(v) + # Incase the url is parameterized + url_params = { + 'INSTANCE_ID': all_keys['instance_id'], + } + url = templater.render_string(url, url_params) try: - uhelp.readurl(url, data=submit_keys, retries=tries, sec_between=3) + uhelp.readurl(url, data=real_submit_keys, retries=tries, sec_between=3) except: - util.logexc(log, "Failed to post phone home data to %s in %s tries", url, tries) + util.logexc(log, ("Failed to post phone home data to" + " %s in %s tries"), url, tries) diff --git a/cloudinit/transforms/cc_puppet.py b/cloudinit/transforms/cc_puppet.py index 0a21a929..76cc9732 100644 --- a/cloudinit/transforms/cc_puppet.py +++ b/cloudinit/transforms/cc_puppet.py @@ -24,31 +24,32 @@ import os import pwd import socket +from cloudinit import cfg as config from cloudinit import util -from cloudinit import cfg def handle(name, cfg, cloud, log, _args): # If there isn't a puppet key in the configuration don't do anything if 'puppet' not in cfg: - log.debug(("Skipping module named %s," + log.debug(("Skipping transform named %s," " no 'puppet' configuration found"), name) return puppet_cfg = cfg['puppet'] # Start by installing the puppet package ... - cloud.distro.install_packages(("puppet",)) + cloud.distro.install_packages(["puppet"]) # ... and then update the puppet configuration if 'conf' in puppet_cfg: # Add all sections from the conf object to puppet.conf contents = util.load_file('/etc/puppet/puppet.conf') # Create object for reading puppet.conf values - puppet_config = cfg.DefaultingConfigParser() + puppet_config = config.DefaultingConfigParser() # Read puppet.conf values from original file in order to be able to # mix the rest up. First clean them up (TODO is this really needed??) - cleaned_contents = '\n'.join([i.lstrip() for i in contents.splitlines()]) + cleaned_lines = [i.lstrip() for i in contents.splitlines()] + cleaned_contents = '\n'.join(cleaned_lines) puppet_config.readfp(StringIO(cleaned_contents), filename='/etc/puppet/puppet.conf') for (cfg_name, cfg) in puppet_cfg['conf'].iteritems(): @@ -81,7 +82,8 @@ def handle(name, cfg, cloud, log, _args): puppet_config.set(cfg_name, o, v) # We got all our config as wanted we'll rename # the previous puppet.conf and create our new one - util.rename('/etc/puppet/puppet.conf', '/etc/puppet/puppet.conf.old') + util.rename('/etc/puppet/puppet.conf', + '/etc/puppet/puppet.conf.old') contents = puppet_config.stringify() util.write_file('/etc/puppet/puppet.conf', contents) @@ -91,7 +93,8 @@ def handle(name, cfg, cloud, log, _args): '-e', 's/^START=.*/START=yes/', '/etc/default/puppet'], capture=False) elif os.path.exists('/bin/systemctl'): - util.subp(['/bin/systemctl', 'enable', 'puppet.service'], capture=False) + util.subp(['/bin/systemctl', 'enable', 'puppet.service'], + capture=False) elif os.path.exists('/sbin/chkconfig'): util.subp(['/sbin/chkconfig', 'puppet', 'on'], capture=False) else: diff --git a/cloudinit/transforms/cc_resizefs.py b/cloudinit/transforms/cc_resizefs.py index daaf4da9..fe012417 100644 --- a/cloudinit/transforms/cc_resizefs.py +++ b/cloudinit/transforms/cc_resizefs.py @@ -18,11 +18,8 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -import errno import os import stat -import sys -import tempfile import time from cloudinit import util @@ -46,15 +43,18 @@ def nodeify_path(devpth, where, log): if util.is_container(): log.debug("Inside container, ignoring mknod failure in resizefs") return - log.warn("Failed to make device node to resize %s at %s", where, devpth) + log.warn("Failed to make device node to resize %s at %s", + where, devpth) raise def get_fs_type(st_dev, path, log): try: - fs_type = util.find_devs_with(tag='TYPE', oformat='value', + dev_entries = util.find_devs_with(tag='TYPE', oformat='value', no_cache=True, path=path) - return fs_type + if not dev_entries: + return None + return dev_entries[0].strip() except util.ProcessExecutionError: util.logexc(log, ("Failed to get filesystem type" " of maj=%s, min=%s for path %s"), @@ -69,12 +69,16 @@ def handle(name, cfg, _cloud, log, args): resize_root = util.get_cfg_option_str(cfg, "resize_rootfs", True) if not util.translate_bool(resize_root): - log.debug("Skipping module named %s, resizing disabled", name) + log.debug("Skipping transform named %s, resizing disabled", name) return # TODO is the directory ok to be used?? resize_root_d = util.get_cfg_option_str(cfg, "resize_rootfs_tmp", "/run") util.ensure_dir(resize_root_d) + + # TODO: allow what is to be resized to + # be configurable?? + resize_what = "/" with util.SilentTemporaryFile(prefix="cloudinit.resizefs.", dir=resize_root_d, delete=True) as tfh: devpth = tfh.name @@ -86,23 +90,25 @@ def handle(name, cfg, _cloud, log, args): # auto deletion tfh.unlink_now() - # TODO: allow what is to be resized to - # be configurable?? - st_dev = nodeify_path(devpth, "/", log) - fs_type = get_fs_type(st_dev, devpath, log) + st_dev = nodeify_path(devpth, resize_what, log) + fs_type = get_fs_type(st_dev, devpth, log) + if not fs_type: + log.warn("Could not determine filesystem type of %s", resize_what) + return resizer = None - fstype_lc = fstype.lower() + fstype_lc = fs_type.lower() for (pfix, root_cmd) in resize_fs_prefixes_cmds: if fstype_lc.startswith(pfix): resizer = root_cmd break if not resizer: - log.warn("Not resizing unknown filesystem type %s", fs_type) + log.warn("Not resizing unknown filesystem type %s for %s", + fs_type, resize_what) return - log.debug("Resizing using %s", resizer) + log.debug("Resizing %s (%s) using %s", resize_what, fs_type, resizer) resize_cmd = [resizer, devpth] if resize_root == "noblock": @@ -125,8 +131,8 @@ def do_resize(resize_cmd, log): start = time.time() try: util.subp(resize_cmd) - except util.ProcessExecutionError as e: - util.logexc(log, "Failed to resize filesystem (using %s)", resize_cmd) + except util.ProcessExecutionError: + util.logexc(log, "Failed to resize filesystem (cmd=%s)", resize_cmd) raise tot_time = int(time.time() - start) log.debug("Resizing took %s seconds", tot_time) diff --git a/cloudinit/transforms/cc_rightscale_userdata.py b/cloudinit/transforms/cc_rightscale_userdata.py index cde11b54..40d76c89 100644 --- a/cloudinit/transforms/cc_rightscale_userdata.py +++ b/cloudinit/transforms/cc_rightscale_userdata.py @@ -53,16 +53,19 @@ def handle(name, _cfg, cloud, log, _args): try: ud = cloud.get_userdata_raw() except: - log.warn("Failed to get raw userdata in module %s", name) + log.warn("Failed to get raw userdata in transform %s", name) return try: mdict = parse_qs(ud) if not mdict or not my_hookname in mdict: - log.debug("Skipping module %s, did not find %s in parsed raw userdata", name, my_hookname) + log.debug(("Skipping transform %s, " + "did not find %s in parsed" + " raw userdata"), name, my_hookname) return except: - log.warn("Failed to parse query string %s into a dictionary", ud) + util.logexc(log, ("Failed to parse query string %s" + " into a dictionary"), ud) raise wrote_fns = [] @@ -83,7 +86,8 @@ def handle(name, _cfg, cloud, log, _args): wrote_fns.append(fname) except Exception as e: captured_excps.append(e) - util.logexc(log, "%s failed to read %s and write %s", my_name, url, fname) + util.logexc(log, "%s failed to read %s and write %s", + my_name, url, fname) if wrote_fns: log.debug("Wrote out rightscale userdata to %s files", len(wrote_fns)) @@ -93,6 +97,6 @@ def handle(name, _cfg, cloud, log, _args): log.debug("%s urls were skipped or failed", skipped) if captured_excps: - log.warn("%s failed with exceptions, re-raising the last one", len(captured_excps)) + log.warn("%s failed with exceptions, re-raising the last one", + len(captured_excps)) raise captured_excps[-1] - diff --git a/cloudinit/transforms/cc_rsyslog.py b/cloudinit/transforms/cc_rsyslog.py index ccbe68ff..71b74711 100644 --- a/cloudinit/transforms/cc_rsyslog.py +++ b/cloudinit/transforms/cc_rsyslog.py @@ -36,7 +36,8 @@ def handle(name, cfg, cloud, log, _args): # process 'rsyslog' if not 'rsyslog' in cfg: - log.debug("Skipping module named %s, no 'rsyslog' key in configuration", name) + log.debug(("Skipping transform named %s," + " no 'rsyslog' key in configuration"), name) return def_dir = cfg.get('rsyslog_dir', DEF_DIR) @@ -62,15 +63,16 @@ def handle(name, cfg, cloud, log, _args): if not filename.startswith("/"): filename = os.path.join(def_dir, filename) + # Truncate filename first time you see it omode = "ab" - # truncate filename first time you see it if filename not in files: omode = "wb" files.append(filename) try: - util.write_file(filename, content + "\n", omode=omode) - except Exception as e: + contents = "%s\n" % (content) + util.write_file(filename, contents, omode=omode) + except Exception: util.logexc(log, "Failed to write to %s", filename) # Attempt to restart syslogd @@ -87,8 +89,8 @@ def handle(name, cfg, cloud, log, _args): log.debug("Restarting rsyslog") util.subp(['service', 'rsyslog', 'restart']) restarted = True - except Exception as e: - util.logexc("Failed restarting rsyslog") + except Exception: + util.logexc(log, "Failed restarting rsyslog") if restarted: # This only needs to run if we *actually* restarted diff --git a/cloudinit/transforms/cc_runcmd.py b/cloudinit/transforms/cc_runcmd.py index 19c0e721..31a254a5 100644 --- a/cloudinit/transforms/cc_runcmd.py +++ b/cloudinit/transforms/cc_runcmd.py @@ -25,13 +25,14 @@ from cloudinit import util def handle(name, cfg, cloud, log, _args): if "runcmd" not in cfg: - log.debug("Skipping module named %s, no 'runcmd' key in configuration", name) + log.debug(("Skipping transform named %s," + " no 'runcmd' key in configuration"), name) return - outfile = os.path.join(cloud.get_ipath('scripts'), "runcmd") + out_fn = os.path.join(cloud.get_ipath('scripts'), "runcmd") cmd = cfg["runcmd"] try: content = util.shellify(cmd) - util.write_file(outfile, content, 0700) + util.write_file(out_fn, content, 0700) except: - util.logexc(log, "Failed to shellify %s into file %s", cmd, outfile) + util.logexc(log, "Failed to shellify %s into file %s", cmd, out_fn) diff --git a/cloudinit/transforms/cc_salt_minion.py b/cloudinit/transforms/cc_salt_minion.py index 47cbc194..d05d2a1e 100644 --- a/cloudinit/transforms/cc_salt_minion.py +++ b/cloudinit/transforms/cc_salt_minion.py @@ -21,16 +21,17 @@ from cloudinit import util # Note: see http://saltstack.org/topics/installation/ -def handle(name, cfg, cloud, _log, _args): +def handle(name, cfg, cloud, log, _args): # If there isn't a salt key in the configuration don't do anything if 'salt_minion' not in cfg: - log.debug("Skipping module named %s, no 'salt_minion' key in configuration", name) + log.debug(("Skipping transform named %s," + " no 'salt_minion' key in configuration"), name) return salt_cfg = cfg['salt_minion'] # Start by installing the salt package ... - cloud.distro.install_packages(("salt",)) + cloud.distro.install_packages(["salt"]) # Ensure we can configure files at the right dir config_dir = salt_cfg.get("config_dir", '/etc/salt') diff --git a/cloudinit/transforms/cc_scripts_per_boot.py b/cloudinit/transforms/cc_scripts_per_boot.py index bcdf4400..364e1d02 100644 --- a/cloudinit/transforms/cc_scripts_per_boot.py +++ b/cloudinit/transforms/cc_scripts_per_boot.py @@ -29,12 +29,13 @@ frequency = PER_ALWAYS script_subdir = 'per-boot' -def handle(_name, _cfg, cloud, log, _args): +def handle(name, _cfg, cloud, log, _args): # Comes from the following: # https://forums.aws.amazon.com/thread.jspa?threadID=96918 runparts_path = os.path.join(cloud.get_cpath(), 'scripts', script_subdir) try: util.runparts(runparts_path) except: - log.warn("Failed to run-parts(%s) in %s", script_subdir, runparts_path) + log.warn("Failed to run transform %s (%s in %s)", + name, script_subdir, runparts_path) raise diff --git a/cloudinit/transforms/cc_scripts_per_instance.py b/cloudinit/transforms/cc_scripts_per_instance.py index 8d6609a1..d75ab47d 100644 --- a/cloudinit/transforms/cc_scripts_per_instance.py +++ b/cloudinit/transforms/cc_scripts_per_instance.py @@ -29,12 +29,13 @@ frequency = PER_INSTANCE script_subdir = 'per-instance' -def handle(_name, _cfg, cloud, log, _args): +def handle(name, _cfg, cloud, log, _args): # Comes from the following: # https://forums.aws.amazon.com/thread.jspa?threadID=96918 runparts_path = os.path.join(cloud.get_cpath(), 'scripts', script_subdir) try: util.runparts(runparts_path) except: - log.warn("Failed to run-parts(%s) in %s", script_subdir, runparts_path) + log.warn("Failed to run transform %s (%s in %s)", + name, script_subdir, runparts_path) raise diff --git a/cloudinit/transforms/cc_scripts_per_once.py b/cloudinit/transforms/cc_scripts_per_once.py index dbcec05d..80f8c325 100644 --- a/cloudinit/transforms/cc_scripts_per_once.py +++ b/cloudinit/transforms/cc_scripts_per_once.py @@ -29,12 +29,13 @@ frequency = PER_ONCE script_subdir = 'per-once' -def handle(_name, _cfg, cloud, log, _args): +def handle(name, _cfg, cloud, log, _args): # Comes from the following: # https://forums.aws.amazon.com/thread.jspa?threadID=96918 runparts_path = os.path.join(cloud.get_cpath(), 'scripts', script_subdir) try: util.runparts(runparts_path) except: - log.warn("Failed to run-parts(%s) in %s", script_subdir, runparts_path) + log.warn("Failed to run transform %s (%s in %s)", + name, script_subdir, runparts_path) raise diff --git a/cloudinit/transforms/cc_scripts_user.py b/cloudinit/transforms/cc_scripts_user.py index 1e438ee6..f4fe3a2a 100644 --- a/cloudinit/transforms/cc_scripts_user.py +++ b/cloudinit/transforms/cc_scripts_user.py @@ -26,14 +26,17 @@ from cloudinit.settings import PER_INSTANCE frequency = PER_INSTANCE +script_subdir = 'scripts' -def handle(_name, _cfg, cloud, log, _args): + +def handle(name, _cfg, cloud, log, _args): # This is written to by the user data handlers # Ie, any custom shell scripts that come down # go here... - runparts_path = os.path.join(cloud.get_ipath_cur(), "scripts") + runparts_path = os.path.join(cloud.get_ipath_cur(), script_subdir) try: util.runparts(runparts_path) except: - log.warn("Failed to run-parts(%s) in %s", "user-data", runparts_path) + log.warn("Failed to run transform %s (%s in %s)", + name, script_subdir, runparts_path) raise diff --git a/cloudinit/transforms/cc_set_hostname.py b/cloudinit/transforms/cc_set_hostname.py index fa2b59c2..3ac8a8fa 100644 --- a/cloudinit/transforms/cc_set_hostname.py +++ b/cloudinit/transforms/cc_set_hostname.py @@ -24,7 +24,7 @@ from cloudinit import util def handle(name, cfg, cloud, log, _args): if util.get_cfg_option_bool(cfg, "preserve_hostname", False): log.debug(("Configuration option 'preserve_hostname' is set," - " not setting the hostname in %s"), name) + " not setting the hostname in transform %s"), name) return (hostname, _fqdn) = util.get_hostname_fqdn(cfg, cloud) diff --git a/cloudinit/transforms/cc_set_passwords.py b/cloudinit/transforms/cc_set_passwords.py index 4f2cdb97..c0cc4e84 100644 --- a/cloudinit/transforms/cc_set_passwords.py +++ b/cloudinit/transforms/cc_set_passwords.py @@ -22,7 +22,7 @@ import sys from cloudinit import util -from string import letters, digits +from string import letters, digits # pylint: disable=W0402 # We are removing certain 'painful' letters/numbers pw_set = (letters.translate(None, 'loLOI') + @@ -71,11 +71,13 @@ def handle(_name, cfg, cloud, log, args): util.subp(['chpasswd'], ch_in) except Exception as e: errors.append(e) - util.logexc(log, "Failed to set passwords with chpasswd for %s", users) + util.logexc(log, + "Failed to set passwords with chpasswd for %s", users) if len(randlist): - sys.stderr.write("%s\n%s\n" % ("Set the following 'random' passwords\n", - '\n'.join(randlist))) + blurb = ("Set the following 'random' passwords\n", + '\n'.join(randlist)) + sys.stderr.write("%s\n%s\n" % blurb) if expire: expired_users = [] diff --git a/cloudinit/transforms/cc_ssh.py b/cloudinit/transforms/cc_ssh.py index db6848d9..3c2b3622 100644 --- a/cloudinit/transforms/cc_ssh.py +++ b/cloudinit/transforms/cc_ssh.py @@ -65,8 +65,7 @@ def handle(_name, cfg, cloud, log, _args): tgt_fn = key2file[key][0] tgt_perms = key2file[key][1] util.write_file(tgt_fn, val, tgt_perms) - - cmd = 'o=$(ssh-keygen -yf "%s") && echo "$o" root@localhost > "%s"' + for priv, pub in priv2pub.iteritems(): if pub in cfg['ssh_keys'] or not priv in cfg['ssh_keys']: continue @@ -78,11 +77,15 @@ def handle(_name, cfg, cloud, log, _args): util.subp(cmd, capture=False) log.debug("Generated a key for %s from %s", pair[0], pair[1]) except: - util.logexc(log, "Failed generated a key for %s from %s", pair[0], pair[1]) + util.logexc(log, ("Failed generated a key" + " for %s from %s"), pair[0], pair[1]) else: # if not, generate them - for keytype in util.get_cfg_option_list_or_str(cfg, 'ssh_genkeytypes', generate_keys): - keyfile = '/etc/ssh/ssh_host_%s_key' % keytype + genkeys = util.get_cfg_option_list(cfg, + 'ssh_genkeytypes', + generate_keys) + for keytype in genkeys: + keyfile = '/etc/ssh/ssh_host_%s_key' % (keytype) if not os.path.exists(keyfile): cmd = ['ssh-keygen', '-t', keytype, '-N', '', '-f', keyfile] try: @@ -90,26 +93,27 @@ def handle(_name, cfg, cloud, log, _args): with util.SeLinuxGuard("/etc/ssh", recursive=True): util.subp(cmd, capture=False) except: - util.logexc(log, "Failed generating key type %s to file %s", keytype, keyfile) + util.logexc(log, ("Failed generating key type" + " %s to file %s"), keytype, keyfile) try: user = util.get_cfg_option_str(cfg, 'user') disable_root = util.get_cfg_option_bool(cfg, "disable_root", True) disable_root_opts = util.get_cfg_option_str(cfg, "disable_root_opts", - DISABLE_ROOT_OPTS) + DISABLE_ROOT_OPTS) keys = cloud.get_public_ssh_keys() or [] if "ssh_authorized_keys" in cfg: cfgkeys = cfg["ssh_authorized_keys"] keys.extend(cfgkeys) - apply_credentials(keys, user, disable_root, disable_root_opts, log) + apply_credentials(keys, user, disable_root, disable_root_opts) except: util.logexc(log, "Applying ssh credentials failed!") def apply_credentials(keys, user, disable_root, - disable_root_opts=DISABLE_ROOT_OPTS, log=None): + disable_root_opts=DISABLE_ROOT_OPTS): keys = set(keys) if user: diff --git a/cloudinit/transforms/cc_ssh_import_id.py b/cloudinit/transforms/cc_ssh_import_id.py index 019413d4..d57e4665 100644 --- a/cloudinit/transforms/cc_ssh_import_id.py +++ b/cloudinit/transforms/cc_ssh_import_id.py @@ -33,10 +33,14 @@ def handle(name, cfg, _cloud, log, args): ids = args[1:] else: user = util.get_cfg_option_str(cfg, "user", "ubuntu") - ids = util.get_cfg_option_list_or_str(cfg, "ssh_import_id", []) + ids = util.get_cfg_option_list(cfg, "ssh_import_id", []) if len(ids) == 0: - log.debug("Skipping module named %s, no ids found to import", name) + log.debug("Skipping transform named %s, no ids found to import", name) + return + + if not user: + log.debug("Skipping transform named %s, no user found to import", name) return cmd = ["sudo", "-Hu", user, "ssh-import-id"] + ids diff --git a/cloudinit/transforms/cc_timezone.py b/cloudinit/transforms/cc_timezone.py index 02cbf2dc..747c436c 100644 --- a/cloudinit/transforms/cc_timezone.py +++ b/cloudinit/transforms/cc_timezone.py @@ -18,20 +18,22 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . +from cloudinit import util + from cloudinit.settings import PER_INSTANCE frequency = PER_INSTANCE -def handle(_name, cfg, cloud, log, args): +def handle(name, cfg, cloud, log, args): if len(args) != 0: timezone = args[0] else: timezone = util.get_cfg_option_str(cfg, "timezone", False) if not timezone: - log.debug("Skipping module named %s, no 'timezone' specified", name) + log.debug("Skipping transform named %s, no 'timezone' specified", name) return - + # Let the distro handle settings its timezone cloud.distro.set_timezone(timezone) diff --git a/cloudinit/transforms/cc_update_etc_hosts.py b/cloudinit/transforms/cc_update_etc_hosts.py index 361097a6..d0e56183 100644 --- a/cloudinit/transforms/cc_update_etc_hosts.py +++ b/cloudinit/transforms/cc_update_etc_hosts.py @@ -30,22 +30,30 @@ def handle(name, cfg, cloud, log, _args): manage_hosts = util.get_cfg_option_str(cfg, "manage_etc_hosts", False) if util.translate_bool(manage_hosts, addons=['template']): (hostname, fqdn) = util.get_hostname_fqdn(cfg, cloud) - # Render from template file if not hostname: - log.warn("Option 'manage_etc_hosts' was set, but no hostname was found") + log.warn(("Option 'manage_etc_hosts' was set," + " but no hostname was found")) return - tpl_fn_name = cloud.get_template_filename("hosts.%s" % (cloud.distro.name())) + + # Render from a template file + distro_n = cloud.distro.name + tpl_fn_name = cloud.get_template_filename("hosts.%s" % (distro_n)) if not tpl_fn_name: - raise Exception("No hosts template could be found for distro %s" % (cloud.distro.name())) + raise Exception(("No hosts template could be" + " found for distro %s") % (distro_n)) + templater.render_to_file(tpl_fn_name, '/etc/hosts', {'hostname': hostname, 'fqdn': fqdn}) + elif manage_hosts == "localhost": - log.debug("Managing localhost in /etc/hosts") (hostname, fqdn) = util.get_hostname_fqdn(cfg, cloud) if not hostname: - log.warn("Option 'manage_etc_hosts' was set, but no hostname was found") + log.warn(("Option 'manage_etc_hosts' was set," + " but no hostname was found")) return + + log.debug("Managing localhost in /etc/hosts") cloud.distro.update_etc_hosts(hostname, fqdn) else: log.debug(("Configuration option 'manage_etc_hosts' is not set," - " not managing /etc/hosts in %s"), name) + " not managing /etc/hosts in transform %s"), name) diff --git a/cloudinit/transforms/cc_update_hostname.py b/cloudinit/transforms/cc_update_hostname.py index 439bdcb3..58444fab 100644 --- a/cloudinit/transforms/cc_update_hostname.py +++ b/cloudinit/transforms/cc_update_hostname.py @@ -18,6 +18,8 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . +import os + from cloudinit import util from cloudinit.settings import PER_ALWAYS @@ -27,7 +29,7 @@ frequency = PER_ALWAYS def handle(name, cfg, cloud, log, _args): if util.get_cfg_option_bool(cfg, "preserve_hostname", False): log.debug(("Configuration option 'preserve_hostname' is set," - " not updating the hostname in %s"), name) + " not updating the hostname in transform %s"), name) return (hostname, _fqdn) = util.get_hostname_fqdn(cfg, cloud) diff --git a/cloudinit/transforms/cc_welcome.py b/cloudinit/transforms/cc_welcome.py index 0db71125..04691d21 100644 --- a/cloudinit/transforms/cc_welcome.py +++ b/cloudinit/transforms/cc_welcome.py @@ -35,9 +35,9 @@ welcome_message_def = ("Cloud-init v. {{version}} starting stage {{stage}} at " frequency = PER_ALWAYS -def handle(name, cfg, cloud, log, args): +def handle(_name, cfg, cloud, log, args): - welcome_msg = util.get_cfg_option_str(cfg, "welcome_msg"): + welcome_msg = util.get_cfg_option_str(cfg, "welcome_msg") if not welcome_msg: tpl_fn = cloud.get_template_filename("welcome_msg") if tpl_fn: @@ -54,7 +54,7 @@ def handle(name, cfg, cloud, log, args): 'stage': stage, 'version': version.version_string(), 'uptime': util.uptime(), - 'timestamp', util.time_rfc2822(), + 'timestamp': util.time_rfc2822(), } try: contents = templater.render_string(welcome_msg, tpl_params) diff --git a/cloudinit/user_data.py b/cloudinit/user_data.py index 64fc2734..9915b8b0 100644 --- a/cloudinit/user_data.py +++ b/cloudinit/user_data.py @@ -22,14 +22,15 @@ import os -import glob import email - +from email.mime.multipart import MIMEMultipart +from email.mime.text import MIMEText from email.mime.base import MIMEBase from cloudinit import importer from cloudinit import log as logging +from cloudinit import url_helper from cloudinit import util from cloudinit.settings import (PER_ALWAYS, PER_INSTANCE, FREQUENCIES) @@ -86,7 +87,7 @@ class UserDataProcessor(object): self.paths = paths def process(self, blob): - base_msg = ud.convert_string(blob) + base_msg = convert_string(blob) process_msg = MIMEMultipart() self._process_msg(base_msg, process_msg) return process_msg @@ -105,7 +106,7 @@ class UserDataProcessor(object): ctype_orig = UNDEF_TYPE if ctype_orig in TYPE_NEEDED: - ctype = ud.type_from_starts_with(payload) + ctype = type_from_starts_with(payload) if ctype is None: ctype = ctype_orig @@ -158,7 +159,7 @@ class UserDataProcessor(object): if not url_helper.ok_http_code(st): content = '' - new_msg = ud.convert_string(content) + new_msg = convert_string(content) self._process_msg(new_msg, append_msg) def _explode_archive(self, archive, append_msg): @@ -179,7 +180,7 @@ class UserDataProcessor(object): content = ent.get('content', '') mtype = ent.get('type') if not mtype: - mtype = ud.type_from_starts_with(content, ARCHIVE_UNDEF_TYPE) + mtype = type_from_starts_with(content, ARCHIVE_UNDEF_TYPE) maintype, subtype = mtype.split('/', 1) if maintype == "text": diff --git a/cloudinit/util.py b/cloudinit/util.py index 7259d933..1f884df8 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -22,8 +22,8 @@ from StringIO import StringIO +import copy as obj_copy import contextlib -import copy import errno import glob import grp @@ -35,12 +35,11 @@ import pwd import random import shutil import socket -import string +import string # pylint: disable=W0402 import subprocess import sys import tempfile import time -import traceback import types import urlparse @@ -171,7 +170,8 @@ def fork_cb(child_cb, *args): child_cb(*args) os._exit(0) # pylint: disable=W0212 except: - logexc(LOG, "Failed forking and calling callback %s", obj_name(child_cb)) + logexc(LOG, ("Failed forking and" + " calling callback %s"), obj_name(child_cb)) os._exit(1) # pylint: disable=W0212 else: LOG.debug("Forked child %s who will run callback %s", @@ -549,10 +549,11 @@ def load_yaml(blob, default=None, allowed=(dict,)): converted = yaml.load(blob) if not isinstance(converted, allowed): # Yes this will just be caught, but thats ok for now... - raise TypeError("Yaml load allows %s root types, but got %s instead" % + raise TypeError(("Yaml load allows %s root types," + " but got %s instead") % (allowed, obj_name(converted))) loaded = converted - except (yaml.YAMLError, TypeError, ValueError) as exc: + except (yaml.YAMLError, TypeError, ValueError): logexc(LOG, "Failed loading yaml blob") return loaded @@ -833,15 +834,12 @@ def find_devs_with(criteria=None, oformat='device', options.append(path) cmd = blk_id_cmd + options (out, _err) = subp(cmd) - if path: - return out.strip() - else: - entries = [] - for line in out.splitlines(): - line = line.strip() - if line: - entries.append(line) - return entries + entries = [] + for line in out.splitlines(): + line = line.strip() + if line: + entries.append(line) + return entries def load_file(fname, read_cb=None, quiet=False): @@ -1109,7 +1107,7 @@ def mount_cb(device, callback, data=None, rw=False, mtype=None): def get_builtin_cfg(): # Deep copy so that others can't modify - return copy.deepcopy(CFG_BUILTIN) + return obj_copy.deepcopy(CFG_BUILTIN) def sym_link(source, link): @@ -1140,16 +1138,14 @@ def time_rfc2822(): def uptime(): + uptime_str = '??' try: - uptimef = load_file("/proc/uptime").strip() - if not uptimef: - uptime = 'na' - else: - uptime = uptimef.split()[0] + contents = load_file("/proc/uptime").strip() + if contents: + uptime_str = contents.split()[0] except: logexc(LOG, "Unable to read uptime from /proc/uptime") - uptime = '??' - return uptime + return uptime_str def ensure_file(path): @@ -1261,7 +1257,8 @@ def shellify(cmdlist, add_header=True): content = "%s%s\n" % (content, args) else: raise RuntimeError(("Unable to shellify type %s" - " which is not a list or string") % (obj_name(args))) + " which is not a list or string") + % (obj_name(args))) LOG.debug("Shellified %s to %s", cmdlist, content) return content @@ -1275,8 +1272,7 @@ def is_container(): try: # try to run a helper program. if it returns true/zero # then we're inside a container. otherwise, no - cmd = [helper] - subp(cmd, allowed_rc=[0]) + subp([helper]) return True except (IOError, OSError): pass -- cgit v1.2.3 From 76f2e0e51dc29b7737ed3aee540749191465ae58 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Sat, 16 Jun 2012 07:34:52 -0700 Subject: Adjust the usage of map items() to iteritems() which will help later translation to python 3. --- cloudinit/sources/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'cloudinit/sources/__init__.py') diff --git a/cloudinit/sources/__init__.py b/cloudinit/sources/__init__.py index beb0f3d7..55900119 100644 --- a/cloudinit/sources/__init__.py +++ b/cloudinit/sources/__init__.py @@ -78,7 +78,7 @@ class DataSource(object): return list(self.metadata['public-keys']) if isinstance(self.metadata['public-keys'], (dict)): - for _keyname, klist in self.metadata['public-keys'].items(): + for (_keyname, klist) in self.metadata['public-keys'].iteritems(): # lp:506332 uec metadata service responds with # data that makes boto populate a string for 'klist' rather # than a list. -- cgit v1.2.3 From a4454ec81a44142db0c51ceb76b979ace4b1176e Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Sat, 16 Jun 2012 09:44:47 -0700 Subject: For the different base classes, ensure we are using the 'abc' module + appropriate annotations + metaclasses. --- cloudinit/sources/__init__.py | 5 +++++ cloudinit/user_data.py | 7 ++++++- 2 files changed, 11 insertions(+), 1 deletion(-) (limited to 'cloudinit/sources/__init__.py') diff --git a/cloudinit/sources/__init__.py b/cloudinit/sources/__init__.py index 55900119..46350255 100644 --- a/cloudinit/sources/__init__.py +++ b/cloudinit/sources/__init__.py @@ -20,6 +20,8 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . +import abc + from cloudinit import importer from cloudinit import log as logging from cloudinit import user_data as ud @@ -37,6 +39,9 @@ class DataSourceNotFoundException(Exception): class DataSource(object): + + __metaclass__ = abc.ABCMeta + def __init__(self, sys_cfg, distro, paths): self.sys_cfg = sys_cfg self.distro = distro diff --git a/cloudinit/user_data.py b/cloudinit/user_data.py index f64a1707..28c9f2ce 100644 --- a/cloudinit/user_data.py +++ b/cloudinit/user_data.py @@ -20,7 +20,7 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . - +import abc import os import email @@ -233,6 +233,9 @@ class UserDataProcessor(object): class PartHandler(object): + + __metaclass__ = abc.ABCMeta + def __init__(self, frequency, version=2): self.handler_version = version self.frequency = frequency @@ -240,12 +243,14 @@ class PartHandler(object): def __repr__(self): return "%s: [%s]" % (util.obj_name(self), self.list_types()) + @abc.abstractmethod def list_types(self): raise NotImplementedError() def handle_part(self, data, ctype, filename, payload, frequency): return self._handle_part(data, ctype, filename, payload, frequency) + @abc.abstractmethod def _handle_part(self, data, ctype, filename, payload, frequency): raise NotImplementedError() -- cgit v1.2.3 From 60f176e13836d77616ae07774fda2139fde790fc Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Sat, 16 Jun 2012 10:57:28 -0700 Subject: Allow the user data processor to be passed in. This could and should be useful for unit testing. --- cloudinit/sources/__init__.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) (limited to 'cloudinit/sources/__init__.py') diff --git a/cloudinit/sources/__init__.py b/cloudinit/sources/__init__.py index 46350255..5842d41b 100644 --- a/cloudinit/sources/__init__.py +++ b/cloudinit/sources/__init__.py @@ -42,7 +42,7 @@ class DataSource(object): __metaclass__ = abc.ABCMeta - def __init__(self, sys_cfg, distro, paths): + def __init__(self, sys_cfg, distro, paths, ud_proc=None): self.sys_cfg = sys_cfg self.distro = distro self.paths = paths @@ -54,11 +54,15 @@ class DataSource(object): name = name[DS_PREFIX:] self.ds_cfg = util.get_cfg_by_path(self.sys_cfg, ("datasource", name), {}) + if not ud_proc: + self.ud_proc = ud.UserDataProcessor(self.paths) + else: + self.ud_proc = ud_proc def get_userdata(self): if self.userdata is None: raw_data = self.get_userdata_raw() - self.userdata = ud.UserDataProcessor(self.paths).process(raw_data) + self.userdata = self.ud_proc.process(raw_data) return self.userdata def get_userdata_raw(self): -- cgit v1.2.3 From 8c9148fda19a83bb6fe9476dff34788ccfc9f639 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Mon, 18 Jun 2012 21:07:17 -0700 Subject: 1. Fixed datasource construction (switched param order) 2. Fixed up importing of modules to handle the failure case better a. Also realized that using the import class we don't have to reimport a module via getattr, so removed that. --- cloudinit/sources/__init__.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) (limited to 'cloudinit/sources/__init__.py') diff --git a/cloudinit/sources/__init__.py b/cloudinit/sources/__init__.py index 5842d41b..831f97ea 100644 --- a/cloudinit/sources/__init__.py +++ b/cloudinit/sources/__init__.py @@ -51,7 +51,7 @@ class DataSource(object): self.userdata_raw = None name = util.obj_name(self) if name.startswith(DS_PREFIX): - name = name[DS_PREFIX:] + name = name[len(DS_PREFIX):] self.ds_cfg = util.get_cfg_by_path(self.sys_cfg, ("datasource", name), {}) if not ud_proc: @@ -171,7 +171,7 @@ def find_source(sys_cfg, distro, paths, ds_deps, cfg_list, pkg_list): for cls in ds_list: ds = util.obj_name(cls) try: - s = cls(distro, sys_cfg, paths) + s = cls(sys_cfg, distro, paths) if s.get_data(): return (s, ds) except Exception: @@ -198,15 +198,13 @@ def list_sources(cfg_list, depends, pkg_list): if pkg: pkg_name.append(str(pkg)) pkg_name.append(ds_name) - mod = importer.import_module(".".join(pkg_name)) - if pkg: - mod = getattr(mod, ds_name, None) - if not mod: + try: + mod = importer.import_module(".".join(pkg_name)) + except RuntimeError: continue lister = getattr(mod, "get_datasource_list", None) if not lister: continue - LOG.debug("Seeing if %s matches using function %s", mod, lister) cls_matches = lister(depends) if not cls_matches: continue -- cgit v1.2.3 From 7eab90c0c88854cc3984ca54d0790dbfe6fd883b Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Tue, 19 Jun 2012 11:06:14 -0700 Subject: Cleanup some of the logging (its a little to verbose). --- cloudinit/sources/__init__.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) (limited to 'cloudinit/sources/__init__.py') diff --git a/cloudinit/sources/__init__.py b/cloudinit/sources/__init__.py index 831f97ea..a227e050 100644 --- a/cloudinit/sources/__init__.py +++ b/cloudinit/sources/__init__.py @@ -169,13 +169,13 @@ def find_source(sys_cfg, distro, paths, ds_deps, cfg_list, pkg_list): LOG.info("Searching for data source in: %s", ds_names) for cls in ds_list: - ds = util.obj_name(cls) try: + LOG.debug("Seeing if we can get any data from %s", cls) s = cls(sys_cfg, distro, paths) if s.get_data(): - return (s, ds) + return (s, util.obj_name(cls)) except Exception: - util.logexc(LOG, "Getting data from %s failed", ds) + util.logexc(LOG, "Getting data from %s failed", cls) msg = "Did not find any data source, searched classes: %s" % (ds_names) raise DataSourceNotFoundException(msg) @@ -188,7 +188,7 @@ def find_source(sys_cfg, distro, paths, ds_deps, cfg_list, pkg_list): def list_sources(cfg_list, depends, pkg_list): src_list = [] LOG.info(("Looking for for data source in: %s," - " %s that matches %s"), cfg_list, pkg_list, depends) + " via packages %s that matches dependencies %s"), cfg_list, pkg_list, depends) for ds_coll in cfg_list: ds_name = str(ds_coll) if not ds_name.startswith(DS_PREFIX): @@ -209,8 +209,8 @@ def list_sources(cfg_list, depends, pkg_list): if not cls_matches: continue src_list.extend(cls_matches) - LOG.debug(("Found a match for data source %s" - " in %s with matches %s"), ds_name, mod, cls_matches) + LOG.debug(("Found a match" + " in %s with matches %s"), mod, cls_matches) break return src_list -- cgit v1.2.3 From e898c7990cbe94d9116dc27d6c051523575c07e0 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Tue, 19 Jun 2012 12:17:06 -0700 Subject: Cleanup no datasource exception to use cleaner csv list. --- cloudinit/sources/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'cloudinit/sources/__init__.py') diff --git a/cloudinit/sources/__init__.py b/cloudinit/sources/__init__.py index a227e050..128d4aad 100644 --- a/cloudinit/sources/__init__.py +++ b/cloudinit/sources/__init__.py @@ -177,7 +177,8 @@ def find_source(sys_cfg, distro, paths, ds_deps, cfg_list, pkg_list): except Exception: util.logexc(LOG, "Getting data from %s failed", cls) - msg = "Did not find any data source, searched classes: %s" % (ds_names) + msg = ("Did not find any data source," + " searched classes: (%s)") % (", ".join(ds_names)) raise DataSourceNotFoundException(msg) -- cgit v1.2.3 From a8241225f408b8df258af0e6ea0820fb6894ea29 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Tue, 19 Jun 2012 15:39:01 -0700 Subject: Fix pylint line length issues. --- cloudinit/sources/__init__.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) (limited to 'cloudinit/sources/__init__.py') diff --git a/cloudinit/sources/__init__.py b/cloudinit/sources/__init__.py index 128d4aad..35fc01df 100644 --- a/cloudinit/sources/__init__.py +++ b/cloudinit/sources/__init__.py @@ -182,14 +182,15 @@ def find_source(sys_cfg, distro, paths, ds_deps, cfg_list, pkg_list): raise DataSourceNotFoundException(msg) -# return a list of classes that have the same depends as 'depends' -# iterate through cfg_list, loading "DataSourceCollections" modules +# Return a list of classes that have the same depends as 'depends' +# iterate through cfg_list, loading "DataSource*" modules # and calling their "get_datasource_list". -# return an ordered list of classes that match +# Return an ordered list of classes that match (if any) def list_sources(cfg_list, depends, pkg_list): src_list = [] LOG.info(("Looking for for data source in: %s," - " via packages %s that matches dependencies %s"), cfg_list, pkg_list, depends) + " via packages %s that matches dependencies %s"), + cfg_list, pkg_list, depends) for ds_coll in cfg_list: ds_name = str(ds_coll) if not ds_name.startswith(DS_PREFIX): -- cgit v1.2.3 From 27349ea69b8ccd3eace67e2b71ae8842e8001020 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Wed, 20 Jun 2012 16:40:16 -0700 Subject: Change to import error instead of runtime error. --- cloudinit/sources/__init__.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'cloudinit/sources/__init__.py') diff --git a/cloudinit/sources/__init__.py b/cloudinit/sources/__init__.py index 35fc01df..8ab7cf54 100644 --- a/cloudinit/sources/__init__.py +++ b/cloudinit/sources/__init__.py @@ -198,11 +198,13 @@ def list_sources(cfg_list, depends, pkg_list): for pkg in pkg_list: pkg_name = [] if pkg: + # Any package name given, this affects + # the lookup path pkg_name.append(str(pkg)) pkg_name.append(ds_name) try: mod = importer.import_module(".".join(pkg_name)) - except RuntimeError: + except ImportError: continue lister = getattr(mod, "get_datasource_list", None) if not lister: -- cgit v1.2.3 From 95e0fa29af3656c1011c41ab0f35dc4e9317269c Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Wed, 20 Jun 2012 23:40:00 -0700 Subject: 1. Add an importer function that will search for a given module in a set of search module 'prefixes' that also has a potential set of required attributes. 2. Use this new importer to find the distro class, the userdata handler modules, the config modules and the datasource modules, if none can be found error out accordingly. --- cloudinit/config/__init__.py | 4 ---- cloudinit/distros/__init__.py | 23 ++++++++++------------ cloudinit/handlers/__init__.py | 8 -------- cloudinit/importer.py | 44 +++++++++++++++++++++++++++++++++--------- cloudinit/sources/__init__.py | 34 +++++++++++--------------------- cloudinit/stages.py | 22 +++++++++++++++++---- 6 files changed, 74 insertions(+), 61 deletions(-) (limited to 'cloudinit/sources/__init__.py') diff --git a/cloudinit/config/__init__.py b/cloudinit/config/__init__.py index 02e32462..ab13045f 100644 --- a/cloudinit/config/__init__.py +++ b/cloudinit/config/__init__.py @@ -51,10 +51,6 @@ def fixup_module(mod, def_freq=PER_INSTANCE): freq = mod.frequency if freq and freq not in FREQUENCIES: LOG.warn("Module %s has an unknown frequency %s", mod, freq) - if not hasattr(mod, 'handle'): - def empty_handle(_name, _cfg, _cloud, _log, _args): - pass - setattr(mod, 'handle', empty_handle) if not hasattr(mod, 'distros'): setattr(mod, 'distros', None) return mod diff --git a/cloudinit/distros/__init__.py b/cloudinit/distros/__init__.py index 25a60c52..e0ef6ee0 100644 --- a/cloudinit/distros/__init__.py +++ b/cloudinit/distros/__init__.py @@ -144,16 +144,13 @@ class Distro(object): return False -def fetch(distro_name, mods=(__name__, )): - mod = None - for m in mods: - mod_name = "%s.%s" % (m, distro_name) - try: - mod = importer.import_module(mod_name) - except ImportError: - pass - if not mod: - raise RuntimeError("No distribution found for distro %s" - % (distro_name)) - distro_cls = getattr(mod, 'Distro') - return distro_cls +def fetch(name): + locs = importer.find_module(name, + ['', __name__], + ['Distro']) + if not locs: + raise ImportError("No distribution found for distro %s" + % (name)) + mod = importer.import_module(locs[0]) + cls = getattr(mod, 'Distro') + return cls diff --git a/cloudinit/handlers/__init__.py b/cloudinit/handlers/__init__.py index d52b1cba..0f7432e5 100644 --- a/cloudinit/handlers/__init__.py +++ b/cloudinit/handlers/__init__.py @@ -202,20 +202,12 @@ def walk(msg, callback, data): def fixup_handler(mod, def_freq=PER_INSTANCE): if not hasattr(mod, "handler_version"): setattr(mod, "handler_version", 1) - if not hasattr(mod, 'list_types'): - def empty_types(): - return [] - setattr(mod, 'list_types', empty_types) if not hasattr(mod, 'frequency'): setattr(mod, 'frequency', def_freq) else: freq = mod.frequency if freq and freq not in FREQUENCIES: LOG.warn("Handler %s has an unknown frequency %s", mod, freq) - if not hasattr(mod, 'handle_part'): - def empty_handler(_data, _ctype, _filename, _payload): - pass - setattr(mod, 'handle_part', empty_handler) return mod diff --git a/cloudinit/importer.py b/cloudinit/importer.py index a36b87bc..71cf2726 100644 --- a/cloudinit/importer.py +++ b/cloudinit/importer.py @@ -23,17 +23,43 @@ import sys from cloudinit import log as logging -from cloudinit import util LOG = logging.getLogger(__name__) -# Simple wrapper that allows us to add more logging in... def import_module(module_name): - try: - LOG.debug("Attempting to import module %s", module_name) - __import__(module_name) - return sys.modules[module_name] - except: - util.logexc(LOG, 'Failed at importing %s', module_name) - raise + __import__(module_name) + return sys.modules[module_name] + + +def find_module(base_name, search_paths, required_attrs=None): + found_places = [] + if not required_attrs: + required_attrs = [] + real_paths = [] + for path in search_paths: + real_path = [] + if path: + real_path.extend(path.split(".")) + real_path.append(base_name) + full_path = '.'.join(real_path) + real_paths.append(full_path) + LOG.debug("Looking for modules %s that have attributes %s", + real_paths, required_attrs) + for full_path in real_paths: + mod = None + try: + mod = import_module(full_path) + except ImportError: + pass + if not mod: + continue + found_attrs = 0 + for attr in required_attrs: + if hasattr(mod, attr): + found_attrs += 1 + if found_attrs == len(required_attrs): + found_places.append(full_path) + LOG.debug("Found %s with attributes %s in %s", base_name, + required_attrs, found_places) + return found_places diff --git a/cloudinit/sources/__init__.py b/cloudinit/sources/__init__.py index 8ab7cf54..42e924b0 100644 --- a/cloudinit/sources/__init__.py +++ b/cloudinit/sources/__init__.py @@ -191,31 +191,19 @@ def list_sources(cfg_list, depends, pkg_list): LOG.info(("Looking for for data source in: %s," " via packages %s that matches dependencies %s"), cfg_list, pkg_list, depends) - for ds_coll in cfg_list: - ds_name = str(ds_coll) + for ds_name in cfg_list: if not ds_name.startswith(DS_PREFIX): ds_name = '%s%s' % (DS_PREFIX, ds_name) - for pkg in pkg_list: - pkg_name = [] - if pkg: - # Any package name given, this affects - # the lookup path - pkg_name.append(str(pkg)) - pkg_name.append(ds_name) - try: - mod = importer.import_module(".".join(pkg_name)) - except ImportError: - continue - lister = getattr(mod, "get_datasource_list", None) - if not lister: - continue - cls_matches = lister(depends) - if not cls_matches: - continue - src_list.extend(cls_matches) - LOG.debug(("Found a match" - " in %s with matches %s"), mod, cls_matches) - break + m_locs = importer.find_module(ds_name, + pkg_list, + ['get_datasource_list']) + for m_loc in m_locs: + mod = importer.import_module(m_loc) + lister = getattr(mod, "get_datasource_list") + matches = lister(depends) + if matches: + src_list.extend(matches) + break return src_list diff --git a/cloudinit/stages.py b/cloudinit/stages.py index 84a965c2..1997301a 100644 --- a/cloudinit/stages.py +++ b/cloudinit/stages.py @@ -205,7 +205,7 @@ class Init(object): # Any config provided??? pkg_list = self.cfg.get('datasource_pkg_list') or [] # Add the defaults at the end - for n in [util.obj_name(sources), '']: + for n in ['', util.obj_name(sources)]: if n not in pkg_list: pkg_list.append(n) cfg_list = self.cfg.get('datasource_list') or [] @@ -334,9 +334,17 @@ class Init(object): # Add handlers in cdir potential_handlers = util.find_modules(cdir) - for (fname, modname) in potential_handlers.iteritems(): + for (fname, mod_name) in potential_handlers.iteritems(): try: - mod = handlers.fixup_handler(importer.import_module(modname)) + mod_locs = importer.find_module(mod_name, [''], + ['list_types', + 'handle_part']) + if not mod_locs: + LOG.warn(("Could not find a valid user-data handler" + " named %s in file %s"), mod_name, fname) + continue + mod = importer.import_module(mod_locs[0]) + mod = handlers.fixup_handler(mod) types = c_handlers.register(mod) LOG.debug("Added handler for %s from %s", types, fname) except: @@ -482,7 +490,13 @@ class Modules(object): " has an unknown frequency %s"), raw_name, freq) # Reset it so when ran it will get set to a known value freq = None - mod = config.fixup_module(importer.import_module(mod_name)) + mod_locs = importer.find_module(mod_name, + ['', util.obj_name(config)], + ['handle']) + if not mod_locs: + LOG.warn("Could not find module named %s", mod_name) + continue + mod = config.fixup_module(importer.import_module(mod_locs[0])) mostly_mods.append([mod, raw_name, freq, run_args]) return mostly_mods -- cgit v1.2.3 From f1cab0d88cbcfa7eaa698db7dcc252bb6543d6c0 Mon Sep 17 00:00:00 2001 From: harlowja Date: Thu, 21 Jun 2012 08:38:12 -0700 Subject: 1. Move all info() logging methods to debug() 2. Adjust comment on sources list from depends 3. For the /etc/timezone 'writing', add a header that says created by cloud-init --- bin/cloud-init | 3 +-- cloudinit/distros/__init__.py | 6 +++--- cloudinit/distros/ubuntu.py | 6 +++++- cloudinit/helpers.py | 2 +- cloudinit/sources/DataSourceCloudStack.py | 2 +- cloudinit/sources/DataSourceConfigDrive.py | 6 ++---- cloudinit/sources/DataSourceEc2.py | 2 +- cloudinit/sources/DataSourceMAAS.py | 2 +- cloudinit/sources/DataSourceNoCloud.py | 2 +- cloudinit/sources/__init__.py | 22 +++++++++++----------- cloudinit/url_helper.py | 14 +++++++------- 11 files changed, 34 insertions(+), 33 deletions(-) (limited to 'cloudinit/sources/__init__.py') diff --git a/bin/cloud-init b/bin/cloud-init index a2f15c4b..d193272e 100755 --- a/bin/cloud-init +++ b/bin/cloud-init @@ -73,8 +73,7 @@ def welcome(action): } welcome_msg = "%s" % (templater.render_string(msg, tpl_params)) sys.stderr.write("%s\n" % (welcome_msg)) - sys.stderr.flush() - LOG.info(welcome_msg) + LOG.debug(welcome_msg) def extract_fns(args): diff --git a/cloudinit/distros/__init__.py b/cloudinit/distros/__init__.py index e0ef6ee0..6325257c 100644 --- a/cloudinit/distros/__init__.py +++ b/cloudinit/distros/__init__.py @@ -133,14 +133,14 @@ class Distro(object): raise NotImplementedError("Unknown interface action %s" % (action)) cmd = IFACE_ACTIONS[action] try: - LOG.info("Attempting to run %s interface action using command %s", - action, cmd) + LOG.debug("Attempting to run %s interface action using command %s", + action, cmd) (_out, err) = util.subp(cmd) if len(err): LOG.warn("Running %s resulted in stderr output: %s", cmd, err) return True except util.ProcessExecutionError: - util.logexc(LOG, "Running %s failed", cmd) + util.logexc(LOG, "Running interface command %s failed", cmd) return False diff --git a/cloudinit/distros/ubuntu.py b/cloudinit/distros/ubuntu.py index fd7b7b8d..15af2e7f 100644 --- a/cloudinit/distros/ubuntu.py +++ b/cloudinit/distros/ubuntu.py @@ -112,7 +112,11 @@ class Distro(distros.Distro): if not os.path.isfile(tz_file): raise Exception(("Invalid timezone %s," " no file found at %s") % (tz, tz_file)) - tz_contents = "%s\n" % tz + tz_lines = [ + "# Created by cloud-init", + str(tz), + ] + tz_contents = "\n".join(tz_lines) tz_fn = self._paths.join(False, "/etc/timezone") util.write_file(tz_fn, tz_contents) util.copy(tz_file, self._paths.join(False, "/etc/localtime")) diff --git a/cloudinit/helpers.py b/cloudinit/helpers.py index b6974f3c..6751f4a5 100644 --- a/cloudinit/helpers.py +++ b/cloudinit/helpers.py @@ -161,7 +161,7 @@ class Runners(object): if not args: args = [] if sem.has_run(name, freq): - LOG.info("%s already ran (freq=%s)", name, freq) + LOG.debug("%s already ran (freq=%s)", name, freq) return (False, None) with sem.lock(name, freq, clear_on_fail) as lk: if not lk: diff --git a/cloudinit/sources/DataSourceCloudStack.py b/cloudinit/sources/DataSourceCloudStack.py index 83c577e6..751bef4f 100644 --- a/cloudinit/sources/DataSourceCloudStack.py +++ b/cloudinit/sources/DataSourceCloudStack.py @@ -98,7 +98,7 @@ class DataSourceCloudStack(sources.DataSource): timeout=timeout, status_cb=LOG.warn) if url: - LOG.info("Using metadata source: '%s'", url) + LOG.debug("Using metadata source: '%s'", url) else: LOG.critical(("Giving up on waiting for the metadata from %s" " after %s seconds"), diff --git a/cloudinit/sources/DataSourceConfigDrive.py b/cloudinit/sources/DataSourceConfigDrive.py index 9905dad4..320dd1d1 100644 --- a/cloudinit/sources/DataSourceConfigDrive.py +++ b/cloudinit/sources/DataSourceConfigDrive.py @@ -87,10 +87,8 @@ class DataSourceConfigDrive(sources.DataSource): # Update interfaces and ifup only on the local datasource # this way the DataSourceConfigDriveNet doesn't do it also. if 'network-interfaces' in md and self.dsmode == "local": - if md['dsmode'] == "pass": - LOG.info("Updating network interfaces from configdrive") - else: - LOG.debug("Updating network interfaces from configdrive") + LOG.debug("Updating network interfaces from config drive (%s)", + md['dsmode']) self.distro.apply_network(md['network-interfaces']) self.seed = found diff --git a/cloudinit/sources/DataSourceEc2.py b/cloudinit/sources/DataSourceEc2.py index 0598dfa2..cb460de1 100644 --- a/cloudinit/sources/DataSourceEc2.py +++ b/cloudinit/sources/DataSourceEc2.py @@ -169,7 +169,7 @@ class DataSourceEc2(sources.DataSource): timeout=timeout, status_cb=LOG.warn) if url: - LOG.info("Using metadata source: '%s'", url2base[url]) + LOG.debug("Using metadata source: '%s'", url2base[url]) else: LOG.critical("Giving up on md from %s after %s seconds", urls, int(time.time() - start_time)) diff --git a/cloudinit/sources/DataSourceMAAS.py b/cloudinit/sources/DataSourceMAAS.py index 104e7a54..22c90b7c 100644 --- a/cloudinit/sources/DataSourceMAAS.py +++ b/cloudinit/sources/DataSourceMAAS.py @@ -128,7 +128,7 @@ class DataSourceMAAS(sources.DataSource): headers_cb=self.md_headers) if url: - LOG.info("Using metadata source: '%s'", url) + LOG.debug("Using metadata source: '%s'", url) else: LOG.critical("Giving up on md from %s after %i seconds", urls, int(time.time() - starttime)) diff --git a/cloudinit/sources/DataSourceNoCloud.py b/cloudinit/sources/DataSourceNoCloud.py index 8499a97c..bed500a2 100644 --- a/cloudinit/sources/DataSourceNoCloud.py +++ b/cloudinit/sources/DataSourceNoCloud.py @@ -152,7 +152,7 @@ class DataSourceNoCloud(sources.DataSource): # ('local' for NoCloud, 'net' for NoCloudNet') if ('network-interfaces' in md and (self.dsmode in ("local", seeded_interfaces))): - LOG.info("Updating network interfaces from %s", self) + LOG.debug("Updating network interfaces from %s", self) self.distro.apply_network(md['network-interfaces']) if md['dsmode'] == self.dsmode: diff --git a/cloudinit/sources/__init__.py b/cloudinit/sources/__init__.py index 42e924b0..b25724a5 100644 --- a/cloudinit/sources/__init__.py +++ b/cloudinit/sources/__init__.py @@ -166,7 +166,7 @@ class DataSource(object): def find_source(sys_cfg, distro, paths, ds_deps, cfg_list, pkg_list): ds_list = list_sources(cfg_list, ds_deps, pkg_list) ds_names = [util.obj_name(f) for f in ds_list] - LOG.info("Searching for data source in: %s", ds_names) + LOG.debug("Searching for data source in: %s", ds_names) for cls in ds_list: try: @@ -188,9 +188,9 @@ def find_source(sys_cfg, distro, paths, ds_deps, cfg_list, pkg_list): # Return an ordered list of classes that match (if any) def list_sources(cfg_list, depends, pkg_list): src_list = [] - LOG.info(("Looking for for data source in: %s," - " via packages %s that matches dependencies %s"), - cfg_list, pkg_list, depends) + LOG.debug(("Looking for for data source in: %s," + " via packages %s that matches dependencies %s"), + cfg_list, pkg_list, depends) for ds_name in cfg_list: if not ds_name.startswith(DS_PREFIX): ds_name = '%s%s' % (DS_PREFIX, ds_name) @@ -207,17 +207,17 @@ def list_sources(cfg_list, depends, pkg_list): return src_list -# depends is a list of dependencies (DEP_FILESYSTEM) -# dslist is a list of 2 item lists -# dslist = [ +# 'depends' is a list of dependencies (DEP_FILESYSTEM) +# ds_list is a list of 2 item lists +# ds_list = [ # ( class, ( depends-that-this-class-needs ) ) # } -# it returns a list of 'class' that matched these deps exactly -# it is a helper function for DataSourceCollections -def list_from_depends(depends, dslist): +# It returns a list of 'class' that matched these deps exactly +# It mainly is a helper function for DataSourceCollections +def list_from_depends(depends, ds_list): ret_list = [] depset = set(depends) - for (cls, deps) in dslist: + for (cls, deps) in ds_list: if depset == set(deps): ret_list.append(cls) return ret_list diff --git a/cloudinit/url_helper.py b/cloudinit/url_helper.py index 223278ce..dbf72392 100644 --- a/cloudinit/url_helper.py +++ b/cloudinit/url_helper.py @@ -88,8 +88,8 @@ def readurl(url, data=None, timeout=None, attempts = retries + 1 excepts = [] - LOG.info(("Attempting to open '%s' with %s attempts" - " (%s retries, timeout=%s) to be performed"), + LOG.debug(("Attempting to open '%s' with %s attempts" + " (%s retries, timeout=%s) to be performed"), url, attempts, retries, timeout) open_args = {} if timeout is not None: @@ -105,8 +105,8 @@ def readurl(url, data=None, timeout=None, headers = {} if rh.headers: headers = dict(rh.headers) - LOG.info("Read from %s (%s, %sb) after %s attempts", - url, status, len(content), (i + 1)) + LOG.debug("Read from %s (%s, %sb) after %s attempts", + url, status, len(content), (i + 1)) return UrlResponse(status, content, headers) except urllib2.HTTPError as e: excepts.append(e) @@ -165,7 +165,7 @@ def wait_for_url(urls, max_wait=None, timeout=None, start_time = time.time() def log_status_cb(msg): - LOG.info(msg) + LOG.debug(msg) if status_cb is None: status_cb = log_status_cb @@ -219,8 +219,8 @@ def wait_for_url(urls, max_wait=None, timeout=None, break loop_n = loop_n + 1 - LOG.info("Please wait %s seconds while we wait to try again", - sleep_time) + LOG.debug("Please wait %s seconds while we wait to try again", + sleep_time) time.sleep(sleep_time) return False -- cgit v1.2.3