diff options
66 files changed, 1044 insertions, 462 deletions
@@ -6,6 +6,23 @@ - cc_landscape: restart landscape after install or config (LP: #1070345) - multipart/archive. do not fail on unknown headers in multipart mime or cloud-archive config (LP: #1065116). + - tools/Z99-cloud-locale-test.sh: avoid warning when user's shell is + zsh (LP: #1073077) + - fix stack trace when unknown user-data input had unicode (LP: #1075756) + - split 'apt-update-upgrade' config module into 'apt-configure' and + 'package-update-upgrade-install'. The 'package-update-upgrade-install' + will be a cross distro module. + - Cleanups: + - Remove usage of paths.join, as all code should run through util helpers + - Fix pylint complaining about tests folder 'helpers.py' not being found + - Add a pylintrc file that is used instead options hidden in 'run_pylint' + - fix bug where cloud-config from user-data could not affect system_info + settings [revno 703] (LP: #1076811) + - for write fqdn to system config for rh/fedora [revno 704] + - add yaml/cloud config examples checking tool [revno 706] + - Fix the merging of group configuration when that group configuration is a + dict => members. [revno 707] + - add yum_add_repo configuration module for adding additional yum repos 0.7.0: - add a 'exception_cb' argument to 'wait_for_url'. If provided, this method will be called back with the exception received and the message. @@ -1,20 +1,24 @@ CWD=$(shell pwd) -PY_FILES=$(shell find cloudinit bin tests tools -name "*.py") +PY_FILES=$(shell find cloudinit bin tests tools -name "*.py" -type f ) PY_FILES+="bin/cloud-init" +YAML_FILES=$(shell find cloudinit bin tests tools -name "*.yaml" -type f ) +YAML_FILES+=$(shell find doc/examples -name "cloud-config*.txt" -type f ) + + all: test pep8: - $(CWD)/tools/run-pep8 $(PY_FILES) + @$(CWD)/tools/run-pep8 $(PY_FILES) pylint: - $(CWD)/tools/run-pylint $(PY_FILES) + @$(CWD)/tools/run-pylint $(PY_FILES) pyflakes: pyflakes $(PY_FILES) test: - nosetests $(noseopts) tests/unittests/ + @nosetests $(noseopts) tests/ 2to3: 2to3 $(PY_FILES) @@ -23,11 +27,14 @@ clean: rm -rf /var/log/cloud-init.log \ /var/lib/cloud/ +yaml: + @$(CWD)/tools/validate-yaml.py $(YAML_FILES) + rpm: ./packages/brpm deb: ./packages/bddeb -.PHONY: test pylint pyflakes 2to3 clean pep8 rpm deb +.PHONY: test pylint pyflakes 2to3 clean pep8 rpm deb yaml diff --git a/cloudinit/config/cc_apt_update_upgrade.py b/cloudinit/config/cc_apt_configure.py index 356bb98d..f8664160 100644 --- a/cloudinit/config/cc_apt_update_upgrade.py +++ b/cloudinit/config/cc_apt_configure.py @@ -20,7 +20,6 @@ import glob import os -import time from cloudinit import templater from cloudinit import util @@ -47,9 +46,6 @@ EXPORT_GPG_KEYID = """ def handle(name, cfg, cloud, log, _args): - update = util.get_cfg_option_bool(cfg, 'apt_update', False) - upgrade = util.get_cfg_option_bool(cfg, 'apt_upgrade', False) - release = get_release() mirrors = find_apt_mirror_info(cloud, cfg) if not mirrors or "primary" not in mirrors: @@ -61,7 +57,7 @@ def handle(name, cfg, cloud, log, _args): mirror = mirrors["primary"] mirrors["mirror"] = mirror - log.debug("mirror info: %s" % mirrors) + log.debug("Mirror info: %s" % mirrors) if not util.get_cfg_option_bool(cfg, 'apt_preserve_sources_list', False): @@ -78,8 +74,7 @@ def handle(name, cfg, cloud, log, _args): try: # See man 'apt.conf' contents = PROXY_TPL % (proxy) - util.write_file(cloud.paths.join(False, proxy_filename), - contents) + util.write_file(proxy_filename, contents) except Exception as e: util.logexc(log, "Failed to write proxy to %s", proxy_filename) elif os.path.isfile(proxy_filename): @@ -90,61 +85,18 @@ def handle(name, cfg, cloud, log, _args): params = mirrors params['RELEASE'] = release params['MIRROR'] = mirror - errors = add_sources(cloud, cfg['apt_sources'], params) + errors = add_sources(cfg['apt_sources'], params) for e in errors: - log.warn("Source Error: %s", ':'.join(e)) + log.warn("Add source error: %s", ':'.join(e)) dconf_sel = util.get_cfg_option_str(cfg, 'debconf_selections', False) if dconf_sel: - log.debug("setting debconf selections per cloud config") + log.debug("Setting debconf selections per cloud config") try: util.subp(('debconf-set-selections', '-'), dconf_sel) - except: + except Exception: util.logexc(log, "Failed to run debconf-set-selections") - pkglist = util.get_cfg_option_list(cfg, 'packages', []) - - errors = [] - if update or len(pkglist) or upgrade: - try: - cloud.distro.update_package_sources() - except Exception as e: - util.logexc(log, "Package update failed") - errors.append(e) - - if upgrade: - try: - cloud.distro.package_command("upgrade") - except Exception as e: - util.logexc(log, "Package upgrade failed") - errors.append(e) - - if len(pkglist): - try: - cloud.distro.install_packages(pkglist) - except Exception as e: - util.logexc(log, "Failed to install packages: %s ", pkglist) - errors.append(e) - - # kernel and openssl (possibly some other packages) - # write a file /var/run/reboot-required after upgrading. - # if that file exists and configured, then just stop right now and reboot - # TODO(smoser): handle this less voilently - reboot_file = "/var/run/reboot-required" - if ((upgrade or pkglist) and cfg.get("apt_reboot_if_required", False) and - os.path.isfile(reboot_file)): - log.warn("rebooting after upgrade or install per %s" % reboot_file) - time.sleep(1) # give the warning time to get out - util.subp(["/sbin/reboot"]) - time.sleep(60) - log.warn("requested reboot did not happen!") - errors.append(Exception("requested reboot did not happen!")) - - if len(errors): - log.warn("%s failed with exceptions, re-raising the last one", - len(errors)) - raise errors[-1] - # get gpg keyid from keyserver def getkeybyid(keyid, keyserver): @@ -196,11 +148,10 @@ def generate_sources_list(codename, mirrors, cloud, log): params = {'codename': codename} for k in mirrors: params[k] = mirrors[k] - out_fn = cloud.paths.join(False, '/etc/apt/sources.list') - templater.render_to_file(template_fn, out_fn, params) + templater.render_to_file(template_fn, '/etc/apt/sources.list', params) -def add_sources(cloud, srclist, template_params=None): +def add_sources(srclist, template_params=None): """ add entries in /etc/apt/sources.list.d for each abbreviated sources.list entry in 'srclist'. When rendering template, also @@ -250,8 +201,7 @@ def add_sources(cloud, srclist, template_params=None): try: contents = "%s\n" % (source) - util.write_file(cloud.paths.join(False, ent['filename']), - contents, omode="ab") + util.write_file(ent['filename'], contents, omode="ab") except: errorlist.append([source, "failed write to file %s" % ent['filename']]) diff --git a/cloudinit/config/cc_apt_pipelining.py b/cloudinit/config/cc_apt_pipelining.py index 02056ee0..e5629175 100644 --- a/cloudinit/config/cc_apt_pipelining.py +++ b/cloudinit/config/cc_apt_pipelining.py @@ -34,26 +34,24 @@ APT_PIPE_TPL = ("//Written by cloud-init per 'apt_pipelining'\n" # on TCP connections - otherwise data corruption will occur. -def handle(_name, cfg, cloud, log, _args): +def handle(_name, cfg, _cloud, log, _args): apt_pipe_value = util.get_cfg_option_str(cfg, "apt_pipelining", False) apt_pipe_value_s = str(apt_pipe_value).lower().strip() if apt_pipe_value_s == "false": - write_apt_snippet(cloud, "0", log, DEFAULT_FILE) + write_apt_snippet("0", log, DEFAULT_FILE) elif apt_pipe_value_s in ("none", "unchanged", "os"): return elif apt_pipe_value_s in [str(b) for b in xrange(0, 6)]: - write_apt_snippet(cloud, apt_pipe_value_s, log, DEFAULT_FILE) + write_apt_snippet(apt_pipe_value_s, log, DEFAULT_FILE) else: log.warn("Invalid option for apt_pipeling: %s", apt_pipe_value) -def write_apt_snippet(cloud, setting, log, f_name): +def write_apt_snippet(setting, log, f_name): """Writes f_name with apt pipeline depth 'setting'.""" file_contents = APT_PIPE_TPL % (setting) - - util.write_file(cloud.paths.join(False, f_name), file_contents) - + util.write_file(f_name, file_contents) log.debug("Wrote %s with apt pipeline depth setting %s", f_name, setting) diff --git a/cloudinit/config/cc_ca_certs.py b/cloudinit/config/cc_ca_certs.py index dc046bda..20f24357 100644 --- a/cloudinit/config/cc_ca_certs.py +++ b/cloudinit/config/cc_ca_certs.py @@ -22,6 +22,7 @@ CA_CERT_PATH = "/usr/share/ca-certificates/" CA_CERT_FILENAME = "cloud-init-ca-certs.crt" CA_CERT_CONFIG = "/etc/ca-certificates.conf" CA_CERT_SYSTEM_PATH = "/etc/ssl/certs/" +CA_CERT_FULL_PATH = os.path.join(CA_CERT_PATH, CA_CERT_FILENAME) distros = ['ubuntu', 'debian'] @@ -33,7 +34,7 @@ def update_ca_certs(): util.subp(["update-ca-certificates"], capture=False) -def add_ca_certs(paths, certs): +def add_ca_certs(certs): """ Adds certificates to the system. To actually apply the new certificates you must also call L{update_ca_certs}. @@ -43,27 +44,24 @@ def add_ca_certs(paths, certs): if certs: # First ensure they are strings... cert_file_contents = "\n".join([str(c) for c in certs]) - cert_file_fullpath = os.path.join(CA_CERT_PATH, CA_CERT_FILENAME) - cert_file_fullpath = paths.join(False, cert_file_fullpath) - util.write_file(cert_file_fullpath, cert_file_contents, mode=0644) + util.write_file(CA_CERT_FULL_PATH, cert_file_contents, mode=0644) # Append cert filename to CA_CERT_CONFIG file. - util.write_file(paths.join(False, CA_CERT_CONFIG), - "\n%s" % CA_CERT_FILENAME, omode="ab") + util.write_file(CA_CERT_CONFIG, "\n%s" % CA_CERT_FILENAME, omode="ab") -def remove_default_ca_certs(paths): +def remove_default_ca_certs(): """ Removes all default trusted CA certificates from the system. To actually apply the change you must also call L{update_ca_certs}. """ - util.delete_dir_contents(paths.join(False, CA_CERT_PATH)) - util.delete_dir_contents(paths.join(False, CA_CERT_SYSTEM_PATH)) - util.write_file(paths.join(False, CA_CERT_CONFIG), "", mode=0644) + util.delete_dir_contents(CA_CERT_PATH) + util.delete_dir_contents(CA_CERT_SYSTEM_PATH) + util.write_file(CA_CERT_CONFIG, "", mode=0644) debconf_sel = "ca-certificates ca-certificates/trust_new_crts select no" util.subp(('debconf-set-selections', '-'), debconf_sel) -def handle(name, cfg, cloud, log, _args): +def handle(name, cfg, _cloud, log, _args): """ Call to handle ca-cert sections in cloud-config file. @@ -85,14 +83,14 @@ def handle(name, cfg, cloud, log, _args): # default trusted CA certs first. if ca_cert_cfg.get("remove-defaults", False): log.debug("Removing default certificates") - remove_default_ca_certs(cloud.paths) + remove_default_ca_certs() # If we are given any new trusted CA certs to add, add them. if "trusted" in ca_cert_cfg: trusted_certs = util.get_cfg_option_list(ca_cert_cfg, "trusted") if trusted_certs: log.debug("Adding %d certificates" % len(trusted_certs)) - add_ca_certs(cloud.paths, trusted_certs) + add_ca_certs(trusted_certs) # Update the system with the new cert configuration. log.debug("Updating certificates") diff --git a/cloudinit/config/cc_chef.py b/cloudinit/config/cc_chef.py index 6f568261..7a3d6a31 100644 --- a/cloudinit/config/cc_chef.py +++ b/cloudinit/config/cc_chef.py @@ -26,6 +26,15 @@ from cloudinit import util RUBY_VERSION_DEFAULT = "1.8" +CHEF_DIRS = [ + '/etc/chef', + '/var/log/chef', + '/var/lib/chef', + '/var/cache/chef', + '/var/backups/chef', + '/var/run/chef', +] + def handle(name, cfg, cloud, log, _args): @@ -37,24 +46,15 @@ def handle(name, cfg, cloud, log, _args): chef_cfg = cfg['chef'] # Ensure the chef directories we use exist - c_dirs = [ - '/etc/chef', - '/var/log/chef', - '/var/lib/chef', - '/var/cache/chef', - '/var/backups/chef', - '/var/run/chef', - ] - for d in c_dirs: - util.ensure_dir(cloud.paths.join(False, d)) + for d in CHEF_DIRS: + util.ensure_dir(d) # Set the validation key based on the presence of either 'validation_key' # or 'validation_cert'. In the case where both exist, 'validation_key' # takes precedence for key in ('validation_key', 'validation_cert'): if key in chef_cfg and chef_cfg[key]: - v_fn = cloud.paths.join(False, '/etc/chef/validation.pem') - util.write_file(v_fn, chef_cfg[key]) + util.write_file('/etc/chef/validation.pem', chef_cfg[key]) break # Create the chef config from template @@ -68,8 +68,7 @@ def handle(name, cfg, cloud, log, _args): '_default'), 'validation_name': chef_cfg['validation_name'] } - out_fn = cloud.paths.join(False, '/etc/chef/client.rb') - templater.render_to_file(template_fn, out_fn, params) + templater.render_to_file(template_fn, '/etc/chef/client.rb', params) else: log.warn("No template found, not rendering to /etc/chef/client.rb") @@ -81,8 +80,7 @@ def handle(name, cfg, cloud, log, _args): initial_attributes = chef_cfg['initial_attributes'] for k in list(initial_attributes.keys()): initial_json[k] = initial_attributes[k] - firstboot_fn = cloud.paths.join(False, '/etc/chef/firstboot.json') - util.write_file(firstboot_fn, json.dumps(initial_json)) + util.write_file('/etc/chef/firstboot.json', json.dumps(initial_json)) # If chef is not installed, we install chef based on 'install_type' if not os.path.isfile('/usr/bin/chef-client'): diff --git a/cloudinit/config/cc_landscape.py b/cloudinit/config/cc_landscape.py index 56ab0ce3..02610dd0 100644 --- a/cloudinit/config/cc_landscape.py +++ b/cloudinit/config/cc_landscape.py @@ -66,22 +66,16 @@ def handle(_name, cfg, cloud, log, _args): merge_data = [ LSC_BUILTIN_CFG, - cloud.paths.join(True, LSC_CLIENT_CFG_FILE), + LSC_CLIENT_CFG_FILE, ls_cloudcfg, ] merged = merge_together(merge_data) - - lsc_client_fn = cloud.paths.join(False, LSC_CLIENT_CFG_FILE) - lsc_dir = cloud.paths.join(False, os.path.dirname(lsc_client_fn)) - if not os.path.isdir(lsc_dir): - util.ensure_dir(lsc_dir) - contents = StringIO() merged.write(contents) - contents.flush() - util.write_file(lsc_client_fn, contents.getvalue()) - log.debug("Wrote landscape config file to %s", lsc_client_fn) + util.ensure_dir(os.path.dirname(LSC_CLIENT_CFG_FILE)) + util.write_file(LSC_CLIENT_CFG_FILE, contents.getvalue()) + log.debug("Wrote landscape config file to %s", LSC_CLIENT_CFG_FILE) util.write_file(LS_DEFAULT_FILE, "RUN=1\n") util.subp(["service", "landscape-client", "restart"]) diff --git a/cloudinit/config/cc_mcollective.py b/cloudinit/config/cc_mcollective.py index 2acdbc6f..b670390d 100644 --- a/cloudinit/config/cc_mcollective.py +++ b/cloudinit/config/cc_mcollective.py @@ -29,6 +29,7 @@ from cloudinit import util PUBCERT_FILE = "/etc/mcollective/ssl/server-public.pem" PRICERT_FILE = "/etc/mcollective/ssl/server-private.pem" +SERVER_CFG = '/etc/mcollective/server.cfg' def handle(name, cfg, cloud, log, _args): @@ -48,26 +49,23 @@ def handle(name, cfg, cloud, log, _args): if 'conf' in mcollective_cfg: # Read server.cfg values from the # original file in order to be able to mix the rest up - server_cfg_fn = cloud.paths.join(True, '/etc/mcollective/server.cfg') - mcollective_config = ConfigObj(server_cfg_fn) + mcollective_config = ConfigObj(SERVER_CFG) # See: http://tiny.cc/jh9agw for (cfg_name, cfg) in mcollective_cfg['conf'].iteritems(): if cfg_name == 'public-cert': - pubcert_fn = cloud.paths.join(True, PUBCERT_FILE) - util.write_file(pubcert_fn, cfg, mode=0644) - mcollective_config['plugin.ssl_server_public'] = pubcert_fn + util.write_file(PUBCERT_FILE, cfg, mode=0644) + mcollective_config['plugin.ssl_server_public'] = PUBCERT_FILE mcollective_config['securityprovider'] = 'ssl' elif cfg_name == 'private-cert': - pricert_fn = cloud.paths.join(True, PRICERT_FILE) - util.write_file(pricert_fn, cfg, mode=0600) - mcollective_config['plugin.ssl_server_private'] = pricert_fn + util.write_file(PRICERT_FILE, cfg, mode=0600) + mcollective_config['plugin.ssl_server_private'] = PRICERT_FILE mcollective_config['securityprovider'] = 'ssl' else: if isinstance(cfg, (basestring, str)): # Just set it in the 'main' section mcollective_config[cfg_name] = cfg elif isinstance(cfg, (dict)): - # Iterate throug the config items, create a section + # Iterate through the config items, create a section # if it is needed and then add/or create items as needed if cfg_name not in mcollective_config.sections: mcollective_config[cfg_name] = {} @@ -78,14 +76,12 @@ def handle(name, cfg, cloud, log, _args): mcollective_config[cfg_name] = str(cfg) # We got all our config as wanted we'll rename # the previous server.cfg and create our new one - old_fn = cloud.paths.join(False, '/etc/mcollective/server.cfg.old') - util.rename(server_cfg_fn, old_fn) + util.rename(SERVER_CFG, "%s.old" % (SERVER_CFG)) # Now we got the whole file, write to disk... contents = StringIO() mcollective_config.write(contents) contents = contents.getvalue() - server_cfg_rw = cloud.paths.join(False, '/etc/mcollective/server.cfg') - util.write_file(server_cfg_rw, contents, mode=0644) + util.write_file(SERVER_CFG, contents, mode=0644) # Start mcollective util.subp(['service', 'mcollective', 'start'], capture=False) diff --git a/cloudinit/config/cc_migrator.py b/cloudinit/config/cc_migrator.py new file mode 100644 index 00000000..58232fc9 --- /dev/null +++ b/cloudinit/config/cc_migrator.py @@ -0,0 +1,83 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2012 Yahoo! Inc. +# +# Author: Joshua Harlow <harlowja@yahoo-inc.com> +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. + +import os +import shutil + +from cloudinit import helpers +from cloudinit import util + +from cloudinit.settings import PER_ALWAYS + +frequency = PER_ALWAYS + + +def _migrate_canon_sems(cloud): + sem_path = cloud.paths.get_ipath('sem') + if not sem_path or not os.path.exists(sem_path): + return 0 + am_adjusted = 0 + for p in os.listdir(sem_path): + full_path = os.path.join(sem_path, p) + if os.path.isfile(full_path): + (name, ext) = os.path.splitext(p) + canon_name = helpers.canon_sem_name(name) + if canon_name != name: + new_path = os.path.join(sem_path, canon_name + ext) + shutil.move(full_path, new_path) + am_adjusted += 1 + return am_adjusted + + +def _migrate_legacy_sems(cloud, log): + sem_path = cloud.paths.get_ipath('sem') + if not sem_path or not os.path.exists(sem_path): + return + legacy_adjust = { + 'apt-update-upgrade': [ + 'apt-configure', + 'package-update-upgrade-install', + ], + } + sem_helper = helpers.FileSemaphores(sem_path) + for (mod_name, migrate_to) in legacy_adjust.items(): + possibles = [mod_name, helpers.canon_sem_name(mod_name)] + old_exists = [] + for p in os.listdir(sem_path): + (name, _ext) = os.path.splitext(p) + if name in possibles and os.path.isfile(p): + old_exists.append(p) + for p in old_exists: + util.del_file(os.path.join(sem_path, p)) + (_name, freq) = os.path.splitext(p) + for m in migrate_to: + log.debug("Migrating %s => %s with the same frequency", + p, m) + with sem_helper.lock(m, freq): + pass + + +def handle(name, cfg, cloud, log, _args): + do_migrate = util.get_cfg_option_str(cfg, "migrate", True) + if not util.translate_bool(do_migrate): + log.debug("Skipping module named %s, migration disabled", name) + return + sems_moved = _migrate_canon_sems(cloud) + log.debug("Migrated %s semaphore files to there canonicalized names", + sems_moved) + _migrate_legacy_sems(cloud, log) diff --git a/cloudinit/config/cc_mounts.py b/cloudinit/config/cc_mounts.py index 14c965bb..cb772c86 100644 --- a/cloudinit/config/cc_mounts.py +++ b/cloudinit/config/cc_mounts.py @@ -28,6 +28,7 @@ from cloudinit import util SHORTNAME_FILTER = r"^[x]{0,1}[shv]d[a-z][0-9]*$" SHORTNAME = re.compile(SHORTNAME_FILTER) WS = re.compile("[%s]+" % (whitespace)) +FSTAB_PATH = "/etc/fstab" def is_mdname(name): @@ -167,8 +168,7 @@ def handle(_name, cfg, cloud, log, _args): cc_lines.append('\t'.join(line)) fstab_lines = [] - fstab = util.load_file(cloud.paths.join(True, "/etc/fstab")) - for line in fstab.splitlines(): + for line in util.load_file(FSTAB_PATH).splitlines(): try: toks = WS.split(line) if toks[3].find(comment) != -1: @@ -179,7 +179,7 @@ def handle(_name, cfg, cloud, log, _args): fstab_lines.extend(cc_lines) contents = "%s\n" % ('\n'.join(fstab_lines)) - util.write_file(cloud.paths.join(False, "/etc/fstab"), contents) + util.write_file(FSTAB_PATH, contents) if needswap: try: @@ -188,9 +188,8 @@ def handle(_name, cfg, cloud, log, _args): util.logexc(log, "Activating swap via 'swapon -a' failed") for d in dirs: - real_dir = cloud.paths.join(False, d) try: - util.ensure_dir(real_dir) + util.ensure_dir(d) except: util.logexc(log, "Failed to make '%s' config-mount", d) diff --git a/cloudinit/config/cc_package_update_upgrade_install.py b/cloudinit/config/cc_package_update_upgrade_install.py new file mode 100644 index 00000000..73b0e30d --- /dev/null +++ b/cloudinit/config/cc_package_update_upgrade_install.py @@ -0,0 +1,99 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2012 Yahoo! Inc. +# +# Author: Joshua Harlow <harlowja@yahoo-inc.com> +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. + +import os +import time + +from cloudinit import log as logging +from cloudinit import util + +REBOOT_FILE = "/var/run/reboot-required" +REBOOT_CMD = ["/sbin/reboot"] + + +def _multi_cfg_bool_get(cfg, *keys): + for k in keys: + if util.get_cfg_option_bool(cfg, k, False): + return True + return False + + +def _fire_reboot(log, wait_attempts=6, initial_sleep=1, backoff=2): + util.subp(REBOOT_CMD) + start = time.time() + wait_time = initial_sleep + for _i in range(0, wait_attempts): + time.sleep(wait_time) + wait_time *= backoff + elapsed = time.time() - start + log.debug("Rebooted, but still running after %s seconds", int(elapsed)) + # If we got here, not good + elapsed = time.time() - start + raise RuntimeError(("Reboot did not happen" + " after %s seconds!") % (int(elapsed))) + + +def handle(_name, cfg, cloud, log, _args): + # Handle the old style + new config names + update = _multi_cfg_bool_get(cfg, 'apt_update', 'package_update') + upgrade = _multi_cfg_bool_get(cfg, 'package_upgrade', 'apt_upgrade') + reboot_if_required = _multi_cfg_bool_get(cfg, 'apt_reboot_if_required', + 'package_reboot_if_required') + pkglist = util.get_cfg_option_list(cfg, 'packages', []) + + errors = [] + if update or len(pkglist) or upgrade: + try: + cloud.distro.update_package_sources() + except Exception as e: + util.logexc(log, "Package update failed") + errors.append(e) + + if upgrade: + try: + cloud.distro.package_command("upgrade") + except Exception as e: + util.logexc(log, "Package upgrade failed") + errors.append(e) + + if len(pkglist): + try: + cloud.distro.install_packages(pkglist) + except Exception as e: + util.logexc(log, "Failed to install packages: %s", pkglist) + errors.append(e) + + # TODO(smoser): handle this less violently + # kernel and openssl (possibly some other packages) + # write a file /var/run/reboot-required after upgrading. + # if that file exists and configured, then just stop right now and reboot + reboot_fn_exists = os.path.isfile(REBOOT_FILE) + if (upgrade or pkglist) and reboot_if_required and reboot_fn_exists: + try: + log.warn("Rebooting after upgrade or install per %s", REBOOT_FILE) + # Flush the above warning + anything else out... + logging.flushLoggers(log) + _fire_reboot(log) + except Exception as e: + util.logexc(log, "Requested reboot did not happen!") + errors.append(e) + + if len(errors): + log.warn("%s failed with exceptions, re-raising the last one", + len(errors)) + raise errors[-1] diff --git a/cloudinit/config/cc_phone_home.py b/cloudinit/config/cc_phone_home.py index ae1349eb..886487f8 100644 --- a/cloudinit/config/cc_phone_home.py +++ b/cloudinit/config/cc_phone_home.py @@ -84,10 +84,10 @@ def handle(name, cfg, cloud, log, args): for (n, path) in pubkeys.iteritems(): try: - all_keys[n] = util.load_file(cloud.paths.join(True, path)) + all_keys[n] = util.load_file(path) except: util.logexc(log, ("%s: failed to open, can not" - " phone home that data"), path) + " phone home that data!"), path) submit_keys = {} for k in post_list: diff --git a/cloudinit/config/cc_puppet.py b/cloudinit/config/cc_puppet.py index 74ee18e1..8fe3af57 100644 --- a/cloudinit/config/cc_puppet.py +++ b/cloudinit/config/cc_puppet.py @@ -21,12 +21,32 @@ from StringIO import StringIO import os -import pwd import socket from cloudinit import helpers from cloudinit import util +PUPPET_CONF_PATH = '/etc/puppet/puppet.conf' +PUPPET_SSL_CERT_DIR = '/var/lib/puppet/ssl/certs/' +PUPPET_SSL_DIR = '/var/lib/puppet/ssl' +PUPPET_SSL_CERT_PATH = '/var/lib/puppet/ssl/certs/ca.pem' + + +def _autostart_puppet(log): + # Set puppet to automatically start + if os.path.exists('/etc/default/puppet'): + util.subp(['sed', '-i', + '-e', 's/^START=.*/START=yes/', + '/etc/default/puppet'], capture=False) + elif os.path.exists('/bin/systemctl'): + util.subp(['/bin/systemctl', 'enable', 'puppet.service'], + capture=False) + elif os.path.exists('/sbin/chkconfig'): + util.subp(['/sbin/chkconfig', 'puppet', 'on'], capture=False) + else: + log.warn(("Sorry we do not know how to enable" + " puppet services on this system")) + def handle(name, cfg, cloud, log, _args): # If there isn't a puppet key in the configuration don't do anything @@ -43,8 +63,7 @@ def handle(name, cfg, cloud, log, _args): # ... and then update the puppet configuration if 'conf' in puppet_cfg: # Add all sections from the conf object to puppet.conf - puppet_conf_fn = cloud.paths.join(True, '/etc/puppet/puppet.conf') - contents = util.load_file(puppet_conf_fn) + contents = util.load_file(PUPPET_CONF_PATH) # Create object for reading puppet.conf values puppet_config = helpers.DefaultingConfigParser() # Read puppet.conf values from original file in order to be able to @@ -53,28 +72,19 @@ def handle(name, cfg, cloud, log, _args): cleaned_lines = [i.lstrip() for i in contents.splitlines()] cleaned_contents = '\n'.join(cleaned_lines) puppet_config.readfp(StringIO(cleaned_contents), - filename=puppet_conf_fn) + filename=PUPPET_CONF_PATH) for (cfg_name, cfg) in puppet_cfg['conf'].iteritems(): # Cert configuration is a special case # Dump the puppet master ca certificate in the correct place if cfg_name == 'ca_cert': # Puppet ssl sub-directory isn't created yet # Create it with the proper permissions and ownership - pp_ssl_dir = cloud.paths.join(False, '/var/lib/puppet/ssl') - util.ensure_dir(pp_ssl_dir, 0771) - util.chownbyid(pp_ssl_dir, - pwd.getpwnam('puppet').pw_uid, 0) - pp_ssl_certs = cloud.paths.join(False, - '/var/lib/puppet/ssl/certs/') - util.ensure_dir(pp_ssl_certs) - util.chownbyid(pp_ssl_certs, - pwd.getpwnam('puppet').pw_uid, 0) - pp_ssl_ca_certs = cloud.paths.join(False, - ('/var/lib/puppet/' - 'ssl/certs/ca.pem')) - util.write_file(pp_ssl_ca_certs, cfg) - util.chownbyid(pp_ssl_ca_certs, - pwd.getpwnam('puppet').pw_uid, 0) + util.ensure_dir(PUPPET_SSL_DIR, 0771) + util.chownbyname(PUPPET_SSL_DIR, 'puppet', 'root') + util.ensure_dir(PUPPET_SSL_CERT_DIR) + util.chownbyname(PUPPET_SSL_CERT_DIR, 'puppet', 'root') + util.write_file(PUPPET_SSL_CERT_PATH, str(cfg)) + util.chownbyname(PUPPET_SSL_CERT_PATH, 'puppet', 'root') else: # Iterate throug the config items, we'll use ConfigParser.set # to overwrite or create new items as needed @@ -90,25 +100,11 @@ def handle(name, cfg, cloud, log, _args): puppet_config.set(cfg_name, o, v) # We got all our config as wanted we'll rename # the previous puppet.conf and create our new one - conf_old_fn = cloud.paths.join(False, - '/etc/puppet/puppet.conf.old') - util.rename(puppet_conf_fn, conf_old_fn) - puppet_conf_rw = cloud.paths.join(False, '/etc/puppet/puppet.conf') - util.write_file(puppet_conf_rw, puppet_config.stringify()) + util.rename(PUPPET_CONF_PATH, "%s.old" % (PUPPET_CONF_PATH)) + util.write_file(PUPPET_CONF_PATH, puppet_config.stringify()) - # Set puppet to automatically start - if os.path.exists('/etc/default/puppet'): - util.subp(['sed', '-i', - '-e', 's/^START=.*/START=yes/', - '/etc/default/puppet'], capture=False) - elif os.path.exists('/bin/systemctl'): - util.subp(['/bin/systemctl', 'enable', 'puppet.service'], - capture=False) - elif os.path.exists('/sbin/chkconfig'): - util.subp(['/sbin/chkconfig', 'puppet', 'on'], capture=False) - else: - log.warn(("Sorry we do not know how to enable" - " puppet services on this system")) + # Set it up so it autostarts + _autostart_puppet(log) # Start puppetd util.subp(['service', 'puppet', 'start'], capture=False) diff --git a/cloudinit/config/cc_resizefs.py b/cloudinit/config/cc_resizefs.py index e7f27944..b958f332 100644 --- a/cloudinit/config/cc_resizefs.py +++ b/cloudinit/config/cc_resizefs.py @@ -62,7 +62,7 @@ def get_fs_type(st_dev, path, log): raise -def handle(name, cfg, cloud, log, args): +def handle(name, cfg, _cloud, log, args): if len(args) != 0: resize_root = args[0] else: @@ -74,11 +74,10 @@ def handle(name, cfg, cloud, log, args): # TODO(harlowja) is the directory ok to be used?? resize_root_d = util.get_cfg_option_str(cfg, "resize_rootfs_tmp", "/run") - resize_root_d = cloud.paths.join(False, resize_root_d) util.ensure_dir(resize_root_d) # TODO(harlowja): allow what is to be resized to be configurable?? - resize_what = cloud.paths.join(False, "/") + resize_what = "/" with util.ExtendedTemporaryFile(prefix="cloudinit.resizefs.", dir=resize_root_d, delete=True) as tfh: devpth = tfh.name diff --git a/cloudinit/config/cc_rsyslog.py b/cloudinit/config/cc_rsyslog.py index 78327526..0c2c6880 100644 --- a/cloudinit/config/cc_rsyslog.py +++ b/cloudinit/config/cc_rsyslog.py @@ -71,8 +71,7 @@ def handle(name, cfg, cloud, log, _args): try: contents = "%s\n" % (content) - util.write_file(cloud.paths.join(False, filename), - contents, omode=omode) + util.write_file(filename, contents, omode=omode) except Exception: util.logexc(log, "Failed to write to %s", filename) diff --git a/cloudinit/config/cc_runcmd.py b/cloudinit/config/cc_runcmd.py index 65064cfb..598c3a3e 100644 --- a/cloudinit/config/cc_runcmd.py +++ b/cloudinit/config/cc_runcmd.py @@ -33,6 +33,6 @@ def handle(name, cfg, cloud, log, _args): cmd = cfg["runcmd"] try: content = util.shellify(cmd) - util.write_file(cloud.paths.join(False, out_fn), content, 0700) + util.write_file(out_fn, content, 0700) except: util.logexc(log, "Failed to shellify %s into file %s", cmd, out_fn) diff --git a/cloudinit/config/cc_salt_minion.py b/cloudinit/config/cc_salt_minion.py index 8a1440d9..f3eede18 100644 --- a/cloudinit/config/cc_salt_minion.py +++ b/cloudinit/config/cc_salt_minion.py @@ -34,8 +34,7 @@ def handle(name, cfg, cloud, log, _args): cloud.distro.install_packages(["salt-minion"]) # Ensure we can configure files at the right dir - config_dir = cloud.paths.join(False, salt_cfg.get("config_dir", - '/etc/salt')) + config_dir = salt_cfg.get("config_dir", '/etc/salt') util.ensure_dir(config_dir) # ... and then update the salt configuration @@ -47,8 +46,7 @@ def handle(name, cfg, cloud, log, _args): # ... copy the key pair if specified if 'public_key' in salt_cfg and 'private_key' in salt_cfg: - pki_dir = cloud.paths.join(False, salt_cfg.get('pki_dir', - '/etc/salt/pki')) + pki_dir = salt_cfg.get('pki_dir', '/etc/salt/pki') with util.umask(077): util.ensure_dir(pki_dir) pub_name = os.path.join(pki_dir, 'minion.pub') diff --git a/cloudinit/config/cc_set_hostname.py b/cloudinit/config/cc_set_hostname.py index b0f27ebf..2b32fc94 100644 --- a/cloudinit/config/cc_set_hostname.py +++ b/cloudinit/config/cc_set_hostname.py @@ -27,9 +27,11 @@ def handle(name, cfg, cloud, log, _args): " not setting the hostname in module %s"), name) return - (hostname, _fqdn) = util.get_hostname_fqdn(cfg, cloud) + (hostname, fqdn) = util.get_hostname_fqdn(cfg, cloud) try: - log.debug("Setting hostname to %s", hostname) - cloud.distro.set_hostname(hostname) + log.debug("Setting the hostname to %s (%s)", fqdn, hostname) + cloud.distro.set_hostname(hostname, fqdn) except Exception: - util.logexc(log, "Failed to set hostname to %s", hostname) + util.logexc(log, "Failed to set the hostname to %s (%s)", + fqdn, hostname) + raise diff --git a/cloudinit/config/cc_set_passwords.py b/cloudinit/config/cc_set_passwords.py index 26c558ad..c6bf62fd 100644 --- a/cloudinit/config/cc_set_passwords.py +++ b/cloudinit/config/cc_set_passwords.py @@ -114,8 +114,7 @@ def handle(_name, cfg, cloud, log, args): replaced_auth = False # See: man sshd_config - conf_fn = cloud.paths.join(True, ssh_util.DEF_SSHD_CFG) - old_lines = ssh_util.parse_ssh_config(conf_fn) + old_lines = ssh_util.parse_ssh_config(ssh_util.DEF_SSHD_CFG) new_lines = [] i = 0 for (i, line) in enumerate(old_lines): @@ -134,8 +133,7 @@ def handle(_name, cfg, cloud, log, args): pw_auth)) lines = [str(e) for e in new_lines] - ssh_rw_fn = cloud.paths.join(False, ssh_util.DEF_SSHD_CFG) - util.write_file(ssh_rw_fn, "\n".join(lines)) + util.write_file(ssh_util.DEF_SSHD_CFG, "\n".join(lines)) try: cmd = ['service'] diff --git a/cloudinit/config/cc_ssh.py b/cloudinit/config/cc_ssh.py index 32e48c30..b623d476 100644 --- a/cloudinit/config/cc_ssh.py +++ b/cloudinit/config/cc_ssh.py @@ -59,7 +59,7 @@ def handle(_name, cfg, cloud, log, _args): # remove the static keys from the pristine image if cfg.get("ssh_deletekeys", True): - key_pth = cloud.paths.join(False, "/etc/ssh/", "ssh_host_*key*") + key_pth = os.path.join("/etc/ssh/", "ssh_host_*key*") for f in glob.glob(key_pth): try: util.del_file(f) @@ -72,8 +72,7 @@ def handle(_name, cfg, cloud, log, _args): if key in KEY_2_FILE: tgt_fn = KEY_2_FILE[key][0] tgt_perms = KEY_2_FILE[key][1] - util.write_file(cloud.paths.join(False, tgt_fn), - val, tgt_perms) + util.write_file(tgt_fn, val, tgt_perms) for (priv, pub) in PRIV_2_PUB.iteritems(): if pub in cfg['ssh_keys'] or not priv in cfg['ssh_keys']: @@ -94,7 +93,7 @@ def handle(_name, cfg, cloud, log, _args): 'ssh_genkeytypes', GENERATE_KEY_NAMES) for keytype in genkeys: - keyfile = cloud.paths.join(False, KEY_FILE_TPL % (keytype)) + keyfile = KEY_FILE_TPL % (keytype) util.ensure_dir(os.path.dirname(keyfile)) if not os.path.exists(keyfile): cmd = ['ssh-keygen', '-t', keytype, '-N', '', '-f', keyfile] @@ -118,17 +117,16 @@ def handle(_name, cfg, cloud, log, _args): cfgkeys = cfg["ssh_authorized_keys"] keys.extend(cfgkeys) - apply_credentials(keys, user, cloud.paths, - disable_root, disable_root_opts) + apply_credentials(keys, user, disable_root, disable_root_opts) except: util.logexc(log, "Applying ssh credentials failed!") -def apply_credentials(keys, user, paths, disable_root, disable_root_opts): +def apply_credentials(keys, user, disable_root, disable_root_opts): keys = set(keys) if user: - ssh_util.setup_user_keys(keys, user, '', paths) + ssh_util.setup_user_keys(keys, user, '') if disable_root: if not user: @@ -137,4 +135,4 @@ def apply_credentials(keys, user, paths, disable_root, disable_root_opts): else: key_prefix = '' - ssh_util.setup_user_keys(keys, 'root', key_prefix, paths) + ssh_util.setup_user_keys(keys, 'root', key_prefix) diff --git a/cloudinit/config/cc_ssh_authkey_fingerprints.py b/cloudinit/config/cc_ssh_authkey_fingerprints.py index 8c9a8806..c38bcea2 100644 --- a/cloudinit/config/cc_ssh_authkey_fingerprints.py +++ b/cloudinit/config/cc_ssh_authkey_fingerprints.py @@ -97,9 +97,8 @@ def handle(name, cfg, cloud, log, _args): "logging of ssh fingerprints disabled"), name) hash_meth = util.get_cfg_option_str(cfg, "authkey_hash", "md5") - extract_func = ssh_util.extract_authorized_keys (users, _groups) = ds.normalize_users_groups(cfg, cloud.distro) for (user_name, _cfg) in users.items(): - (auth_key_fn, auth_key_entries) = extract_func(user_name, cloud.paths) - _pprint_key_entries(user_name, auth_key_fn, - auth_key_entries, hash_meth) + (key_fn, key_entries) = ssh_util.extract_authorized_keys(user_name) + _pprint_key_entries(user_name, key_fn, + key_entries, hash_meth) diff --git a/cloudinit/config/cc_update_etc_hosts.py b/cloudinit/config/cc_update_etc_hosts.py index 4d75000f..96103615 100644 --- a/cloudinit/config/cc_update_etc_hosts.py +++ b/cloudinit/config/cc_update_etc_hosts.py @@ -42,8 +42,7 @@ def handle(name, cfg, cloud, log, _args): raise RuntimeError(("No hosts template could be" " found for distro %s") % (cloud.distro.name)) - out_fn = cloud.paths.join(False, '/etc/hosts') - templater.render_to_file(tpl_fn_name, out_fn, + templater.render_to_file(tpl_fn_name, '/etc/hosts', {'hostname': hostname, 'fqdn': fqdn}) elif manage_hosts == "localhost": diff --git a/cloudinit/config/cc_update_hostname.py b/cloudinit/config/cc_update_hostname.py index 1d6679ea..52225cd8 100644 --- a/cloudinit/config/cc_update_hostname.py +++ b/cloudinit/config/cc_update_hostname.py @@ -32,10 +32,12 @@ def handle(name, cfg, cloud, log, _args): " not updating the hostname in module %s"), name) return - (hostname, _fqdn) = util.get_hostname_fqdn(cfg, cloud) + (hostname, fqdn) = util.get_hostname_fqdn(cfg, cloud) try: prev_fn = os.path.join(cloud.get_cpath('data'), "previous-hostname") - cloud.distro.update_hostname(hostname, prev_fn) + log.debug("Updating hostname to %s (%s)", fqdn, hostname) + cloud.distro.update_hostname(hostname, fqdn, prev_fn) except Exception: - util.logexc(log, "Failed to set the hostname to %s", hostname) + util.logexc(log, "Failed to update the hostname to %s (%s)", + fqdn, hostname) raise diff --git a/cloudinit/config/cc_yum_add_repo.py b/cloudinit/config/cc_yum_add_repo.py new file mode 100644 index 00000000..5c273825 --- /dev/null +++ b/cloudinit/config/cc_yum_add_repo.py @@ -0,0 +1,106 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2012 Yahoo! Inc. +# +# Author: Joshua Harlow <harlowja@yahoo-inc.com> +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. + +import os + +from cloudinit import util + +import configobj + + +def _canonicalize_id(repo_id): + repo_id = repo_id.lower().replace("-", "_") + repo_id = repo_id.replace(" ", "_") + return repo_id + + +def _format_repo_value(val): + if isinstance(val, (bool)): + # Seems like yum prefers 1/0 + return str(int(val)) + if isinstance(val, (list, tuple)): + # Can handle 'lists' in certain cases + # See: http://bit.ly/Qqrf1t + return "\n ".join([_format_repo_value(v) for v in val]) + if not isinstance(val, (basestring, str)): + return str(val) + return val + + +## TODO(harlowja): move to distro? +# See man yum.conf +def _format_repository_config(repo_id, repo_config): + to_be = configobj.ConfigObj() + to_be[repo_id] = {} + # Do basic translation of the items -> values + for (k, v) in repo_config.items(): + # For now assume that people using this know + # the format of yum and don't verify keys/values further + to_be[repo_id][k] = _format_repo_value(v) + lines = to_be.write() + lines.insert(0, "# Created by cloud-init on %s" % (util.time_rfc2822())) + return "\n".join(lines) + + +def handle(name, cfg, _cloud, log, _args): + repos = cfg.get('yum_repos') + if not repos: + log.debug(("Skipping module named %s," + " no 'yum_repos' configuration found"), name) + return + repo_base_path = util.get_cfg_option_str(cfg, 'yum_repo_dir', + '/etc/yum.repos.d/') + repo_locations = {} + repo_configs = {} + for (repo_id, repo_config) in repos.items(): + canon_repo_id = _canonicalize_id(repo_id) + repo_fn_pth = os.path.join(repo_base_path, "%s.repo" % (canon_repo_id)) + if os.path.exists(repo_fn_pth): + log.info("Skipping repo %s, file %s already exists!", + repo_id, repo_fn_pth) + continue + elif canon_repo_id in repo_locations: + log.info("Skipping repo %s, file %s already pending!", + repo_id, repo_fn_pth) + continue + if not repo_config: + repo_config = {} + # Do some basic sanity checks/cleaning + n_repo_config = {} + for (k, v) in repo_config.items(): + k = k.lower().strip().replace("-", "_") + if k: + n_repo_config[k] = v + repo_config = n_repo_config + missing_required = 0 + for req_field in ['baseurl']: + if not req_field in repo_config: + log.warn(("Repository %s does not contain a %s" + " configuration 'required' entry"), + repo_id, req_field) + missing_required += 1 + if not missing_required: + repo_configs[canon_repo_id] = repo_config + repo_locations[canon_repo_id] = repo_fn_pth + else: + log.warn("Repository %s is missing %s required fields, skipping!", + repo_id, missing_required) + for (c_repo_id, path) in repo_locations.items(): + repo_blob = _format_repository_config(c_repo_id, + repo_configs.get(c_repo_id)) + util.write_file(path, repo_blob) diff --git a/cloudinit/distros/__init__.py b/cloudinit/distros/__init__.py index 2fbb0e9b..8a98e334 100644 --- a/cloudinit/distros/__init__.py +++ b/cloudinit/distros/__init__.py @@ -58,11 +58,11 @@ class Distro(object): return self._cfg.get(opt_name, default) @abc.abstractmethod - def set_hostname(self, hostname): + def set_hostname(self, hostname, fqdn=None): raise NotImplementedError() @abc.abstractmethod - def update_hostname(self, hostname, prev_hostname_fn): + def update_hostname(self, hostname, fqdn, prev_hostname_fn): raise NotImplementedError() @abc.abstractmethod @@ -81,7 +81,7 @@ class Distro(object): def _get_arch_package_mirror_info(self, arch=None): mirror_info = self.get_option("package_mirrors", []) - if arch == None: + if not arch: arch = self.get_primary_arch() return _get_arch_package_mirror_info(mirror_info, arch) @@ -122,8 +122,7 @@ class Distro(object): new_etchosts = StringIO() need_write = False need_change = True - hosts_ro_fn = self._paths.join(True, "/etc/hosts") - for line in util.load_file(hosts_ro_fn).splitlines(): + for line in util.load_file("/etc/hosts").splitlines(): if line.strip().startswith(header): continue if not line.strip() or line.strip().startswith("#"): @@ -147,8 +146,7 @@ class Distro(object): need_write = True if need_write: contents = new_etchosts.getvalue() - util.write_file(self._paths.join(False, "/etc/hosts"), - contents, mode=0644) + util.write_file("/etc/hosts", contents, mode=0644) def _bring_up_interface(self, device_name): cmd = ['ifup', device_name] @@ -189,23 +187,23 @@ class Distro(object): # inputs. If something goes wrong, we can end up with a system # that nobody can login to. adduser_opts = { - "gecos": '--comment', - "homedir": '--home', - "primary_group": '--gid', - "groups": '--groups', - "passwd": '--password', - "shell": '--shell', - "expiredate": '--expiredate', - "inactive": '--inactive', - "selinux_user": '--selinux-user', - } + "gecos": '--comment', + "homedir": '--home', + "primary_group": '--gid', + "groups": '--groups', + "passwd": '--password', + "shell": '--shell', + "expiredate": '--expiredate', + "inactive": '--inactive', + "selinux_user": '--selinux-user', + } adduser_opts_flags = { - "no_user_group": '--no-user-group', - "system": '--system', - "no_log_init": '--no-log-init', - "no_create_home": "-M", - } + "no_user_group": '--no-user-group', + "system": '--system', + "no_log_init": '--no-log-init', + "no_create_home": "-M", + } # Now check the value and create the command for option in kwargs: @@ -262,7 +260,7 @@ class Distro(object): # Import SSH keys if 'ssh_authorized_keys' in kwargs: keys = set(kwargs['ssh_authorized_keys']) or [] - ssh_util.setup_user_keys(keys, name, None, self._paths) + ssh_util.setup_user_keys(keys, name, key_prefix=None) return True @@ -285,8 +283,10 @@ class Distro(object): # Ensure the dir is included and that # it actually exists as a directory sudoers_contents = '' + base_exists = False if os.path.exists(sudo_base): sudoers_contents = util.load_file(sudo_base) + base_exists = True found_include = False for line in sudoers_contents.splitlines(): line = line.strip() @@ -301,24 +301,28 @@ class Distro(object): found_include = True break if not found_include: - sudoers_contents += "\n#includedir %s\n" % (path) try: - if not os.path.exists(sudo_base): + if not base_exists: + lines = [('# See sudoers(5) for more information' + ' on "#include" directives:'), '', + '# Added by cloud-init', + "#includedir %s" % (path), ''] + sudoers_contents = "\n".join(lines) util.write_file(sudo_base, sudoers_contents, 0440) else: - with open(sudo_base, 'a') as f: - f.write(sudoers_contents) - LOG.debug("added '#includedir %s' to %s" % (path, sudo_base)) + lines = ['', '# Added by cloud-init', + "#includedir %s" % (path), ''] + sudoers_contents = "\n".join(lines) + util.append_file(sudo_base, sudoers_contents) + LOG.debug("Added '#includedir %s' to %s" % (path, sudo_base)) except IOError as e: util.logexc(LOG, "Failed to write %s" % sudo_base, e) raise e - util.ensure_dir(path, 0755) + util.ensure_dir(path, 0750) - def write_sudo_rules(self, - user, - rules, - sudo_file="/etc/sudoers.d/90-cloud-init-users", - ): + def write_sudo_rules(self, user, rules, sudo_file=None): + if not sudo_file: + sudo_file = "/etc/sudoers.d/90-cloud-init-users" content_header = "# user rules for %s" % user content = "%s\n%s %s\n\n" % (content_header, user, rules) @@ -427,12 +431,36 @@ def _get_arch_package_mirror_info(package_mirrors, arch): # is the standard form used in the rest # of cloud-init def _normalize_groups(grp_cfg): - if isinstance(grp_cfg, (str, basestring, list)): + if isinstance(grp_cfg, (str, basestring)): + grp_cfg = grp_cfg.strip().split(",") + if isinstance(grp_cfg, (list)): c_grp_cfg = {} - for i in util.uniq_merge(grp_cfg): - c_grp_cfg[i] = [] + for i in grp_cfg: + if isinstance(i, (dict)): + for k, v in i.items(): + if k not in c_grp_cfg: + if isinstance(v, (list)): + c_grp_cfg[k] = list(v) + elif isinstance(v, (basestring, str)): + c_grp_cfg[k] = [v] + else: + raise TypeError("Bad group member type %s" % + util.obj_name(v)) + else: + if isinstance(v, (list)): + c_grp_cfg[k].extend(v) + elif isinstance(v, (basestring, str)): + c_grp_cfg[k].append(v) + else: + raise TypeError("Bad group member type %s" % + util.obj_name(v)) + elif isinstance(i, (str, basestring)): + if i not in c_grp_cfg: + c_grp_cfg[i] = [] + else: + raise TypeError("Unknown group name type %s" % + util.obj_name(i)) grp_cfg = c_grp_cfg - groups = {} if isinstance(grp_cfg, (dict)): for (grp_name, grp_members) in grp_cfg.items(): diff --git a/cloudinit/distros/debian.py b/cloudinit/distros/debian.py index 88f4e978..ed4070b4 100644 --- a/cloudinit/distros/debian.py +++ b/cloudinit/distros/debian.py @@ -43,7 +43,7 @@ class Distro(distros.Distro): def apply_locale(self, locale, out_fn=None): if not out_fn: - out_fn = self._paths.join(False, '/etc/default/locale') + out_fn = '/etc/default/locale' util.subp(['locale-gen', locale], capture=False) util.subp(['update-locale', locale], capture=False) lines = ["# Created by cloud-init", 'LANG="%s"' % (locale), ""] @@ -54,8 +54,7 @@ class Distro(distros.Distro): self.package_command('install', pkglist) def _write_network(self, settings): - net_fn = self._paths.join(False, "/etc/network/interfaces") - util.write_file(net_fn, settings) + util.write_file("/etc/network/interfaces", settings) return ['all'] def _bring_up_interfaces(self, device_names): @@ -68,30 +67,25 @@ class Distro(distros.Distro): else: return distros.Distro._bring_up_interfaces(self, device_names) - def set_hostname(self, hostname): - out_fn = self._paths.join(False, "/etc/hostname") - self._write_hostname(hostname, out_fn) - if out_fn == '/etc/hostname': - # Only do this if we are running in non-adjusted root mode - LOG.debug("Setting hostname to %s", hostname) - util.subp(['hostname', hostname]) + def set_hostname(self, hostname, fqdn=None): + self._write_hostname(hostname, "/etc/hostname") + LOG.debug("Setting hostname to %s", hostname) + util.subp(['hostname', hostname]) def _write_hostname(self, hostname, out_fn): # "" gives trailing newline. util.write_file(out_fn, "%s\n" % str(hostname), 0644) - def update_hostname(self, hostname, prev_fn): + def update_hostname(self, hostname, fqdn, prev_fn): hostname_prev = self._read_hostname(prev_fn) - read_fn = self._paths.join(True, "/etc/hostname") - hostname_in_etc = self._read_hostname(read_fn) + hostname_in_etc = self._read_hostname("/etc/hostname") update_files = [] if not hostname_prev or hostname_prev != hostname: update_files.append(prev_fn) if (not hostname_in_etc or (hostname_in_etc == hostname_prev and hostname_in_etc != hostname)): - write_fn = self._paths.join(False, "/etc/hostname") - update_files.append(write_fn) + update_files.append("/etc/hostname") for fn in update_files: try: self._write_hostname(hostname, fn) @@ -103,7 +97,6 @@ class Distro(distros.Distro): LOG.debug(("%s differs from /etc/hostname." " Assuming user maintained hostname."), prev_fn) if "/etc/hostname" in update_files: - # Only do this if we are running in non-adjusted root mode LOG.debug("Setting hostname to %s", hostname) util.subp(['hostname', hostname]) @@ -130,9 +123,8 @@ class Distro(distros.Distro): " no file found at %s") % (tz, tz_file)) # "" provides trailing newline during join tz_lines = ["# Created by cloud-init", str(tz), ""] - tz_fn = self._paths.join(False, "/etc/timezone") - util.write_file(tz_fn, "\n".join(tz_lines)) - util.copy(tz_file, self._paths.join(False, "/etc/localtime")) + util.write_file("/etc/timezone", "\n".join(tz_lines)) + util.copy(tz_file, "/etc/localtime") def package_command(self, command, args=None): e = os.environ.copy() diff --git a/cloudinit/distros/rhel.py b/cloudinit/distros/rhel.py index bf3c18d2..e4c27216 100644 --- a/cloudinit/distros/rhel.py +++ b/cloudinit/distros/rhel.py @@ -146,8 +146,13 @@ class Distro(distros.Distro): lines.insert(0, _make_header()) util.write_file(fn, "\n".join(lines), 0644) - def set_hostname(self, hostname): - self._write_hostname(hostname, '/etc/sysconfig/network') + def set_hostname(self, hostname, fqdn=None): + # See: http://bit.ly/TwitgL + # Should be fqdn if we can use it + sysconfig_hostname = fqdn + if not sysconfig_hostname: + sysconfig_hostname = hostname + self._write_hostname(sysconfig_hostname, '/etc/sysconfig/network') LOG.debug("Setting hostname to %s", hostname) util.subp(['hostname', hostname]) @@ -165,28 +170,32 @@ class Distro(distros.Distro): } self._update_sysconfig_file(out_fn, host_cfg) - def update_hostname(self, hostname, prev_file): + def update_hostname(self, hostname, fqdn, prev_file): + # See: http://bit.ly/TwitgL + # Should be fqdn if we can use it + sysconfig_hostname = fqdn + if not sysconfig_hostname: + sysconfig_hostname = hostname hostname_prev = self._read_hostname(prev_file) hostname_in_sys = self._read_hostname("/etc/sysconfig/network") update_files = [] - if not hostname_prev or hostname_prev != hostname: + if not hostname_prev or hostname_prev != sysconfig_hostname: update_files.append(prev_file) if (not hostname_in_sys or (hostname_in_sys == hostname_prev - and hostname_in_sys != hostname)): + and hostname_in_sys != sysconfig_hostname)): update_files.append("/etc/sysconfig/network") for fn in update_files: try: - self._write_hostname(hostname, fn) + self._write_hostname(sysconfig_hostname, fn) except: util.logexc(LOG, "Failed to write hostname %s to %s", - hostname, fn) + sysconfig_hostname, fn) if (hostname_in_sys and hostname_prev and hostname_in_sys != hostname_prev): LOG.debug(("%s differs from /etc/sysconfig/network." " Assuming user maintained hostname."), prev_file) if "/etc/sysconfig/network" in update_files: - # Only do this if we are running in non-adjusted root mode LOG.debug("Setting hostname to %s", hostname) util.subp(['hostname', hostname]) diff --git a/cloudinit/handlers/__init__.py b/cloudinit/handlers/__init__.py index 99caed1f..8d6dcd4d 100644 --- a/cloudinit/handlers/__init__.py +++ b/cloudinit/handlers/__init__.py @@ -160,6 +160,19 @@ def _extract_first_or_bytes(blob, size): return start +def _escape_string(text): + try: + return text.encode("string-escape") + except TypeError: + try: + # Unicode doesn't support string-escape... + return text.encode('unicode-escape') + except TypeError: + # Give up... + pass + return text + + def walker_callback(pdata, ctype, filename, payload): if ctype in PART_CONTENT_TYPES: walker_handle_handler(pdata, ctype, filename, payload) @@ -171,7 +184,7 @@ def walker_callback(pdata, ctype, filename, payload): elif payload: # Extract the first line or 24 bytes for displaying in the log start = _extract_first_or_bytes(payload, 24) - details = "'%s...'" % (start.encode("string-escape")) + details = "'%s...'" % (_escape_string(start)) if ctype == NOT_MULTIPART_TYPE: LOG.warning("Unhandled non-multipart (%s) userdata: %s", ctype, details) diff --git a/cloudinit/helpers.py b/cloudinit/helpers.py index a4b20208..794f00ec 100644 --- a/cloudinit/helpers.py +++ b/cloudinit/helpers.py @@ -71,12 +71,17 @@ class FileLock(object): return "<%s using file %r>" % (util.obj_name(self), self.fn) +def canon_sem_name(name): + return name.replace("-", "_") + + class FileSemaphores(object): def __init__(self, sem_path): self.sem_path = sem_path @contextlib.contextmanager def lock(self, name, freq, clear_on_fail=False): + name = canon_sem_name(name) try: yield self._acquire(name, freq) except: @@ -85,6 +90,7 @@ class FileSemaphores(object): raise def clear(self, name, freq): + name = canon_sem_name(name) sem_file = self._get_path(name, freq) try: util.del_file(sem_file) @@ -119,6 +125,7 @@ class FileSemaphores(object): def has_run(self, name, freq): if not freq or freq == PER_ALWAYS: return False + name = canon_sem_name(name) sem_file = self._get_path(name, freq) # This isn't really a good atomic check # but it suffices for where and when cloudinit runs @@ -302,14 +309,10 @@ class Paths(object): def __init__(self, path_cfgs, ds=None): self.cfgs = path_cfgs # Populate all the initial paths - self.cloud_dir = self.join(False, - path_cfgs.get('cloud_dir', - '/var/lib/cloud')) + self.cloud_dir = path_cfgs.get('cloud_dir', '/var/lib/cloud') self.instance_link = os.path.join(self.cloud_dir, 'instance') self.boot_finished = os.path.join(self.instance_link, "boot-finished") self.upstart_conf_d = path_cfgs.get('upstart_dir') - if self.upstart_conf_d: - self.upstart_conf_d = self.join(False, self.upstart_conf_d) self.seed_dir = os.path.join(self.cloud_dir, 'seed') # This one isn't joined, since it should just be read-only template_dir = path_cfgs.get('templates_dir', '/etc/cloud/templates/') @@ -328,29 +331,6 @@ class Paths(object): # Set when a datasource becomes active self.datasource = ds - # joins the paths but also appends a read - # or write root if available - def join(self, read_only, *paths): - if read_only: - root = self.cfgs.get('read_root') - else: - root = self.cfgs.get('write_root') - if not paths: - return root - if len(paths) > 1: - joined = os.path.join(*paths) - else: - joined = paths[0] - if root: - pre_joined = joined - # Need to remove any starting '/' since this - # will confuse os.path.join - joined = joined.lstrip("/") - joined = os.path.join(root, joined) - LOG.debug("Translated %s to adjusted path %s (read-only=%s)", - pre_joined, joined, read_only) - return joined - # get_ipath_cur: get the current instance path for an item def get_ipath_cur(self, name=None): ipath = self.instance_link diff --git a/cloudinit/log.py b/cloudinit/log.py index 2333e5ee..da6c2851 100644 --- a/cloudinit/log.py +++ b/cloudinit/log.py @@ -53,6 +53,18 @@ def setupBasicLogging(): root.setLevel(DEBUG) +def flushLoggers(root): + if not root: + return + for h in root.handlers: + if isinstance(h, (logging.StreamHandler)): + try: + h.flush() + except IOError: + pass + flushLoggers(root.parent) + + def setupLogging(cfg=None): # See if the config provides any logging conf... if not cfg: diff --git a/cloudinit/sources/DataSourceAltCloud.py b/cloudinit/sources/DataSourceAltCloud.py index d7e1204f..9812bdcb 100644 --- a/cloudinit/sources/DataSourceAltCloud.py +++ b/cloudinit/sources/DataSourceAltCloud.py @@ -47,7 +47,7 @@ META_DATA_NOT_SUPPORTED = { 'instance-id': 455, 'local-hostname': 'localhost', 'placement': {}, - } +} def read_user_data_callback(mount_dir): diff --git a/cloudinit/sources/DataSourceConfigDrive.py b/cloudinit/sources/DataSourceConfigDrive.py index 9729cfb9..dbbedce1 100644 --- a/cloudinit/sources/DataSourceConfigDrive.py +++ b/cloudinit/sources/DataSourceConfigDrive.py @@ -218,11 +218,6 @@ class DataSourceConfigDrive(sources.DataSource): return True - def get_public_ssh_keys(self): - if not 'public-keys' in self.metadata: - return [] - return self.metadata['public-keys'] - class DataSourceConfigDriveNet(DataSourceConfigDrive): def __init__(self, sys_cfg, distro, paths): @@ -331,6 +326,13 @@ def read_config_drive_dir_v2(source_dir, version="2012-08-10"): except KeyError: raise BrokenConfigDriveDir("No uuid entry in metadata") + # other datasources (and config-drive-v1) populate metadata['public-keys'] + # where as with config-drive-v2, that would be 'public_keys'. So, just + # copy the field if it is present + if ('public_keys' in results['metadata'] and not + 'public-keys' in results['metadata']): + results['public-keys'] = results['public_keys'] + def read_content_path(item): # do not use os.path.join here, as content_path starts with / cpath = os.path.sep.join((source_dir, "openstack", diff --git a/cloudinit/sources/DataSourceEc2.py b/cloudinit/sources/DataSourceEc2.py index 0fc79b32..6f51dfae 100644 --- a/cloudinit/sources/DataSourceEc2.py +++ b/cloudinit/sources/DataSourceEc2.py @@ -85,9 +85,6 @@ class DataSourceEc2(sources.DataSource): def get_instance_id(self): return self.metadata['instance-id'] - def get_availability_zone(self): - return self.metadata['placement']['availability-zone'] - def _get_url_settings(self): mcfg = self.ds_cfg if not mcfg: @@ -197,19 +194,6 @@ class DataSourceEc2(sources.DataSource): return None return ofound - def is_vpc(self): - # See: https://bugs.launchpad.net/ubuntu/+source/cloud-init/+bug/615545 - # Detect that the machine was launched in a VPC. - # But I did notice that when in a VPC, meta-data - # does not have public-ipv4 and public-hostname - # listed as a possibility. - ph = "public-hostname" - p4 = "public-ipv4" - if ((ph not in self.metadata or self.metadata[ph] == "") and - (p4 not in self.metadata or self.metadata[p4] == "")): - return True - return False - @property def availability_zone(self): try: diff --git a/cloudinit/sources/__init__.py b/cloudinit/sources/__init__.py index b22369a8..745627d0 100644 --- a/cloudinit/sources/__init__.py +++ b/cloudinit/sources/__init__.py @@ -20,8 +20,6 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see <http://www.gnu.org/licenses/>. -from email.mime.multipart import MIMEMultipart - import abc import os diff --git a/cloudinit/ssh_util.py b/cloudinit/ssh_util.py index 88a11a1a..dd6b742f 100644 --- a/cloudinit/ssh_util.py +++ b/cloudinit/ssh_util.py @@ -212,17 +212,15 @@ def update_authorized_keys(old_entries, keys): return '\n'.join(lines) -def users_ssh_info(username, paths): +def users_ssh_info(username): pw_ent = pwd.getpwnam(username) - if not pw_ent: + if not pw_ent or not pw_ent.pw_dir: raise RuntimeError("Unable to get ssh info for user %r" % (username)) - ssh_dir = paths.join(False, os.path.join(pw_ent.pw_dir, '.ssh')) - return (ssh_dir, pw_ent) + return (os.path.join(pw_ent.pw_dir, '.ssh'), pw_ent) -def extract_authorized_keys(username, paths): - (ssh_dir, pw_ent) = users_ssh_info(username, paths) - sshd_conf_fn = paths.join(True, DEF_SSHD_CFG) +def extract_authorized_keys(username): + (ssh_dir, pw_ent) = users_ssh_info(username) auth_key_fn = None with util.SeLinuxGuard(ssh_dir, recursive=True): try: @@ -231,7 +229,7 @@ def extract_authorized_keys(username, paths): # The following tokens are defined: %% is replaced by a literal # '%', %h is replaced by the home directory of the user being # authenticated and %u is replaced by the username of that user. - ssh_cfg = parse_ssh_config_map(sshd_conf_fn) + ssh_cfg = parse_ssh_config_map(DEF_SSHD_CFG) auth_key_fn = ssh_cfg.get("authorizedkeysfile", '').strip() if not auth_key_fn: auth_key_fn = "%h/.ssh/authorized_keys" @@ -240,7 +238,6 @@ def extract_authorized_keys(username, paths): auth_key_fn = auth_key_fn.replace("%%", '%') if not auth_key_fn.startswith('/'): auth_key_fn = os.path.join(pw_ent.pw_dir, auth_key_fn) - auth_key_fn = paths.join(False, auth_key_fn) except (IOError, OSError): # Give up and use a default key filename auth_key_fn = os.path.join(ssh_dir, 'authorized_keys') @@ -248,14 +245,13 @@ def extract_authorized_keys(username, paths): " in ssh config" " from %r, using 'AuthorizedKeysFile' file" " %r instead"), - sshd_conf_fn, auth_key_fn) - auth_key_entries = parse_authorized_keys(auth_key_fn) - return (auth_key_fn, auth_key_entries) + DEF_SSHD_CFG, auth_key_fn) + return (auth_key_fn, parse_authorized_keys(auth_key_fn)) -def setup_user_keys(keys, username, key_prefix, paths): +def setup_user_keys(keys, username, key_prefix): # Make sure the users .ssh dir is setup accordingly - (ssh_dir, pwent) = users_ssh_info(username, paths) + (ssh_dir, pwent) = users_ssh_info(username) if not os.path.isdir(ssh_dir): util.ensure_dir(ssh_dir, mode=0700) util.chownbyid(ssh_dir, pwent.pw_uid, pwent.pw_gid) @@ -267,7 +263,7 @@ def setup_user_keys(keys, username, key_prefix, paths): key_entries.append(parser.parse(str(k), def_opt=key_prefix)) # Extract the old and make the new - (auth_key_fn, auth_key_entries) = extract_authorized_keys(username, paths) + (auth_key_fn, auth_key_entries) = extract_authorized_keys(username) with util.SeLinuxGuard(ssh_dir, recursive=True): content = update_authorized_keys(auth_key_entries, key_entries) util.ensure_dir(os.path.dirname(auth_key_fn), mode=0700) diff --git a/cloudinit/stages.py b/cloudinit/stages.py index 4ed1a750..e0cf1cbe 100644 --- a/cloudinit/stages.py +++ b/cloudinit/stages.py @@ -36,6 +36,8 @@ from cloudinit.handlers import cloud_config as cc_part from cloudinit.handlers import shell_script as ss_part from cloudinit.handlers import upstart_job as up_part +from cloudinit.sources import DataSourceNone + from cloudinit import cloud from cloudinit import config from cloudinit import distros @@ -58,8 +60,16 @@ class Init(object): self._cfg = None self._paths = None self._distro = None - # Created only when a fetch occurs - self.datasource = None + # Changed only when a fetch occurs + self.datasource = DataSourceNone.DataSourceNone({}, None, None) + + def _reset(self, ds=False): + # Recreated on access + self._cfg = None + self._paths = None + self._distro = None + if ds: + self.datasource = DataSourceNone.DataSourceNone({}, None, None) @property def distro(self): @@ -236,7 +246,7 @@ class Init(object): self.datasource = ds # Ensure we adjust our path members datasource # now that we have one (thus allowing ipath to be used) - self.paths.datasource = ds + self._reset() return ds def _get_instance_subdirs(self): @@ -296,6 +306,10 @@ class Init(object): util.write_file(iid_fn, "%s\n" % iid) util.write_file(os.path.join(dp, 'previous-instance-id'), "%s\n" % (previous_iid)) + # Ensure needed components are regenerated + # after change of instance which may cause + # change of configuration + self._reset() return iid def fetch(self): @@ -409,6 +423,17 @@ class Init(object): handlers.call_end(mod, data, frequency) called.append(mod) + # Perform post-consumption adjustments so that + # modules that run during the init stage reflect + # this consumed set. + # + # They will be recreated on future access... + self._reset() + # Note(harlowja): the 'active' datasource will have + # references to the previous config, distro, paths + # objects before the load of the userdata happened, + # this is expected. + class Modules(object): def __init__(self, init, cfg_files=None): diff --git a/cloudinit/util.py b/cloudinit/util.py index 7890a3d6..4f5b15ee 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -1193,8 +1193,7 @@ def yaml_dumps(obj): indent=4, explicit_start=True, explicit_end=True, - default_flow_style=False, - ) + default_flow_style=False) return formatted diff --git a/config/cloud.cfg b/config/cloud.cfg index f6c9065a..ad100fff 100644 --- a/config/cloud.cfg +++ b/config/cloud.cfg @@ -23,6 +23,7 @@ preserve_hostname: false # The modules that run in the 'init' stage cloud_init_modules: + - migrator - bootcmd - write-files - resizefs @@ -45,7 +46,8 @@ cloud_config_modules: - set-passwords - grub-dpkg - apt-pipelining - - apt-update-upgrade + - apt-configure + - package-update-upgrade-install - landscape - timezone - puppet diff --git a/doc/examples/cloud-config-yum-repo.txt b/doc/examples/cloud-config-yum-repo.txt new file mode 100644 index 00000000..ab2c031e --- /dev/null +++ b/doc/examples/cloud-config-yum-repo.txt @@ -0,0 +1,20 @@ +#cloud-config +# vim: syntax=yaml +# +# Add yum repository configuration to the system +# +# The following example adds the file /etc/yum.repos.d/epel_testing.repo +# which can then subsequently be used by yum for later operations. +yum_repos: + # The name of the repository + epel-testing: + # Any repository configuration options + # See: man yum.conf + # + # This one is required! + baseurl: http://download.fedoraproject.org/pub/epel/testing/5/$basearch + enabled: false + failovermethod: priority + gpgcheck: true + gpgkey: file:///etc/pki/rpm-gpg/RPM-GPG-KEY-EPEL + name: Extra Packages for Enterprise Linux 5 - Testing diff --git a/doc/examples/cloud-config.txt b/doc/examples/cloud-config.txt index 56a6c35a..12bf2c91 100644 --- a/doc/examples/cloud-config.txt +++ b/doc/examples/cloud-config.txt @@ -3,15 +3,20 @@ # (ie run apt-get update) # # Default: true -# -apt_update: false +# Aliases: apt_update +package_update: false # Upgrade the instance on first boot # (ie run apt-get upgrade) # # Default: false -# -apt_upgrade: true +# Aliases: apt_upgrade +package_upgrade: true + +# Reboot after package install/update if necessary +# Default: false +# Aliases: apt_reboot_if_required +package_reboot_if_required: true # Add apt repositories # @@ -350,8 +355,7 @@ rsyslog: - ':syslogtag, isequal, "[CLOUDINIT]" /var/log/cloud-foo.log' - content: "*.* @@192.0.2.1:10514" - filename: 01-examplecom.conf - content: | - *.* @@syslogd.example.com + content: "*.* @@syslogd.example.com" # resize_rootfs should the / filesytem be resized on first boot # this allows you to launch an instance with a larger disk / partition diff --git a/pylintrc b/pylintrc new file mode 100644 index 00000000..ee886510 --- /dev/null +++ b/pylintrc @@ -0,0 +1,19 @@ +[General] +init-hook='import sys; sys.path.append("tests/")' + +[MESSAGES CONTROL] +# See: http://pylint-messages.wikidot.com/all-codes +# W0142: *args and **kwargs are fine. +# W0511: TODOs in code comments are fine. +# W0702: No exception type(s) specified +# W0703: Catch "Exception" +# C0103: Invalid name +# C0111: Missing docstring +disable=W0142,W0511,W0702,W0703,C0103,C0111 + +[REPORTS] +reports=no +include-ids=yes + +[FORMAT] +max-line-length=79 diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 00000000..e69de29b --- /dev/null +++ b/tests/__init__.py diff --git a/tests/configs/sample1.yaml b/tests/configs/sample1.yaml index 24e874ee..6231f293 100644 --- a/tests/configs/sample1.yaml +++ b/tests/configs/sample1.yaml @@ -50,4 +50,3 @@ runcmd: byobu_by_default: user -output: {all: '| tee -a /var/log/cloud-init-output.log'} diff --git a/tests/data/user_data.1.txt b/tests/data/user_data.1.txt new file mode 100644 index 00000000..4c4543de --- /dev/null +++ b/tests/data/user_data.1.txt @@ -0,0 +1,15 @@ +#cloud-config +write_files: +- content: blah + path: /etc/blah.ini + permissions: 493 + +system_info: + package_mirrors: + - arches: [i386, amd64, blah] + failsafe: + primary: http://my.archive.mydomain.com/ubuntu + security: http://my.security.mydomain.com/ubuntu + search: + primary: [] + security: [] diff --git a/tests/unittests/__init__.py b/tests/unittests/__init__.py new file mode 100644 index 00000000..e69de29b --- /dev/null +++ b/tests/unittests/__init__.py diff --git a/tests/unittests/helpers.py b/tests/unittests/helpers.py index 2c5dcad2..e8080668 100644 --- a/tests/unittests/helpers.py +++ b/tests/unittests/helpers.py @@ -103,6 +103,7 @@ class FilesystemMockingTestCase(ResourceUsingTestCase): def patchUtils(self, new_root): patch_funcs = { util: [('write_file', 1), + ('append_file', 1), ('load_file', 1), ('ensure_dir', 1), ('chmod', 1), diff --git a/tests/unittests/test_datasource/__init__.py b/tests/unittests/test_datasource/__init__.py new file mode 100644 index 00000000..e69de29b --- /dev/null +++ b/tests/unittests/test_datasource/__init__.py diff --git a/tests/unittests/test_distros/__init__.py b/tests/unittests/test_distros/__init__.py new file mode 100644 index 00000000..e69de29b --- /dev/null +++ b/tests/unittests/test_distros/__init__.py diff --git a/tests/unittests/test_distros/test_generic.py b/tests/unittests/test_distros/test_generic.py index 2df4c2f0..704699b5 100644 --- a/tests/unittests/test_distros/test_generic.py +++ b/tests/unittests/test_distros/test_generic.py @@ -1,6 +1,9 @@ -from mocker import MockerTestCase - from cloudinit import distros +from cloudinit import util + +from tests.unittests import helpers + +import os unknown_arch_info = { 'arches': ['default'], @@ -27,7 +30,7 @@ gpmi = distros._get_package_mirror_info # pylint: disable=W0212 gapmi = distros._get_arch_package_mirror_info # pylint: disable=W0212 -class TestGenericDistro(MockerTestCase): +class TestGenericDistro(helpers.FilesystemMockingTestCase): def return_first(self, mlist): if not mlist: @@ -52,6 +55,29 @@ class TestGenericDistro(MockerTestCase): # Make a temp directoy for tests to use. self.tmp = self.makeDir() + def test_sudoers_ensure_new(self): + cls = distros.fetch("ubuntu") + d = cls("ubuntu", {}, None) + self.patchOS(self.tmp) + self.patchUtils(self.tmp) + d.ensure_sudo_dir("/b") + contents = util.load_file("/etc/sudoers") + self.assertIn("includedir /b", contents) + self.assertTrue(os.path.isdir("/b")) + + def test_sudoers_ensure_append(self): + cls = distros.fetch("ubuntu") + d = cls("ubuntu", {}, None) + self.patchOS(self.tmp) + self.patchUtils(self.tmp) + util.write_file("/etc/sudoers", "josh, josh\n") + d.ensure_sudo_dir("/b") + contents = util.load_file("/etc/sudoers") + self.assertIn("includedir /b", contents) + self.assertTrue(os.path.isdir("/b")) + self.assertIn("josh", contents) + self.assertEquals(2, contents.count("josh")) + def test_arch_package_mirror_info_unknown(self): """for an unknown arch, we should get back that with arch 'default'.""" arch_mirrors = gapmi(package_mirrors, arch="unknown") diff --git a/tests/unittests/test_distros/test_user_data_normalize.py b/tests/unittests/test_distros/test_user_data_normalize.py index 8f0d8896..5d9d4311 100644 --- a/tests/unittests/test_distros/test_user_data_normalize.py +++ b/tests/unittests/test_distros/test_user_data_normalize.py @@ -30,6 +30,28 @@ class TestUGNormalize(MockerTestCase): def _norm(self, cfg, distro): return distros.normalize_users_groups(cfg, distro) + def test_group_dict(self): + distro = self._make_distro('ubuntu') + g = {'groups': [ + { + 'ubuntu': ['foo', 'bar'], + 'bob': 'users', + }, + 'cloud-users', + { + 'bob': 'users2', + }, + ] + } + (_users, groups) = self._norm(g, distro) + self.assertIn('ubuntu', groups) + ub_members = groups['ubuntu'] + self.assertEquals(sorted(['foo', 'bar']), sorted(ub_members)) + self.assertIn('bob', groups) + b_members = groups['bob'] + self.assertEquals(sorted(['users', 'users2']), + sorted(b_members)) + def test_basic_groups(self): distro = self._make_distro('ubuntu') ug_cfg = { diff --git a/tests/unittests/test_filters/__init__.py b/tests/unittests/test_filters/__init__.py new file mode 100644 index 00000000..e69de29b --- /dev/null +++ b/tests/unittests/test_filters/__init__.py diff --git a/tests/unittests/test_filters/test_launch_index.py b/tests/unittests/test_filters/test_launch_index.py index 1e9b9053..773bb312 100644 --- a/tests/unittests/test_filters/test_launch_index.py +++ b/tests/unittests/test_filters/test_launch_index.py @@ -1,14 +1,6 @@ import copy -import os -import sys -top_dir = os.path.join(os.path.dirname(__file__), os.pardir, "helpers.py") -top_dir = os.path.abspath(top_dir) -if os.path.exists(top_dir): - sys.path.insert(0, os.path.dirname(top_dir)) - - -import helpers +from tests.unittests import helpers import itertools diff --git a/tests/unittests/test_handler/__init__.py b/tests/unittests/test_handler/__init__.py new file mode 100644 index 00000000..e69de29b --- /dev/null +++ b/tests/unittests/test_handler/__init__.py diff --git a/tests/unittests/test_handler/test_handler_ca_certs.py b/tests/unittests/test_handler/test_handler_ca_certs.py index d3df5c50..d73c9fa9 100644 --- a/tests/unittests/test_handler/test_handler_ca_certs.py +++ b/tests/unittests/test_handler/test_handler_ca_certs.py @@ -77,7 +77,7 @@ class TestConfig(MockerTestCase): """Test that a single cert gets passed to add_ca_certs.""" config = {"ca-certs": {"trusted": ["CERT1"]}} - self.mock_add(self.paths, ["CERT1"]) + self.mock_add(["CERT1"]) self.mock_update() self.mocker.replay() @@ -87,7 +87,7 @@ class TestConfig(MockerTestCase): """Test that multiple certs get passed to add_ca_certs.""" config = {"ca-certs": {"trusted": ["CERT1", "CERT2"]}} - self.mock_add(self.paths, ["CERT1", "CERT2"]) + self.mock_add(["CERT1", "CERT2"]) self.mock_update() self.mocker.replay() @@ -97,7 +97,7 @@ class TestConfig(MockerTestCase): """Test remove_defaults works as expected.""" config = {"ca-certs": {"remove-defaults": True}} - self.mock_remove(self.paths) + self.mock_remove() self.mock_update() self.mocker.replay() @@ -116,8 +116,8 @@ class TestConfig(MockerTestCase): """Test remove_defaults is not called when config value is False.""" config = {"ca-certs": {"remove-defaults": True, "trusted": ["CERT1"]}} - self.mock_remove(self.paths) - self.mock_add(self.paths, ["CERT1"]) + self.mock_remove() + self.mock_add(["CERT1"]) self.mock_update() self.mocker.replay() @@ -136,7 +136,7 @@ class TestAddCaCerts(MockerTestCase): """Test that no certificate are written if not provided.""" self.mocker.replace(util.write_file, passthrough=False) self.mocker.replay() - cc_ca_certs.add_ca_certs(self.paths, []) + cc_ca_certs.add_ca_certs([]) def test_single_cert(self): """Test adding a single certificate to the trusted CAs.""" @@ -149,7 +149,7 @@ class TestAddCaCerts(MockerTestCase): "\ncloud-init-ca-certs.crt", omode="ab") self.mocker.replay() - cc_ca_certs.add_ca_certs(self.paths, [cert]) + cc_ca_certs.add_ca_certs([cert]) def test_multiple_certs(self): """Test adding multiple certificates to the trusted CAs.""" @@ -163,7 +163,7 @@ class TestAddCaCerts(MockerTestCase): "\ncloud-init-ca-certs.crt", omode="ab") self.mocker.replay() - cc_ca_certs.add_ca_certs(self.paths, certs) + cc_ca_certs.add_ca_certs(certs) class TestUpdateCaCerts(MockerTestCase): @@ -198,4 +198,4 @@ class TestRemoveDefaultCaCerts(MockerTestCase): "ca-certificates ca-certificates/trust_new_crts select no") self.mocker.replay() - cc_ca_certs.remove_default_ca_certs(self.paths) + cc_ca_certs.remove_default_ca_certs() diff --git a/tests/unittests/test_handler/test_handler_set_hostname.py b/tests/unittests/test_handler/test_handler_set_hostname.py new file mode 100644 index 00000000..a1aba62f --- /dev/null +++ b/tests/unittests/test_handler/test_handler_set_hostname.py @@ -0,0 +1,58 @@ +from cloudinit.config import cc_set_hostname + +from cloudinit import cloud +from cloudinit import distros +from cloudinit import helpers +from cloudinit import util + +from tests.unittests import helpers as t_help + +import logging + +from StringIO import StringIO + +from configobj import ConfigObj + +LOG = logging.getLogger(__name__) + + +class TestHostname(t_help.FilesystemMockingTestCase): + def setUp(self): + super(TestHostname, self).setUp() + self.tmp = self.makeDir(prefix="unittest_") + + def _fetch_distro(self, kind): + cls = distros.fetch(kind) + paths = helpers.Paths({}) + return cls(kind, {}, paths) + + def test_write_hostname_rhel(self): + cfg = { + 'hostname': 'blah.blah.blah.yahoo.com', + } + distro = self._fetch_distro('rhel') + paths = helpers.Paths({}) + ds = None + cc = cloud.Cloud(ds, paths, {}, distro, None) + self.patchUtils(self.tmp) + self.patchOS(self.tmp) + cc_set_hostname.handle('cc_set_hostname', + cfg, cc, LOG, []) + contents = util.load_file("/etc/sysconfig/network") + n_cfg = ConfigObj(StringIO(contents)) + self.assertEquals({'HOSTNAME': 'blah.blah.blah.yahoo.com'}, + dict(n_cfg)) + + def test_write_hostname_debian(self): + cfg = { + 'hostname': 'blah.blah.blah.yahoo.com', + } + distro = self._fetch_distro('debian') + paths = helpers.Paths({}) + ds = None + cc = cloud.Cloud(ds, paths, {}, distro, None) + self.patchUtils(self.tmp) + cc_set_hostname.handle('cc_set_hostname', + cfg, cc, LOG, []) + contents = util.load_file("/etc/hostname") + self.assertEquals('blah', contents.strip()) diff --git a/tests/unittests/test_handler/test_handler_yum_add_repo.py b/tests/unittests/test_handler/test_handler_yum_add_repo.py new file mode 100644 index 00000000..c7fcddc4 --- /dev/null +++ b/tests/unittests/test_handler/test_handler_yum_add_repo.py @@ -0,0 +1,68 @@ +from cloudinit import helpers +from cloudinit import util + +from cloudinit.config import cc_yum_add_repo + +from tests.unittests import helpers + +import logging + +from StringIO import StringIO + +import configobj + +LOG = logging.getLogger(__name__) + + +class TestConfig(helpers.FilesystemMockingTestCase): + def setUp(self): + super(TestConfig, self).setUp() + self.tmp = self.makeDir(prefix="unittest_") + + def test_bad_config(self): + cfg = { + 'yum_repos': { + 'epel-testing': { + 'name': 'Extra Packages for Enterprise Linux 5 - Testing', + # Missing this should cause the repo not to be written + #'baseurl': 'http://blah.org/pub/epel/testing/5/$basearch', + 'enabled': False, + 'gpgcheck': True, + 'gpgkey': 'file:///etc/pki/rpm-gpg/RPM-GPG-KEY-EPEL', + 'failovermethod': 'priority', + }, + }, + } + self.patchUtils(self.tmp) + cc_yum_add_repo.handle('yum_add_repo', cfg, None, LOG, []) + with self.assertRaises(IOError): + util.load_file("/etc/yum.repos.d/epel_testing.repo") + + def test_write_config(self): + cfg = { + 'yum_repos': { + 'epel-testing': { + 'name': 'Extra Packages for Enterprise Linux 5 - Testing', + 'baseurl': 'http://blah.org/pub/epel/testing/5/$basearch', + 'enabled': False, + 'gpgcheck': True, + 'gpgkey': 'file:///etc/pki/rpm-gpg/RPM-GPG-KEY-EPEL', + 'failovermethod': 'priority', + }, + }, + } + self.patchUtils(self.tmp) + cc_yum_add_repo.handle('yum_add_repo', cfg, None, LOG, []) + contents = util.load_file("/etc/yum.repos.d/epel_testing.repo") + contents = configobj.ConfigObj(StringIO(contents)) + expected = { + 'epel_testing': { + 'name': 'Extra Packages for Enterprise Linux 5 - Testing', + 'failovermethod': 'priority', + 'gpgkey': 'file:///etc/pki/rpm-gpg/RPM-GPG-KEY-EPEL', + 'enabled': '0', + 'baseurl': 'http://blah.org/pub/epel/testing/5/$basearch', + 'gpgcheck': '1', + } + } + self.assertEquals(expected, dict(contents)) diff --git a/tests/unittests/test_merging.py b/tests/unittests/test_merging.py new file mode 100644 index 00000000..0037b966 --- /dev/null +++ b/tests/unittests/test_merging.py @@ -0,0 +1,62 @@ +from mocker import MockerTestCase + +from cloudinit import util + + +class TestMergeDict(MockerTestCase): + def test_simple_merge(self): + """Test simple non-conflict merge.""" + source = {"key1": "value1"} + candidate = {"key2": "value2"} + result = util.mergedict(source, candidate) + self.assertEqual({"key1": "value1", "key2": "value2"}, result) + + def test_nested_merge(self): + """Test nested merge.""" + source = {"key1": {"key1.1": "value1.1"}} + candidate = {"key1": {"key1.2": "value1.2"}} + result = util.mergedict(source, candidate) + self.assertEqual( + {"key1": {"key1.1": "value1.1", "key1.2": "value1.2"}}, result) + + def test_merge_does_not_override(self): + """Test that candidate doesn't override source.""" + source = {"key1": "value1", "key2": "value2"} + candidate = {"key1": "value2", "key2": "NEW VALUE"} + result = util.mergedict(source, candidate) + self.assertEqual(source, result) + + def test_empty_candidate(self): + """Test empty candidate doesn't change source.""" + source = {"key": "value"} + candidate = {} + result = util.mergedict(source, candidate) + self.assertEqual(source, result) + + def test_empty_source(self): + """Test empty source is replaced by candidate.""" + source = {} + candidate = {"key": "value"} + result = util.mergedict(source, candidate) + self.assertEqual(candidate, result) + + def test_non_dict_candidate(self): + """Test non-dict candidate is discarded.""" + source = {"key": "value"} + candidate = "not a dict" + result = util.mergedict(source, candidate) + self.assertEqual(source, result) + + def test_non_dict_source(self): + """Test non-dict source is not modified with a dict candidate.""" + source = "not a dict" + candidate = {"key": "value"} + result = util.mergedict(source, candidate) + self.assertEqual(source, result) + + def test_neither_dict(self): + """Test if neither candidate or source is dict source wins.""" + source = "source" + candidate = "candidate" + result = util.mergedict(source, candidate) + self.assertEqual(source, result) diff --git a/tests/unittests/test_runs/__init__.py b/tests/unittests/test_runs/__init__.py new file mode 100644 index 00000000..e69de29b --- /dev/null +++ b/tests/unittests/test_runs/__init__.py diff --git a/tests/unittests/test_runs/test_merge_run.py b/tests/unittests/test_runs/test_merge_run.py new file mode 100644 index 00000000..d9c3a455 --- /dev/null +++ b/tests/unittests/test_runs/test_merge_run.py @@ -0,0 +1,52 @@ +import os + +from tests.unittests import helpers + +from cloudinit.settings import (PER_INSTANCE) +from cloudinit import stages +from cloudinit import util + + +class TestMergeRun(helpers.FilesystemMockingTestCase): + def _patchIn(self, root): + self.restore() + self.patchOS(root) + self.patchUtils(root) + + def test_none_ds(self): + new_root = self.makeDir() + self.replicateTestRoot('simple_ubuntu', new_root) + cfg = { + 'datasource_list': ['None'], + 'cloud_init_modules': ['write-files'], + } + ud = self.readResource('user_data.1.txt') + cloud_cfg = util.yaml_dumps(cfg) + util.ensure_dir(os.path.join(new_root, 'etc', 'cloud')) + util.write_file(os.path.join(new_root, 'etc', + 'cloud', 'cloud.cfg'), cloud_cfg) + self._patchIn(new_root) + + # Now start verifying whats created + initer = stages.Init() + initer.read_cfg() + initer.initialize() + initer.fetch() + initer.datasource.userdata_raw = ud + _iid = initer.instancify() + initer.update() + initer.cloudify().run('consume_userdata', + initer.consume_userdata, + args=[PER_INSTANCE], + freq=PER_INSTANCE) + mirrors = initer.distro.get_option('package_mirrors') + self.assertEquals(1, len(mirrors)) + mirror = mirrors[0] + self.assertEquals(mirror['arches'], ['i386', 'amd64', 'blah']) + mods = stages.Modules(initer) + (which_ran, failures) = mods.run_section('cloud_init_modules') + self.assertTrue(len(failures) == 0) + self.assertTrue(os.path.exists('/etc/blah.ini')) + self.assertIn('write-files', which_ran) + contents = util.load_file('/etc/blah.ini') + self.assertEquals(contents, 'blah') diff --git a/tests/unittests/test_runs/test_simple_run.py b/tests/unittests/test_runs/test_simple_run.py index 1e852e1e..60ef812a 100644 --- a/tests/unittests/test_runs/test_simple_run.py +++ b/tests/unittests/test_runs/test_simple_run.py @@ -1,14 +1,6 @@ import os -import sys -# Allow running this test individually -top_dir = os.path.join(os.path.dirname(__file__), os.pardir, "helpers.py") -top_dir = os.path.abspath(top_dir) -if os.path.exists(top_dir): - sys.path.insert(0, os.path.dirname(top_dir)) - - -import helpers +from tests.unittests import helpers from cloudinit.settings import (PER_INSTANCE) from cloudinit import stages @@ -45,11 +37,13 @@ class TestSimpleRun(helpers.FilesystemMockingTestCase): self.replicateTestRoot('simple_ubuntu', new_root) cfg = { 'datasource_list': ['None'], - 'write_files': [{ + 'write_files': [ + { 'path': '/etc/blah.ini', 'content': 'blah', 'permissions': 0755, - }], + }, + ], 'cloud_init_modules': ['write-files'], } cloud_cfg = util.yaml_dumps(cfg) diff --git a/tests/unittests/test_util.py b/tests/unittests/test_util.py index 96962b91..02611581 100644 --- a/tests/unittests/test_util.py +++ b/tests/unittests/test_util.py @@ -28,65 +28,6 @@ class FakeSelinux(object): self.restored.append(path) -class TestMergeDict(MockerTestCase): - def test_simple_merge(self): - """Test simple non-conflict merge.""" - source = {"key1": "value1"} - candidate = {"key2": "value2"} - result = util.mergedict(source, candidate) - self.assertEqual({"key1": "value1", "key2": "value2"}, result) - - def test_nested_merge(self): - """Test nested merge.""" - source = {"key1": {"key1.1": "value1.1"}} - candidate = {"key1": {"key1.2": "value1.2"}} - result = util.mergedict(source, candidate) - self.assertEqual( - {"key1": {"key1.1": "value1.1", "key1.2": "value1.2"}}, result) - - def test_merge_does_not_override(self): - """Test that candidate doesn't override source.""" - source = {"key1": "value1", "key2": "value2"} - candidate = {"key1": "value2", "key2": "NEW VALUE"} - result = util.mergedict(source, candidate) - self.assertEqual(source, result) - - def test_empty_candidate(self): - """Test empty candidate doesn't change source.""" - source = {"key": "value"} - candidate = {} - result = util.mergedict(source, candidate) - self.assertEqual(source, result) - - def test_empty_source(self): - """Test empty source is replaced by candidate.""" - source = {} - candidate = {"key": "value"} - result = util.mergedict(source, candidate) - self.assertEqual(candidate, result) - - def test_non_dict_candidate(self): - """Test non-dict candidate is discarded.""" - source = {"key": "value"} - candidate = "not a dict" - result = util.mergedict(source, candidate) - self.assertEqual(source, result) - - def test_non_dict_source(self): - """Test non-dict source is not modified with a dict candidate.""" - source = "not a dict" - candidate = {"key": "value"} - result = util.mergedict(source, candidate) - self.assertEqual(source, result) - - def test_neither_dict(self): - """Test if neither candidate or source is dict source wins.""" - source = "source" - candidate = "candidate" - result = util.mergedict(source, candidate) - self.assertEqual(source, result) - - class TestGetCfgOptionListOrStr(TestCase): def test_not_found_no_default(self): """None is returned if key is not found and no default given.""" diff --git a/tools/Z99-cloud-locale-test.sh b/tools/Z99-cloud-locale-test.sh index 8ad485e8..3c51f22d 100755 --- a/tools/Z99-cloud-locale-test.sh +++ b/tools/Z99-cloud-locale-test.sh @@ -10,10 +10,13 @@ # locale_warn() { - local cr=" -" - local bad_names="" bad_lcs="" key="" value="" var="" + local bad_names="" bad_lcs="" key="" val="" var="" vars="" local w1 w2 w3 w4 remain + + # if shell is zsh, act like sh only for this function (-L). + # The behavior change will not permenently affect user's shell. + [ "${ZSH_NAME+zsh}" = "zsh" ] && emulate -L sh + # locale is expected to output either: # VARIABLE= # VARIABLE="value" @@ -32,9 +35,9 @@ locale_warn() { for bad in $bad_names; do for var in ${vars}; do [ "${bad}" = "${var%=*}" ] || continue - value=${var#*=} - [ "${bad_lcs#* ${value}}" = "${bad_lcs}" ] && - bad_lcs="${bad_lcs} ${value}" + val=${var#*=} + [ "${bad_lcs#* ${val}}" = "${bad_lcs}" ] && + bad_lcs="${bad_lcs} ${val}" break done done diff --git a/tools/hacking.py b/tools/hacking.py index 11163df3..26a07c53 100755 --- a/tools/hacking.py +++ b/tools/hacking.py @@ -66,8 +66,8 @@ def cloud_import_alphabetical(physical_line, line_number, lines): # handle import x # use .lower since capitalization shouldn't dictate order split_line = import_normalize(physical_line.strip()).lower().split() - split_previous = import_normalize(lines[line_number - 2] - ).strip().lower().split() + split_previous = import_normalize(lines[line_number - 2]) + split_previous = split_previous.strip().lower().split() # with or without "as y" length = [2, 4] if (len(split_line) in length and len(split_previous) in length and diff --git a/tools/run-pep8 b/tools/run-pep8 index ad55d420..20e594bc 100755 --- a/tools/run-pep8 +++ b/tools/run-pep8 @@ -21,10 +21,22 @@ else base=`pwd`/tools/ fi +IGNORE="E501" # Line too long (these are caught by pylint) + +# King Arthur: Be quiet! ... Be Quiet! I Order You to Be Quiet. +IGNORE="$IGNORE,E121" # Continuation line indentation is not a multiple of four +IGNORE="$IGNORE,E123" # Closing bracket does not match indentation of opening bracket's line +IGNORE="$IGNORE,E124" # Closing bracket missing visual indentation +IGNORE="$IGNORE,E125" # Continuation line does not distinguish itself from next logical line +IGNORE="$IGNORE,E126" # Continuation line over-indented for hanging indent +IGNORE="$IGNORE,E127" # Continuation line over-indented for visual indent +IGNORE="$IGNORE,E128" # Continuation line under-indented for visual indent +IGNORE="$IGNORE,E502" # The backslash is redundant between brackets + cmd=( ${base}/hacking.py - --ignore=E501 # Line too long (these are caught by pylint) + --ignore="$IGNORE" "${files[@]}" ) diff --git a/tools/run-pylint b/tools/run-pylint index 7ef44ac5..b74efda9 100755 --- a/tools/run-pylint +++ b/tools/run-pylint @@ -6,23 +6,16 @@ else files=( "$@" ); fi +RC_FILE="pylintrc" +if [ ! -f $RC_FILE ]; then + RC_FILE="../pylintrc" +fi + cmd=( pylint - --reports=n - --include-ids=y - --max-line-length=79 - + --rcfile=$RC_FILE --disable=R --disable=I - - --disable=W0142 # Used * or ** magic - --disable=W0511 # TODO/FIXME note - --disable=W0702 # No exception type(s) specified - --disable=W0703 # Catch "Exception" - - --disable=C0103 # Invalid name - --disable=C0111 # Missing docstring - "${files[@]}" ) diff --git a/tools/validate-yaml.py b/tools/validate-yaml.py new file mode 100755 index 00000000..eda59cb8 --- /dev/null +++ b/tools/validate-yaml.py @@ -0,0 +1,26 @@ +#!/usr/bin/env python + +"""Try to read a YAML file and report any errors. +""" + +import sys + +import yaml + + +if __name__ == "__main__": + bads = 0 + for fn in sys.argv[1:]: + sys.stdout.write("%s" % (fn)) + try: + fh = open(fn, 'r') + yaml.safe_load(fh.read()) + fh.close() + sys.stdout.write(" - ok\n") + except Exception, e: + sys.stdout.write(" - bad (%s)\n" % (e)) + bads += 1 + if bads > 0: + sys.exit(1) + else: + sys.exit(0) |