diff options
Diffstat (limited to 'cloudinit')
50 files changed, 710 insertions, 323 deletions
diff --git a/cloudinit/config/cc_apt_configure.py b/cloudinit/config/cc_apt_configure.py index 29c13a3d..f10b76a3 100644 --- a/cloudinit/config/cc_apt_configure.py +++ b/cloudinit/config/cc_apt_configure.py @@ -235,7 +235,7 @@ def find_apt_mirror_info(cloud, cfg): mirror = util.search_for_mirror(search) if (not mirror and - util.get_cfg_option_bool(cfg, "apt_mirror_search_dns", False)): + util.get_cfg_option_bool(cfg, "apt_mirror_search_dns", False)): mydom = "" doms = [] diff --git a/cloudinit/config/cc_byobu.py b/cloudinit/config/cc_byobu.py index 92d428b7..ef0ce7ab 100644 --- a/cloudinit/config/cc_byobu.py +++ b/cloudinit/config/cc_byobu.py @@ -43,7 +43,7 @@ def handle(name, cfg, cloud, log, args): valid = ("enable-user", "enable-system", "enable", "disable-user", "disable-system", "disable") - if not value in valid: + if value not in valid: log.warn("Unknown value %s for byobu_by_default", value) mod_user = value.endswith("-user") diff --git a/cloudinit/config/cc_chef.py b/cloudinit/config/cc_chef.py index 727769cd..806deed9 100644 --- a/cloudinit/config/cc_chef.py +++ b/cloudinit/config/cc_chef.py @@ -87,7 +87,8 @@ def handle(name, cfg, cloud, log, _args): # If chef is not installed, we install chef based on 'install_type' if (not os.path.isfile('/usr/bin/chef-client') or - util.get_cfg_option_bool(chef_cfg, 'force_install', default=False)): + util.get_cfg_option_bool(chef_cfg, + 'force_install', default=False)): install_type = util.get_cfg_option_str(chef_cfg, 'install_type', 'packages') diff --git a/cloudinit/config/cc_disk_setup.py b/cloudinit/config/cc_disk_setup.py index 0b970e4e..1660832b 100644 --- a/cloudinit/config/cc_disk_setup.py +++ b/cloudinit/config/cc_disk_setup.py @@ -271,7 +271,7 @@ def find_device_node(device, fs_type=None, label=None, valid_targets=None, return ('/dev/%s' % d['name'], False) if (d['fstype'] == fs_type and - ((label_match and d['label'] == label) or not label_match)): + ((label_match and d['label'] == label) or not label_match)): # If we find a matching device, we return that return ('/dev/%s' % d['name'], True) @@ -447,7 +447,7 @@ def get_partition_mbr_layout(size, layout): return "0," if ((len(layout) == 0 and isinstance(layout, list)) or - not isinstance(layout, list)): + not isinstance(layout, list)): raise Exception("Partition layout is invalid") last_part_num = len(layout) @@ -484,7 +484,7 @@ def get_partition_mbr_layout(size, layout): def purge_disk_ptable(device): # wipe the first and last megabyte of a disk (or file) # gpt stores partition table both at front and at end. - null = '\0' # pylint: disable=W1401 + null = '\0' start_len = 1024 * 1024 end_len = 1024 * 1024 with open(device, "rb+") as fp: diff --git a/cloudinit/config/cc_grub_dpkg.py b/cloudinit/config/cc_grub_dpkg.py index d96e85cf..e3219e81 100644 --- a/cloudinit/config/cc_grub_dpkg.py +++ b/cloudinit/config/cc_grub_dpkg.py @@ -36,7 +36,8 @@ def handle(_name, cfg, _cloud, log, _args): "grub-pc/install_devices_empty", None) if ((os.path.exists("/dev/sda1") and not os.path.exists("/dev/sda")) or - (os.path.exists("/dev/xvda1") and not os.path.exists("/dev/xvda"))): + (os.path.exists("/dev/xvda1") + and not os.path.exists("/dev/xvda"))): if idevs is None: idevs = "" if idevs_empty is None: diff --git a/cloudinit/config/cc_mounts.py b/cloudinit/config/cc_mounts.py index 80590118..ba1303d1 100644 --- a/cloudinit/config/cc_mounts.py +++ b/cloudinit/config/cc_mounts.py @@ -18,7 +18,7 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see <http://www.gnu.org/licenses/>. -from string import whitespace # pylint: disable=W0402 +from string import whitespace import logging import os.path diff --git a/cloudinit/config/cc_phone_home.py b/cloudinit/config/cc_phone_home.py index 2e058ccd..5bc68b83 100644 --- a/cloudinit/config/cc_phone_home.py +++ b/cloudinit/config/cc_phone_home.py @@ -47,7 +47,7 @@ def handle(name, cfg, cloud, log, args): if len(args) != 0: ph_cfg = util.read_conf(args[0]) else: - if not 'phone_home' in cfg: + if 'phone_home' not in cfg: log.debug(("Skipping module named %s, " "no 'phone_home' configuration found"), name) return diff --git a/cloudinit/config/cc_power_state_change.py b/cloudinit/config/cc_power_state_change.py index 8f99e887..09d37371 100644 --- a/cloudinit/config/cc_power_state_change.py +++ b/cloudinit/config/cc_power_state_change.py @@ -89,8 +89,9 @@ def load_power_state(cfg): mode = pstate.get("mode") if mode not in opt_map: - raise TypeError("power_state[mode] required, must be one of: %s." % - ','.join(opt_map.keys())) + raise TypeError( + "power_state[mode] required, must be one of: %s. found: '%s'." % + (','.join(opt_map.keys()), mode)) delay = pstate.get("delay", "now") # convert integer 30 or string '30' to '+30' @@ -100,7 +101,9 @@ def load_power_state(cfg): pass if delay != "now" and not re.match(r"\+[0-9]+", delay): - raise TypeError("power_state[delay] must be 'now' or '+m' (minutes).") + raise TypeError( + "power_state[delay] must be 'now' or '+m' (minutes)." + " found '%s'." % delay) args = ["shutdown", opt_map[mode], delay] if pstate.get("message"): @@ -116,7 +119,7 @@ def load_power_state(cfg): def doexit(sysexit): - os._exit(sysexit) # pylint: disable=W0212 + os._exit(sysexit) def execmd(exe_args, output=None, data_in=None): @@ -124,7 +127,7 @@ def execmd(exe_args, output=None, data_in=None): proc = subprocess.Popen(exe_args, stdin=subprocess.PIPE, stdout=output, stderr=subprocess.STDOUT) proc.communicate(data_in) - ret = proc.returncode # pylint: disable=E1101 + ret = proc.returncode except Exception: doexit(EXIT_FAIL) doexit(ret) diff --git a/cloudinit/config/cc_resizefs.py b/cloudinit/config/cc_resizefs.py index be406034..cbc07853 100644 --- a/cloudinit/config/cc_resizefs.py +++ b/cloudinit/config/cc_resizefs.py @@ -28,19 +28,19 @@ from cloudinit import util frequency = PER_ALWAYS -def _resize_btrfs(mount_point, devpth): # pylint: disable=W0613 +def _resize_btrfs(mount_point, devpth): return ('btrfs', 'filesystem', 'resize', 'max', mount_point) -def _resize_ext(mount_point, devpth): # pylint: disable=W0613 +def _resize_ext(mount_point, devpth): return ('resize2fs', devpth) -def _resize_xfs(mount_point, devpth): # pylint: disable=W0613 +def _resize_xfs(mount_point, devpth): return ('xfs_growfs', devpth) -def _resize_ufs(mount_point, devpth): # pylint: disable=W0613 +def _resize_ufs(mount_point, devpth): return ('growfs', devpth) # Do not use a dictionary as these commands should be able to be used @@ -98,14 +98,14 @@ def handle(name, cfg, _cloud, log, args): (devpth, fs_type, mount_point) = result - # Ensure the path is a block device. info = "dev=%s mnt_point=%s path=%s" % (devpth, mount_point, resize_what) log.debug("resize_info: %s" % info) container = util.is_container() + # Ensure the path is a block device. if (devpth == "/dev/root" and not os.path.exists(devpth) and - not container): + not container): devpth = rootdev_from_cmdline(util.get_cmdline()) if devpth is None: log.warn("Unable to find device '/dev/root'") @@ -117,14 +117,22 @@ def handle(name, cfg, _cloud, log, args): except OSError as exc: if container and exc.errno == errno.ENOENT: log.debug("Device '%s' did not exist in container. " - "cannot resize: %s" % (devpth, info)) + "cannot resize: %s", devpth, info) elif exc.errno == errno.ENOENT: - log.warn("Device '%s' did not exist. cannot resize: %s" % - (devpth, info)) + log.warn("Device '%s' did not exist. cannot resize: %s", + devpth, info) else: raise exc return + if not os.access(devpth, os.W_OK): + if container: + log.debug("'%s' not writable in container. cannot resize: %s", + devpth, info) + else: + log.warn("'%s' not writable. cannot resize: %s", devpth, info) + return + if not stat.S_ISBLK(statret.st_mode) and not stat.S_ISCHR(statret.st_mode): if container: log.debug("device '%s' not a block device in container." @@ -154,8 +162,8 @@ def handle(name, cfg, _cloud, log, args): # Fork to a child that will run # the resize command util.fork_cb( - util.log_time(logfunc=log.debug, msg="backgrounded Resizing", - func=do_resize, args=(resize_cmd, log))) + util.log_time, logfunc=log.debug, msg="backgrounded Resizing", + func=do_resize, args=(resize_cmd, log)) else: util.log_time(logfunc=log.debug, msg="Resizing", func=do_resize, args=(resize_cmd, log)) diff --git a/cloudinit/config/cc_resolv_conf.py b/cloudinit/config/cc_resolv_conf.py index 879b62b1..bbaa6c63 100644 --- a/cloudinit/config/cc_resolv_conf.py +++ b/cloudinit/config/cc_resolv_conf.py @@ -49,23 +49,22 @@ # +from cloudinit import log as logging from cloudinit.settings import PER_INSTANCE from cloudinit import templater from cloudinit import util +LOG = logging.getLogger(__name__) + frequency = PER_INSTANCE distros = ['fedora', 'rhel', 'sles'] -def generate_resolv_conf(cloud, log, params): - template_fn = cloud.get_template_filename('resolv.conf') - if not template_fn: - log.warn("No template found, not rendering /etc/resolv.conf") - return - +def generate_resolv_conf(template_fn, params, target_fname="/etc/resolv.conf"): flags = [] false_flags = [] + if 'options' in params: for key, val in params['options'].iteritems(): if type(val) == bool: @@ -77,12 +76,15 @@ def generate_resolv_conf(cloud, log, params): for flag in flags + false_flags: del params['options'][flag] + if not params.get('options'): + params['options'] = {} + params['flags'] = flags - log.debug("Writing resolv.conf from template %s" % template_fn) - templater.render_to_file(template_fn, '/etc/resolv.conf', params) + LOG.debug("Writing resolv.conf from template %s" % template_fn) + templater.render_to_file(template_fn, target_fname, params) -def handle(name, cfg, _cloud, log, _args): +def handle(name, cfg, cloud, log, _args): """ Handler for resolv.conf @@ -102,8 +104,13 @@ def handle(name, cfg, _cloud, log, _args): " 'manage_resolv_conf' present but set to False"), name) return - if not "resolv_conf" in cfg: + if "resolv_conf" not in cfg: log.warn("manage_resolv_conf True but no parameters provided!") - generate_resolv_conf(_cloud, log, cfg["resolv_conf"]) + template_fn = cloud.get_template_filename('resolv.conf') + if not template_fn: + log.warn("No template found, not rendering /etc/resolv.conf") + return + + generate_resolv_conf(template_fn=template_fn, params=cfg["resolv_conf"]) return diff --git a/cloudinit/config/cc_rightscale_userdata.py b/cloudinit/config/cc_rightscale_userdata.py index c771728d..7d2ec10a 100644 --- a/cloudinit/config/cc_rightscale_userdata.py +++ b/cloudinit/config/cc_rightscale_userdata.py @@ -18,22 +18,22 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see <http://www.gnu.org/licenses/>. -## -## The purpose of this script is to allow cloud-init to consume -## rightscale style userdata. rightscale user data is key-value pairs -## in a url-query-string like format. -## -## for cloud-init support, there will be a key named -## 'CLOUD_INIT_REMOTE_HOOK'. -## -## This cloud-config module will -## - read the blob of data from raw user data, and parse it as key/value -## - for each key that is found, download the content to -## the local instance/scripts directory and set them executable. -## - the files in that directory will be run by the user-scripts module -## Therefore, this must run before that. -## -## +# +# The purpose of this script is to allow cloud-init to consume +# rightscale style userdata. rightscale user data is key-value pairs +# in a url-query-string like format. +# +# for cloud-init support, there will be a key named +# 'CLOUD_INIT_REMOTE_HOOK'. +# +# This cloud-config module will +# - read the blob of data from raw user data, and parse it as key/value +# - for each key that is found, download the content to +# the local instance/scripts directory and set them executable. +# - the files in that directory will be run by the user-scripts module +# Therefore, this must run before that. +# +# import os @@ -58,7 +58,7 @@ def handle(name, _cfg, cloud, log, _args): try: mdict = parse_qs(ud) - if not mdict or not MY_HOOKNAME in mdict: + if mdict or MY_HOOKNAME not in mdict: log.debug(("Skipping module %s, " "did not find %s in parsed" " raw userdata"), name, MY_HOOKNAME) diff --git a/cloudinit/config/cc_rsyslog.py b/cloudinit/config/cc_rsyslog.py index 0c2c6880..57486edc 100644 --- a/cloudinit/config/cc_rsyslog.py +++ b/cloudinit/config/cc_rsyslog.py @@ -35,7 +35,7 @@ def handle(name, cfg, cloud, log, _args): # *.* @@syslogd.example.com # process 'rsyslog' - if not 'rsyslog' in cfg: + if 'rsyslog' not in cfg: log.debug(("Skipping module named %s," " no 'rsyslog' key in configuration"), name) return @@ -46,7 +46,7 @@ def handle(name, cfg, cloud, log, _args): files = [] for i, ent in enumerate(cfg['rsyslog']): if isinstance(ent, dict): - if not "content" in ent: + if "content" not in ent: log.warn("No 'content' entry in config entry %s", i + 1) continue content = ent['content'] diff --git a/cloudinit/config/cc_set_passwords.py b/cloudinit/config/cc_set_passwords.py index 4a3b21af..4ca85e21 100644 --- a/cloudinit/config/cc_set_passwords.py +++ b/cloudinit/config/cc_set_passwords.py @@ -28,7 +28,7 @@ from cloudinit import distros as ds from cloudinit import ssh_util from cloudinit import util -from string import letters, digits # pylint: disable=W0402 +from string import letters, digits # We are removing certain 'painful' letters/numbers PW_SET = (letters.translate(None, 'loLOI') + @@ -132,7 +132,7 @@ def handle(_name, cfg, cloud, log, args): 'PasswordAuthentication', pw_auth)) - lines = [str(e) for e in new_lines] + lines = [str(l) for l in new_lines] util.write_file(ssh_util.DEF_SSHD_CFG, "\n".join(lines)) try: diff --git a/cloudinit/config/cc_ssh.py b/cloudinit/config/cc_ssh.py index 64a5e3cb..4c76581c 100644 --- a/cloudinit/config/cc_ssh.py +++ b/cloudinit/config/cc_ssh.py @@ -75,7 +75,7 @@ def handle(_name, cfg, cloud, log, _args): util.write_file(tgt_fn, val, tgt_perms) for (priv, pub) in PRIV_2_PUB.iteritems(): - if pub in cfg['ssh_keys'] or not priv in cfg['ssh_keys']: + if pub in cfg['ssh_keys'] or priv not in cfg['ssh_keys']: continue pair = (KEY_2_FILE[priv][0], KEY_2_FILE[pub][0]) cmd = ['sh', '-xc', KEY_GEN_TPL % pair] diff --git a/cloudinit/config/cc_ssh_authkey_fingerprints.py b/cloudinit/config/cc_ssh_authkey_fingerprints.py index be8083db..51580633 100644 --- a/cloudinit/config/cc_ssh_authkey_fingerprints.py +++ b/cloudinit/config/cc_ssh_authkey_fingerprints.py @@ -55,7 +55,7 @@ def _gen_fingerprint(b64_text, hash_meth='md5'): def _is_printable_key(entry): if any([entry.keytype, entry.base64, entry.comment, entry.options]): if (entry.keytype and - entry.keytype.lower().strip() in ['ssh-dss', 'ssh-rsa']): + entry.keytype.lower().strip() in ['ssh-dss', 'ssh-rsa']): return True return False @@ -92,9 +92,10 @@ def _pprint_key_entries(user, key_fn, key_entries, hash_meth='md5', def handle(name, cfg, cloud, log, _args): - if 'no_ssh_fingerprints' in cfg: + if util.is_true(cfg.get('no_ssh_fingerprints', False)): log.debug(("Skipping module named %s, " "logging of ssh fingerprints disabled"), name) + return hash_meth = util.get_cfg_option_str(cfg, "authkey_hash", "md5") (users, _groups) = ds.normalize_users_groups(cfg, cloud.distro) diff --git a/cloudinit/config/cc_ssh_import_id.py b/cloudinit/config/cc_ssh_import_id.py index 76c1663d..2d480d7e 100644 --- a/cloudinit/config/cc_ssh_import_id.py +++ b/cloudinit/config/cc_ssh_import_id.py @@ -85,7 +85,7 @@ def import_ssh_ids(ids, user, log): return try: - _check = pwd.getpwnam(user) + pwd.getpwnam(user) except KeyError as exc: raise exc diff --git a/cloudinit/config/cc_ubuntu_init_switch.py b/cloudinit/config/cc_ubuntu_init_switch.py new file mode 100644 index 00000000..6f994bff --- /dev/null +++ b/cloudinit/config/cc_ubuntu_init_switch.py @@ -0,0 +1,164 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2014 Canonical Ltd. +# +# Author: Scott Moser <scott.moser@canonical.com> +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. + +""" +ubuntu_init_switch: reboot system into another init + +This provides a way for the user to boot with systemd even if the +image is set to boot with upstart. It should be run as one of the first +cloud_init_modules, and will switch the init system and then issue a reboot. +The next boot will come up in the target init system and no action will +be taken. + +This should be inert on non-ubuntu systems, and also exit quickly. + +config is comes under the top level 'init_switch' dictionary. + +#cloud-config +init_switch: + target: systemd + reboot: true + +'target' can be 'systemd' or 'upstart'. Best effort is made, but its possible +this system will break, and probably won't interact well with any other +mechanism you've used to switch the init system. + +'reboot': [default=true]. + true: reboot if a change was made. + false: do not reboot. +""" + +from cloudinit.settings import PER_INSTANCE +from cloudinit import log as logging +from cloudinit import util +from cloudinit.distros import ubuntu + +import os +import time + +frequency = PER_INSTANCE +REBOOT_CMD = ["/sbin/reboot", "--force"] + +DEFAULT_CONFIG = { + 'init_switch': {'target': None, 'reboot': True} +} + +SWITCH_INIT = """ +#!/bin/sh +# switch_init: [upstart | systemd] + +is_systemd() { + [ "$(dpkg-divert --listpackage /sbin/init)" = "systemd-sysv" ] +} +debug() { echo "$@" 1>&2; } +fail() { echo "$@" 1>&2; exit 1; } + +if [ "$1" = "systemd" ]; then + if is_systemd; then + debug "already systemd, nothing to do" + else + [ -f /lib/systemd/systemd ] || fail "no systemd available"; + dpkg-divert --package systemd-sysv --divert /sbin/init.diverted \\ + --rename /sbin/init + fi + [ -f /sbin/init ] || ln /lib/systemd/systemd /sbin/init +elif [ "$1" = "upstart" ]; then + if is_systemd; then + rm -f /sbin/init + dpkg-divert --package systemd-sysv --rename --remove /sbin/init + else + debug "already upstart, nothing to do." + fi +else + fail "Error. expect 'upstart' or 'systemd'" +fi +""" + + +def handle(name, cfg, cloud, log, args): + + if not isinstance(cloud.distro, ubuntu.Distro): + log.debug("%s: distro is '%s', not ubuntu. returning", + name, cloud.distro.__class__) + return + + cfg = util.mergemanydict([cfg, DEFAULT_CONFIG]) + target = cfg['init_switch']['target'] + reboot = cfg['init_switch']['reboot'] + + if len(args) != 0: + target = args[0] + if len(args) > 1: + reboot = util.is_true(args[1]) + + if not target: + log.debug("%s: target=%s. nothing to do", name, target) + return + + if not util.which('dpkg'): + log.warn("%s: 'dpkg' not available. Assuming not ubuntu", name) + return + + supported = ('upstart', 'systemd') + if target not in supported: + log.warn("%s: target set to %s, expected one of: %s", + name, target, str(supported)) + + if os.path.exists("/run/systemd/system"): + current = "systemd" + else: + current = "upstart" + + if current == target: + log.debug("%s: current = target = %s. nothing to do", name, target) + return + + try: + util.subp(['sh', '-s', target], data=SWITCH_INIT) + except util.ProcessExecutionError as e: + log.warn("%s: Failed to switch to init '%s'. %s", name, target, e) + return + + if util.is_false(reboot): + log.info("%s: switched '%s' to '%s'. reboot=false, not rebooting.", + name, current, target) + return + + try: + log.warn("%s: switched '%s' to '%s'. rebooting.", + name, current, target) + logging.flushLoggers(log) + _fire_reboot(log, wait_attempts=4, initial_sleep=4) + except Exception as e: + util.logexc(log, "Requested reboot did not happen!") + raise + + +def _fire_reboot(log, wait_attempts=6, initial_sleep=1, backoff=2): + util.subp(REBOOT_CMD) + start = time.time() + wait_time = initial_sleep + for _i in range(0, wait_attempts): + time.sleep(wait_time) + wait_time *= backoff + elapsed = time.time() - start + log.debug("Rebooted, but still running after %s seconds", int(elapsed)) + # If we got here, not good + elapsed = time.time() - start + raise RuntimeError(("Reboot did not happen" + " after %s seconds!") % (int(elapsed))) diff --git a/cloudinit/config/cc_yum_add_repo.py b/cloudinit/config/cc_yum_add_repo.py index 5c273825..0d836f28 100644 --- a/cloudinit/config/cc_yum_add_repo.py +++ b/cloudinit/config/cc_yum_add_repo.py @@ -42,7 +42,7 @@ def _format_repo_value(val): return val -## TODO(harlowja): move to distro? +# TODO(harlowja): move to distro? # See man yum.conf def _format_repository_config(repo_id, repo_config): to_be = configobj.ConfigObj() @@ -89,7 +89,7 @@ def handle(name, cfg, _cloud, log, _args): repo_config = n_repo_config missing_required = 0 for req_field in ['baseurl']: - if not req_field in repo_config: + if req_field not in repo_config: log.warn(("Repository %s does not contain a %s" " configuration 'required' entry"), repo_id, req_field) diff --git a/cloudinit/distros/__init__.py b/cloudinit/distros/__init__.py index 55d6bcbc..2599d9f2 100644 --- a/cloudinit/distros/__init__.py +++ b/cloudinit/distros/__init__.py @@ -167,7 +167,7 @@ class Distro(object): def expand_osfamily(family_list): distros = [] for family in family_list: - if not family in OSFAMILIES: + if family not in OSFAMILIES: raise ValueError("No distibutions found for osfamily %s" % (family)) distros.extend(OSFAMILIES[family]) @@ -218,7 +218,7 @@ class Distro(object): fn) if (sys_hostname and prev_hostname and - sys_hostname != prev_hostname): + sys_hostname != prev_hostname): LOG.debug("%s differs from %s, assuming user maintained hostname.", prev_hostname_fn, sys_fn) @@ -847,12 +847,19 @@ def extract_default(users, default_name=None, default_config=None): def fetch(name): - locs = importer.find_module(name, - ['', __name__], - ['Distro']) + locs, looked_locs = importer.find_module(name, ['', __name__], ['Distro']) if not locs: - raise ImportError("No distribution found for distro %s" - % (name)) + raise ImportError("No distribution found for distro %s (searched %s)" + % (name, looked_locs)) mod = importer.import_module(locs[0]) cls = getattr(mod, 'Distro') return cls + + +def set_etc_timezone(tz, tz_file=None, tz_conf="/etc/timezone", + tz_local="/etc/localtime"): + util.write_file(tz_conf, str(tz).rstrip() + "\n") + # This ensures that the correct tz will be used for the system + if tz_local and tz_file: + util.copy(tz_file, tz_local) + return diff --git a/cloudinit/distros/arch.py b/cloudinit/distros/arch.py index 310c3dff..005a0dd4 100644 --- a/cloudinit/distros/arch.py +++ b/cloudinit/distros/arch.py @@ -32,8 +32,6 @@ LOG = logging.getLogger(__name__) class Distro(distros.Distro): locale_conf_fn = "/etc/locale.gen" network_conf_dir = "/etc/netctl" - tz_conf_fn = "/etc/timezone" - tz_local_fn = "/etc/localtime" resolve_conf_fn = "/etc/resolv.conf" init_cmd = ['systemctl'] # init scripts @@ -161,16 +159,7 @@ class Distro(distros.Distro): return hostname def set_timezone(self, tz): - tz_file = self._find_tz_file(tz) - # Note: "" provides trailing newline during join - tz_lines = [ - util.make_header(), - str(tz), - "", - ] - util.write_file(self.tz_conf_fn, "\n".join(tz_lines)) - # This ensures that the correct tz will be used for the system - util.copy(tz_file, self.tz_local_fn) + distros.set_etc_timezone(tz=tz, tz_file=self._find_tz_file(tz)) def package_command(self, command, args=None, pkgs=None): if pkgs is None: diff --git a/cloudinit/distros/debian.py b/cloudinit/distros/debian.py index 1ae232fd..010be67d 100644 --- a/cloudinit/distros/debian.py +++ b/cloudinit/distros/debian.py @@ -46,8 +46,6 @@ class Distro(distros.Distro): hostname_conf_fn = "/etc/hostname" locale_conf_fn = "/etc/default/locale" network_conf_fn = "/etc/network/interfaces" - tz_conf_fn = "/etc/timezone" - tz_local_fn = "/etc/localtime" def __init__(self, name, cfg, paths): distros.Distro.__init__(self, name, cfg, paths) @@ -133,16 +131,7 @@ class Distro(distros.Distro): return "127.0.1.1" def set_timezone(self, tz): - tz_file = self._find_tz_file(tz) - # Note: "" provides trailing newline during join - tz_lines = [ - util.make_header(), - str(tz), - "", - ] - util.write_file(self.tz_conf_fn, "\n".join(tz_lines)) - # This ensures that the correct tz will be used for the system - util.copy(tz_file, self.tz_local_fn) + distros.set_etc_timezone(tz=tz, tz_file=self._find_tz_file(tz)) def package_command(self, command, args=None, pkgs=None): if pkgs is None: diff --git a/cloudinit/distros/freebsd.py b/cloudinit/distros/freebsd.py index d98f9578..cff10387 100644 --- a/cloudinit/distros/freebsd.py +++ b/cloudinit/distros/freebsd.py @@ -26,6 +26,9 @@ from cloudinit import log as logging from cloudinit import ssh_util from cloudinit import util +from cloudinit.distros import net_util +from cloudinit.distros.parsers.resolv_conf import ResolvConf + LOG = logging.getLogger(__name__) @@ -33,6 +36,8 @@ class Distro(distros.Distro): rc_conf_fn = "/etc/rc.conf" login_conf_fn = '/etc/login.conf' login_conf_fn_bak = '/etc/login.conf.orig' + resolv_conf_fn = '/etc/resolv.conf' + ci_sudoers_fn = '/usr/local/etc/sudoers.d/90-cloud-init-users' def __init__(self, name, cfg, paths): distros.Distro.__init__(self, name, cfg, paths) @@ -44,30 +49,53 @@ class Distro(distros.Distro): # Updates a key in /etc/rc.conf. def updatercconf(self, key, value): - LOG.debug("updatercconf: %s => %s", key, value) + LOG.debug("Checking %s for: %s = %s", self.rc_conf_fn, key, value) conf = self.loadrcconf() config_changed = False - for item in conf: - if item == key and conf[item] != value: - conf[item] = value - LOG.debug("[rc.conf]: Value %s for key %s needs to be changed", - value, key) - config_changed = True + if key not in conf: + LOG.debug("Adding key in %s: %s = %s", self.rc_conf_fn, key, + value) + conf[key] = value + config_changed = True + else: + for item in conf.keys(): + if item == key and conf[item] != value: + conf[item] = value + LOG.debug("Changing key in %s: %s = %s", self.rc_conf_fn, + key, value) + config_changed = True if config_changed: - LOG.debug("Writing new %s file", self.rc_conf_fn) + LOG.info("Writing %s", self.rc_conf_fn) buf = StringIO() for keyval in conf.items(): - buf.write("%s=%s\n" % keyval) + buf.write('%s="%s"\n' % keyval) util.write_file(self.rc_conf_fn, buf.getvalue()) - # Load the contents of /etc/rc.conf and store all keys in a dict. + # Load the contents of /etc/rc.conf and store all keys in a dict. Make sure + # quotes are ignored: + # hostname="bla" def loadrcconf(self): + RE_MATCH = re.compile(r'^(\w+)\s*=\s*(.*)\s*') conf = {} lines = util.load_file(self.rc_conf_fn).splitlines() for line in lines: - tok = line.split('=') - conf[tok[0]] = tok[1].rstrip() + m = RE_MATCH.match(line) + if not m: + LOG.debug("Skipping line from /etc/rc.conf: %s", line) + continue + key = m.group(1).rstrip() + val = m.group(2).rstrip() + # Kill them quotes (not completely correct, aka won't handle + # quoted values, but should be ok ...) + if val[0] in ('"', "'"): + val = val[1:] + if val[-1] in ('"', "'"): + val = val[0:-1] + if len(val) == 0: + LOG.debug("Skipping empty value from /etc/rc.conf: %s", line) + continue + conf[key] = val return conf def readrcconf(self, key): @@ -192,10 +220,6 @@ class Distro(distros.Distro): util.logexc(LOG, "Failed to lock user %s", name) raise e - # TODO: - def write_sudo_rules(self, name, rules, sudo_file=None): - LOG.debug("[write_sudo_rules] Name: %s", name) - def create_user(self, name, **kwargs): self.add_user(name, **kwargs) @@ -218,7 +242,60 @@ class Distro(distros.Distro): ssh_util.setup_user_keys(keys, name, options=None) def _write_network(self, settings): - return + entries = net_util.translate_network(settings) + nameservers = [] + searchdomains = [] + dev_names = entries.keys() + for (dev, info) in entries.iteritems(): + # Skip the loopback interface. + if dev.startswith('lo'): + continue + + LOG.info('Configuring interface %s', dev) + + if info.get('bootproto') == 'static': + LOG.debug('Configuring dev %s with %s / %s', dev, info.get('address'), info.get('netmask')) + # Configure an ipv4 address. + ifconfig = info.get('address') + ' netmask ' + info.get('netmask') + + # Configure the gateway. + self.updatercconf('defaultrouter', info.get('gateway')) + + if 'dns-nameservers' in info: + nameservers.extend(info['dns-nameservers']) + if 'dns-search' in info: + searchdomains.extend(info['dns-search']) + else: + ifconfig = 'DHCP' + + self.updatercconf('ifconfig_' + dev, ifconfig) + + # Try to read the /etc/resolv.conf or just start from scratch if that + # fails. + try: + resolvconf = ResolvConf(util.load_file(self.resolv_conf_fn)) + resolvconf.parse() + except IOError: + util.logexc(LOG, "Failed to parse %s, use new empty file", self.resolv_conf_fn) + resolvconf = ResolvConf('') + resolvconf.parse() + + # Add some nameservers + for server in nameservers: + try: + resolvconf.add_nameserver(server) + except ValueError: + util.logexc(LOG, "Failed to add nameserver %s", server) + + # And add any searchdomains. + for domain in searchdomains: + try: + resolvconf.add_search_domain(domain) + except ValueError: + util.logexc(LOG, "Failed to add search domain %s", domain) + util.write_file(self.resolv_conf_fn, str(resolvconf), 0644) + + return dev_names def apply_locale(self, locale, out_fn=None): # Adjust the locals value to the new value diff --git a/cloudinit/distros/gentoo.py b/cloudinit/distros/gentoo.py index 09f8d8ea..45c2e658 100644 --- a/cloudinit/distros/gentoo.py +++ b/cloudinit/distros/gentoo.py @@ -31,8 +31,6 @@ LOG = logging.getLogger(__name__) class Distro(distros.Distro): locale_conf_fn = "/etc/locale.gen" network_conf_fn = "/etc/conf.d/net" - tz_conf_fn = "/etc/timezone" - tz_local_fn = "/etc/localtime" init_cmd = [''] # init scripts def __init__(self, name, cfg, paths): @@ -140,16 +138,7 @@ class Distro(distros.Distro): return hostname def set_timezone(self, tz): - tz_file = self._find_tz_file(tz) - # Note: "" provides trailing newline during join - tz_lines = [ - util.make_header(), - str(tz), - "", - ] - util.write_file(self.tz_conf_fn, "\n".join(tz_lines)) - # This ensures that the correct tz will be used for the system - util.copy(tz_file, self.tz_local_fn) + distros.set_etc_timezone(tz=tz, tz_file=self._find_tz_file(tz)) def package_command(self, command, args=None, pkgs=None): if pkgs is None: diff --git a/cloudinit/distros/parsers/resolv_conf.py b/cloudinit/distros/parsers/resolv_conf.py index 1be9d46b..5733c25a 100644 --- a/cloudinit/distros/parsers/resolv_conf.py +++ b/cloudinit/distros/parsers/resolv_conf.py @@ -137,8 +137,8 @@ class ResolvConf(object): self._contents.append(('option', ['search', s_list, ''])) return flat_sds - @local_domain.setter # pl51222 pylint: disable=E1101 - def local_domain(self, domain): # pl51222 pylint: disable=E0102 + @local_domain.setter + def local_domain(self, domain): self.parse() self._remove_option('domain') self._contents.append(('option', ['domain', str(domain), ''])) diff --git a/cloudinit/ec2_utils.py b/cloudinit/ec2_utils.py index a7c9c9ab..0c751140 100644 --- a/cloudinit/ec2_utils.py +++ b/cloudinit/ec2_utils.py @@ -166,7 +166,9 @@ def get_instance_metadata(api_version='latest', metadata_address='http://169.254.169.254', ssl_details=None, timeout=5, retries=5): md_url = url_helper.combine_url(metadata_address, api_version) - md_url = url_helper.combine_url(md_url, 'meta-data') + # Note, 'meta-data' explicitly has trailing /. + # this is required for CloudStack (LP: #1356855) + md_url = url_helper.combine_url(md_url, 'meta-data/') caller = functools.partial(util.read_file_or_url, ssl_details=ssl_details, timeout=timeout, retries=retries) diff --git a/cloudinit/handlers/boot_hook.py b/cloudinit/handlers/boot_hook.py index 1848ce2c..3a50cf87 100644 --- a/cloudinit/handlers/boot_hook.py +++ b/cloudinit/handlers/boot_hook.py @@ -53,8 +53,7 @@ class BootHookPartHandler(handlers.Handler): util.write_file(filepath, contents.lstrip(), 0700) return filepath - def handle_part(self, _data, ctype, filename, # pylint: disable=W0221 - payload, frequency): # pylint: disable=W0613 + def handle_part(self, data, ctype, filename, payload, frequency): if ctype in handlers.CONTENT_SIGNALS: return diff --git a/cloudinit/handlers/cloud_config.py b/cloudinit/handlers/cloud_config.py index 4232700f..bf994e33 100644 --- a/cloudinit/handlers/cloud_config.py +++ b/cloudinit/handlers/cloud_config.py @@ -138,8 +138,7 @@ class CloudConfigPartHandler(handlers.Handler): self.file_names = [] self.cloud_buf = None - def handle_part(self, _data, ctype, filename, # pylint: disable=W0221 - payload, _frequency, headers): # pylint: disable=W0613 + def handle_part(self, data, ctype, filename, payload, frequency, headers): if ctype == handlers.CONTENT_START: self._reset() return diff --git a/cloudinit/handlers/shell_script.py b/cloudinit/handlers/shell_script.py index 30c1ed89..9755ab05 100644 --- a/cloudinit/handlers/shell_script.py +++ b/cloudinit/handlers/shell_script.py @@ -44,8 +44,7 @@ class ShellScriptPartHandler(handlers.Handler): handlers.type_from_starts_with(SHELL_PREFIX), ] - def handle_part(self, _data, ctype, filename, # pylint: disable=W0221 - payload, frequency): # pylint: disable=W0613 + def handle_part(self, data, ctype, filename, payload, frequency): if ctype in handlers.CONTENT_SIGNALS: # TODO(harlowja): maybe delete existing things here return diff --git a/cloudinit/handlers/upstart_job.py b/cloudinit/handlers/upstart_job.py index bac4cad2..50d193c4 100644 --- a/cloudinit/handlers/upstart_job.py +++ b/cloudinit/handlers/upstart_job.py @@ -44,8 +44,7 @@ class UpstartJobPartHandler(handlers.Handler): handlers.type_from_starts_with(UPSTART_PREFIX), ] - def handle_part(self, _data, ctype, filename, # pylint: disable=W0221 - payload, frequency): + def handle_part(self, data, ctype, filename, payload, frequency): if ctype in handlers.CONTENT_SIGNALS: return diff --git a/cloudinit/importer.py b/cloudinit/importer.py index a094141a..fb57253c 100644 --- a/cloudinit/importer.py +++ b/cloudinit/importer.py @@ -22,10 +22,6 @@ import sys -from cloudinit import log as logging - -LOG = logging.getLogger(__name__) - def import_module(module_name): __import__(module_name) @@ -33,27 +29,24 @@ def import_module(module_name): def find_module(base_name, search_paths, required_attrs=None): - found_places = [] if not required_attrs: required_attrs = [] # NOTE(harlowja): translate the search paths to include the base name. - real_paths = [] + lookup_paths = [] for path in search_paths: real_path = [] if path: real_path.extend(path.split(".")) real_path.append(base_name) full_path = '.'.join(real_path) - real_paths.append(full_path) - LOG.debug("Looking for modules %s that have attributes %s", - real_paths, required_attrs) - for full_path in real_paths: + lookup_paths.append(full_path) + found_paths = [] + for full_path in lookup_paths: mod = None try: mod = import_module(full_path) - except ImportError as e: - LOG.debug("Failed at attempted import of '%s' due to: %s", - full_path, e) + except ImportError: + pass if not mod: continue found_attrs = 0 @@ -61,7 +54,5 @@ def find_module(base_name, search_paths, required_attrs=None): if hasattr(mod, attr): found_attrs += 1 if found_attrs == len(required_attrs): - found_places.append(full_path) - LOG.debug("Found %s with attributes %s in %s", base_name, - required_attrs, found_places) - return found_places + found_paths.append(full_path) + return (found_paths, lookup_paths) diff --git a/cloudinit/mergers/__init__.py b/cloudinit/mergers/__init__.py index 0978b2c6..03aa1ee1 100644 --- a/cloudinit/mergers/__init__.py +++ b/cloudinit/mergers/__init__.py @@ -55,9 +55,6 @@ class UnknownMerger(object): if not meth: meth = self._handle_unknown args.insert(0, method_name) - LOG.debug("Merging '%s' into '%s' using method '%s' of '%s'", - type_name, type_utils.obj_name(merge_with), - meth.__name__, self) return meth(*args) @@ -84,8 +81,6 @@ class LookupMerger(UnknownMerger): # First one that has that method/attr gets to be # the one that will be called meth = getattr(merger, meth_wanted) - LOG.debug(("Merging using located merger '%s'" - " since it had method '%s'"), merger, meth_wanted) break if not meth: return UnknownMerger._handle_unknown(self, meth_wanted, @@ -148,12 +143,14 @@ def construct(parsed_mergers): for (m_name, m_ops) in parsed_mergers: if not m_name.startswith(MERGER_PREFIX): m_name = MERGER_PREFIX + str(m_name) - merger_locs = importer.find_module(m_name, - [__name__], - [MERGER_ATTR]) + merger_locs, looked_locs = importer.find_module(m_name, + [__name__], + [MERGER_ATTR]) if not merger_locs: msg = ("Could not find merger module named '%s' " - "with attribute '%s'") % (m_name, MERGER_ATTR) + "with attribute '%s' (searched %s)") % (m_name, + MERGER_ATTR, + looked_locs) raise ImportError(msg) else: mod = importer.import_module(merger_locs[0]) diff --git a/cloudinit/mergers/m_list.py b/cloudinit/mergers/m_list.py index 62999b4e..3b87b0fc 100644 --- a/cloudinit/mergers/m_list.py +++ b/cloudinit/mergers/m_list.py @@ -53,7 +53,7 @@ class Merger(object): def _on_list(self, value, merge_with): if (self._method == 'replace' and - not isinstance(merge_with, (tuple, list))): + not isinstance(merge_with, (tuple, list))): return merge_with # Ok we now know that what we are merging with is a list or tuple. diff --git a/cloudinit/netinfo.py b/cloudinit/netinfo.py index 30b6f3b3..8d4df342 100644 --- a/cloudinit/netinfo.py +++ b/cloudinit/netinfo.py @@ -21,10 +21,13 @@ # along with this program. If not, see <http://www.gnu.org/licenses/>. import cloudinit.util as util +from cloudinit.log import logging import re from prettytable import PrettyTable +LOG = logging.getLogger() + def netdev_info(empty=""): fields = ("hwaddr", "addr", "bcast", "mask") @@ -56,8 +59,8 @@ def netdev_info(empty=""): # newer (freebsd and fedora) show 'inet xx.yy' # just skip this 'inet' entry. (LP: #1285185) try: - if (toks[i] in ("inet", "inet6") and - toks[i + 1].startswith("addr:")): + if ((toks[i] in ("inet", "inet6") and + toks[i + 1].startswith("addr:"))): continue except IndexError: pass @@ -168,8 +171,9 @@ def route_pformat(): lines = [] try: routes = route_info() - except Exception: + except Exception as e: lines.append(util.center('Route info failed', '!', 80)) + util.logexc(LOG, "Route info failed: %s" % e) routes = None if routes is not None: fields = ['Route', 'Destination', 'Gateway', diff --git a/cloudinit/patcher.py b/cloudinit/patcher.py index 0f3c034e..f6609d6f 100644 --- a/cloudinit/patcher.py +++ b/cloudinit/patcher.py @@ -41,7 +41,7 @@ def _patch_logging(): fallback_handler = QuietStreamHandler(sys.stderr) fallback_handler.setFormatter(logging.Formatter(FALL_FORMAT)) - def handleError(self, record): # pylint: disable=W0613 + def handleError(self, record): try: fallback_handler.handle(record) fallback_handler.flush() diff --git a/cloudinit/settings.py b/cloudinit/settings.py index 37d4958b..5efcb0b0 100644 --- a/cloudinit/settings.py +++ b/cloudinit/settings.py @@ -37,7 +37,7 @@ CFG_BUILTIN = { 'OVF', 'MAAS', 'GCE', - 'OpenStack' + 'OpenStack', 'Ec2', 'CloudSigma', 'CloudStack', diff --git a/cloudinit/sources/DataSourceAzure.py b/cloudinit/sources/DataSourceAzure.py index bd75e6d8..09bc196d 100644 --- a/cloudinit/sources/DataSourceAzure.py +++ b/cloudinit/sources/DataSourceAzure.py @@ -452,7 +452,7 @@ def load_azure_ovf_pubkeys(sshnode): continue if (len(child.childNodes) != 1 or - child.childNodes[0].nodeType != text_node): + child.childNodes[0].nodeType != text_node): continue cur[name] = child.childNodes[0].wholeText.strip() @@ -521,7 +521,7 @@ def read_azure_ovf(contents): simple = False value = "" if (len(child.childNodes) == 1 and - child.childNodes[0].nodeType == dom.TEXT_NODE): + child.childNodes[0].nodeType == dom.TEXT_NODE): simple = True value = child.childNodes[0].wholeText diff --git a/cloudinit/sources/DataSourceCloudStack.py b/cloudinit/sources/DataSourceCloudStack.py index 08f661e4..1bbeca59 100644 --- a/cloudinit/sources/DataSourceCloudStack.py +++ b/cloudinit/sources/DataSourceCloudStack.py @@ -78,7 +78,8 @@ class DataSourceCloudStack(sources.DataSource): (max_wait, timeout) = self._get_url_settings() - urls = [self.metadata_address + "/latest/meta-data/instance-id"] + urls = [uhelp.combine_url(self.metadata_address, + 'latest/meta-data/instance-id')] start_time = time.time() url = uhelp.wait_for_url(urls=urls, max_wait=max_wait, timeout=timeout, status_cb=LOG.warn) diff --git a/cloudinit/sources/DataSourceConfigDrive.py b/cloudinit/sources/DataSourceConfigDrive.py index 0c35f83a..4e5d90de 100644 --- a/cloudinit/sources/DataSourceConfigDrive.py +++ b/cloudinit/sources/DataSourceConfigDrive.py @@ -125,7 +125,15 @@ class DataSourceConfigDrive(openstack.SourceMixin, sources.DataSource): self.userdata_raw = results.get('userdata') self.version = results['version'] self.files.update(results.get('files', {})) - self.vendordata_raw = results.get('vendordata') + + vd = results.get('vendordata') + self.vendordata_pure = vd + try: + self.vendordata_raw = openstack.convert_vendordata_json(vd) + except ValueError as e: + LOG.warn("Invalid content in vendor-data: %s", e) + self.vendordata_raw = None + return True @@ -160,10 +168,10 @@ def get_ds_mode(cfgdrv_ver, ds_cfg=None, user=None): return "net" -def read_config_drive(source_dir, version="2012-08-10"): +def read_config_drive(source_dir): reader = openstack.ConfigDriveReader(source_dir) finders = [ - (reader.read_v2, [], {'version': version}), + (reader.read_v2, [], {}), (reader.read_v1, [], {}), ] excps = [] diff --git a/cloudinit/sources/DataSourceNoCloud.py b/cloudinit/sources/DataSourceNoCloud.py index a315aae0..c26a645c 100644 --- a/cloudinit/sources/DataSourceNoCloud.py +++ b/cloudinit/sources/DataSourceNoCloud.py @@ -181,7 +181,7 @@ class DataSourceNoCloud(sources.DataSource): # and the source of the seed was self.dsmode # ('local' for NoCloud, 'net' for NoCloudNet') if ('network-interfaces' in mydata['meta-data'] and - (self.dsmode in ("local", seeded_interfaces))): + (self.dsmode in ("local", seeded_interfaces))): LOG.debug("Updating network interfaces from %s", self) self.distro.apply_network( mydata['meta-data']['network-interfaces']) diff --git a/cloudinit/sources/DataSourceOVF.py b/cloudinit/sources/DataSourceOVF.py index 77b43e17..2f53c1ba 100644 --- a/cloudinit/sources/DataSourceOVF.py +++ b/cloudinit/sources/DataSourceOVF.py @@ -107,7 +107,7 @@ class DataSourceOVF(sources.DataSource): return True def get_public_ssh_keys(self): - if not 'public-keys' in self.metadata: + if 'public-keys' not in self.metadata: return [] pks = self.metadata['public-keys'] if isinstance(pks, (list)): @@ -205,7 +205,7 @@ def transport_iso9660(require_iso=True): fullp = os.path.join("/dev/", dev) if (fullp in mounts or - not cdmatch.match(dev) or os.path.isdir(fullp)): + not cdmatch.match(dev) or os.path.isdir(fullp)): continue try: diff --git a/cloudinit/sources/DataSourceOpenNebula.py b/cloudinit/sources/DataSourceOpenNebula.py index 34557f8b..e2469f6e 100644 --- a/cloudinit/sources/DataSourceOpenNebula.py +++ b/cloudinit/sources/DataSourceOpenNebula.py @@ -28,7 +28,7 @@ import base64 import os import pwd import re -import string # pylint: disable=W0402 +import string from cloudinit import log as logging from cloudinit import sources diff --git a/cloudinit/sources/DataSourceOpenStack.py b/cloudinit/sources/DataSourceOpenStack.py index 0970d07b..469c2e2a 100644 --- a/cloudinit/sources/DataSourceOpenStack.py +++ b/cloudinit/sources/DataSourceOpenStack.py @@ -88,11 +88,9 @@ class DataSourceOpenStack(openstack.SourceMixin, sources.DataSource): md_urls = [] url2base = {} for url in urls: - for version in openstack.OS_VERSIONS + (openstack.OS_LATEST,): - md_url = url_helper.combine_url(url, 'openstack', - version, 'meta_data.json') - md_urls.append(md_url) - url2base[md_url] = url + md_url = url_helper.combine_url(url, 'openstack') + md_urls.append(md_url) + url2base[md_url] = url (max_wait, timeout) = self._get_url_settings() start_time = time.time() @@ -119,8 +117,7 @@ class DataSourceOpenStack(openstack.SourceMixin, sources.DataSource): 'Crawl of openstack metadata service', read_metadata_service, args=[self.metadata_address], - kwargs={'ssl_details': self.ssl_details, - 'version': openstack.OS_HAVANA}) + kwargs={'ssl_details': self.ssl_details}) except openstack.NonReadable: return False except (openstack.BrokenMetadata, IOError): @@ -143,20 +140,20 @@ class DataSourceOpenStack(openstack.SourceMixin, sources.DataSource): self.version = results['version'] self.files.update(results.get('files', {})) - # if vendordata includes 'cloud-init', then read that explicitly - # for cloud-init (for namespacing). vd = results.get('vendordata') - if isinstance(vd, dict) and 'cloud-init' in vd: - self.vendordata_raw = vd['cloud-init'] - else: - self.vendordata_raw = vd + self.vendordata_pure = vd + try: + self.vendordata_raw = openstack.convert_vendordata_json(vd) + except ValueError as e: + LOG.warn("Invalid content in vendor-data: %s", e) + self.vendordata_raw = None return True -def read_metadata_service(base_url, version=None, ssl_details=None): +def read_metadata_service(base_url, ssl_details=None): reader = openstack.MetadataReader(base_url, ssl_details=ssl_details) - return reader.read_v2(version=version) + return reader.read_v2() # Used to match classes to dependencies diff --git a/cloudinit/sources/DataSourceSmartOS.py b/cloudinit/sources/DataSourceSmartOS.py index 65ec0339..2733a2f6 100644 --- a/cloudinit/sources/DataSourceSmartOS.py +++ b/cloudinit/sources/DataSourceSmartOS.py @@ -41,7 +41,7 @@ import serial LOG = logging.getLogger(__name__) SMARTOS_ATTRIB_MAP = { - #Cloud-init Key : (SmartOS Key, Strip line endings) + # Cloud-init Key : (SmartOS Key, Strip line endings) 'local-hostname': ('hostname', True), 'public-keys': ('root_authorized_keys', True), 'user-script': ('user-script', False), @@ -96,21 +96,21 @@ BUILTIN_CLOUD_CONFIG = { 'device': 'ephemeral0'}], } -## builtin vendor-data is a boothook that writes a script into -## /var/lib/cloud/scripts/per-boot. *That* script then handles -## executing the 'operator-script' and 'user-script' files -## that cloud-init writes into /var/lib/cloud/instance/data/ -## if they exist. -## -## This is all very indirect, but its done like this so that at -## some point in the future, perhaps cloud-init wouldn't do it at -## all, but rather the vendor actually provide vendor-data that accomplished -## their desires. (That is the point of vendor-data). -## -## cloud-init does cheat a bit, and write the operator-script and user-script -## itself. It could have the vendor-script do that, but it seems better -## to not require the image to contain a tool (mdata-get) to read those -## keys when we have a perfectly good one inside cloud-init. +# builtin vendor-data is a boothook that writes a script into +# /var/lib/cloud/scripts/per-boot. *That* script then handles +# executing the 'operator-script' and 'user-script' files +# that cloud-init writes into /var/lib/cloud/instance/data/ +# if they exist. +# +# This is all very indirect, but its done like this so that at +# some point in the future, perhaps cloud-init wouldn't do it at +# all, but rather the vendor actually provide vendor-data that accomplished +# their desires. (That is the point of vendor-data). +# +# cloud-init does cheat a bit, and write the operator-script and user-script +# itself. It could have the vendor-script do that, but it seems better +# to not require the image to contain a tool (mdata-get) to read those +# keys when we have a perfectly good one inside cloud-init. BUILTIN_VENDOR_DATA = """\ #cloud-boothook #!/bin/sh diff --git a/cloudinit/sources/__init__.py b/cloudinit/sources/__init__.py index fef4d460..7c7ef9ab 100644 --- a/cloudinit/sources/__init__.py +++ b/cloudinit/sources/__init__.py @@ -66,7 +66,7 @@ class DataSource(object): name = name[0:-3] self.ds_cfg = util.get_cfg_by_path(self.sys_cfg, - ("datasource", name), {}) + ("datasource", name), {}) if not ud_proc: self.ud_proc = ud.UserDataProcessor(self.paths) else: @@ -166,7 +166,7 @@ class DataSource(object): defhost = "localhost" domain = defdomain - if not self.metadata or not 'local-hostname' in self.metadata: + if not self.metadata or 'local-hostname' not in self.metadata: # this is somewhat questionable really. # the cloud datasource was asked for a hostname # and didn't have one. raising error might be more appropriate @@ -272,9 +272,9 @@ def list_sources(cfg_list, depends, pkg_list): for ds_name in cfg_list: if not ds_name.startswith(DS_PREFIX): ds_name = '%s%s' % (DS_PREFIX, ds_name) - m_locs = importer.find_module(ds_name, - pkg_list, - ['get_datasource_list']) + m_locs, _looked_locs = importer.find_module(ds_name, + pkg_list, + ['get_datasource_list']) for m_loc in m_locs: mod = importer.import_module(m_loc) lister = getattr(mod, "get_datasource_list") diff --git a/cloudinit/sources/helpers/openstack.py b/cloudinit/sources/helpers/openstack.py index 0fac0335..b7e19314 100644 --- a/cloudinit/sources/helpers/openstack.py +++ b/cloudinit/sources/helpers/openstack.py @@ -21,6 +21,7 @@ import abc import base64 import copy +import functools import os from cloudinit import ec2_utils @@ -48,6 +49,7 @@ OS_LATEST = 'latest' OS_FOLSOM = '2012-08-10' OS_GRIZZLY = '2013-04-04' OS_HAVANA = '2013-10-17' +# keep this in chronological order. new supported versions go at the end. OS_VERSIONS = ( OS_FOLSOM, OS_GRIZZLY, @@ -150,17 +152,40 @@ class BaseReader(object): pass @abc.abstractmethod - def _path_exists(self, path): + def _path_read(self, path): pass @abc.abstractmethod - def _path_read(self, path): + def _fetch_available_versions(self): pass @abc.abstractmethod def _read_ec2_metadata(self): pass + def _find_working_version(self): + try: + versions_available = self._fetch_available_versions() + except Exception as e: + LOG.debug("Unable to read openstack versions from %s due to: %s", + self.base_path, e) + versions_available = [] + + # openstack.OS_VERSIONS is stored in chronological order, so + # reverse it to check newest first. + supported = [v for v in reversed(list(OS_VERSIONS))] + selected_version = OS_LATEST + + for potential_version in supported: + if potential_version not in versions_available: + continue + selected_version = potential_version + break + + LOG.debug("Selected version '%s' from %s", selected_version, + versions_available) + return selected_version + def _read_content_path(self, item): path = item.get('content_path', '').lstrip("/") path_pieces = path.split("/") @@ -170,24 +195,7 @@ class BaseReader(object): path = self._path_join(self.base_path, "openstack", *path_pieces) return self._path_read(path) - def _find_working_version(self, version): - search_versions = [version] + list(OS_VERSIONS) - for potential_version in search_versions: - if not potential_version: - continue - path = self._path_join(self.base_path, "openstack", - potential_version) - if self._path_exists(path): - if potential_version != version: - LOG.debug("Version '%s' not available, attempting to use" - " version '%s' instead", version, - potential_version) - return potential_version - LOG.debug("Version '%s' not available, attempting to use '%s'" - " instead", version, OS_LATEST) - return OS_LATEST - - def read_v2(self, version=None): + def read_v2(self): """Reads a version 2 formatted location. Return a dict with metadata, userdata, ec2-metadata, dsmode, @@ -196,6 +204,9 @@ class BaseReader(object): If not a valid location, raise a NonReadable exception. """ + load_json_anytype = functools.partial( + util.load_json, root_types=(dict, basestring, list)) + def datafiles(version): files = {} files['metadata'] = ( @@ -214,29 +225,32 @@ class BaseReader(object): files['vendordata'] = ( self._path_join("openstack", version, 'vendor_data.json'), False, - util.load_json, + load_json_anytype, ) return files - version = self._find_working_version(version) results = { 'userdata': '', 'version': 2, } - data = datafiles(version) + data = datafiles(self._find_working_version()) for (name, (path, required, translator)) in data.iteritems(): path = self._path_join(self.base_path, path) data = None found = False - if self._path_exists(path): - try: - data = self._path_read(path) - except IOError: - raise NonReadable("Failed to read: %s" % path) - found = True + try: + data = self._path_read(path) + except IOError as e: + if not required: + LOG.debug("Failed reading optional path %s due" + " to: %s", path, e) + else: + LOG.debug("Failed reading mandatory path %s due" + " to: %s", path, e) else: - if required: - raise NonReadable("Missing mandatory path: %s" % path) + found = True + if required and not found: + raise NonReadable("Missing mandatory path: %s" % path) if found and translator: try: data = translator(data) @@ -304,21 +318,27 @@ class BaseReader(object): class ConfigDriveReader(BaseReader): def __init__(self, base_path): super(ConfigDriveReader, self).__init__(base_path) + self._versions = None def _path_join(self, base, *add_ons): components = [base] + list(add_ons) return os.path.join(*components) - def _path_exists(self, path): - return os.path.exists(path) - def _path_read(self, path): return util.load_file(path) + def _fetch_available_versions(self): + if self._versions is None: + path = self._path_join(self.base_path, 'openstack') + found = [d for d in os.listdir(path) + if os.path.isdir(os.path.join(path))] + self._versions = found + return self._versions + def _read_ec2_metadata(self): path = self._path_join(self.base_path, 'ec2', 'latest', 'meta-data.json') - if not self._path_exists(path): + if not os.path.exists(path): return {} else: try: @@ -338,7 +358,7 @@ class ConfigDriveReader(BaseReader): found = {} for name in FILES_V1.keys(): path = self._path_join(self.base_path, name) - if self._path_exists(path): + if os.path.exists(path): found[name] = path if len(found) == 0: raise NonReadable("%s: no files found" % (self.base_path)) @@ -400,17 +420,26 @@ class MetadataReader(BaseReader): self.ssl_details = ssl_details self.timeout = float(timeout) self.retries = int(retries) + self._versions = None + + def _fetch_available_versions(self): + # <baseurl>/openstack/ returns a newline separated list of versions + if self._versions is not None: + return self._versions + found = [] + version_path = self._path_join(self.base_path, "openstack") + content = self._path_read(version_path) + for line in content.splitlines(): + line = line.strip() + if not line: + continue + found.append(line) + self._versions = found + return self._versions def _path_read(self, path): - response = url_helper.readurl(path, - retries=self.retries, - ssl_details=self.ssl_details, - timeout=self.timeout) - return response.contents - def _path_exists(self, path): - - def should_retry_cb(request, cause): + def should_retry_cb(_request_args, cause): try: code = int(cause.code) if code >= 400: @@ -420,15 +449,12 @@ class MetadataReader(BaseReader): pass return True - try: - response = url_helper.readurl(path, - retries=self.retries, - ssl_details=self.ssl_details, - timeout=self.timeout, - exception_cb=should_retry_cb) - return response.ok() - except IOError: - return False + response = url_helper.readurl(path, + retries=self.retries, + ssl_details=self.ssl_details, + timeout=self.timeout, + exception_cb=should_retry_cb) + return response.contents def _path_join(self, base, *add_ons): return url_helper.combine_url(base, *add_ons) @@ -437,3 +463,28 @@ class MetadataReader(BaseReader): return ec2_utils.get_instance_metadata(ssl_details=self.ssl_details, timeout=self.timeout, retries=self.retries) + + +def convert_vendordata_json(data, recurse=True): + """ data: a loaded json *object* (strings, arrays, dicts). + return something suitable for cloudinit vendordata_raw. + + if data is: + None: return None + string: return string + list: return data + the list is then processed in UserDataProcessor + dict: return convert_vendordata_json(data.get('cloud-init')) + """ + if not data: + return None + if isinstance(data, (str, unicode, basestring)): + return data + if isinstance(data, list): + return copy.deepcopy(data) + if isinstance(data, dict): + if recurse is True: + return convert_vendordata_json(data.get('cloud-init'), + recurse=False) + raise ValueError("vendordata['cloud-init'] cannot be dict") + raise ValueError("Unknown data type for vendordata: %s" % type(data)) diff --git a/cloudinit/stages.py b/cloudinit/stages.py index 58349ffc..67f467f7 100644 --- a/cloudinit/stages.py +++ b/cloudinit/stages.py @@ -386,19 +386,19 @@ class Init(object): potential_handlers = util.find_modules(path) for (fname, mod_name) in potential_handlers.iteritems(): try: - mod_locs = importer.find_module(mod_name, [''], - ['list_types', - 'handle_part']) + mod_locs, looked_locs = importer.find_module( + mod_name, [''], ['list_types', 'handle_part']) if not mod_locs: - LOG.warn(("Could not find a valid user-data handler" - " named %s in file %s"), mod_name, fname) + LOG.warn("Could not find a valid user-data handler" + " named %s in file %s (searched %s)", + mod_name, fname, looked_locs) continue mod = importer.import_module(mod_locs[0]) mod = handlers.fixup_handler(mod) types = c_handlers.register(mod) if types: - LOG.debug("Added custom handler for %s from %s", - types, fname) + LOG.debug("Added custom handler for %s [%s] from %s", + types, mod, fname) except Exception: util.logexc(LOG, "Failed to register handler from %s", fname) @@ -621,11 +621,11 @@ class Modules(object): " has an unknown frequency %s"), raw_name, freq) # Reset it so when ran it will get set to a known value freq = None - mod_locs = importer.find_module(mod_name, - ['', type_utils.obj_name(config)], - ['handle']) + mod_locs, looked_locs = importer.find_module( + mod_name, ['', type_utils.obj_name(config)], ['handle']) if not mod_locs: - LOG.warn("Could not find module named %s", mod_name) + LOG.warn("Could not find module named %s (searched %s)", + mod_name, looked_locs) continue mod = config.fixup_module(importer.import_module(mod_locs[0])) mostly_mods.append([mod, raw_name, freq, run_args]) @@ -642,8 +642,10 @@ class Modules(object): # Try the modules frequency, otherwise fallback to a known one if not freq: freq = mod.frequency - if not freq in FREQUENCIES: + if freq not in FREQUENCIES: freq = PER_INSTANCE + LOG.debug("Running module %s (%s) with frequency %s", + name, mod, freq) # Use the configs logger and not our own # TODO(harlowja): possibly check the module @@ -657,7 +659,7 @@ class Modules(object): run_name = "config-%s" % (name) cc.run(run_name, mod.handle, func_args, freq=freq) except Exception as e: - util.logexc(LOG, "Running %s (%s) failed", name, mod) + util.logexc(LOG, "Running module %s (%s) failed", name, mod) failures.append((name, e)) return (which_ran, failures) diff --git a/cloudinit/templater.py b/cloudinit/templater.py index 77af1270..02f6261d 100644 --- a/cloudinit/templater.py +++ b/cloudinit/templater.py @@ -20,13 +20,119 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see <http://www.gnu.org/licenses/>. -from Cheetah.Template import Template +import collections +import re +try: + from Cheetah.Template import Template as CTemplate + CHEETAH_AVAILABLE = True +except (ImportError, AttributeError): + CHEETAH_AVAILABLE = False + +try: + import jinja2 + from jinja2 import Template as JTemplate + JINJA_AVAILABLE = True +except (ImportError, AttributeError): + JINJA_AVAILABLE = False + +from cloudinit import log as logging +from cloudinit import type_utils as tu from cloudinit import util +LOG = logging.getLogger(__name__) +TYPE_MATCHER = re.compile(r"##\s*template:(.*)", re.I) +BASIC_MATCHER = re.compile(r'\$\{([A-Za-z0-9_.]+)\}|\$([A-Za-z0-9_.]+)') + + +def basic_render(content, params): + """This does simple replacement of bash variable like templates. + + It identifies patterns like ${a} or $a and can also identify patterns like + ${a.b} or $a.b which will look for a key 'b' in the dictionary rooted + by key 'a'. + """ + + def replacer(match): + # Only 1 of the 2 groups will actually have a valid entry. + name = match.group(1) + if name is None: + name = match.group(2) + if name is None: + raise RuntimeError("Match encountered but no valid group present") + path = collections.deque(name.split(".")) + selected_params = params + while len(path) > 1: + key = path.popleft() + if not isinstance(selected_params, dict): + raise TypeError("Can not traverse into" + " non-dictionary '%s' of type %s while" + " looking for subkey '%s'" + % (selected_params, + tu.obj_name(selected_params), + key)) + selected_params = selected_params[key] + key = path.popleft() + if not isinstance(selected_params, dict): + raise TypeError("Can not extract key '%s' from non-dictionary" + " '%s' of type %s" + % (key, selected_params, + tu.obj_name(selected_params))) + return str(selected_params[key]) + + return BASIC_MATCHER.sub(replacer, content) + + +def detect_template(text): + + def cheetah_render(content, params): + return CTemplate(content, searchList=[params]).respond() + + def jinja_render(content, params): + return JTemplate(content, + undefined=jinja2.StrictUndefined, + trim_blocks=True).render(**params) + + if text.find("\n") != -1: + ident, rest = text.split("\n", 1) + else: + ident = text + rest = '' + type_match = TYPE_MATCHER.match(ident) + if not type_match: + if not CHEETAH_AVAILABLE: + LOG.warn("Cheetah not available as the default renderer for" + " unknown template, reverting to the basic renderer.") + return ('basic', basic_render, text) + else: + return ('cheetah', cheetah_render, text) + else: + template_type = type_match.group(1).lower().strip() + if template_type not in ('jinja', 'cheetah', 'basic'): + raise ValueError("Unknown template rendering type '%s' requested" + % template_type) + if template_type == 'jinja' and not JINJA_AVAILABLE: + LOG.warn("Jinja not available as the selected renderer for" + " desired template, reverting to the basic renderer.") + return ('basic', basic_render, rest) + elif template_type == 'jinja' and JINJA_AVAILABLE: + return ('jinja', jinja_render, rest) + if template_type == 'cheetah' and not CHEETAH_AVAILABLE: + LOG.warn("Cheetah not available as the selected renderer for" + " desired template, reverting to the basic renderer.") + return ('basic', basic_render, rest) + elif template_type == 'cheetah' and CHEETAH_AVAILABLE: + return ('cheetah', cheetah_render, rest) + # Only thing left over is the basic renderer (it is always available). + return ('basic', basic_render, rest) + def render_from_file(fn, params): - return render_string(util.load_file(fn), params) + if not params: + params = {} + template_type, renderer, content = detect_template(util.load_file(fn)) + LOG.debug("Rendering content of '%s' using renderer %s", fn, template_type) + return renderer(content, params) def render_to_file(fn, outfn, params, mode=0644): @@ -37,4 +143,5 @@ def render_to_file(fn, outfn, params, mode=0644): def render_string(content, params): if not params: params = {} - return Template(content, searchList=[params]).respond() + template_type, renderer, content = detect_template(content) + return renderer(content, params) diff --git a/cloudinit/type_utils.py b/cloudinit/type_utils.py index 2decbfc5..cc3d9495 100644 --- a/cloudinit/type_utils.py +++ b/cloudinit/type_utils.py @@ -19,8 +19,6 @@ # # You should have received a copy of the GNU General Public License # along with this program. If not, see <http://www.gnu.org/licenses/>. -# -# pylint: disable=C0302 import types diff --git a/cloudinit/url_helper.py b/cloudinit/url_helper.py index 4a83169a..3074dd08 100644 --- a/cloudinit/url_helper.py +++ b/cloudinit/url_helper.py @@ -44,7 +44,7 @@ try: from distutils.version import LooseVersion import pkg_resources _REQ = pkg_resources.get_distribution('requests') - _REQ_VER = LooseVersion(_REQ.version) # pylint: disable=E1103 + _REQ_VER = LooseVersion(_REQ.version) if _REQ_VER >= LooseVersion('0.8.8'): SSL_ENABLED = True if _REQ_VER >= LooseVersion('0.7.0') and _REQ_VER < LooseVersion('1.0.0'): @@ -54,7 +54,7 @@ except: def _cleanurl(url): - parsed_url = list(urlparse(url, scheme='http')) # pylint: disable=E1123 + parsed_url = list(urlparse(url, scheme='http')) if not parsed_url[1] and parsed_url[2]: # Swap these since this seems to be a common # occurrence when given urls like 'www.google.com' @@ -90,7 +90,7 @@ class StringResponse(object): self.contents = contents self.url = None - def ok(self, *args, **kwargs): # pylint: disable=W0613 + def ok(self, *args, **kwargs): if self.code != 200: return False return True @@ -150,7 +150,7 @@ class UrlError(IOError): def _get_ssl_args(url, ssl_details): ssl_args = {} - scheme = urlparse(url).scheme # pylint: disable=E1101 + scheme = urlparse(url).scheme if scheme == 'https' and ssl_details: if not SSL_ENABLED: LOG.warn("SSL is not supported in requests v%s, " @@ -227,18 +227,17 @@ def readurl(url, data=None, timeout=None, retries=0, sec_between=1, r = requests.request(**req_args) if check_status: - r.raise_for_status() # pylint: disable=E1103 + r.raise_for_status() LOG.debug("Read from %s (%s, %sb) after %s attempts", url, - r.status_code, len(r.content), # pylint: disable=E1103 - (i + 1)) + r.status_code, len(r.content), (i + 1)) # Doesn't seem like we can make it use a different # subclass for responses, so add our own backward-compat # attrs return UrlResponse(r) except exceptions.RequestException as e: if (isinstance(e, (exceptions.HTTPError)) - and hasattr(e, 'response') # This appeared in v 0.10.8 - and hasattr(e.response, 'status_code')): + and hasattr(e, 'response') # This appeared in v 0.10.8 + and hasattr(e.response, 'status_code')): excps.append(UrlError(e, code=e.response.status_code, headers=e.response.headers)) else: diff --git a/cloudinit/util.py b/cloudinit/util.py index 06039ee2..946059e9 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -19,8 +19,6 @@ # # You should have received a copy of the GNU General Public License # along with this program. If not, see <http://www.gnu.org/licenses/>. -# -# pylint: disable=C0302 from StringIO import StringIO @@ -42,7 +40,7 @@ import re import shutil import socket import stat -import string # pylint: disable=W0402 +import string import subprocess import sys import tempfile @@ -146,23 +144,23 @@ class SeLinuxGuard(object): return False def __exit__(self, excp_type, excp_value, excp_traceback): - if self.selinux and self.selinux.is_selinux_enabled(): - path = os.path.realpath(os.path.expanduser(self.path)) - # path should be a string, not unicode - path = str(path) - do_restore = False - try: - # See if even worth restoring?? - stats = os.lstat(path) - if stat.ST_MODE in stats: - self.selinux.matchpathcon(path, stats[stat.ST_MODE]) - do_restore = True - except OSError: - pass - if do_restore: - LOG.debug("Restoring selinux mode for %s (recursive=%s)", - path, self.recursive) - self.selinux.restorecon(path, recursive=self.recursive) + if not self.selinux or not self.selinux.is_selinux_enabled(): + return + if not os.path.lexists(self.path): + return + + path = os.path.realpath(self.path) + # path should be a string, not unicode + path = str(path) + try: + stats = os.lstat(path) + self.selinux.matchpathcon(path, stats[stat.ST_MODE]) + except OSError: + return + + LOG.debug("Restoring selinux mode for %s (recursive=%s)", + path, self.recursive) + self.selinux.restorecon(path, recursive=self.recursive) class MountFailedError(Exception): @@ -193,16 +191,16 @@ def ExtendedTemporaryFile(**kwargs): return fh -def fork_cb(child_cb, *args): +def fork_cb(child_cb, *args, **kwargs): fid = os.fork() if fid == 0: try: - child_cb(*args) - os._exit(0) # pylint: disable=W0212 + child_cb(*args, **kwargs) + os._exit(0) except: logexc(LOG, "Failed forking and calling callback %s", type_utils.obj_name(child_cb)) - os._exit(1) # pylint: disable=W0212 + os._exit(1) else: LOG.debug("Forked child %s who will run callback %s", fid, type_utils.obj_name(child_cb)) @@ -423,7 +421,7 @@ def get_cfg_option_list(yobj, key, default=None): @return: The configuration option as a list of strings or default if key is not found. """ - if not key in yobj: + if key not in yobj: return default if yobj[key] is None: return [] @@ -487,7 +485,7 @@ def redirect_output(outfmt, errfmt, o_out=None, o_err=None): new_fp = open(arg, owith) elif mode == "|": proc = subprocess.Popen(arg, shell=True, stdin=subprocess.PIPE) - new_fp = proc.stdin # pylint: disable=E1101 + new_fp = proc.stdin else: raise TypeError("Invalid type for output format: %s" % outfmt) @@ -509,7 +507,7 @@ def redirect_output(outfmt, errfmt, o_out=None, o_err=None): new_fp = open(arg, owith) elif mode == "|": proc = subprocess.Popen(arg, shell=True, stdin=subprocess.PIPE) - new_fp = proc.stdin # pylint: disable=E1101 + new_fp = proc.stdin else: raise TypeError("Invalid type for error format: %s" % errfmt) @@ -937,7 +935,7 @@ def is_resolvable(name): should also not exist. The random entry will be resolved inside the search list. """ - global _DNS_REDIRECT_IP # pylint: disable=W0603 + global _DNS_REDIRECT_IP if _DNS_REDIRECT_IP is None: badips = set() badnames = ("does-not-exist.example.com.", "example.invalid.", @@ -1148,7 +1146,7 @@ def chownbyname(fname, user=None, group=None): # this returns the specific 'mode' entry, cleanly formatted, with value def get_output_cfg(cfg, mode): ret = [None, None] - if not cfg or not 'output' in cfg: + if cfg or 'output' not in cfg: return ret outcfg = cfg['output'] @@ -1532,7 +1530,7 @@ def subp(args, data=None, rcs=None, env=None, capture=True, shell=False, (out, err) = sp.communicate(data) except OSError as e: raise ProcessExecutionError(cmd=args, reason=e) - rc = sp.returncode # pylint: disable=E1101 + rc = sp.returncode if rc not in rcs: raise ProcessExecutionError(stdout=out, stderr=err, exit_code=rc, @@ -1745,7 +1743,7 @@ def parse_mount_info(path, mountinfo_lines, log=LOG): # Ignore mount points higher than an already seen mount # point. if (match_mount_point_elements is not None and - len(match_mount_point_elements) > len(mount_point_elements)): + len(match_mount_point_elements) > len(mount_point_elements)): continue # Find the '-' which terminates a list of optional columns to |