diff options
Diffstat (limited to 'cloudinit')
24 files changed, 953 insertions, 286 deletions
diff --git a/cloudinit/config/__init__.py b/cloudinit/config/__init__.py index 69a8cc68..d57453be 100644 --- a/cloudinit/config/__init__.py +++ b/cloudinit/config/__init__.py @@ -52,5 +52,7 @@ def fixup_module(mod, def_freq=PER_INSTANCE): if freq and freq not in FREQUENCIES: LOG.warn("Module %s has an unknown frequency %s", mod, freq) if not hasattr(mod, 'distros'): - setattr(mod, 'distros', None) + setattr(mod, 'distros', []) + if not hasattr(mod, 'osfamilies'): + setattr(mod, 'osfamilies', []) return mod diff --git a/cloudinit/config/cc_apt_configure.py b/cloudinit/config/cc_apt_configure.py index f8664160..3ce3b351 100644 --- a/cloudinit/config/cc_apt_configure.py +++ b/cloudinit/config/cc_apt_configure.py @@ -140,10 +140,13 @@ def get_release(): def generate_sources_list(codename, mirrors, cloud, log): - template_fn = cloud.get_template_filename('sources.list') + template_fn = cloud.get_template_filename('sources.list.%s' % + (cloud.distro.name)) if not template_fn: - log.warn("No template found, not rendering /etc/apt/sources.list") - return + template_fn = cloud.get_template_filename('sources.list') + if not template_fn: + log.warn("No template found, not rendering /etc/apt/sources.list") + return params = {'codename': codename} for k in mirrors: diff --git a/cloudinit/config/cc_ca_certs.py b/cloudinit/config/cc_ca_certs.py index 20f24357..4f2a46a1 100644 --- a/cloudinit/config/cc_ca_certs.py +++ b/cloudinit/config/cc_ca_certs.py @@ -45,8 +45,15 @@ def add_ca_certs(certs): # First ensure they are strings... cert_file_contents = "\n".join([str(c) for c in certs]) util.write_file(CA_CERT_FULL_PATH, cert_file_contents, mode=0644) + # Append cert filename to CA_CERT_CONFIG file. - util.write_file(CA_CERT_CONFIG, "\n%s" % CA_CERT_FILENAME, omode="ab") + # We have to strip the content because blank lines in the file + # causes subsequent entries to be ignored. (LP: #1077020) + orig = util.load_file(CA_CERT_CONFIG) + cur_cont = '\n'.join([l for l in orig.splitlines() + if l != CA_CERT_FILENAME]) + out = "%s\n%s\n" % (cur_cont.rstrip(), CA_CERT_FILENAME) + util.write_file(CA_CERT_CONFIG, out, omode="wb") def remove_default_ca_certs(): diff --git a/cloudinit/config/cc_chef.py b/cloudinit/config/cc_chef.py index 7a3d6a31..607f789e 100644 --- a/cloudinit/config/cc_chef.py +++ b/cloudinit/config/cc_chef.py @@ -22,6 +22,7 @@ import json import os from cloudinit import templater +from cloudinit import url_helper from cloudinit import util RUBY_VERSION_DEFAULT = "1.8" @@ -35,6 +36,8 @@ CHEF_DIRS = [ '/var/run/chef', ] +OMNIBUS_URL = "https://www.opscode.com/chef/install.sh" + def handle(name, cfg, cloud, log, _args): @@ -83,7 +86,9 @@ def handle(name, cfg, cloud, log, _args): util.write_file('/etc/chef/firstboot.json', json.dumps(initial_json)) # If chef is not installed, we install chef based on 'install_type' - if not os.path.isfile('/usr/bin/chef-client'): + if (not os.path.isfile('/usr/bin/chef-client') or + util.get_cfg_option_bool(chef_cfg, 'force_install', default=False)): + install_type = util.get_cfg_option_str(chef_cfg, 'install_type', 'packages') if install_type == "gems": @@ -99,6 +104,14 @@ def handle(name, cfg, cloud, log, _args): elif install_type == 'packages': # this will install and run the chef-client from packages cloud.distro.install_packages(('chef',)) + elif install_type == 'omnibus': + url = util.get_cfg_option_str(chef_cfg, "omnibus_url", OMNIBUS_URL) + content = url_helper.readurl(url=url, retries=5) + with util.tempdir() as tmpd: + # use tmpd over tmpfile to avoid 'Text file busy' on execute + tmpf = "%s/chef-omnibus-install" % tmpd + util.write_file(tmpf, content, mode=0700) + util.subp([tmpf], capture=False) else: log.warn("Unknown chef install type %s", install_type) diff --git a/cloudinit/config/cc_growpart.py b/cloudinit/config/cc_growpart.py new file mode 100644 index 00000000..b6e1fd37 --- /dev/null +++ b/cloudinit/config/cc_growpart.py @@ -0,0 +1,272 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2011 Canonical Ltd. +# +# Author: Scott Moser <scott.moser@canonical.com> +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. + +import os +import os.path +import re +import stat + +from cloudinit import log as logging +from cloudinit.settings import PER_ALWAYS +from cloudinit import util + +frequency = PER_ALWAYS + +DEFAULT_CONFIG = { + 'mode': 'auto', + 'devices': ['/'], +} + + +def enum(**enums): + return type('Enum', (), enums) + + +RESIZE = enum(SKIPPED="SKIPPED", CHANGED="CHANGED", NOCHANGE="NOCHANGE", + FAILED="FAILED") + +LOG = logging.getLogger(__name__) + + +def resizer_factory(mode): + resize_class = None + if mode == "auto": + for (_name, resizer) in RESIZERS: + cur = resizer() + if cur.available(): + resize_class = cur + break + + if not resize_class: + raise ValueError("No resizers available") + + else: + mmap = {} + for (k, v) in RESIZERS: + mmap[k] = v + + if mode not in mmap: + raise TypeError("unknown resize mode %s" % mode) + + mclass = mmap[mode]() + if mclass.available(): + resize_class = mclass + + if not resize_class: + raise ValueError("mode %s not available" % mode) + + return resize_class + + +class ResizeFailedException(Exception): + pass + + +class ResizeParted(object): + def available(self): + myenv = os.environ.copy() + myenv['LANG'] = 'C' + + try: + (out, _err) = util.subp(["parted", "--help"], env=myenv) + if re.search(r"COMMAND.*resizepart\s+", out, re.DOTALL): + return True + + except util.ProcessExecutionError: + pass + return False + + def resize(self, diskdev, partnum, partdev): + before = get_size(partdev) + try: + util.subp(["parted", "resizepart", diskdev, partnum]) + except util.ProcessExecutionError as e: + raise ResizeFailedException(e) + + return (before, get_size(partdev)) + + +class ResizeGrowPart(object): + def available(self): + myenv = os.environ.copy() + myenv['LANG'] = 'C' + + try: + (out, _err) = util.subp(["growpart", "--help"], env=myenv) + if re.search(r"--update\s+", out, re.DOTALL): + return True + + except util.ProcessExecutionError: + pass + return False + + def resize(self, diskdev, partnum, partdev): + before = get_size(partdev) + try: + util.subp(["growpart", '--dry-run', diskdev, partnum]) + except util.ProcessExecutionError as e: + if e.exit_code != 1: + util.logexc(LOG, ("Failed growpart --dry-run for (%s, %s)" % + (diskdev, partnum))) + raise ResizeFailedException(e) + return (before, before) + + try: + util.subp(["growpart", diskdev, partnum]) + except util.ProcessExecutionError as e: + util.logexc(LOG, "Failed: growpart %s %s" % (diskdev, partnum)) + raise ResizeFailedException(e) + + return (before, get_size(partdev)) + + +def get_size(filename): + fd = os.open(filename, os.O_RDONLY) + try: + return os.lseek(fd, 0, os.SEEK_END) + finally: + os.close(fd) + + +def device_part_info(devpath): + # convert an entry in /dev/ to parent disk and partition number + + # input of /dev/vdb or /dev/disk/by-label/foo + # rpath is hopefully a real-ish path in /dev (vda, sdb..) + rpath = os.path.realpath(devpath) + + bname = os.path.basename(rpath) + syspath = "/sys/class/block/%s" % bname + + if not os.path.exists(syspath): + raise ValueError("%s had no syspath (%s)" % (devpath, syspath)) + + ptpath = os.path.join(syspath, "partition") + if not os.path.exists(ptpath): + raise TypeError("%s not a partition" % devpath) + + ptnum = util.load_file(ptpath).rstrip() + + # for a partition, real syspath is something like: + # /sys/devices/pci0000:00/0000:00:04.0/virtio1/block/vda/vda1 + rsyspath = os.path.realpath(syspath) + disksyspath = os.path.dirname(rsyspath) + + diskmajmin = util.load_file(os.path.join(disksyspath, "dev")).rstrip() + diskdevpath = os.path.realpath("/dev/block/%s" % diskmajmin) + + # diskdevpath has something like 253:0 + # and udev has put links in /dev/block/253:0 to the device name in /dev/ + return (diskdevpath, ptnum) + + +def devent2dev(devent): + if devent.startswith("/dev/"): + return devent + else: + result = util.get_mount_info(devent) + if not result: + raise ValueError("Could not determine device of '%s' % dev_ent") + return result[0] + + +def resize_devices(resizer, devices): + # returns a tuple of tuples containing (entry-in-devices, action, message) + info = [] + for devent in devices: + try: + blockdev = devent2dev(devent) + except ValueError as e: + info.append((devent, RESIZE.SKIPPED, + "unable to convert to device: %s" % e,)) + continue + + try: + statret = os.stat(blockdev) + except OSError as e: + info.append((devent, RESIZE.SKIPPED, + "stat of '%s' failed: %s" % (blockdev, e),)) + continue + + if not stat.S_ISBLK(statret.st_mode): + info.append((devent, RESIZE.SKIPPED, + "device '%s' not a block device" % blockdev,)) + continue + + try: + (disk, ptnum) = device_part_info(blockdev) + except (TypeError, ValueError) as e: + info.append((devent, RESIZE.SKIPPED, + "device_part_info(%s) failed: %s" % (blockdev, e),)) + continue + + try: + (old, new) = resizer.resize(disk, ptnum, blockdev) + if old == new: + info.append((devent, RESIZE.NOCHANGE, + "no change necessary (%s, %s)" % (disk, ptnum),)) + else: + info.append((devent, RESIZE.CHANGED, + "changed (%s, %s) from %s to %s" % + (disk, ptnum, old, new),)) + + except ResizeFailedException as e: + info.append((devent, RESIZE.FAILED, + "failed to resize: disk=%s, ptnum=%s: %s" % + (disk, ptnum, e),)) + + return info + + +def handle(_name, cfg, _cloud, log, _args): + if 'growpart' not in cfg: + log.debug("No 'growpart' entry in cfg. Using default: %s" % + DEFAULT_CONFIG) + cfg['growpart'] = DEFAULT_CONFIG + + mycfg = cfg.get('growpart') + if not isinstance(mycfg, dict): + log.warn("'growpart' in config was not a dict") + return + + mode = mycfg.get('mode', "auto") + if util.is_false(mode): + log.debug("growpart disabled: mode=%s" % mode) + return + + devices = util.get_cfg_option_list(cfg, "devices", ["/"]) + if not len(devices): + log.debug("growpart: empty device list") + return + + try: + resizer = resizer_factory(mode) + except (ValueError, TypeError) as e: + log.debug("growpart unable to find resizer for '%s': %s" % (mode, e)) + if mode != "auto": + raise e + return + + resized = resize_devices(resizer, devices) + for (entry, action, msg) in resized: + if action == RESIZE.CHANGED: + log.info("'%s' resized: %s" % (entry, msg)) + else: + log.debug("'%s' %s: %s" % (entry, action, msg)) + +RESIZERS = (('parted', ResizeParted), ('growpart', ResizeGrowPart)) diff --git a/cloudinit/config/cc_landscape.py b/cloudinit/config/cc_landscape.py index 6734efee..47c10a97 100644 --- a/cloudinit/config/cc_landscape.py +++ b/cloudinit/config/cc_landscape.py @@ -63,7 +63,7 @@ def handle(_name, cfg, cloud, log, _args): if not ls_cloudcfg: return - cloud.distro.install_packages(["landscape-client"]) + cloud.distro.install_packages(('landscape-client',)) merge_data = [ LSC_BUILTIN_CFG, diff --git a/cloudinit/config/cc_mounts.py b/cloudinit/config/cc_mounts.py index 6ebe563d..390ba711 100644 --- a/cloudinit/config/cc_mounts.py +++ b/cloudinit/config/cc_mounts.py @@ -25,8 +25,8 @@ import re from cloudinit import type_utils from cloudinit import util -# Shortname matches 'sda', 'sda1', 'xvda', 'hda', 'sdb', xvdb, vda, vdd1 -SHORTNAME_FILTER = r"^[x]{0,1}[shv]d[a-z][0-9]*$" +# Shortname matches 'sda', 'sda1', 'xvda', 'hda', 'sdb', xvdb, vda, vdd1, sr0 +SHORTNAME_FILTER = r"^([x]{0,1}[shv]d[a-z][0-9]*|sr[0-9]+)$" SHORTNAME = re.compile(SHORTNAME_FILTER) WS = re.compile("[%s]+" % (whitespace)) FSTAB_PATH = "/etc/fstab" diff --git a/cloudinit/config/cc_power_state_change.py b/cloudinit/config/cc_power_state_change.py index aefa3aff..de0c0bbd 100644 --- a/cloudinit/config/cc_power_state_change.py +++ b/cloudinit/config/cc_power_state_change.py @@ -75,7 +75,7 @@ def load_power_state(cfg): ','.join(opt_map.keys())) delay = pstate.get("delay", "now") - if delay != "now" and not re.match("\+[0-9]+", delay): + if delay != "now" and not re.match(r"\+[0-9]+", delay): raise TypeError("power_state[delay] must be 'now' or '+m' (minutes).") args = ["shutdown", opt_map[mode], delay] diff --git a/cloudinit/config/cc_puppet.py b/cloudinit/config/cc_puppet.py index 8fe3af57..471a1a8a 100644 --- a/cloudinit/config/cc_puppet.py +++ b/cloudinit/config/cc_puppet.py @@ -57,8 +57,16 @@ def handle(name, cfg, cloud, log, _args): puppet_cfg = cfg['puppet'] - # Start by installing the puppet package ... - cloud.distro.install_packages(["puppet"]) + # Start by installing the puppet package if necessary... + install = util.get_cfg_option_bool(puppet_cfg, 'install', True) + version = util.get_cfg_option_str(puppet_cfg, 'version', None) + if not install and version: + log.warn(("Puppet install set false but version supplied," + " doing nothing.")) + elif install: + log.debug(("Attempting to install puppet %s,"), + version if version else 'latest') + cloud.distro.install_packages(('puppet', version)) # ... and then update the puppet configuration if 'conf' in puppet_cfg: diff --git a/cloudinit/config/cc_resizefs.py b/cloudinit/config/cc_resizefs.py index 70294eda..51dead2f 100644 --- a/cloudinit/config/cc_resizefs.py +++ b/cloudinit/config/cc_resizefs.py @@ -27,41 +27,28 @@ from cloudinit import util frequency = PER_ALWAYS -RESIZE_FS_PREFIXES_CMDS = [ - ('ext', 'resize2fs'), - ('xfs', 'xfs_growfs'), -] -NOBLOCK = "noblock" +def _resize_btrfs(mount_point, devpth): # pylint: disable=W0613 + return ('btrfs', 'filesystem', 'resize', 'max', mount_point) -def nodeify_path(devpth, where, log): - try: - st_dev = os.stat(where).st_dev - dev = os.makedev(os.major(st_dev), os.minor(st_dev)) - os.mknod(devpth, 0400 | stat.S_IFBLK, dev) - return st_dev - except: - if util.is_container(): - log.debug("Inside container, ignoring mknod failure in resizefs") - return - log.warn("Failed to make device node to resize %s at %s", - where, devpth) - raise +def _resize_ext(mount_point, devpth): # pylint: disable=W0613 + return ('resize2fs', devpth) -def get_fs_type(st_dev, path, log): - try: - dev_entries = util.find_devs_with(tag='TYPE', oformat='value', - no_cache=True, path=path) - if not dev_entries: - return None - return dev_entries[0].strip() - except util.ProcessExecutionError: - util.logexc(log, ("Failed to get filesystem type" - " of maj=%s, min=%s for path %s"), - os.major(st_dev), os.minor(st_dev), path) - raise +def _resize_xfs(mount_point, devpth): # pylint: disable=W0613 + return ('xfs_growfs', devpth) + +# Do not use a dictionary as these commands should be able to be used +# for multiple filesystem types if possible, e.g. one command for +# ext2, ext3 and ext4. +RESIZE_FS_PREFIXES_CMDS = [ + ('btrfs', _resize_btrfs), + ('ext', _resize_ext), + ('xfs', _resize_xfs), +] + +NOBLOCK = "noblock" def handle(name, cfg, _cloud, log, args): @@ -80,52 +67,47 @@ def handle(name, cfg, _cloud, log, args): # TODO(harlowja): allow what is to be resized to be configurable?? resize_what = "/" - with util.ExtendedTemporaryFile(prefix="cloudinit.resizefs.", - dir=resize_root_d, delete=True) as tfh: - devpth = tfh.name - - # Delete the file so that mknod will work - # but don't change the file handle to know that its - # removed so that when a later call that recreates - # occurs this temporary file will still benefit from - # auto deletion - tfh.unlink_now() - - st_dev = nodeify_path(devpth, resize_what, log) - fs_type = get_fs_type(st_dev, devpth, log) - if not fs_type: - log.warn("Could not determine filesystem type of %s", resize_what) - return - - resizer = None - fstype_lc = fs_type.lower() - for (pfix, root_cmd) in RESIZE_FS_PREFIXES_CMDS: - if fstype_lc.startswith(pfix): - resizer = root_cmd - break - - if not resizer: - log.warn("Not resizing unknown filesystem type %s for %s", - fs_type, resize_what) - return - - log.debug("Resizing %s (%s) using %s", resize_what, fs_type, resizer) - resize_cmd = [resizer, devpth] - - if resize_root == NOBLOCK: - # Fork to a child that will run - # the resize command - util.fork_cb(do_resize, resize_cmd, log) - # Don't delete the file now in the parent - tfh.delete = False - else: - do_resize(resize_cmd, log) + result = util.get_mount_info(resize_what, log) + if not result: + log.warn("Could not determine filesystem type of %s", resize_what) + return + + (devpth, fs_type, mount_point) = result + + # Ensure the path is a block device. + if not stat.S_ISBLK(os.stat(devpth).st_mode): + log.debug("The %s device which was found for mount point %s for %s " + "is not a block device" % (devpth, mount_point, resize_what)) + return + + resizer = None + fstype_lc = fs_type.lower() + for (pfix, root_cmd) in RESIZE_FS_PREFIXES_CMDS: + if fstype_lc.startswith(pfix): + resizer = root_cmd + break + + if not resizer: + log.warn("Not resizing unknown filesystem type %s for %s", + fs_type, resize_what) + return + + resize_cmd = resizer(resize_what, devpth) + log.debug("Resizing %s (%s) using %s", resize_what, fs_type, + ' '.join(resize_cmd)) + + if resize_root == NOBLOCK: + # Fork to a child that will run + # the resize command + util.fork_cb(do_resize, resize_cmd, log) + else: + do_resize(resize_cmd, log) action = 'Resized' if resize_root == NOBLOCK: action = 'Resizing (via forking)' - log.debug("%s root filesystem (type=%s, maj=%i, min=%i, val=%s)", - action, fs_type, os.major(st_dev), os.minor(st_dev), resize_root) + log.debug("%s root filesystem (type=%s, val=%s)", action, fs_type, + resize_root) def do_resize(resize_cmd, log): diff --git a/cloudinit/config/cc_resolv_conf.py b/cloudinit/config/cc_resolv_conf.py new file mode 100644 index 00000000..8a460f7e --- /dev/null +++ b/cloudinit/config/cc_resolv_conf.py @@ -0,0 +1,107 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2013 Craig Tracey +# +# Author: Craig Tracey <craigtracey@gmail.com> +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. + +# Note: +# This module is intended to manage resolv.conf in environments where +# early configuration of resolv.conf is necessary for further +# bootstrapping and/or where configuration management such as puppet or +# chef own dns configuration. As Debian/Ubuntu will, by default, utilize +# resovlconf, and similarly RedHat will use sysconfig, this module is +# likely to be of little use unless those are configured correctly. +# +# For RedHat with sysconfig, be sure to set PEERDNS=no for all DHCP +# enabled NICs. And, in Ubuntu/Debian it is recommended that DNS +# be configured via the standard /etc/network/interfaces configuration +# file. +# +# +# Usage Example: +# +# #cloud-config +# manage_resolv_conf: true +# +# resolv_conf: +# nameservers: ['8.8.4.4', '8.8.8.8'] +# searchdomains: +# - foo.example.com +# - bar.example.com +# domain: example.com +# options: +# rotate: true +# timeout: 1 +# + + +from cloudinit.settings import PER_INSTANCE +from cloudinit import templater +from cloudinit import util + +frequency = PER_INSTANCE + +distros = ['fedora', 'rhel'] + + +def generate_resolv_conf(cloud, log, params): + template_fn = cloud.get_template_filename('resolv.conf') + if not template_fn: + log.warn("No template found, not rendering /etc/resolv.conf") + return + + flags = [] + false_flags = [] + if 'options' in params: + for key, val in params['options'].iteritems(): + if type(val) == bool: + if val: + flags.append(key) + else: + false_flags.append(key) + + for flag in flags + false_flags: + del params['options'][flag] + + params['flags'] = flags + log.debug("Writing resolv.conf from template %s" % template_fn) + templater.render_to_file(template_fn, '/etc/resolv.conf', params) + + +def handle(name, cfg, _cloud, log, _args): + """ + Handler for resolv.conf + + @param name: The module name "resolv-conf" from cloud.cfg + @param cfg: A nested dict containing the entire cloud config contents. + @param cloud: The L{CloudInit} object in use. + @param log: Pre-initialized Python logger object to use for logging. + @param args: Any module arguments from cloud.cfg + """ + if "manage_resolv_conf" not in cfg: + log.debug(("Skipping module named %s," + " no 'manage_resolv_conf' key in configuration"), name) + return + + if not util.get_cfg_option_bool(cfg, "manage_resolv_conf", False): + log.debug(("Skipping module named %s," + " 'manage_resolv_conf' present but set to False"), name) + return + + if not "resolv_conf" in cfg: + log.warn("manage_resolv_conf True but no parameters provided!") + + generate_resolv_conf(_cloud, log, cfg["resolv_conf"]) + return diff --git a/cloudinit/config/cc_salt_minion.py b/cloudinit/config/cc_salt_minion.py index f3eede18..53013dcb 100644 --- a/cloudinit/config/cc_salt_minion.py +++ b/cloudinit/config/cc_salt_minion.py @@ -31,7 +31,7 @@ def handle(name, cfg, cloud, log, _args): salt_cfg = cfg['salt_minion'] # Start by installing the salt package ... - cloud.distro.install_packages(["salt-minion"]) + cloud.distro.install_packages(('salt-minion',)) # Ensure we can configure files at the right dir config_dir = salt_cfg.get("config_dir", '/etc/salt') diff --git a/cloudinit/config/cc_ssh.py b/cloudinit/config/cc_ssh.py index b623d476..7ef20d9f 100644 --- a/cloudinit/config/cc_ssh.py +++ b/cloudinit/config/cc_ssh.py @@ -126,7 +126,7 @@ def apply_credentials(keys, user, disable_root, disable_root_opts): keys = set(keys) if user: - ssh_util.setup_user_keys(keys, user, '') + ssh_util.setup_user_keys(keys, user) if disable_root: if not user: @@ -135,4 +135,4 @@ def apply_credentials(keys, user, disable_root, disable_root_opts): else: key_prefix = '' - ssh_util.setup_user_keys(keys, 'root', key_prefix) + ssh_util.setup_user_keys(keys, 'root', options=key_prefix) diff --git a/cloudinit/config/cc_update_etc_hosts.py b/cloudinit/config/cc_update_etc_hosts.py index 96103615..d3dd1f32 100644 --- a/cloudinit/config/cc_update_etc_hosts.py +++ b/cloudinit/config/cc_update_etc_hosts.py @@ -37,10 +37,11 @@ def handle(name, cfg, cloud, log, _args): # Render from a template file tpl_fn_name = cloud.get_template_filename("hosts.%s" % - (cloud.distro.name)) + (cloud.distro.osfamily)) if not tpl_fn_name: raise RuntimeError(("No hosts template could be" - " found for distro %s") % (cloud.distro.name)) + " found for distro %s") % + (cloud.distro.osfamily)) templater.render_to_file(tpl_fn_name, '/etc/hosts', {'hostname': hostname, 'fqdn': fqdn}) diff --git a/cloudinit/distros/__init__.py b/cloudinit/distros/__init__.py index eeea6af1..7b6276c5 100644 --- a/cloudinit/distros/__init__.py +++ b/cloudinit/distros/__init__.py @@ -36,6 +36,11 @@ from cloudinit import util from cloudinit.distros.parsers import hosts +OSFAMILIES = { + 'debian': ['debian', 'ubuntu'], + 'redhat': ['fedora', 'rhel'] +} + LOG = logging.getLogger(__name__) @@ -69,7 +74,7 @@ class Distro(object): self._apply_hostname(hostname) @abc.abstractmethod - def package_command(self, cmd, args=None): + def package_command(self, cmd, args=None, pkgs=None): raise NotImplementedError() @abc.abstractmethod @@ -144,6 +149,16 @@ class Distro(object): def _select_hostname(self, hostname, fqdn): raise NotImplementedError() + @staticmethod + def expand_osfamily(family_list): + distros = [] + for family in family_list: + if not family in OSFAMILIES: + raise ValueError("No distibutions found for osfamily %s" + % (family)) + distros.extend(OSFAMILIES[family]) + return distros + def update_hostname(self, hostname, fqdn, prev_hostname_fn): applying_hostname = hostname @@ -298,22 +313,26 @@ class Distro(object): "no_create_home": "-M", } + redact_fields = ['passwd'] + # Now check the value and create the command for option in kwargs: value = kwargs[option] if option in adduser_opts and value \ and isinstance(value, str): adduser_cmd.extend([adduser_opts[option], value]) - - # Redact the password field from the logs - if option != "password": - x_adduser_cmd.extend([adduser_opts[option], value]) - else: + # Redact certain fields from the logs + if option in redact_fields: x_adduser_cmd.extend([adduser_opts[option], 'REDACTED']) - + else: + x_adduser_cmd.extend([adduser_opts[option], value]) elif option in adduser_opts_flags and value: adduser_cmd.append(adduser_opts_flags[option]) - x_adduser_cmd.append(adduser_opts_flags[option]) + # Redact certain fields from the logs + if option in redact_fields: + x_adduser_cmd.append('REDACTED') + else: + x_adduser_cmd.append(adduser_opts_flags[option]) # Default to creating home directory unless otherwise directed # Also, we do not create home directories for system users. @@ -335,10 +354,9 @@ class Distro(object): if 'plain_text_passwd' in kwargs and kwargs['plain_text_passwd']: self.set_passwd(name, kwargs['plain_text_passwd']) - # Default locking down the account. - if ('lock_passwd' not in kwargs and - ('lock_passwd' in kwargs and kwargs['lock_passwd']) or - 'system' not in kwargs): + # Default locking down the account. 'lock_passwd' defaults to True. + # lock account unless lock_password is False. + if kwargs.get('lock_passwd', True): try: util.subp(['passwd', '--lock', name]) except Exception as e: @@ -353,7 +371,7 @@ class Distro(object): # Import SSH keys if 'ssh_authorized_keys' in kwargs: keys = set(kwargs['ssh_authorized_keys']) or [] - ssh_util.setup_user_keys(keys, name, key_prefix=None) + ssh_util.setup_user_keys(keys, name, options=None) return True @@ -703,41 +721,68 @@ def _normalize_users(u_cfg, def_user_cfg=None): def normalize_users_groups(cfg, distro): if not cfg: cfg = {} + users = {} groups = {} if 'groups' in cfg: groups = _normalize_groups(cfg['groups']) - # Handle the previous style of doing this... - old_user = None + # Handle the previous style of doing this where the first user + # overrides the concept of the default user if provided in the user: XYZ + # format. + old_user = {} if 'user' in cfg and cfg['user']: - old_user = str(cfg['user']) - if not 'users' in cfg: - cfg['users'] = old_user - old_user = None - if 'users' in cfg: - default_user_config = None - try: - default_user_config = distro.get_default_user() - except NotImplementedError: - LOG.warn(("Distro has not implemented default user " - "access. No default user will be normalized.")) - base_users = cfg['users'] - if old_user: - if isinstance(base_users, (list)): - if len(base_users): - # The old user replaces user[0] - base_users[0] = {'name': old_user} - else: - # Just add it on at the end... - base_users.append({'name': old_user}) - elif isinstance(base_users, (dict)): - if old_user not in base_users: - base_users[old_user] = True - elif isinstance(base_users, (str, basestring)): - # Just append it on to be re-parsed later - base_users += ",%s" % (old_user) - users = _normalize_users(base_users, default_user_config) + old_user = cfg['user'] + # Translate it into the format that is more useful + # going forward + if isinstance(old_user, (basestring, str)): + old_user = { + 'name': old_user, + } + if not isinstance(old_user, (dict)): + LOG.warn(("Format for 'user' key must be a string or " + "dictionary and not %s"), util.obj_name(old_user)) + old_user = {} + + # If no old user format, then assume the distro + # provides what the 'default' user maps to, but notice + # that if this is provided, we won't automatically inject + # a 'default' user into the users list, while if a old user + # format is provided we will. + distro_user_config = {} + try: + distro_user_config = distro.get_default_user() + except NotImplementedError: + LOG.warn(("Distro has not implemented default user " + "access. No distribution provided default user" + " will be normalized.")) + + # Merge the old user (which may just be an empty dict when not + # present with the distro provided default user configuration so + # that the old user style picks up all the distribution specific + # attributes (if any) + default_user_config = util.mergemanydict([old_user, distro_user_config]) + + base_users = cfg.get('users', []) + if not isinstance(base_users, (list, dict, str, basestring)): + LOG.warn(("Format for 'users' key must be a comma separated string" + " or a dictionary or a list and not %s"), + util.obj_name(base_users)) + base_users = [] + + if old_user: + # Ensure that when user: is provided that this user + # always gets added (as the default user) + if isinstance(base_users, (list)): + # Just add it on at the end... + base_users.append({'name': 'default'}) + elif isinstance(base_users, (dict)): + base_users['default'] = dict(base_users).get('default', True) + elif isinstance(base_users, (str, basestring)): + # Just append it on to be re-parsed later + base_users += ",default" + + users = _normalize_users(base_users, default_user_config) return (users, groups) diff --git a/cloudinit/distros/debian.py b/cloudinit/distros/debian.py index 7422f4f0..4b779d57 100644 --- a/cloudinit/distros/debian.py +++ b/cloudinit/distros/debian.py @@ -33,6 +33,10 @@ from cloudinit.settings import PER_INSTANCE LOG = logging.getLogger(__name__) +APT_GET_COMMAND = ('apt-get', '--option=Dpkg::Options::=--force-confold', + '--option=Dpkg::options::=--force-unsafe-io', + '--assume-yes', '--quiet') + class Distro(distros.Distro): hostname_conf_fn = "/etc/hostname" @@ -48,6 +52,7 @@ class Distro(distros.Distro): # calls from repeatly happening (when they # should only happen say once per instance...) self._runner = helpers.Runners(paths) + self.osfamily = 'debian' def apply_locale(self, locale, out_fn=None): if not out_fn: @@ -64,7 +69,7 @@ class Distro(distros.Distro): def install_packages(self, pkglist): self.update_package_sources() - self.package_command('install', pkglist) + self.package_command('install', pkgs=pkglist) def _write_network(self, settings): util.write_file(self.network_conf_fn, settings) @@ -141,15 +146,26 @@ class Distro(distros.Distro): # This ensures that the correct tz will be used for the system util.copy(tz_file, self.tz_local_fn) - def package_command(self, command, args=None): + def package_command(self, command, args=None, pkgs=None): + if pkgs is None: + pkgs = [] + e = os.environ.copy() # See: http://tiny.cc/kg91fw # Or: http://tiny.cc/mh91fw e['DEBIAN_FRONTEND'] = 'noninteractive' - cmd = ['apt-get', '--option', 'Dpkg::Options::=--force-confold', - '--assume-yes', '--quiet', command] - if args: + cmd = list(self.get_option("apt_get_command", APT_GET_COMMAND)) + + if args and isinstance(args, str): + cmd.append(args) + elif args and isinstance(args, list): cmd.extend(args) + + cmd.append(command) + + pkglist = util.expand_package_list('%s=%s', pkgs) + cmd.extend(pkglist) + # Allow the output of this to flow outwards (ie not be captured) util.subp(cmd, env=e, capture=False) diff --git a/cloudinit/distros/rhel.py b/cloudinit/distros/rhel.py index bc0877d5..9fee5fd1 100644 --- a/cloudinit/distros/rhel.py +++ b/cloudinit/distros/rhel.py @@ -60,9 +60,10 @@ class Distro(distros.Distro): # calls from repeatly happening (when they # should only happen say once per instance...) self._runner = helpers.Runners(paths) + self.osfamily = 'redhat' def install_packages(self, pkglist): - self.package_command('install', pkglist) + self.package_command('install', pkgs=pkglist) def _adjust_resolve(self, dns_servers, search_servers): try: @@ -207,7 +208,10 @@ class Distro(distros.Distro): # This ensures that the correct tz will be used for the system util.copy(tz_file, self.tz_local_fn) - def package_command(self, command, args=None): + def package_command(self, command, args=None, pkgs=None): + if pkgs is None: + pkgs = [] + cmd = ['yum'] # If enabled, then yum will be tolerant of errors on the command line # with regard to packages. @@ -218,9 +222,17 @@ class Distro(distros.Distro): # Determines whether or not yum prompts for confirmation # of critical actions. We don't want to prompt... cmd.append("-y") - cmd.append(command) - if args: + + if args and isinstance(args, str): + cmd.append(args) + elif args and isinstance(args, list): cmd.extend(args) + + cmd.append(command) + + pkglist = util.expand_package_list('%s-%s', pkgs) + cmd.extend(pkglist) + # Allow the output of this to flow outwards (ie not be captured) util.subp(cmd, capture=False) diff --git a/cloudinit/handlers/upstart_job.py b/cloudinit/handlers/upstart_job.py index a5cb9b0c..3d8833a1 100644 --- a/cloudinit/handlers/upstart_job.py +++ b/cloudinit/handlers/upstart_job.py @@ -64,3 +64,15 @@ class UpstartJobPartHandler(handlers.Handler): payload = util.dos2unix(payload) path = os.path.join(self.upstart_dir, filename) util.write_file(path, payload, 0644) + + # FIXME LATER (LP: #1124384) + # a bug in upstart means that invoking reload-configuration + # at this stage in boot causes havoc. So, until that is fixed + # we will not do that. However, I'd like to be able to easily + # test to see if this bug is still present in an image with + # a newer upstart. So, a boot hook could easiliy write this file. + if os.path.exists("/run/cloud-init-upstart-reload"): + # if inotify support is not present in the root filesystem + # (overlayroot) then we need to tell upstart to re-read /etc + + util.subp(["initctl", "reload-configuration"], capture=False) diff --git a/cloudinit/sources/DataSourceCloudStack.py b/cloudinit/sources/DataSourceCloudStack.py index c0e1a23c..81c8cda9 100644 --- a/cloudinit/sources/DataSourceCloudStack.py +++ b/cloudinit/sources/DataSourceCloudStack.py @@ -3,10 +3,12 @@ # Copyright (C) 2012 Canonical Ltd. # Copyright (C) 2012 Cosmin Luta # Copyright (C) 2012 Yahoo! Inc. +# Copyright (C) 2012 Gerard Dethier # # Author: Cosmin Luta <q4break@gmail.com> # Author: Scott Moser <scott.moser@canonical.com> # Author: Joshua Harlow <harlowja@yahoo-inc.com> +# Author: Gerard Dethier <g.dethier@gmail.com> # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License version 3, as @@ -20,9 +22,6 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see <http://www.gnu.org/licenses/>. -from socket import inet_ntoa -from struct import pack - import os import time @@ -31,6 +30,8 @@ from cloudinit import log as logging from cloudinit import sources from cloudinit import url_helper as uhelp from cloudinit import util +from socket import inet_ntoa +from struct import pack LOG = logging.getLogger(__name__) @@ -40,24 +41,12 @@ class DataSourceCloudStack(sources.DataSource): sources.DataSource.__init__(self, sys_cfg, distro, paths) self.seed_dir = os.path.join(paths.seed_dir, 'cs') # Cloudstack has its metadata/userdata URLs located at - # http://<default-gateway-ip>/latest/ + # http://<virtual-router-ip>/latest/ self.api_ver = 'latest' - gw_addr = self.get_default_gateway() - if not gw_addr: - raise RuntimeError("No default gateway found!") - self.metadata_address = "http://%s/" % (gw_addr) - - def get_default_gateway(self): - """Returns the default gateway ip address in the dotted format.""" - lines = util.load_file("/proc/net/route").splitlines() - for line in lines: - items = line.split("\t") - if items[1] == "00000000": - # Found the default route, get the gateway - gw = inet_ntoa(pack("<L", int(items[2], 16))) - LOG.debug("Found default route, gateway is %s", gw) - return gw - return None + vr_addr = get_vr_address() + if not vr_addr: + raise RuntimeError("No virtual router found!") + self.metadata_address = "http://%s/" % (vr_addr) def _get_url_settings(self): mcfg = self.ds_cfg @@ -87,7 +76,7 @@ class DataSourceCloudStack(sources.DataSource): (max_wait, timeout) = self._get_url_settings() - urls = [self.metadata_address] + urls = [self.metadata_address + "/latest/meta-data/instance-id"] start_time = time.time() url = uhelp.wait_for_url(urls=urls, max_wait=max_wait, timeout=timeout, status_cb=LOG.warn) @@ -132,6 +121,72 @@ class DataSourceCloudStack(sources.DataSource): return self.metadata['availability-zone'] +def get_default_gateway(): + # Returns the default gateway ip address in the dotted format. + lines = util.load_file("/proc/net/route").splitlines() + for line in lines: + items = line.split("\t") + if items[1] == "00000000": + # Found the default route, get the gateway + gw = inet_ntoa(pack("<L", int(items[2], 16))) + LOG.debug("Found default route, gateway is %s", gw) + return gw + return None + + +def get_dhclient_d(): + # find lease files directory + supported_dirs = ["/var/lib/dhclient", "/var/lib/dhcp"] + for d in supported_dirs: + if os.path.exists(d): + LOG.debug("Using %s lease directory", d) + return d + return None + + +def get_latest_lease(): + # find latest lease file + lease_d = get_dhclient_d() + if not lease_d: + return None + lease_files = os.listdir(lease_d) + latest_mtime = -1 + latest_file = None + for file_name in lease_files: + if file_name.endswith(".lease") or file_name.endswith(".leases"): + abs_path = os.path.join(lease_d, file_name) + mtime = os.path.getmtime(abs_path) + if mtime > latest_mtime: + latest_mtime = mtime + latest_file = abs_path + return latest_file + + +def get_vr_address(): + # Get the address of the virtual router via dhcp leases + # see http://bit.ly/T76eKC for documentation on the virtual router. + # If no virtual router is detected, fallback on default gateway. + lease_file = get_latest_lease() + if not lease_file: + LOG.debug("No lease file found, using default gateway") + return get_default_gateway() + + latest_address = None + with open(lease_file, "r") as fd: + for line in fd: + if "dhcp-server-identifier" in line: + words = line.strip(" ;\r\n").split(" ") + if len(words) > 2: + dhcp = words[2] + LOG.debug("Found DHCP identifier %s", dhcp) + latest_address = dhcp + if not latest_address: + # No virtual router found, fallback on default gateway + LOG.debug("No DHCP found, using default gateway") + return get_default_gateway() + return latest_address + + # Used to match classes to dependencies datasources = [ (DataSourceCloudStack, (sources.DEP_FILESYSTEM, sources.DEP_NETWORK)), diff --git a/cloudinit/sources/DataSourceConfigDrive.py b/cloudinit/sources/DataSourceConfigDrive.py index 0216ed07..5f152299 100644 --- a/cloudinit/sources/DataSourceConfigDrive.py +++ b/cloudinit/sources/DataSourceConfigDrive.py @@ -272,7 +272,7 @@ def find_candidate_devs(): combined = (by_label + [d for d in by_fstype if d not in by_label]) # We are looking for block device (sda, not sda1), ignore partitions - combined = [d for d in combined if d[-1] not in "0123456789"] + combined = [d for d in combined if not util.is_partition(d)] return combined diff --git a/cloudinit/sources/DataSourceNoCloud.py b/cloudinit/sources/DataSourceNoCloud.py index 7800812b..08a853cc 100644 --- a/cloudinit/sources/DataSourceNoCloud.py +++ b/cloudinit/sources/DataSourceNoCloud.py @@ -76,37 +76,47 @@ class DataSourceNoCloud(sources.DataSource): found.append("ds_config") md["seedfrom"] = self.ds_cfg['seedfrom'] - fslist = util.find_devs_with("TYPE=vfat") - fslist.extend(util.find_devs_with("TYPE=iso9660")) - - label_list = util.find_devs_with("LABEL=cidata") - devlist = list(set(fslist) & set(label_list)) - devlist.sort(reverse=True) - - for dev in devlist: - try: - LOG.debug("Attempting to use data from %s", dev) - - (newmd, newud) = util.mount_cb(dev, util.read_seeded) - md = util.mergemanydict([newmd, md]) - ud = newud - - # For seed from a device, the default mode is 'net'. - # that is more likely to be what is desired. - # If they want dsmode of local, then they must - # specify that. - if 'dsmode' not in md: - md['dsmode'] = "net" - - LOG.debug("Using data from %s", dev) - found.append(dev) - break - except OSError as e: - if e.errno != errno.ENOENT: - raise - except util.MountFailedError: - util.logexc(LOG, ("Failed to mount %s" - " when looking for data"), dev) + # if ds_cfg has 'user-data' and 'meta-data' + if 'user-data' in self.ds_cfg and 'meta-data' in self.ds_cfg: + if self.ds_cfg['user-data']: + ud = self.ds_cfg['user-data'] + if self.ds_cfg['meta-data'] is not False: + md = util.mergemanydict([md, self.ds_cfg['meta-data']]) + if 'ds_config' not in found: + found.append("ds_config") + + label = self.ds_cfg.get('fs_label', "cidata") + if label is not None: + fslist = util.find_devs_with("TYPE=vfat") + fslist.extend(util.find_devs_with("TYPE=iso9660")) + + label_list = util.find_devs_with("LABEL=%s" % label) + devlist = list(set(fslist) & set(label_list)) + devlist.sort(reverse=True) + + for dev in devlist: + try: + LOG.debug("Attempting to use data from %s", dev) + + (newmd, newud) = util.mount_cb(dev, util.read_seeded) + md = util.mergemanydict([newmd, md]) + ud = newud + + # For seed from a device, the default mode is 'net'. + # that is more likely to be what is desired. If they want + # dsmode of local, then they must specify that. + if 'dsmode' not in md: + md['dsmode'] = "net" + + LOG.debug("Using data from %s", dev) + found.append(dev) + break + except OSError as e: + if e.errno != errno.ENOENT: + raise + except util.MountFailedError: + util.logexc(LOG, ("Failed to mount %s" + " when looking for data"), dev) # There was no indication on kernel cmdline or data # in the seeddir suggesting this handler should be used. @@ -194,6 +204,8 @@ def parse_cmdline_data(ds_id, fill, cmdline=None): # short2long mapping to save cmdline typing s2l = {"h": "local-hostname", "i": "instance-id", "s": "seedfrom"} for item in kvpairs: + if item == "": + continue try: (k, v) = item.split("=", 1) except: diff --git a/cloudinit/ssh_util.py b/cloudinit/ssh_util.py index dd6b742f..95133236 100644 --- a/cloudinit/ssh_util.py +++ b/cloudinit/ssh_util.py @@ -19,9 +19,6 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see <http://www.gnu.org/licenses/>. -from StringIO import StringIO - -import csv import os import pwd @@ -33,6 +30,15 @@ LOG = logging.getLogger(__name__) # See: man sshd_config DEF_SSHD_CFG = "/etc/ssh/sshd_config" +# taken from openssh source key.c/key_type_from_name +VALID_KEY_TYPES = ("rsa", "dsa", "ssh-rsa", "ssh-dss", "ecdsa", + "ssh-rsa-cert-v00@openssh.com", "ssh-dss-cert-v00@openssh.com", + "ssh-rsa-cert-v00@openssh.com", "ssh-dss-cert-v00@openssh.com", + "ssh-rsa-cert-v01@openssh.com", "ssh-dss-cert-v01@openssh.com", + "ecdsa-sha2-nistp256-cert-v01@openssh.com", + "ecdsa-sha2-nistp384-cert-v01@openssh.com", + "ecdsa-sha2-nistp521-cert-v01@openssh.com") + class AuthKeyLine(object): def __init__(self, source, keytype=None, base64=None, @@ -43,11 +49,8 @@ class AuthKeyLine(object): self.keytype = keytype self.source = source - def empty(self): - if (not self.base64 and - not self.comment and not self.keytype and not self.options): - return True - return False + def valid(self): + return (self.base64 and self.keytype) def __str__(self): toks = [] @@ -107,62 +110,47 @@ class AuthKeyLineParser(object): i = i + 1 options = ent[0:i] - options_lst = [] - - # Now use a csv parser to pull the options - # out of the above string that we just found an endpoint for. - # - # No quoting so we don't mess up any of the quoting that - # is already there. - reader = csv.reader(StringIO(options), quoting=csv.QUOTE_NONE) - for row in reader: - for e in row: - # Only keep non-empty csv options - e = e.strip() - if e: - options_lst.append(e) - - # Now take the rest of the items before the string - # as long as there is room to do this... - toks = [] - if i + 1 < len(ent): - rest = ent[i + 1:] - toks = rest.split(None, 2) - return (options_lst, toks) - - def _form_components(self, src_line, toks, options=None): - components = {} - if len(toks) == 1: - components['base64'] = toks[0] - elif len(toks) == 2: - components['base64'] = toks[0] - components['comment'] = toks[1] - elif len(toks) == 3: - components['keytype'] = toks[0] - components['base64'] = toks[1] - components['comment'] = toks[2] - components['options'] = options - if not components: - return AuthKeyLine(src_line) - else: - return AuthKeyLine(src_line, **components) - def parse(self, src_line, def_opt=None): + # Return the rest of the string in 'remain' + remain = ent[i:].lstrip() + return (options, remain) + + def parse(self, src_line, options=None): + # modeled after opensshes auth2-pubkey.c:user_key_allowed2 line = src_line.rstrip("\r\n") if line.startswith("#") or line.strip() == '': return AuthKeyLine(src_line) - else: - ent = line.strip() - toks = ent.split(None, 3) - if len(toks) < 4: - return self._form_components(src_line, toks, def_opt) - else: - (options, toks) = self._extract_options(ent) - if options: - options = ",".join(options) - else: - options = def_opt - return self._form_components(src_line, toks, options) + + def parse_ssh_key(ent): + # return ketype, key, [comment] + toks = ent.split(None, 2) + if len(toks) < 2: + raise TypeError("To few fields: %s" % len(toks)) + if toks[0] not in VALID_KEY_TYPES: + raise TypeError("Invalid keytype %s" % toks[0]) + + # valid key type and 2 or 3 fields: + if len(toks) == 2: + # no comment in line + toks.append("") + + return toks + + ent = line.strip() + try: + (keytype, base64, comment) = parse_ssh_key(ent) + except TypeError: + (keyopts, remain) = self._extract_options(ent) + if options is None: + options = keyopts + + try: + (keytype, base64, comment) = parse_ssh_key(remain) + except TypeError: + return AuthKeyLine(src_line) + + return AuthKeyLine(src_line, keytype=keytype, base64=base64, + comment=comment, options=options) def parse_authorized_keys(fname): @@ -186,11 +174,11 @@ def update_authorized_keys(old_entries, keys): for i in range(0, len(old_entries)): ent = old_entries[i] - if ent.empty() or not ent.base64: + if not ent.valid(): continue # Replace those with the same base64 for k in keys: - if k.empty() or not k.base64: + if not ent.valid(): continue if k.base64 == ent.base64: # Replace it with our better one @@ -249,7 +237,7 @@ def extract_authorized_keys(username): return (auth_key_fn, parse_authorized_keys(auth_key_fn)) -def setup_user_keys(keys, username, key_prefix): +def setup_user_keys(keys, username, options=None): # Make sure the users .ssh dir is setup accordingly (ssh_dir, pwent) = users_ssh_info(username) if not os.path.isdir(ssh_dir): @@ -260,7 +248,7 @@ def setup_user_keys(keys, username, key_prefix): parser = AuthKeyLineParser() key_entries = [] for k in keys: - key_entries.append(parser.parse(str(k), def_opt=key_prefix)) + key_entries.append(parser.parse(str(k), options=options)) # Extract the old and make the new (auth_key_fn, auth_key_entries) = extract_authorized_keys(username) diff --git a/cloudinit/stages.py b/cloudinit/stages.py index 531e7997..543d247f 100644 --- a/cloudinit/stages.py +++ b/cloudinit/stages.py @@ -64,23 +64,29 @@ class Init(object): # Changed only when a fetch occurs self.datasource = NULL_DATA_SOURCE - def _reset(self, ds=False): + def _reset(self, reset_ds=False): # Recreated on access self._cfg = None self._paths = None self._distro = None - if ds: + if reset_ds: self.datasource = NULL_DATA_SOURCE @property def distro(self): if not self._distro: # Try to find the right class to use - scfg = self._extract_cfg('system') - name = scfg.pop('distro', 'ubuntu') - cls = distros.fetch(name) - LOG.debug("Using distro class %s", cls) - self._distro = cls(name, scfg, self.paths) + system_config = self._extract_cfg('system') + distro_name = system_config.pop('distro', 'ubuntu') + distro_cls = distros.fetch(distro_name) + LOG.debug("Using distro class %s", distro_cls) + self._distro = distro_cls(distro_name, system_config, self.paths) + # If we have an active datasource we need to adjust + # said datasource and move its distro/system config + # from whatever it was to a new set... + if self.datasource is not NULL_DATA_SOURCE: + self.datasource.distro = self._distro + self.datasource.sys_cfg = system_config return self._distro @property @@ -158,27 +164,12 @@ class Init(object): self._cfg = self._read_cfg(extra_fns) # LOG.debug("Loaded 'init' config %s", self._cfg) - def _read_base_cfg(self): - base_cfgs = [] - default_cfg = util.get_builtin_cfg() - kern_contents = util.read_cc_from_cmdline() - # Kernel/cmdline parameters override system config - if kern_contents: - base_cfgs.append(util.load_yaml(kern_contents, default={})) - # Anything in your conf.d location?? - # or the 'default' cloud.cfg location??? - base_cfgs.append(util.read_conf_with_confd(CLOUD_CONFIG)) - # And finally the default gets to play - if default_cfg: - base_cfgs.append(default_cfg) - return util.mergemanydict(base_cfgs) - def _read_cfg(self, extra_fns): no_cfg_paths = helpers.Paths({}, self.datasource) merger = helpers.ConfigMerger(paths=no_cfg_paths, datasource=self.datasource, additional_fns=extra_fns, - base_cfg=self._read_base_cfg()) + base_cfg=fetch_base_config()) return merger.cfg def _restore_from_cache(self): @@ -539,11 +530,16 @@ class Modules(object): freq = mod.frequency if not freq in FREQUENCIES: freq = PER_INSTANCE - worked_distros = mod.distros + + worked_distros = set(mod.distros) + worked_distros.update( + distros.Distro.expand_osfamily(mod.osfamilies)) + if (worked_distros and d_name not in worked_distros): LOG.warn(("Module %s is verified on %s distros" " but not on %s distro. It may or may not work" - " correctly."), name, worked_distros, d_name) + " correctly."), name, list(worked_distros), + d_name) # Use the configs logger and not our own # TODO(harlowja): possibly check the module # for having a LOG attr and just give it back @@ -576,3 +572,23 @@ class Modules(object): raw_mods = self._read_modules(section_name) mostly_mods = self._fixup_modules(raw_mods) return self._run_modules(mostly_mods) + + +def fetch_base_config(): + base_cfgs = [] + default_cfg = util.get_builtin_cfg() + kern_contents = util.read_cc_from_cmdline() + + # Kernel/cmdline parameters override system config + if kern_contents: + base_cfgs.append(util.load_yaml(kern_contents, default={})) + + # Anything in your conf.d location?? + # or the 'default' cloud.cfg location??? + base_cfgs.append(util.read_conf_with_confd(CLOUD_CONFIG)) + + # And finally the default gets to play + if default_cfg: + base_cfgs.append(default_cfg) + + return util.mergemanydict(base_cfgs) diff --git a/cloudinit/util.py b/cloudinit/util.py index e5c6f4ea..709d5cca 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -404,10 +404,9 @@ def get_cfg_option_list(yobj, key, default=None): return [] val = yobj[key] if isinstance(val, (list)): - # Should we ensure they are all strings?? - cval = [str(v) for v in val] + cval = [v for v in val] return cval - if not isinstance(val, (str, basestring)): + if not isinstance(val, (basestring)): val = str(val) return [val] @@ -1519,7 +1518,7 @@ def get_proc_env(pid): fn = os.path.join("/proc/", str(pid), "environ") try: contents = load_file(fn) - toks = contents.split("\0") + toks = contents.split("\x00") for tok in toks: if tok == "": continue @@ -1541,3 +1540,120 @@ def keyval_str_to_dict(kvstring): val = True ret[key] = val return ret + + +def is_partition(device): + if device.startswith("/dev/"): + device = device[5:] + + return os.path.isfile("/sys/class/block/%s/partition" % device) + + +def expand_package_list(version_fmt, pkgs): + # we will accept tuples, lists of tuples, or just plain lists + if not isinstance(pkgs, list): + pkgs = [pkgs] + + pkglist = [] + for pkg in pkgs: + if isinstance(pkg, basestring): + pkglist.append(pkg) + continue + + if isinstance(pkg, (tuple, list)): + if len(pkg) < 1 or len(pkg) > 2: + raise RuntimeError("Invalid package & version tuple.") + + if len(pkg) == 2 and pkg[1]: + pkglist.append(version_fmt % tuple(pkg)) + continue + + pkglist.append(pkg[0]) + + else: + raise RuntimeError("Invalid package type.") + + return pkglist + + +def get_mount_info(path, log=LOG): + # Use /proc/$$/mountinfo to find the device where path is mounted. + # This is done because with a btrfs filesystem using os.stat(path) + # does not return the ID of the device. + # + # Here, / has a device of 18 (decimal). + # + # $ stat / + # File: '/' + # Size: 234 Blocks: 0 IO Block: 4096 directory + # Device: 12h/18d Inode: 256 Links: 1 + # Access: (0755/drwxr-xr-x) Uid: ( 0/ root) Gid: ( 0/ root) + # Access: 2013-01-13 07:31:04.358011255 +0000 + # Modify: 2013-01-13 18:48:25.930011255 +0000 + # Change: 2013-01-13 18:48:25.930011255 +0000 + # Birth: - + # + # Find where / is mounted: + # + # $ mount | grep ' / ' + # /dev/vda1 on / type btrfs (rw,subvol=@,compress=lzo) + # + # And the device ID for /dev/vda1 is not 18: + # + # $ ls -l /dev/vda1 + # brw-rw---- 1 root disk 253, 1 Jan 13 08:29 /dev/vda1 + # + # So use /proc/$$/mountinfo to find the device underlying the + # input path. + path_elements = [e for e in path.split('/') if e] + devpth = None + fs_type = None + match_mount_point = None + match_mount_point_elements = None + mountinfo_path = '/proc/%s/mountinfo' % os.getpid() + for line in load_file(mountinfo_path).splitlines(): + parts = line.split() + + mount_point = parts[4] + mount_point_elements = [e for e in mount_point.split('/') if e] + + # Ignore mounts deeper than the path in question. + if len(mount_point_elements) > len(path_elements): + continue + + # Ignore mounts where the common path is not the same. + l = min(len(mount_point_elements), len(path_elements)) + if mount_point_elements[0:l] != path_elements[0:l]: + continue + + # Ignore mount points higher than an already seen mount + # point. + if (match_mount_point_elements is not None and + len(match_mount_point_elements) > len(mount_point_elements)): + continue + + # Find the '-' which terminates a list of optional columns to + # find the filesystem type and the path to the device. See + # man 5 proc for the format of this file. + try: + i = parts.index('-') + except ValueError: + log.debug("Did not find column named '-' in %s", + mountinfo_path) + return None + + # Get the path to the device. + try: + fs_type = parts[i + 1] + devpth = parts[i + 2] + except IndexError: + log.debug("Too few columns in %s after '-' column", mountinfo_path) + return None + + match_mount_point = mount_point + match_mount_point_elements = mount_point_elements + + if devpth and fs_type and match_mount_point: + return (devpth, fs_type, match_mount_point) + else: + return None |