summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorScott Moser <smoser@ubuntu.com>2013-03-19 08:22:30 -0400
committerScott Moser <smoser@ubuntu.com>2013-03-19 08:22:30 -0400
commita904de054f7818137c864e0f3ff728b3764cf457 (patch)
treee28f4807469f2c39554d4353bf11685c03a1b4aa
parent204e79b93c882e17df63b24f7f682c0dbefb482d (diff)
parentae0f94c8f39a234d73ab8e2caf24d73439c8b5ee (diff)
downloadvyos-cloud-init-a904de054f7818137c864e0f3ff728b3764cf457.tar.gz
vyos-cloud-init-a904de054f7818137c864e0f3ff728b3764cf457.zip
merge from trunk at revno 799
-rw-r--r--ChangeLog9
-rw-r--r--cloudinit/config/cc_growpart.py272
-rw-r--r--cloudinit/config/cc_landscape.py4
-rw-r--r--cloudinit/config/cc_mounts.py3
-rw-r--r--cloudinit/config/cc_power_state_change.py2
-rw-r--r--cloudinit/config/cc_resizefs.py126
-rw-r--r--cloudinit/config/cc_ssh.py4
-rw-r--r--cloudinit/distros/__init__.py25
-rw-r--r--cloudinit/distros/debian.py12
-rw-r--r--cloudinit/distros/rhel.py5
-rw-r--r--cloudinit/handlers/__init__.py75
-rw-r--r--cloudinit/handlers/boot_hook.py3
-rw-r--r--cloudinit/handlers/cloud_config.py92
-rw-r--r--cloudinit/handlers/shell_script.py3
-rw-r--r--cloudinit/handlers/upstart_job.py17
-rw-r--r--cloudinit/helpers.py3
-rw-r--r--cloudinit/mergers/__init__.py154
-rw-r--r--cloudinit/mergers/dict.py48
-rw-r--r--cloudinit/mergers/list.py50
-rw-r--r--cloudinit/mergers/str.py39
-rw-r--r--cloudinit/sources/DataSourceAltCloud.py5
-rw-r--r--cloudinit/sources/DataSourceCloudStack.py3
-rw-r--r--cloudinit/sources/DataSourceConfigDrive.py6
-rw-r--r--cloudinit/sources/DataSourceEc2.py3
-rw-r--r--cloudinit/sources/DataSourceMAAS.py3
-rw-r--r--cloudinit/sources/DataSourceNoCloud.py19
-rw-r--r--cloudinit/sources/DataSourceNone.py4
-rw-r--r--cloudinit/sources/DataSourceOVF.py7
-rw-r--r--cloudinit/sources/__init__.py10
-rw-r--r--cloudinit/ssh_util.py116
-rw-r--r--cloudinit/stages.py9
-rw-r--r--cloudinit/type_utils.py34
-rw-r--r--cloudinit/util.py161
-rw-r--r--config/cloud.cfg1
-rw-r--r--doc/examples/cloud-config-growpart.txt24
-rw-r--r--doc/merging.rst188
-rw-r--r--doc/rtd/conf.py8
-rw-r--r--doc/rtd/index.rst1
-rw-r--r--doc/rtd/topics/merging.rst5
-rw-r--r--tests/unittests/helpers.py2
-rw-r--r--tests/unittests/test__init__.py26
-rw-r--r--tests/unittests/test_builtin_handlers.py23
-rw-r--r--tests/unittests/test_datasource/test_nocloud.py2
-rw-r--r--tests/unittests/test_handler/test_handler_growpart.py255
-rw-r--r--tests/unittests/test_merging.py204
-rw-r--r--tests/unittests/test_sshutil.py101
-rw-r--r--tests/unittests/test_userdata.py107
-rw-r--r--tests/unittests/test_util.py95
-rw-r--r--upstart/cloud-init-nonet.conf58
49 files changed, 2032 insertions, 394 deletions
diff --git a/ChangeLog b/ChangeLog
index 04d55617..54f49da2 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -42,6 +42,15 @@
in config (LP: #1115833).
- improve debian support in sysvinit scripts, package build scripts, and
split sources.list template to be distro specific.
+ - support for resizing btrfs root filesystems [Blair Zajac]
+ - fix issue when writing ssh keys to .ssh/authorized_keys (LP: #1136343)
+ - upstart: cloud-init-nonet.conf trap the TERM signal, so that dmesg or other
+ output does not get a 'killed by TERM signal' message.
+ - support resizing partitions via growpart or parted (LP: #1136936)
+ - allow specifying apt-get command in distro config ('apt_get_command')
+ - support different and user-suppliable merging algorithms for cloud-config
+ (LP: #1023179)
+
0.7.1:
- sysvinit: fix missing dependency in cloud-init job for RHEL 5.6
- config-drive: map hostname to local-hostname (LP: #1061964)
diff --git a/cloudinit/config/cc_growpart.py b/cloudinit/config/cc_growpart.py
new file mode 100644
index 00000000..b6e1fd37
--- /dev/null
+++ b/cloudinit/config/cc_growpart.py
@@ -0,0 +1,272 @@
+# vi: ts=4 expandtab
+#
+# Copyright (C) 2011 Canonical Ltd.
+#
+# Author: Scott Moser <scott.moser@canonical.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 3, as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+import os
+import os.path
+import re
+import stat
+
+from cloudinit import log as logging
+from cloudinit.settings import PER_ALWAYS
+from cloudinit import util
+
+frequency = PER_ALWAYS
+
+DEFAULT_CONFIG = {
+ 'mode': 'auto',
+ 'devices': ['/'],
+}
+
+
+def enum(**enums):
+ return type('Enum', (), enums)
+
+
+RESIZE = enum(SKIPPED="SKIPPED", CHANGED="CHANGED", NOCHANGE="NOCHANGE",
+ FAILED="FAILED")
+
+LOG = logging.getLogger(__name__)
+
+
+def resizer_factory(mode):
+ resize_class = None
+ if mode == "auto":
+ for (_name, resizer) in RESIZERS:
+ cur = resizer()
+ if cur.available():
+ resize_class = cur
+ break
+
+ if not resize_class:
+ raise ValueError("No resizers available")
+
+ else:
+ mmap = {}
+ for (k, v) in RESIZERS:
+ mmap[k] = v
+
+ if mode not in mmap:
+ raise TypeError("unknown resize mode %s" % mode)
+
+ mclass = mmap[mode]()
+ if mclass.available():
+ resize_class = mclass
+
+ if not resize_class:
+ raise ValueError("mode %s not available" % mode)
+
+ return resize_class
+
+
+class ResizeFailedException(Exception):
+ pass
+
+
+class ResizeParted(object):
+ def available(self):
+ myenv = os.environ.copy()
+ myenv['LANG'] = 'C'
+
+ try:
+ (out, _err) = util.subp(["parted", "--help"], env=myenv)
+ if re.search(r"COMMAND.*resizepart\s+", out, re.DOTALL):
+ return True
+
+ except util.ProcessExecutionError:
+ pass
+ return False
+
+ def resize(self, diskdev, partnum, partdev):
+ before = get_size(partdev)
+ try:
+ util.subp(["parted", "resizepart", diskdev, partnum])
+ except util.ProcessExecutionError as e:
+ raise ResizeFailedException(e)
+
+ return (before, get_size(partdev))
+
+
+class ResizeGrowPart(object):
+ def available(self):
+ myenv = os.environ.copy()
+ myenv['LANG'] = 'C'
+
+ try:
+ (out, _err) = util.subp(["growpart", "--help"], env=myenv)
+ if re.search(r"--update\s+", out, re.DOTALL):
+ return True
+
+ except util.ProcessExecutionError:
+ pass
+ return False
+
+ def resize(self, diskdev, partnum, partdev):
+ before = get_size(partdev)
+ try:
+ util.subp(["growpart", '--dry-run', diskdev, partnum])
+ except util.ProcessExecutionError as e:
+ if e.exit_code != 1:
+ util.logexc(LOG, ("Failed growpart --dry-run for (%s, %s)" %
+ (diskdev, partnum)))
+ raise ResizeFailedException(e)
+ return (before, before)
+
+ try:
+ util.subp(["growpart", diskdev, partnum])
+ except util.ProcessExecutionError as e:
+ util.logexc(LOG, "Failed: growpart %s %s" % (diskdev, partnum))
+ raise ResizeFailedException(e)
+
+ return (before, get_size(partdev))
+
+
+def get_size(filename):
+ fd = os.open(filename, os.O_RDONLY)
+ try:
+ return os.lseek(fd, 0, os.SEEK_END)
+ finally:
+ os.close(fd)
+
+
+def device_part_info(devpath):
+ # convert an entry in /dev/ to parent disk and partition number
+
+ # input of /dev/vdb or /dev/disk/by-label/foo
+ # rpath is hopefully a real-ish path in /dev (vda, sdb..)
+ rpath = os.path.realpath(devpath)
+
+ bname = os.path.basename(rpath)
+ syspath = "/sys/class/block/%s" % bname
+
+ if not os.path.exists(syspath):
+ raise ValueError("%s had no syspath (%s)" % (devpath, syspath))
+
+ ptpath = os.path.join(syspath, "partition")
+ if not os.path.exists(ptpath):
+ raise TypeError("%s not a partition" % devpath)
+
+ ptnum = util.load_file(ptpath).rstrip()
+
+ # for a partition, real syspath is something like:
+ # /sys/devices/pci0000:00/0000:00:04.0/virtio1/block/vda/vda1
+ rsyspath = os.path.realpath(syspath)
+ disksyspath = os.path.dirname(rsyspath)
+
+ diskmajmin = util.load_file(os.path.join(disksyspath, "dev")).rstrip()
+ diskdevpath = os.path.realpath("/dev/block/%s" % diskmajmin)
+
+ # diskdevpath has something like 253:0
+ # and udev has put links in /dev/block/253:0 to the device name in /dev/
+ return (diskdevpath, ptnum)
+
+
+def devent2dev(devent):
+ if devent.startswith("/dev/"):
+ return devent
+ else:
+ result = util.get_mount_info(devent)
+ if not result:
+ raise ValueError("Could not determine device of '%s' % dev_ent")
+ return result[0]
+
+
+def resize_devices(resizer, devices):
+ # returns a tuple of tuples containing (entry-in-devices, action, message)
+ info = []
+ for devent in devices:
+ try:
+ blockdev = devent2dev(devent)
+ except ValueError as e:
+ info.append((devent, RESIZE.SKIPPED,
+ "unable to convert to device: %s" % e,))
+ continue
+
+ try:
+ statret = os.stat(blockdev)
+ except OSError as e:
+ info.append((devent, RESIZE.SKIPPED,
+ "stat of '%s' failed: %s" % (blockdev, e),))
+ continue
+
+ if not stat.S_ISBLK(statret.st_mode):
+ info.append((devent, RESIZE.SKIPPED,
+ "device '%s' not a block device" % blockdev,))
+ continue
+
+ try:
+ (disk, ptnum) = device_part_info(blockdev)
+ except (TypeError, ValueError) as e:
+ info.append((devent, RESIZE.SKIPPED,
+ "device_part_info(%s) failed: %s" % (blockdev, e),))
+ continue
+
+ try:
+ (old, new) = resizer.resize(disk, ptnum, blockdev)
+ if old == new:
+ info.append((devent, RESIZE.NOCHANGE,
+ "no change necessary (%s, %s)" % (disk, ptnum),))
+ else:
+ info.append((devent, RESIZE.CHANGED,
+ "changed (%s, %s) from %s to %s" %
+ (disk, ptnum, old, new),))
+
+ except ResizeFailedException as e:
+ info.append((devent, RESIZE.FAILED,
+ "failed to resize: disk=%s, ptnum=%s: %s" %
+ (disk, ptnum, e),))
+
+ return info
+
+
+def handle(_name, cfg, _cloud, log, _args):
+ if 'growpart' not in cfg:
+ log.debug("No 'growpart' entry in cfg. Using default: %s" %
+ DEFAULT_CONFIG)
+ cfg['growpart'] = DEFAULT_CONFIG
+
+ mycfg = cfg.get('growpart')
+ if not isinstance(mycfg, dict):
+ log.warn("'growpart' in config was not a dict")
+ return
+
+ mode = mycfg.get('mode', "auto")
+ if util.is_false(mode):
+ log.debug("growpart disabled: mode=%s" % mode)
+ return
+
+ devices = util.get_cfg_option_list(cfg, "devices", ["/"])
+ if not len(devices):
+ log.debug("growpart: empty device list")
+ return
+
+ try:
+ resizer = resizer_factory(mode)
+ except (ValueError, TypeError) as e:
+ log.debug("growpart unable to find resizer for '%s': %s" % (mode, e))
+ if mode != "auto":
+ raise e
+ return
+
+ resized = resize_devices(resizer, devices)
+ for (entry, action, msg) in resized:
+ if action == RESIZE.CHANGED:
+ log.info("'%s' resized: %s" % (entry, msg))
+ else:
+ log.debug("'%s' %s: %s" % (entry, action, msg))
+
+RESIZERS = (('parted', ResizeParted), ('growpart', ResizeGrowPart))
diff --git a/cloudinit/config/cc_landscape.py b/cloudinit/config/cc_landscape.py
index 2efdff79..8a709677 100644
--- a/cloudinit/config/cc_landscape.py
+++ b/cloudinit/config/cc_landscape.py
@@ -24,6 +24,7 @@ from StringIO import StringIO
from configobj import ConfigObj
+from cloudinit import type_utils
from cloudinit import util
from cloudinit.settings import PER_INSTANCE
@@ -58,7 +59,8 @@ def handle(_name, cfg, cloud, log, _args):
if not isinstance(ls_cloudcfg, (dict)):
raise RuntimeError(("'landscape' key existed in config,"
" but not a dictionary type,"
- " is a %s instead"), util.obj_name(ls_cloudcfg))
+ " is a %s instead"),
+ type_utils.obj_name(ls_cloudcfg))
if not ls_cloudcfg:
return
diff --git a/cloudinit/config/cc_mounts.py b/cloudinit/config/cc_mounts.py
index 9010d97f..390ba711 100644
--- a/cloudinit/config/cc_mounts.py
+++ b/cloudinit/config/cc_mounts.py
@@ -22,6 +22,7 @@ from string import whitespace # pylint: disable=W0402
import re
+from cloudinit import type_utils
from cloudinit import util
# Shortname matches 'sda', 'sda1', 'xvda', 'hda', 'sdb', xvdb, vda, vdd1, sr0
@@ -60,7 +61,7 @@ def handle(_name, cfg, cloud, log, _args):
# skip something that wasn't a list
if not isinstance(cfgmnt[i], list):
log.warn("Mount option %s not a list, got a %s instead",
- (i + 1), util.obj_name(cfgmnt[i]))
+ (i + 1), type_utils.obj_name(cfgmnt[i]))
continue
startname = str(cfgmnt[i][0])
diff --git a/cloudinit/config/cc_power_state_change.py b/cloudinit/config/cc_power_state_change.py
index aefa3aff..de0c0bbd 100644
--- a/cloudinit/config/cc_power_state_change.py
+++ b/cloudinit/config/cc_power_state_change.py
@@ -75,7 +75,7 @@ def load_power_state(cfg):
','.join(opt_map.keys()))
delay = pstate.get("delay", "now")
- if delay != "now" and not re.match("\+[0-9]+", delay):
+ if delay != "now" and not re.match(r"\+[0-9]+", delay):
raise TypeError("power_state[delay] must be 'now' or '+m' (minutes).")
args = ["shutdown", opt_map[mode], delay]
diff --git a/cloudinit/config/cc_resizefs.py b/cloudinit/config/cc_resizefs.py
index 70294eda..51dead2f 100644
--- a/cloudinit/config/cc_resizefs.py
+++ b/cloudinit/config/cc_resizefs.py
@@ -27,41 +27,28 @@ from cloudinit import util
frequency = PER_ALWAYS
-RESIZE_FS_PREFIXES_CMDS = [
- ('ext', 'resize2fs'),
- ('xfs', 'xfs_growfs'),
-]
-NOBLOCK = "noblock"
+def _resize_btrfs(mount_point, devpth): # pylint: disable=W0613
+ return ('btrfs', 'filesystem', 'resize', 'max', mount_point)
-def nodeify_path(devpth, where, log):
- try:
- st_dev = os.stat(where).st_dev
- dev = os.makedev(os.major(st_dev), os.minor(st_dev))
- os.mknod(devpth, 0400 | stat.S_IFBLK, dev)
- return st_dev
- except:
- if util.is_container():
- log.debug("Inside container, ignoring mknod failure in resizefs")
- return
- log.warn("Failed to make device node to resize %s at %s",
- where, devpth)
- raise
+def _resize_ext(mount_point, devpth): # pylint: disable=W0613
+ return ('resize2fs', devpth)
-def get_fs_type(st_dev, path, log):
- try:
- dev_entries = util.find_devs_with(tag='TYPE', oformat='value',
- no_cache=True, path=path)
- if not dev_entries:
- return None
- return dev_entries[0].strip()
- except util.ProcessExecutionError:
- util.logexc(log, ("Failed to get filesystem type"
- " of maj=%s, min=%s for path %s"),
- os.major(st_dev), os.minor(st_dev), path)
- raise
+def _resize_xfs(mount_point, devpth): # pylint: disable=W0613
+ return ('xfs_growfs', devpth)
+
+# Do not use a dictionary as these commands should be able to be used
+# for multiple filesystem types if possible, e.g. one command for
+# ext2, ext3 and ext4.
+RESIZE_FS_PREFIXES_CMDS = [
+ ('btrfs', _resize_btrfs),
+ ('ext', _resize_ext),
+ ('xfs', _resize_xfs),
+]
+
+NOBLOCK = "noblock"
def handle(name, cfg, _cloud, log, args):
@@ -80,52 +67,47 @@ def handle(name, cfg, _cloud, log, args):
# TODO(harlowja): allow what is to be resized to be configurable??
resize_what = "/"
- with util.ExtendedTemporaryFile(prefix="cloudinit.resizefs.",
- dir=resize_root_d, delete=True) as tfh:
- devpth = tfh.name
-
- # Delete the file so that mknod will work
- # but don't change the file handle to know that its
- # removed so that when a later call that recreates
- # occurs this temporary file will still benefit from
- # auto deletion
- tfh.unlink_now()
-
- st_dev = nodeify_path(devpth, resize_what, log)
- fs_type = get_fs_type(st_dev, devpth, log)
- if not fs_type:
- log.warn("Could not determine filesystem type of %s", resize_what)
- return
-
- resizer = None
- fstype_lc = fs_type.lower()
- for (pfix, root_cmd) in RESIZE_FS_PREFIXES_CMDS:
- if fstype_lc.startswith(pfix):
- resizer = root_cmd
- break
-
- if not resizer:
- log.warn("Not resizing unknown filesystem type %s for %s",
- fs_type, resize_what)
- return
-
- log.debug("Resizing %s (%s) using %s", resize_what, fs_type, resizer)
- resize_cmd = [resizer, devpth]
-
- if resize_root == NOBLOCK:
- # Fork to a child that will run
- # the resize command
- util.fork_cb(do_resize, resize_cmd, log)
- # Don't delete the file now in the parent
- tfh.delete = False
- else:
- do_resize(resize_cmd, log)
+ result = util.get_mount_info(resize_what, log)
+ if not result:
+ log.warn("Could not determine filesystem type of %s", resize_what)
+ return
+
+ (devpth, fs_type, mount_point) = result
+
+ # Ensure the path is a block device.
+ if not stat.S_ISBLK(os.stat(devpth).st_mode):
+ log.debug("The %s device which was found for mount point %s for %s "
+ "is not a block device" % (devpth, mount_point, resize_what))
+ return
+
+ resizer = None
+ fstype_lc = fs_type.lower()
+ for (pfix, root_cmd) in RESIZE_FS_PREFIXES_CMDS:
+ if fstype_lc.startswith(pfix):
+ resizer = root_cmd
+ break
+
+ if not resizer:
+ log.warn("Not resizing unknown filesystem type %s for %s",
+ fs_type, resize_what)
+ return
+
+ resize_cmd = resizer(resize_what, devpth)
+ log.debug("Resizing %s (%s) using %s", resize_what, fs_type,
+ ' '.join(resize_cmd))
+
+ if resize_root == NOBLOCK:
+ # Fork to a child that will run
+ # the resize command
+ util.fork_cb(do_resize, resize_cmd, log)
+ else:
+ do_resize(resize_cmd, log)
action = 'Resized'
if resize_root == NOBLOCK:
action = 'Resizing (via forking)'
- log.debug("%s root filesystem (type=%s, maj=%i, min=%i, val=%s)",
- action, fs_type, os.major(st_dev), os.minor(st_dev), resize_root)
+ log.debug("%s root filesystem (type=%s, val=%s)", action, fs_type,
+ resize_root)
def do_resize(resize_cmd, log):
diff --git a/cloudinit/config/cc_ssh.py b/cloudinit/config/cc_ssh.py
index b623d476..7ef20d9f 100644
--- a/cloudinit/config/cc_ssh.py
+++ b/cloudinit/config/cc_ssh.py
@@ -126,7 +126,7 @@ def apply_credentials(keys, user, disable_root, disable_root_opts):
keys = set(keys)
if user:
- ssh_util.setup_user_keys(keys, user, '')
+ ssh_util.setup_user_keys(keys, user)
if disable_root:
if not user:
@@ -135,4 +135,4 @@ def apply_credentials(keys, user, disable_root, disable_root_opts):
else:
key_prefix = ''
- ssh_util.setup_user_keys(keys, 'root', key_prefix)
+ ssh_util.setup_user_keys(keys, 'root', options=key_prefix)
diff --git a/cloudinit/distros/__init__.py b/cloudinit/distros/__init__.py
index 0db4aac7..50d52594 100644
--- a/cloudinit/distros/__init__.py
+++ b/cloudinit/distros/__init__.py
@@ -31,6 +31,7 @@ import re
from cloudinit import importer
from cloudinit import log as logging
from cloudinit import ssh_util
+from cloudinit import type_utils
from cloudinit import util
from cloudinit.distros.parsers import hosts
@@ -73,7 +74,7 @@ class Distro(object):
self._apply_hostname(hostname)
@abc.abstractmethod
- def package_command(self, cmd, args=None):
+ def package_command(self, cmd, args=None, pkgs=None):
raise NotImplementedError()
@abc.abstractmethod
@@ -370,7 +371,7 @@ class Distro(object):
# Import SSH keys
if 'ssh_authorized_keys' in kwargs:
keys = set(kwargs['ssh_authorized_keys']) or []
- ssh_util.setup_user_keys(keys, name, key_prefix=None)
+ ssh_util.setup_user_keys(keys, name, options=None)
return True
@@ -445,7 +446,7 @@ class Distro(object):
lines.append("%s %s" % (user, rules))
else:
msg = "Can not create sudoers rule addition with type %r"
- raise TypeError(msg % (util.obj_name(rules)))
+ raise TypeError(msg % (type_utils.obj_name(rules)))
content = "\n".join(lines)
content += "\n" # trailing newline
@@ -568,7 +569,7 @@ def _normalize_groups(grp_cfg):
c_grp_cfg[k] = [v]
else:
raise TypeError("Bad group member type %s" %
- util.obj_name(v))
+ type_utils.obj_name(v))
else:
if isinstance(v, (list)):
c_grp_cfg[k].extend(v)
@@ -576,13 +577,13 @@ def _normalize_groups(grp_cfg):
c_grp_cfg[k].append(v)
else:
raise TypeError("Bad group member type %s" %
- util.obj_name(v))
+ type_utils.obj_name(v))
elif isinstance(i, (str, basestring)):
if i not in c_grp_cfg:
c_grp_cfg[i] = []
else:
raise TypeError("Unknown group name type %s" %
- util.obj_name(i))
+ type_utils.obj_name(i))
grp_cfg = c_grp_cfg
groups = {}
if isinstance(grp_cfg, (dict)):
@@ -591,7 +592,7 @@ def _normalize_groups(grp_cfg):
else:
raise TypeError(("Group config must be list, dict "
" or string types only and not %s") %
- util.obj_name(grp_cfg))
+ type_utils.obj_name(grp_cfg))
return groups
@@ -622,7 +623,7 @@ def _normalize_users(u_cfg, def_user_cfg=None):
ad_ucfg.append(v)
else:
raise TypeError(("Unmappable user value type %s"
- " for key %s") % (util.obj_name(v), k))
+ " for key %s") % (type_utils.obj_name(v), k))
u_cfg = ad_ucfg
elif isinstance(u_cfg, (str, basestring)):
u_cfg = util.uniq_merge_sorted(u_cfg)
@@ -647,7 +648,7 @@ def _normalize_users(u_cfg, def_user_cfg=None):
else:
raise TypeError(("User config must be dictionary/list "
" or string types only and not %s") %
- util.obj_name(user_config))
+ type_utils.obj_name(user_config))
# Ensure user options are in the right python friendly format
if users:
@@ -740,7 +741,7 @@ def normalize_users_groups(cfg, distro):
}
if not isinstance(old_user, (dict)):
LOG.warn(("Format for 'user' key must be a string or "
- "dictionary and not %s"), util.obj_name(old_user))
+ "dictionary and not %s"), type_utils.obj_name(old_user))
old_user = {}
# If no old user format, then assume the distro
@@ -766,7 +767,7 @@ def normalize_users_groups(cfg, distro):
if not isinstance(base_users, (list, dict, str, basestring)):
LOG.warn(("Format for 'users' key must be a comma separated string"
" or a dictionary or a list and not %s"),
- util.obj_name(base_users))
+ type_utils.obj_name(base_users))
base_users = []
if old_user:
@@ -776,7 +777,7 @@ def normalize_users_groups(cfg, distro):
# Just add it on at the end...
base_users.append({'name': 'default'})
elif isinstance(base_users, (dict)):
- base_users['default'] = base_users.get('default', True)
+ base_users['default'] = dict(base_users).get('default', True)
elif isinstance(base_users, (str, basestring)):
# Just append it on to be re-parsed later
base_users += ",default"
diff --git a/cloudinit/distros/debian.py b/cloudinit/distros/debian.py
index 1a8e927b..4b779d57 100644
--- a/cloudinit/distros/debian.py
+++ b/cloudinit/distros/debian.py
@@ -33,6 +33,10 @@ from cloudinit.settings import PER_INSTANCE
LOG = logging.getLogger(__name__)
+APT_GET_COMMAND = ('apt-get', '--option=Dpkg::Options::=--force-confold',
+ '--option=Dpkg::options::=--force-unsafe-io',
+ '--assume-yes', '--quiet')
+
class Distro(distros.Distro):
hostname_conf_fn = "/etc/hostname"
@@ -142,13 +146,15 @@ class Distro(distros.Distro):
# This ensures that the correct tz will be used for the system
util.copy(tz_file, self.tz_local_fn)
- def package_command(self, command, args=None, pkgs=[]):
+ def package_command(self, command, args=None, pkgs=None):
+ if pkgs is None:
+ pkgs = []
+
e = os.environ.copy()
# See: http://tiny.cc/kg91fw
# Or: http://tiny.cc/mh91fw
e['DEBIAN_FRONTEND'] = 'noninteractive'
- cmd = ['apt-get', '--option', 'Dpkg::Options::=--force-confold',
- '--assume-yes', '--quiet']
+ cmd = list(self.get_option("apt_get_command", APT_GET_COMMAND))
if args and isinstance(args, str):
cmd.append(args)
diff --git a/cloudinit/distros/rhel.py b/cloudinit/distros/rhel.py
index 2f91e386..9fee5fd1 100644
--- a/cloudinit/distros/rhel.py
+++ b/cloudinit/distros/rhel.py
@@ -208,7 +208,10 @@ class Distro(distros.Distro):
# This ensures that the correct tz will be used for the system
util.copy(tz_file, self.tz_local_fn)
- def package_command(self, command, args=None, pkgs=[]):
+ def package_command(self, command, args=None, pkgs=None):
+ if pkgs is None:
+ pkgs = []
+
cmd = ['yum']
# If enabled, then yum will be tolerant of errors on the command line
# with regard to packages.
diff --git a/cloudinit/handlers/__init__.py b/cloudinit/handlers/__init__.py
index 8d6dcd4d..924463ce 100644
--- a/cloudinit/handlers/__init__.py
+++ b/cloudinit/handlers/__init__.py
@@ -27,6 +27,7 @@ from cloudinit.settings import (PER_ALWAYS, PER_INSTANCE, FREQUENCIES)
from cloudinit import importer
from cloudinit import log as logging
+from cloudinit import type_utils
from cloudinit import util
LOG = logging.getLogger(__name__)
@@ -69,7 +70,6 @@ INCLUSION_SRCH = sorted(list(INCLUSION_TYPES_MAP.keys()),
class Handler(object):
-
__metaclass__ = abc.ABCMeta
def __init__(self, frequency, version=2):
@@ -77,53 +77,66 @@ class Handler(object):
self.frequency = frequency
def __repr__(self):
- return "%s: [%s]" % (util.obj_name(self), self.list_types())
+ return "%s: [%s]" % (type_utils.obj_name(self), self.list_types())
@abc.abstractmethod
def list_types(self):
raise NotImplementedError()
- def handle_part(self, data, ctype, filename, payload, frequency):
- return self._handle_part(data, ctype, filename, payload, frequency)
-
@abc.abstractmethod
- def _handle_part(self, data, ctype, filename, payload, frequency):
+ def handle_part(self, *args, **kwargs):
raise NotImplementedError()
-def run_part(mod, data, ctype, filename, payload, frequency):
+def run_part(mod, data, filename, payload, frequency, headers):
mod_freq = mod.frequency
if not (mod_freq == PER_ALWAYS or
(frequency == PER_INSTANCE and mod_freq == PER_INSTANCE)):
return
- mod_ver = mod.handler_version
# Sanity checks on version (should be an int convertable)
try:
+ mod_ver = mod.handler_version
mod_ver = int(mod_ver)
- except:
+ except (TypeError, ValueError, AttributeError):
mod_ver = 1
+ content_type = headers['Content-Type']
try:
LOG.debug("Calling handler %s (%s, %s, %s) with frequency %s",
- mod, ctype, filename, mod_ver, frequency)
- if mod_ver >= 2:
+ mod, content_type, filename, mod_ver, frequency)
+ if mod_ver == 3:
+ # Treat as v. 3 which does get a frequency + headers
+ mod.handle_part(data, content_type, filename,
+ payload, frequency, headers)
+ elif mod_ver == 2:
# Treat as v. 2 which does get a frequency
- mod.handle_part(data, ctype, filename, payload, frequency)
- else:
+ mod.handle_part(data, content_type, filename,
+ payload, frequency)
+ elif mod_ver == 1:
# Treat as v. 1 which gets no frequency
- mod.handle_part(data, ctype, filename, payload)
+ mod.handle_part(data, content_type, filename, payload)
+ else:
+ raise ValueError("Unknown module version %s" % (mod_ver))
except:
util.logexc(LOG, ("Failed calling handler %s (%s, %s, %s)"
" with frequency %s"),
- mod, ctype, filename,
+ mod, content_type, filename,
mod_ver, frequency)
def call_begin(mod, data, frequency):
- run_part(mod, data, CONTENT_START, None, None, frequency)
+ # Create a fake header set
+ headers = {
+ 'Content-Type': CONTENT_START,
+ }
+ run_part(mod, data, None, None, frequency, headers)
def call_end(mod, data, frequency):
- run_part(mod, data, CONTENT_END, None, None, frequency)
+ # Create a fake header set
+ headers = {
+ 'Content-Type': CONTENT_END,
+ }
+ run_part(mod, data, None, None, frequency, headers)
def walker_handle_handler(pdata, _ctype, _filename, payload):
@@ -173,26 +186,27 @@ def _escape_string(text):
return text
-def walker_callback(pdata, ctype, filename, payload):
- if ctype in PART_CONTENT_TYPES:
- walker_handle_handler(pdata, ctype, filename, payload)
+def walker_callback(data, filename, payload, headers):
+ content_type = headers['Content-Type']
+ if content_type in PART_CONTENT_TYPES:
+ walker_handle_handler(data, content_type, filename, payload)
return
- handlers = pdata['handlers']
- if ctype in pdata['handlers']:
- run_part(handlers[ctype], pdata['data'], ctype, filename,
- payload, pdata['frequency'])
+ handlers = data['handlers']
+ if content_type in handlers:
+ run_part(handlers[content_type], data['data'], filename,
+ payload, data['frequency'], headers)
elif payload:
# Extract the first line or 24 bytes for displaying in the log
start = _extract_first_or_bytes(payload, 24)
details = "'%s...'" % (_escape_string(start))
- if ctype == NOT_MULTIPART_TYPE:
+ if content_type == NOT_MULTIPART_TYPE:
LOG.warning("Unhandled non-multipart (%s) userdata: %s",
- ctype, details)
+ content_type, details)
else:
LOG.warning("Unhandled unknown content-type (%s) userdata: %s",
- ctype, details)
+ content_type, details)
else:
- LOG.debug("empty payload of type %s" % ctype)
+ LOG.debug("Empty payload of type %s", content_type)
# Callback is a function that will be called with
@@ -212,7 +226,10 @@ def walk(msg, callback, data):
if not filename:
filename = PART_FN_TPL % (partnum)
- callback(data, ctype, filename, part.get_payload(decode=True))
+ headers = dict(part)
+ LOG.debug(headers)
+ headers['Content-Type'] = ctype
+ callback(data, filename, part.get_payload(decode=True), headers)
partnum = partnum + 1
diff --git a/cloudinit/handlers/boot_hook.py b/cloudinit/handlers/boot_hook.py
index 456b8020..bf2899ab 100644
--- a/cloudinit/handlers/boot_hook.py
+++ b/cloudinit/handlers/boot_hook.py
@@ -56,7 +56,8 @@ class BootHookPartHandler(handlers.Handler):
util.write_file(filepath, contents, 0700)
return filepath
- def _handle_part(self, _data, ctype, filename, payload, _frequency):
+ def handle_part(self, _data, ctype, filename, # pylint: disable=W0221
+ payload, frequency): # pylint: disable=W0613
if ctype in handlers.CONTENT_SIGNALS:
return
diff --git a/cloudinit/handlers/cloud_config.py b/cloudinit/handlers/cloud_config.py
index f6d95244..d30d6338 100644
--- a/cloudinit/handlers/cloud_config.py
+++ b/cloudinit/handlers/cloud_config.py
@@ -22,41 +22,103 @@
from cloudinit import handlers
from cloudinit import log as logging
+from cloudinit import mergers
from cloudinit import util
from cloudinit.settings import (PER_ALWAYS)
LOG = logging.getLogger(__name__)
+MERGE_HEADER = 'Merge-Type'
+DEF_MERGERS = mergers.default_mergers()
+
class CloudConfigPartHandler(handlers.Handler):
def __init__(self, paths, **_kwargs):
- handlers.Handler.__init__(self, PER_ALWAYS)
- self.cloud_buf = []
+ handlers.Handler.__init__(self, PER_ALWAYS, version=3)
+ self.cloud_buf = None
self.cloud_fn = paths.get_ipath("cloud_config")
+ self.file_names = []
+ self.mergers = [DEF_MERGERS]
def list_types(self):
return [
handlers.type_from_starts_with("#cloud-config"),
]
- def _write_cloud_config(self, buf):
+ def _write_cloud_config(self):
if not self.cloud_fn:
return
- lines = [str(b) for b in buf]
- payload = "\n".join(lines)
- util.write_file(self.cloud_fn, payload, 0600)
+ # Capture which files we merged from...
+ file_lines = []
+ if self.file_names:
+ file_lines.append("# from %s files" % (len(self.file_names)))
+ for fn in self.file_names:
+ file_lines.append("# %s" % (fn))
+ file_lines.append("")
+ if self.cloud_buf is not None:
+ # Something was actually gathered....
+ lines = [
+ "#cloud-config",
+ '',
+ ]
+ lines.extend(file_lines)
+ lines.append(util.yaml_dumps(self.cloud_buf))
+ else:
+ lines = []
+ util.write_file(self.cloud_fn, "\n".join(lines), 0600)
+
+ def _extract_mergers(self, payload, headers):
+ merge_header_headers = ''
+ for h in [MERGE_HEADER, 'X-%s' % (MERGE_HEADER)]:
+ tmp_h = headers.get(h, '')
+ if tmp_h:
+ merge_header_headers = tmp_h
+ break
+ # Select either the merge-type from the content
+ # or the merge type from the headers or default to our own set
+ # if neither exists (or is empty) from the later.
+ payload_yaml = util.load_yaml(payload)
+ mergers_yaml = mergers.dict_extract_mergers(payload_yaml)
+ mergers_header = mergers.string_extract_mergers(merge_header_headers)
+ all_mergers = []
+ all_mergers.extend(mergers_yaml)
+ all_mergers.extend(mergers_header)
+ if not all_mergers:
+ all_mergers = DEF_MERGERS
+ return all_mergers
+
+ def _merge_part(self, payload, headers):
+ next_mergers = self._extract_mergers(payload, headers)
+ # Use the merger list from the last call, since it is the one
+ # that will be defining how to merge with the next payload.
+ curr_mergers = list(self.mergers[-1])
+ LOG.debug("Merging by applying %s", curr_mergers)
+ self.mergers.append(next_mergers)
+ merger = mergers.construct(curr_mergers)
+ if self.cloud_buf is None:
+ # First time through, merge with an empty dict...
+ self.cloud_buf = {}
+ self.cloud_buf = merger.merge(self.cloud_buf,
+ util.load_yaml(payload))
- def _handle_part(self, _data, ctype, filename, payload, _frequency):
+ def _reset(self):
+ self.file_names = []
+ self.cloud_buf = None
+ self.mergers = [DEF_MERGERS]
+
+ def handle_part(self, _data, ctype, filename, # pylint: disable=W0221
+ payload, _frequency, headers): # pylint: disable=W0613
if ctype == handlers.CONTENT_START:
- self.cloud_buf = []
+ self._reset()
return
if ctype == handlers.CONTENT_END:
- self._write_cloud_config(self.cloud_buf)
- self.cloud_buf = []
+ self._write_cloud_config()
+ self._reset()
return
-
- filename = util.clean_filename(filename)
- if not filename:
- filename = '??'
- self.cloud_buf.extend(["#%s" % (filename), str(payload)])
+ try:
+ self._merge_part(payload, headers)
+ self.file_names.append(filename)
+ except:
+ util.logexc(LOG, "Failed at merging in cloud config part from %s",
+ filename)
diff --git a/cloudinit/handlers/shell_script.py b/cloudinit/handlers/shell_script.py
index 6c5c11ca..b185c374 100644
--- a/cloudinit/handlers/shell_script.py
+++ b/cloudinit/handlers/shell_script.py
@@ -41,7 +41,8 @@ class ShellScriptPartHandler(handlers.Handler):
handlers.type_from_starts_with("#!"),
]
- def _handle_part(self, _data, ctype, filename, payload, _frequency):
+ def handle_part(self, _data, ctype, filename, # pylint: disable=W0221
+ payload, frequency): # pylint: disable=W0613
if ctype in handlers.CONTENT_SIGNALS:
# TODO(harlowja): maybe delete existing things here
return
diff --git a/cloudinit/handlers/upstart_job.py b/cloudinit/handlers/upstart_job.py
index 4684f7f2..edd56527 100644
--- a/cloudinit/handlers/upstart_job.py
+++ b/cloudinit/handlers/upstart_job.py
@@ -42,7 +42,8 @@ class UpstartJobPartHandler(handlers.Handler):
handlers.type_from_starts_with("#upstart-job"),
]
- def _handle_part(self, _data, ctype, filename, payload, frequency):
+ def handle_part(self, _data, ctype, filename, # pylint: disable=W0221
+ payload, frequency):
if ctype in handlers.CONTENT_SIGNALS:
return
@@ -65,6 +66,14 @@ class UpstartJobPartHandler(handlers.Handler):
path = os.path.join(self.upstart_dir, filename)
util.write_file(path, payload, 0644)
- # if inotify support is not present in the root filesystem
- # (overlayroot) then we need to tell upstart to re-read /etc
- util.subp(["initctl", "reload-configuration"], capture=False)
+ # FIXME LATER (LP: #1124384)
+ # a bug in upstart means that invoking reload-configuration
+ # at this stage in boot causes havoc. So, until that is fixed
+ # we will not do that. However, I'd like to be able to easily
+ # test to see if this bug is still present in an image with
+ # a newer upstart. So, a boot hook could easiliy write this file.
+ if os.path.exists("/run/cloud-init-upstart-reload"):
+ # if inotify support is not present in the root filesystem
+ # (overlayroot) then we need to tell upstart to re-read /etc
+
+ util.subp(["initctl", "reload-configuration"], capture=False)
diff --git a/cloudinit/helpers.py b/cloudinit/helpers.py
index 2077401c..a4e6fb03 100644
--- a/cloudinit/helpers.py
+++ b/cloudinit/helpers.py
@@ -32,6 +32,7 @@ from cloudinit.settings import (PER_INSTANCE, PER_ALWAYS, PER_ONCE,
CFG_ENV_NAME)
from cloudinit import log as logging
+from cloudinit import type_utils
from cloudinit import util
LOG = logging.getLogger(__name__)
@@ -68,7 +69,7 @@ class FileLock(object):
self.fn = fn
def __str__(self):
- return "<%s using file %r>" % (util.obj_name(self), self.fn)
+ return "<%s using file %r>" % (type_utils.obj_name(self), self.fn)
def canon_sem_name(name):
diff --git a/cloudinit/mergers/__init__.py b/cloudinit/mergers/__init__.py
new file mode 100644
index 00000000..e1ff57ba
--- /dev/null
+++ b/cloudinit/mergers/__init__.py
@@ -0,0 +1,154 @@
+# vi: ts=4 expandtab
+#
+# Copyright (C) 2012 Yahoo! Inc.
+#
+# Author: Joshua Harlow <harlowja@yahoo-inc.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 3, as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+import re
+
+from cloudinit import importer
+from cloudinit import log as logging
+from cloudinit import type_utils
+
+NAME_MTCH = re.compile(r"(^[a-zA-Z_][A-Za-z0-9_]*)\((.*?)\)$")
+
+LOG = logging.getLogger(__name__)
+DEF_MERGE_TYPE = "list()+dict()+str()"
+
+
+class UnknownMerger(object):
+ # Named differently so auto-method finding
+ # doesn't pick this up if there is ever a type
+ # named "unknown"
+ def _handle_unknown(self, _meth_wanted, value, _merge_with):
+ return value
+
+ # This merging will attempt to look for a '_on_X' method
+ # in our own object for a given object Y with type X,
+ # if found it will be called to perform the merge of a source
+ # object and a object to merge_with.
+ #
+ # If not found the merge will be given to a '_handle_unknown'
+ # function which can decide what to do wit the 2 values.
+ def merge(self, source, merge_with):
+ type_name = type_utils.obj_name(source)
+ type_name = type_name.lower()
+ method_name = "_on_%s" % (type_name)
+ meth = None
+ args = [source, merge_with]
+ if hasattr(self, method_name):
+ meth = getattr(self, method_name)
+ if not meth:
+ meth = self._handle_unknown
+ args.insert(0, method_name)
+ return meth(*args)
+
+
+class LookupMerger(UnknownMerger):
+ def __init__(self, lookups=None):
+ UnknownMerger.__init__(self)
+ if lookups is None:
+ self._lookups = []
+ else:
+ self._lookups = lookups
+
+ # For items which can not be merged by the parent this object
+ # will lookup in a internally maintained set of objects and
+ # find which one of those objects can perform the merge. If
+ # any of the contained objects have the needed method, they
+ # will be called to perform the merge.
+ def _handle_unknown(self, meth_wanted, value, merge_with):
+ meth = None
+ for merger in self._lookups:
+ if hasattr(merger, meth_wanted):
+ # First one that has that method/attr gets to be
+ # the one that will be called
+ meth = getattr(merger, meth_wanted)
+ break
+ if not meth:
+ return UnknownMerger._handle_unknown(self, meth_wanted,
+ value, merge_with)
+ return meth(value, merge_with)
+
+
+def dict_extract_mergers(config):
+ parsed_mergers = []
+ raw_mergers = config.get('merge_how')
+ if raw_mergers is None:
+ raw_mergers = config.get('merge_type')
+ if raw_mergers is None:
+ return parsed_mergers
+ if isinstance(raw_mergers, (str, basestring)):
+ return string_extract_mergers(raw_mergers)
+ for m in raw_mergers:
+ if isinstance(m, (dict)):
+ name = m['name']
+ name = name.replace("-", "_").strip()
+ opts = m['settings']
+ else:
+ name = m[0]
+ if len(m) >= 2:
+ opts = m[1:]
+ else:
+ opts = []
+ if name:
+ parsed_mergers.append((name, opts))
+ return parsed_mergers
+
+
+def string_extract_mergers(merge_how):
+ parsed_mergers = []
+ for m_name in merge_how.split("+"):
+ # Canonicalize the name (so that it can be found
+ # even when users alter it in various ways)
+ m_name = m_name.lower().strip()
+ m_name = m_name.replace("-", "_")
+ if not m_name:
+ continue
+ match = NAME_MTCH.match(m_name)
+ if not match:
+ msg = ("Matcher identifer '%s' is not in the right format" %
+ (m_name))
+ raise ValueError(msg)
+ (m_name, m_ops) = match.groups()
+ m_ops = m_ops.strip().split(",")
+ m_ops = [m.strip().lower() for m in m_ops if m.strip()]
+ parsed_mergers.append((m_name, m_ops))
+ return parsed_mergers
+
+
+def default_mergers():
+ return tuple(string_extract_mergers(DEF_MERGE_TYPE))
+
+
+def construct(parsed_mergers):
+ mergers_to_be = []
+ for (m_name, m_ops) in parsed_mergers:
+ merger_locs = importer.find_module(m_name,
+ [__name__],
+ ['Merger'])
+ if not merger_locs:
+ msg = "Could not find merger named '%s'" % (m_name)
+ raise ImportError(msg)
+ else:
+ mod = importer.import_module(merger_locs[0])
+ mod_attr = getattr(mod, 'Merger')
+ mergers_to_be.append((mod_attr, m_ops))
+ # Now form them...
+ mergers = []
+ root = LookupMerger(mergers)
+ for (attr, opts) in mergers_to_be:
+ mergers.append(attr(root, opts))
+ return root
diff --git a/cloudinit/mergers/dict.py b/cloudinit/mergers/dict.py
new file mode 100644
index 00000000..45a7d3a5
--- /dev/null
+++ b/cloudinit/mergers/dict.py
@@ -0,0 +1,48 @@
+# vi: ts=4 expandtab
+#
+# Copyright (C) 2012 Yahoo! Inc.
+#
+# Author: Joshua Harlow <harlowja@yahoo-inc.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 3, as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+
+class Merger(object):
+ def __init__(self, merger, opts):
+ self._merger = merger
+ self._overwrite = 'overwrite' in opts
+
+ # This merging algorithm will attempt to merge with
+ # another dictionary, on encountering any other type of object
+ # it will not merge with said object, but will instead return
+ # the original value
+ #
+ # On encountering a dictionary, it will create a new dictionary
+ # composed of the original and the one to merge with, if 'overwrite'
+ # is enabled then keys that exist in the original will be overwritten
+ # by keys in the one to merge with (and associated values). Otherwise
+ # if not in overwrite mode the 2 conflicting keys themselves will
+ # be merged.
+ def _on_dict(self, value, merge_with):
+ if not isinstance(merge_with, (dict)):
+ return value
+ merged = dict(value)
+ for (k, v) in merge_with.items():
+ if k in merged:
+ if not self._overwrite:
+ merged[k] = self._merger.merge(merged[k], v)
+ else:
+ merged[k] = v
+ else:
+ merged[k] = v
+ return merged
diff --git a/cloudinit/mergers/list.py b/cloudinit/mergers/list.py
new file mode 100644
index 00000000..a56ff007
--- /dev/null
+++ b/cloudinit/mergers/list.py
@@ -0,0 +1,50 @@
+# vi: ts=4 expandtab
+#
+# Copyright (C) 2012 Yahoo! Inc.
+#
+# Author: Joshua Harlow <harlowja@yahoo-inc.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 3, as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+
+class Merger(object):
+ def __init__(self, merger, opts):
+ self._merger = merger
+ self._discard_non = 'discard_non_list' in opts
+ self._extend = 'extend' in opts
+
+ def _on_tuple(self, value, merge_with):
+ return self._on_list(list(value), merge_with)
+
+ # On encountering a list or tuple type this action will be applied
+ # a new list will be returned, if the value to merge with is itself
+ # a list and we have been told to 'extend', then the value here will
+ # be extended with the other list. If in 'extend' mode then we will
+ # attempt to merge instead, which means that values from the list
+ # to merge with will replace values in te original list (they will
+ # also be merged recursively).
+ #
+ # If the value to merge with is not a list, and we are set to discared
+ # then no modifications will take place, otherwise we will just append
+ # the value to merge with onto the end of our own list.
+ def _on_list(self, value, merge_with):
+ new_value = list(value)
+ if isinstance(merge_with, (tuple, list)):
+ if self._extend:
+ new_value.extend(merge_with)
+ else:
+ return new_value
+ else:
+ if not self._discard_non:
+ new_value.append(merge_with)
+ return new_value
diff --git a/cloudinit/mergers/str.py b/cloudinit/mergers/str.py
new file mode 100644
index 00000000..291c91c2
--- /dev/null
+++ b/cloudinit/mergers/str.py
@@ -0,0 +1,39 @@
+# vi: ts=4 expandtab
+#
+# Copyright (C) 2012 Yahoo! Inc.
+#
+# Author: Joshua Harlow <harlowja@yahoo-inc.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 3, as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+
+class Merger(object):
+ def __init__(self, _merger, opts):
+ self._append = 'append' in opts
+
+ # On encountering a unicode object to merge value with
+ # we will for now just proxy into the string method to let it handle it.
+ def _on_unicode(self, value, merge_with):
+ return self._on_str(value, merge_with)
+
+ # On encountering a string object to merge with we will
+ # perform the following action, if appending we will
+ # merge them together, otherwise we will just return value.
+ def _on_str(self, value, merge_with):
+ if not self._append:
+ return value
+ else:
+ if isinstance(value, (unicode)):
+ return value + unicode(merge_with)
+ else:
+ return value + str(merge_with)
diff --git a/cloudinit/sources/DataSourceAltCloud.py b/cloudinit/sources/DataSourceAltCloud.py
index 9812bdcb..64548d43 100644
--- a/cloudinit/sources/DataSourceAltCloud.py
+++ b/cloudinit/sources/DataSourceAltCloud.py
@@ -30,6 +30,7 @@ import os.path
from cloudinit import log as logging
from cloudinit import sources
from cloudinit import util
+
from cloudinit.util import ProcessExecutionError
LOG = logging.getLogger(__name__)
@@ -91,8 +92,8 @@ class DataSourceAltCloud(sources.DataSource):
self.supported_seed_starts = ("/", "file://")
def __str__(self):
- mstr = "%s [seed=%s]" % (util.obj_name(self), self.seed)
- return mstr
+ root = sources.DataSource.__str__(self)
+ return "%s [seed=%s]" % (root, self.seed)
def get_cloud_type(self):
'''
diff --git a/cloudinit/sources/DataSourceCloudStack.py b/cloudinit/sources/DataSourceCloudStack.py
index 1335b63d..7fd38982 100644
--- a/cloudinit/sources/DataSourceCloudStack.py
+++ b/cloudinit/sources/DataSourceCloudStack.py
@@ -48,9 +48,6 @@ class DataSourceCloudStack(sources.DataSource):
raise RuntimeError("No virtual router found!")
self.metadata_address = "http://%s/" % (vr_addr)
- def __str__(self):
- return util.obj_name(self)
-
def _get_url_settings(self):
mcfg = self.ds_cfg
if not mcfg:
diff --git a/cloudinit/sources/DataSourceConfigDrive.py b/cloudinit/sources/DataSourceConfigDrive.py
index ec016a1d..5f152299 100644
--- a/cloudinit/sources/DataSourceConfigDrive.py
+++ b/cloudinit/sources/DataSourceConfigDrive.py
@@ -51,7 +51,9 @@ class DataSourceConfigDrive(sources.DataSource):
self.ec2_metadata = None
def __str__(self):
- mstr = "%s [%s,ver=%s]" % (util.obj_name(self), self.dsmode,
+ root = sources.DataSource.__str__(self)
+ mstr = "%s [%s,ver=%s]" % (root,
+ self.dsmode,
self.version)
mstr += "[source=%s]" % (self.source)
return mstr
@@ -152,7 +154,7 @@ class DataSourceConfigDrive(sources.DataSource):
return False
md = results['metadata']
- md = util.mergedict(md, DEFAULT_METADATA)
+ md = util.mergemanydict([md, DEFAULT_METADATA])
# Perform some metadata 'fixups'
#
diff --git a/cloudinit/sources/DataSourceEc2.py b/cloudinit/sources/DataSourceEc2.py
index 0a58820b..0d62cf01 100644
--- a/cloudinit/sources/DataSourceEc2.py
+++ b/cloudinit/sources/DataSourceEc2.py
@@ -49,9 +49,6 @@ class DataSourceEc2(sources.DataSource):
self.seed_dir = os.path.join(paths.seed_dir, "ec2")
self.api_ver = DEF_MD_VERSION
- def __str__(self):
- return util.obj_name(self)
-
def get_data(self):
seed_ret = {}
if util.read_optional_seed(seed_ret, base=(self.seed_dir + "/")):
diff --git a/cloudinit/sources/DataSourceMAAS.py b/cloudinit/sources/DataSourceMAAS.py
index f3759b4b..de822924 100644
--- a/cloudinit/sources/DataSourceMAAS.py
+++ b/cloudinit/sources/DataSourceMAAS.py
@@ -50,7 +50,8 @@ class DataSourceMAAS(sources.DataSource):
self.oauth_clockskew = None
def __str__(self):
- return "%s [%s]" % (util.obj_name(self), self.base_url)
+ root = sources.DataSource.__str__(self)
+ return "%s [%s]" % (root, self.base_url)
def get_data(self):
mcfg = self.ds_cfg
diff --git a/cloudinit/sources/DataSourceNoCloud.py b/cloudinit/sources/DataSourceNoCloud.py
index 097bbc52..08a853cc 100644
--- a/cloudinit/sources/DataSourceNoCloud.py
+++ b/cloudinit/sources/DataSourceNoCloud.py
@@ -40,9 +40,8 @@ class DataSourceNoCloud(sources.DataSource):
self.supported_seed_starts = ("/", "file://")
def __str__(self):
- mstr = "%s [seed=%s][dsmode=%s]" % (util.obj_name(self),
- self.seed, self.dsmode)
- return mstr
+ root = sources.DataSource.__str__(self)
+ return "%s [seed=%s][dsmode=%s]" % (root, self.seed, self.dsmode)
def get_data(self):
defaults = {
@@ -65,7 +64,7 @@ class DataSourceNoCloud(sources.DataSource):
# Check to see if the seed dir has data.
seedret = {}
if util.read_optional_seed(seedret, base=self.seed_dir + "/"):
- md = util.mergedict(md, seedret['meta-data'])
+ md = util.mergemanydict([md, seedret['meta-data']])
ud = seedret['user-data']
found.append(self.seed_dir)
LOG.debug("Using seeded cache data from %s", self.seed_dir)
@@ -82,15 +81,15 @@ class DataSourceNoCloud(sources.DataSource):
if self.ds_cfg['user-data']:
ud = self.ds_cfg['user-data']
if self.ds_cfg['meta-data'] is not False:
- md = util.mergedict(md, self.ds_cfg['meta-data'])
+ md = util.mergemanydict([md, self.ds_cfg['meta-data']])
if 'ds_config' not in found:
found.append("ds_config")
- if self.ds_cfg.get('fs_label', "cidata"):
+ label = self.ds_cfg.get('fs_label', "cidata")
+ if label is not None:
fslist = util.find_devs_with("TYPE=vfat")
fslist.extend(util.find_devs_with("TYPE=iso9660"))
- label = self.ds_cfg.get('fs_label')
label_list = util.find_devs_with("LABEL=%s" % label)
devlist = list(set(fslist) & set(label_list))
devlist.sort(reverse=True)
@@ -100,7 +99,7 @@ class DataSourceNoCloud(sources.DataSource):
LOG.debug("Attempting to use data from %s", dev)
(newmd, newud) = util.mount_cb(dev, util.read_seeded)
- md = util.mergedict(newmd, md)
+ md = util.mergemanydict([newmd, md])
ud = newud
# For seed from a device, the default mode is 'net'.
@@ -150,11 +149,11 @@ class DataSourceNoCloud(sources.DataSource):
LOG.debug("Using seeded cache data from %s", seedfrom)
# Values in the command line override those from the seed
- md = util.mergedict(md, md_seed)
+ md = util.mergemanydict([md, md_seed])
found.append(seedfrom)
# Now that we have exhausted any other places merge in the defaults
- md = util.mergedict(md, defaults)
+ md = util.mergemanydict([md, defaults])
# Update the network-interfaces if metadata had 'network-interfaces'
# entry and this is the local datasource, or 'seedfrom' was used
diff --git a/cloudinit/sources/DataSourceNone.py b/cloudinit/sources/DataSourceNone.py
index c2125bee..12a8a992 100644
--- a/cloudinit/sources/DataSourceNone.py
+++ b/cloudinit/sources/DataSourceNone.py
@@ -18,7 +18,6 @@
from cloudinit import log as logging
from cloudinit import sources
-from cloudinit import util
LOG = logging.getLogger(__name__)
@@ -41,9 +40,6 @@ class DataSourceNone(sources.DataSource):
def get_instance_id(self):
return 'iid-datasource-none'
- def __str__(self):
- return util.obj_name(self)
-
@property
def is_disconnected(self):
return True
diff --git a/cloudinit/sources/DataSourceOVF.py b/cloudinit/sources/DataSourceOVF.py
index e90150c6..0530c4b7 100644
--- a/cloudinit/sources/DataSourceOVF.py
+++ b/cloudinit/sources/DataSourceOVF.py
@@ -43,7 +43,8 @@ class DataSourceOVF(sources.DataSource):
self.supported_seed_starts = ("/", "file://")
def __str__(self):
- return "%s [seed=%s]" % (util.obj_name(self), self.seed)
+ root = sources.DataSource.__str__(self)
+ return "%s [seed=%s]" % (root, self.seed)
def get_data(self):
found = []
@@ -93,11 +94,11 @@ class DataSourceOVF(sources.DataSource):
(md_seed, ud) = util.read_seeded(seedfrom, timeout=None)
LOG.debug("Using seeded cache data from %s", seedfrom)
- md = util.mergedict(md, md_seed)
+ md = util.mergemanydict([md, md_seed])
found.append(seedfrom)
# Now that we have exhausted any other places merge in the defaults
- md = util.mergedict(md, defaults)
+ md = util.mergemanydict([md, defaults])
self.seed = ",".join(found)
self.metadata = md
diff --git a/cloudinit/sources/__init__.py b/cloudinit/sources/__init__.py
index 96baff90..d8fbacdd 100644
--- a/cloudinit/sources/__init__.py
+++ b/cloudinit/sources/__init__.py
@@ -25,6 +25,7 @@ import os
from cloudinit import importer
from cloudinit import log as logging
+from cloudinit import type_utils
from cloudinit import user_data as ud
from cloudinit import util
@@ -52,7 +53,7 @@ class DataSource(object):
self.userdata = None
self.metadata = None
self.userdata_raw = None
- name = util.obj_name(self)
+ name = type_utils.obj_name(self)
if name.startswith(DS_PREFIX):
name = name[len(DS_PREFIX):]
self.ds_cfg = util.get_cfg_by_path(self.sys_cfg,
@@ -62,6 +63,9 @@ class DataSource(object):
else:
self.ud_proc = ud_proc
+ def __str__(self):
+ return type_utils.obj_name(self)
+
def get_userdata(self, apply_filter=False):
if self.userdata is None:
self.userdata = self.ud_proc.process(self.get_userdata_raw())
@@ -214,7 +218,7 @@ def normalize_pubkey_data(pubkey_data):
def find_source(sys_cfg, distro, paths, ds_deps, cfg_list, pkg_list):
ds_list = list_sources(cfg_list, ds_deps, pkg_list)
- ds_names = [util.obj_name(f) for f in ds_list]
+ ds_names = [type_utils.obj_name(f) for f in ds_list]
LOG.debug("Searching for data source in: %s", ds_names)
for cls in ds_list:
@@ -222,7 +226,7 @@ def find_source(sys_cfg, distro, paths, ds_deps, cfg_list, pkg_list):
LOG.debug("Seeing if we can get any data from %s", cls)
s = cls(sys_cfg, distro, paths)
if s.get_data():
- return (s, util.obj_name(cls))
+ return (s, type_utils.obj_name(cls))
except Exception:
util.logexc(LOG, "Getting data from %s failed", cls)
diff --git a/cloudinit/ssh_util.py b/cloudinit/ssh_util.py
index dd6b742f..95133236 100644
--- a/cloudinit/ssh_util.py
+++ b/cloudinit/ssh_util.py
@@ -19,9 +19,6 @@
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
-from StringIO import StringIO
-
-import csv
import os
import pwd
@@ -33,6 +30,15 @@ LOG = logging.getLogger(__name__)
# See: man sshd_config
DEF_SSHD_CFG = "/etc/ssh/sshd_config"
+# taken from openssh source key.c/key_type_from_name
+VALID_KEY_TYPES = ("rsa", "dsa", "ssh-rsa", "ssh-dss", "ecdsa",
+ "ssh-rsa-cert-v00@openssh.com", "ssh-dss-cert-v00@openssh.com",
+ "ssh-rsa-cert-v00@openssh.com", "ssh-dss-cert-v00@openssh.com",
+ "ssh-rsa-cert-v01@openssh.com", "ssh-dss-cert-v01@openssh.com",
+ "ecdsa-sha2-nistp256-cert-v01@openssh.com",
+ "ecdsa-sha2-nistp384-cert-v01@openssh.com",
+ "ecdsa-sha2-nistp521-cert-v01@openssh.com")
+
class AuthKeyLine(object):
def __init__(self, source, keytype=None, base64=None,
@@ -43,11 +49,8 @@ class AuthKeyLine(object):
self.keytype = keytype
self.source = source
- def empty(self):
- if (not self.base64 and
- not self.comment and not self.keytype and not self.options):
- return True
- return False
+ def valid(self):
+ return (self.base64 and self.keytype)
def __str__(self):
toks = []
@@ -107,62 +110,47 @@ class AuthKeyLineParser(object):
i = i + 1
options = ent[0:i]
- options_lst = []
-
- # Now use a csv parser to pull the options
- # out of the above string that we just found an endpoint for.
- #
- # No quoting so we don't mess up any of the quoting that
- # is already there.
- reader = csv.reader(StringIO(options), quoting=csv.QUOTE_NONE)
- for row in reader:
- for e in row:
- # Only keep non-empty csv options
- e = e.strip()
- if e:
- options_lst.append(e)
-
- # Now take the rest of the items before the string
- # as long as there is room to do this...
- toks = []
- if i + 1 < len(ent):
- rest = ent[i + 1:]
- toks = rest.split(None, 2)
- return (options_lst, toks)
-
- def _form_components(self, src_line, toks, options=None):
- components = {}
- if len(toks) == 1:
- components['base64'] = toks[0]
- elif len(toks) == 2:
- components['base64'] = toks[0]
- components['comment'] = toks[1]
- elif len(toks) == 3:
- components['keytype'] = toks[0]
- components['base64'] = toks[1]
- components['comment'] = toks[2]
- components['options'] = options
- if not components:
- return AuthKeyLine(src_line)
- else:
- return AuthKeyLine(src_line, **components)
- def parse(self, src_line, def_opt=None):
+ # Return the rest of the string in 'remain'
+ remain = ent[i:].lstrip()
+ return (options, remain)
+
+ def parse(self, src_line, options=None):
+ # modeled after opensshes auth2-pubkey.c:user_key_allowed2
line = src_line.rstrip("\r\n")
if line.startswith("#") or line.strip() == '':
return AuthKeyLine(src_line)
- else:
- ent = line.strip()
- toks = ent.split(None, 3)
- if len(toks) < 4:
- return self._form_components(src_line, toks, def_opt)
- else:
- (options, toks) = self._extract_options(ent)
- if options:
- options = ",".join(options)
- else:
- options = def_opt
- return self._form_components(src_line, toks, options)
+
+ def parse_ssh_key(ent):
+ # return ketype, key, [comment]
+ toks = ent.split(None, 2)
+ if len(toks) < 2:
+ raise TypeError("To few fields: %s" % len(toks))
+ if toks[0] not in VALID_KEY_TYPES:
+ raise TypeError("Invalid keytype %s" % toks[0])
+
+ # valid key type and 2 or 3 fields:
+ if len(toks) == 2:
+ # no comment in line
+ toks.append("")
+
+ return toks
+
+ ent = line.strip()
+ try:
+ (keytype, base64, comment) = parse_ssh_key(ent)
+ except TypeError:
+ (keyopts, remain) = self._extract_options(ent)
+ if options is None:
+ options = keyopts
+
+ try:
+ (keytype, base64, comment) = parse_ssh_key(remain)
+ except TypeError:
+ return AuthKeyLine(src_line)
+
+ return AuthKeyLine(src_line, keytype=keytype, base64=base64,
+ comment=comment, options=options)
def parse_authorized_keys(fname):
@@ -186,11 +174,11 @@ def update_authorized_keys(old_entries, keys):
for i in range(0, len(old_entries)):
ent = old_entries[i]
- if ent.empty() or not ent.base64:
+ if not ent.valid():
continue
# Replace those with the same base64
for k in keys:
- if k.empty() or not k.base64:
+ if not ent.valid():
continue
if k.base64 == ent.base64:
# Replace it with our better one
@@ -249,7 +237,7 @@ def extract_authorized_keys(username):
return (auth_key_fn, parse_authorized_keys(auth_key_fn))
-def setup_user_keys(keys, username, key_prefix):
+def setup_user_keys(keys, username, options=None):
# Make sure the users .ssh dir is setup accordingly
(ssh_dir, pwent) = users_ssh_info(username)
if not os.path.isdir(ssh_dir):
@@ -260,7 +248,7 @@ def setup_user_keys(keys, username, key_prefix):
parser = AuthKeyLineParser()
key_entries = []
for k in keys:
- key_entries.append(parser.parse(str(k), def_opt=key_prefix))
+ key_entries.append(parser.parse(str(k), options=options))
# Extract the old and make the new
(auth_key_fn, auth_key_entries) = extract_authorized_keys(username)
diff --git a/cloudinit/stages.py b/cloudinit/stages.py
index d7d1dea0..543d247f 100644
--- a/cloudinit/stages.py
+++ b/cloudinit/stages.py
@@ -43,6 +43,7 @@ from cloudinit import helpers
from cloudinit import importer
from cloudinit import log as logging
from cloudinit import sources
+from cloudinit import type_utils
from cloudinit import util
LOG = logging.getLogger(__name__)
@@ -211,7 +212,7 @@ class Init(object):
# Any config provided???
pkg_list = self.cfg.get('datasource_pkg_list') or []
# Add the defaults at the end
- for n in ['', util.obj_name(sources)]:
+ for n in ['', type_utils.obj_name(sources)]:
if n not in pkg_list:
pkg_list.append(n)
cfg_list = self.cfg.get('datasource_list') or []
@@ -271,7 +272,7 @@ class Init(object):
dp = self.paths.get_cpath('data')
# Write what the datasource was and is..
- ds = "%s: %s" % (util.obj_name(self.datasource), self.datasource)
+ ds = "%s: %s" % (type_utils.obj_name(self.datasource), self.datasource)
previous_ds = None
ds_fn = os.path.join(idir, 'datasource')
try:
@@ -488,7 +489,7 @@ class Modules(object):
else:
raise TypeError(("Failed to read '%s' item in config,"
" unknown type %s") %
- (item, util.obj_name(item)))
+ (item, type_utils.obj_name(item)))
return module_list
def _fixup_modules(self, raw_mods):
@@ -506,7 +507,7 @@ class Modules(object):
# Reset it so when ran it will get set to a known value
freq = None
mod_locs = importer.find_module(mod_name,
- ['', util.obj_name(config)],
+ ['', type_utils.obj_name(config)],
['handle'])
if not mod_locs:
LOG.warn("Could not find module named %s", mod_name)
diff --git a/cloudinit/type_utils.py b/cloudinit/type_utils.py
new file mode 100644
index 00000000..2decbfc5
--- /dev/null
+++ b/cloudinit/type_utils.py
@@ -0,0 +1,34 @@
+# vi: ts=4 expandtab
+#
+# Copyright (C) 2012 Canonical Ltd.
+# Copyright (C) 2012 Hewlett-Packard Development Company, L.P.
+# Copyright (C) 2012 Yahoo! Inc.
+#
+# Author: Scott Moser <scott.moser@canonical.com>
+# Author: Juerg Haefliger <juerg.haefliger@hp.com>
+# Author: Joshua Harlow <harlowja@yahoo-inc.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 3, as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+#
+# pylint: disable=C0302
+
+import types
+
+
+def obj_name(obj):
+ if isinstance(obj, (types.TypeType,
+ types.ModuleType,
+ types.FunctionType,
+ types.LambdaType)):
+ return str(obj.__name__)
+ return obj_name(obj.__class__)
diff --git a/cloudinit/util.py b/cloudinit/util.py
index dc3c5639..c961bbc7 100644
--- a/cloudinit/util.py
+++ b/cloudinit/util.py
@@ -43,15 +43,16 @@ import subprocess
import sys
import tempfile
import time
-import types
import urlparse
import yaml
from cloudinit import importer
from cloudinit import log as logging
+from cloudinit import mergers
from cloudinit import safeyaml
from cloudinit import url_helper
+from cloudinit import type_utils
from cloudinit import version
from cloudinit.settings import (CFG_BUILTIN)
@@ -219,11 +220,12 @@ def fork_cb(child_cb, *args):
os._exit(0) # pylint: disable=W0212
except:
logexc(LOG, ("Failed forking and"
- " calling callback %s"), obj_name(child_cb))
+ " calling callback %s"),
+ type_utils.obj_name(child_cb))
os._exit(1) # pylint: disable=W0212
else:
LOG.debug("Forked child %s who will run callback %s",
- fid, obj_name(child_cb))
+ fid, type_utils.obj_name(child_cb))
def is_true(val, addons=None):
@@ -537,40 +539,26 @@ def make_url(scheme, host, port=None,
return urlparse.urlunparse(pieces)
-def obj_name(obj):
- if isinstance(obj, (types.TypeType,
- types.ModuleType,
- types.FunctionType,
- types.LambdaType)):
- return str(obj.__name__)
- return obj_name(obj.__class__)
-
-
def mergemanydict(srcs, reverse=False):
if reverse:
srcs = reversed(srcs)
m_cfg = {}
+ merge_how = [mergers.default_mergers()]
for a_cfg in srcs:
if a_cfg:
- m_cfg = mergedict(m_cfg, a_cfg)
+ # Take the last merger as the one that
+ # will define how to merge next...
+ mergers_to_apply = list(merge_how[-1])
+ merger = mergers.construct(mergers_to_apply)
+ m_cfg = merger.merge(m_cfg, a_cfg)
+ # If the config has now has new merger set,
+ # extract them to be used next time...
+ new_mergers = mergers.dict_extract_mergers(m_cfg)
+ if new_mergers:
+ merge_how.append(new_mergers)
return m_cfg
-def mergedict(src, cand):
- """
- Merge values from C{cand} into C{src}.
- If C{src} has a key C{cand} will not override.
- Nested dictionaries are merged recursively.
- """
- if isinstance(src, dict) and isinstance(cand, dict):
- for (k, v) in cand.iteritems():
- if k not in src:
- src[k] = v
- else:
- src[k] = mergedict(src[k], v)
- return src
-
-
@contextlib.contextmanager
def chdir(ndir):
curr = os.getcwd()
@@ -713,7 +701,7 @@ def load_yaml(blob, default=None, allowed=(dict,)):
# Yes this will just be caught, but thats ok for now...
raise TypeError(("Yaml load allows %s root types,"
" but got %s instead") %
- (allowed, obj_name(converted)))
+ (allowed, type_utils.obj_name(converted)))
loaded = converted
except (yaml.YAMLError, TypeError, ValueError):
if len(blob) == 0:
@@ -782,7 +770,7 @@ def read_conf_with_confd(cfgfile):
if not isinstance(confd, (str, basestring)):
raise TypeError(("Config file %s contains 'conf_d' "
"with non-string type %s") %
- (cfgfile, obj_name(confd)))
+ (cfgfile, type_utils.obj_name(confd)))
else:
confd = str(confd).strip()
elif os.path.isdir("%s.d" % cfgfile):
@@ -793,7 +781,7 @@ def read_conf_with_confd(cfgfile):
# Conf.d settings override input configuration
confd_cfg = read_conf_d(confd)
- return mergedict(confd_cfg, cfg)
+ return mergemanydict([confd_cfg, cfg])
def read_cc_from_cmdline(cmdline=None):
@@ -948,7 +936,7 @@ def is_resolvable(name):
for (_fam, _stype, _proto, cname, sockaddr) in result:
badresults[iname].append("%s: %s" % (cname, sockaddr[0]))
badips.add(sockaddr[0])
- except socket.gaierror:
+ except (socket.gaierror, socket.error):
pass
_DNS_REDIRECT_IP = badips
if badresults:
@@ -961,7 +949,7 @@ def is_resolvable(name):
if addr in _DNS_REDIRECT_IP:
return False
return True
- except socket.gaierror:
+ except (socket.gaierror, socket.error):
return False
@@ -1540,7 +1528,7 @@ def shellify(cmdlist, add_header=True):
else:
raise RuntimeError(("Unable to shellify type %s"
" which is not a list or string")
- % (obj_name(args)))
+ % (type_utils.obj_name(args)))
LOG.debug("Shellified %s commands.", cmds_made)
return content
@@ -1599,7 +1587,7 @@ def get_proc_env(pid):
fn = os.path.join("/proc/", str(pid), "environ")
try:
contents = load_file(fn)
- toks = contents.split("\0")
+ toks = contents.split("\x00")
for tok in toks:
if tok == "":
continue
@@ -1655,3 +1643,106 @@ def expand_package_list(version_fmt, pkgs):
raise RuntimeError("Invalid package type.")
return pkglist
+
+
+def parse_mount_info(path, mountinfo_lines, log=LOG):
+ """Return the mount information for PATH given the lines from
+ /proc/$$/mountinfo."""
+
+ path_elements = [e for e in path.split('/') if e]
+ devpth = None
+ fs_type = None
+ match_mount_point = None
+ match_mount_point_elements = None
+ for i, line in enumerate(mountinfo_lines):
+ parts = line.split()
+
+ # Completely fail if there is anything in any line that is
+ # unexpected, as continuing to parse past a bad line could
+ # cause an incorrect result to be returned, so it's better
+ # return nothing than an incorrect result.
+
+ # The minimum number of elements in a valid line is 10.
+ if len(parts) < 10:
+ log.debug("Line %d has two few columns (%d): %s",
+ i + 1, len(parts), line)
+ return None
+
+ mount_point = parts[4]
+ mount_point_elements = [e for e in mount_point.split('/') if e]
+
+ # Ignore mounts deeper than the path in question.
+ if len(mount_point_elements) > len(path_elements):
+ continue
+
+ # Ignore mounts where the common path is not the same.
+ l = min(len(mount_point_elements), len(path_elements))
+ if mount_point_elements[0:l] != path_elements[0:l]:
+ continue
+
+ # Ignore mount points higher than an already seen mount
+ # point.
+ if (match_mount_point_elements is not None and
+ len(match_mount_point_elements) > len(mount_point_elements)):
+ continue
+
+ # Find the '-' which terminates a list of optional columns to
+ # find the filesystem type and the path to the device. See
+ # man 5 proc for the format of this file.
+ try:
+ i = parts.index('-')
+ except ValueError:
+ log.debug("Did not find column named '-' in line %d: %s",
+ i + 1, line)
+ return None
+
+ # Get the path to the device.
+ try:
+ fs_type = parts[i + 1]
+ devpth = parts[i + 2]
+ except IndexError:
+ log.debug("Too few columns after '-' column in line %d: %s",
+ i + 1, line)
+ return None
+
+ match_mount_point = mount_point
+ match_mount_point_elements = mount_point_elements
+
+ if devpth and fs_type and match_mount_point:
+ return (devpth, fs_type, match_mount_point)
+ else:
+ return None
+
+
+def get_mount_info(path, log=LOG):
+ # Use /proc/$$/mountinfo to find the device where path is mounted.
+ # This is done because with a btrfs filesystem using os.stat(path)
+ # does not return the ID of the device.
+ #
+ # Here, / has a device of 18 (decimal).
+ #
+ # $ stat /
+ # File: '/'
+ # Size: 234 Blocks: 0 IO Block: 4096 directory
+ # Device: 12h/18d Inode: 256 Links: 1
+ # Access: (0755/drwxr-xr-x) Uid: ( 0/ root) Gid: ( 0/ root)
+ # Access: 2013-01-13 07:31:04.358011255 +0000
+ # Modify: 2013-01-13 18:48:25.930011255 +0000
+ # Change: 2013-01-13 18:48:25.930011255 +0000
+ # Birth: -
+ #
+ # Find where / is mounted:
+ #
+ # $ mount | grep ' / '
+ # /dev/vda1 on / type btrfs (rw,subvol=@,compress=lzo)
+ #
+ # And the device ID for /dev/vda1 is not 18:
+ #
+ # $ ls -l /dev/vda1
+ # brw-rw---- 1 root disk 253, 1 Jan 13 08:29 /dev/vda1
+ #
+ # So use /proc/$$/mountinfo to find the device underlying the
+ # input path.
+ mountinfo_path = '/proc/%s/mountinfo' % os.getpid()
+ lines = load_file(mountinfo_path).splitlines()
+ return parse_mount_info(path, lines, log)
diff --git a/config/cloud.cfg b/config/cloud.cfg
index a8c74486..b61b8a7d 100644
--- a/config/cloud.cfg
+++ b/config/cloud.cfg
@@ -26,6 +26,7 @@ cloud_init_modules:
- migrator
- bootcmd
- write-files
+ - growpart
- resizefs
- set_hostname
- update_hostname
diff --git a/doc/examples/cloud-config-growpart.txt b/doc/examples/cloud-config-growpart.txt
new file mode 100644
index 00000000..705f02c2
--- /dev/null
+++ b/doc/examples/cloud-config-growpart.txt
@@ -0,0 +1,24 @@
+#cloud-config
+#
+# growpart entry is a dict, if it is not present at all
+# in config, then the default is used ({'mode': 'auto', 'devices': ['/']})
+#
+# mode:
+# values:
+# * auto: use any option possible (growpart or parted)
+# if none are available, do not warn, but debug.
+# * growpart: use growpart to grow partitions
+# if growpart is not available, this is an error.
+# * parted: use parted (parted resizepart) to resize partitions
+# if parted is not available, this is an error.
+# * off, false
+#
+# devices:
+# a list of things to resize.
+# items can be filesystem paths or devices (in /dev)
+# examples:
+# devices: [/, /dev/vdb1]
+#
+growpart:
+ mode: auto
+ devices: ['/']
diff --git a/doc/merging.rst b/doc/merging.rst
new file mode 100644
index 00000000..6344facd
--- /dev/null
+++ b/doc/merging.rst
@@ -0,0 +1,188 @@
+Overview
+--------
+
+This was done because it has been a common feature request that there be a
+way to specify how cloud-config yaml "dictionaries" are merged together when
+there are multiple yamls to merge together (say when performing an #include).
+
+Since previously the merging algorithm was very simple and would only overwrite
+and not append lists, or strings, and so on it was decided to create a new and
+improved way to merge dictionaries (and there contained objects) together in a
+way that is customizable, thus allowing for users who provide cloud-config data
+to determine exactly how there objects will be merged.
+
+For example.
+
+.. code-block:: yaml
+
+ #cloud-config (1)
+ run_cmd:
+ - bash1
+ - bash2
+
+ #cloud-config (2)
+ run_cmd:
+ - bash3
+ - bash4
+
+The previous way of merging the following 2 objects would result in a final
+cloud-config object that contains the following.
+
+.. code-block:: yaml
+
+ #cloud-config (merged)
+ run_cmd:
+ - bash3
+ - bash4
+
+Typically this is not what users want, instead they would likely prefer:
+
+.. code-block:: yaml
+
+ #cloud-config (merged)
+ run_cmd:
+ - bash1
+ - bash2
+ - bash3
+ - bash4
+
+This way makes it easier to combine the various cloud-config objects you have
+into a more useful list, thus reducing duplication that would have had to
+occur in the previous method to accomplish the same result.
+
+Customizability
+---------------
+
+Since the above merging algorithm may not always be the desired merging
+algorithm (like how the previous merging algorithm was not always the preferred
+one) the concept of customizing how merging can be done was introduced through
+a new concept call 'merge classes'.
+
+A merge class is a class defintion which provides functions that can be used
+to merge a given type with another given type.
+
+An example of one of these merging classes is the following:
+
+.. code-block:: python
+
+ class Merger(object):
+ def __init__(self, merger, opts):
+ self._merger = merger
+ self._overwrite = 'overwrite' in opts
+
+ # This merging algorithm will attempt to merge with
+ # another dictionary, on encountering any other type of object
+ # it will not merge with said object, but will instead return
+ # the original value
+ #
+ # On encountering a dictionary, it will create a new dictionary
+ # composed of the original and the one to merge with, if 'overwrite'
+ # is enabled then keys that exist in the original will be overwritten
+ # by keys in the one to merge with (and associated values). Otherwise
+ # if not in overwrite mode the 2 conflicting keys themselves will
+ # be merged.
+ def _on_dict(self, value, merge_with):
+ if not isinstance(merge_with, (dict)):
+ return value
+ merged = dict(value)
+ for (k, v) in merge_with.items():
+ if k in merged:
+ if not self._overwrite:
+ merged[k] = self._merger.merge(merged[k], v)
+ else:
+ merged[k] = v
+ else:
+ merged[k] = v
+ return merged
+
+As you can see there is a '_on_dict' method here that will be given a source value
+and a value to merge with. The result will be the merged object. This code itself
+is called by another merging class which 'directs' the merging to happen by
+analyzing the types of the objects to merge and attempting to find a know object
+that will merge that type. I will avoid pasting that here, but it can be found
+in the `mergers/__init__.py` file (see `LookupMerger` and `UnknownMerger`).
+
+So following the typical cloud-init way of allowing source code to be downloaded
+and used dynamically, it is possible for users to inject there own merging files
+to handle specific types of merging as they choose (the basic ones included will
+handle lists, dicts, and strings). Note how each merge can have options associated
+with it which affect how the merging is performed, for example a dictionary merger
+can be told to overwrite instead of attempt to merge, or a string merger can be
+told to append strings instead of discarding other strings to merge with.
+
+How to activate
+---------------
+
+There are a few ways to activate the merging algorithms, and to customize them
+for your own usage.
+
+1. The first way involves the usage of MIME messages in cloud-init to specify
+ multipart documents (this is one way in which multiple cloud-config is joined
+ together into a single cloud-config). Two new headers are looked for, both
+ of which can define the way merging is done (the first header to exist wins).
+ These new headers (in lookup order) are 'Merge-Type' and 'X-Merge-Type'. The value
+ should be a string which will satisfy the new merging format defintion (see
+ below for this format).
+2. The second way is actually specifying the merge-type in the body of the
+ cloud-config dictionary. There are 2 ways to specify this, either as a string
+ or as a dictionary (see format below). The keys that are looked up for this
+ definition are the following (in order), 'merge_how', 'merge_type'.
+
+String format
+********
+
+The string format that is expected is the following.
+
+::
+
+ classname1(option1,option2)+classname2(option3,option4)....
+
+The class name there will be connected to class names used when looking for the
+class that can be used to merge and options provided will be given to the class
+on construction of that class.
+
+For example, the default string that is used when none is provided is the following:
+
+::
+
+ list(extend)+dict()+str(append)
+
+Dictionary format
+********
+
+In cases where a dictionary can be used to specify the same information as the
+string format (ie option #2 of above) it can be used, for example.
+
+.. code-block:: python
+
+ {'merge_how': [{'name': 'list', 'settings': ['extend']},
+ {'name': 'dict', 'settings': []},
+ {'name': 'str', 'settings': ['append']}]}
+
+This would be the equivalent format for default string format but in dictionary
+form instead of string form.
+
+Specifying multiple types and its effect
+----------------------------------------
+
+Now you may be asking yourself, if I specify a merge-type header or dictionary
+for every cloud-config that I provide, what exactly happens?
+
+The answer is that when merging, a stack of 'merging classes' is kept, the
+first one on that stack is the default merging classes, this set of mergers
+will be used when the first cloud-config is merged with the initial empty
+cloud-config dictionary. If the cloud-config that was just merged provided a
+set of merging classes (via the above formats) then those merging classes will
+be pushed onto the stack. Now if there is a second cloud-config to be merged then
+the merging classes from the cloud-config before the first will be used (not the
+default) and so on. This way a cloud-config can decide how it will merge with a
+cloud-config dictionary coming after it.
+
+Other uses
+----------
+
+The default merging algorithm for merging 'conf.d' yaml files (which form a initial
+yaml config for cloud-init) was also changed to use this mechanism so its full
+benefits (and customization) can also be used there as well. Other places that
+used the previous merging are also similar now extensible (metadata merging for
+example).
diff --git a/doc/rtd/conf.py b/doc/rtd/conf.py
index 87fc40ab..c9ae79f4 100644
--- a/doc/rtd/conf.py
+++ b/doc/rtd/conf.py
@@ -17,13 +17,13 @@ from cloudinit import version
# General information about the project.
project = 'Cloud-Init'
-# -- General configuration -----------------------------------------------------
+# -- General configuration ----------------------------------------------------
# If your documentation needs a minimal Sphinx version, state it here.
#needs_sphinx = '1.0'
-# Add any Sphinx extension module names here, as strings. They can be extensions
-# coming with Sphinx (named 'sphinx.ext.*') or your custom ones.
+# Add any Sphinx extension module names here, as strings. They can be
+# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom ones.
extensions = [
'sphinx.ext.intersphinx',
]
@@ -55,7 +55,7 @@ exclude_patterns = []
# output. They are ignored by default.
show_authors = False
-# -- Options for HTML output ---------------------------------------------------
+# -- Options for HTML output --------------------------------------------------
# The theme to use for HTML and HTML Help pages. See the documentation for
# a list of builtin themes.
diff --git a/doc/rtd/index.rst b/doc/rtd/index.rst
index 619bb5dc..fe04b1a9 100644
--- a/doc/rtd/index.rst
+++ b/doc/rtd/index.rst
@@ -24,6 +24,7 @@ Summary
topics/examples
topics/datasources
topics/modules
+ topics/merging
topics/moreinfo
topics/hacking
diff --git a/doc/rtd/topics/merging.rst b/doc/rtd/topics/merging.rst
new file mode 100644
index 00000000..8a03f3c7
--- /dev/null
+++ b/doc/rtd/topics/merging.rst
@@ -0,0 +1,5 @@
+=========
+Merging
+=========
+
+.. include:: ../../merging.rst
diff --git a/tests/unittests/helpers.py b/tests/unittests/helpers.py
index 91a50e18..e020a3ec 100644
--- a/tests/unittests/helpers.py
+++ b/tests/unittests/helpers.py
@@ -175,6 +175,7 @@ class FilesystemMockingTestCase(ResourceUsingTestCase):
def patchOS(self, new_root):
patch_funcs = {
os.path: ['isfile', 'exists', 'islink', 'isdir'],
+ os: ['listdir'],
}
for (mod, funcs) in patch_funcs.items():
for f in funcs:
@@ -183,6 +184,7 @@ class FilesystemMockingTestCase(ResourceUsingTestCase):
setattr(mod, f, trap_func)
self.patched_funcs.append((mod, f, func))
+
def populate_dir(path, files):
os.makedirs(path)
for (name, content) in files.iteritems():
diff --git a/tests/unittests/test__init__.py b/tests/unittests/test__init__.py
index d707afa9..b4b20e51 100644
--- a/tests/unittests/test__init__.py
+++ b/tests/unittests/test__init__.py
@@ -22,7 +22,8 @@ class FakeModule(handlers.Handler):
def list_types(self):
return self.types
- def _handle_part(self, data, ctype, filename, payload, frequency):
+ def handle_part(self, _data, ctype, filename, # pylint: disable=W0221
+ payload, frequency):
pass
@@ -103,6 +104,9 @@ class TestHandlerHandlePart(MockerTestCase):
self.filename = "fake filename"
self.payload = "fake payload"
self.frequency = settings.PER_INSTANCE
+ self.headers = {
+ 'Content-Type': self.ctype,
+ }
def test_normal_version_1(self):
"""
@@ -118,8 +122,8 @@ class TestHandlerHandlePart(MockerTestCase):
self.payload)
self.mocker.replay()
- handlers.run_part(mod_mock, self.data, self.ctype, self.filename,
- self.payload, self.frequency)
+ handlers.run_part(mod_mock, self.data, self.filename,
+ self.payload, self.frequency, self.headers)
def test_normal_version_2(self):
"""
@@ -135,8 +139,8 @@ class TestHandlerHandlePart(MockerTestCase):
self.payload, self.frequency)
self.mocker.replay()
- handlers.run_part(mod_mock, self.data, self.ctype, self.filename,
- self.payload, self.frequency)
+ handlers.run_part(mod_mock, self.data, self.filename,
+ self.payload, self.frequency, self.headers)
def test_modfreq_per_always(self):
"""
@@ -152,8 +156,8 @@ class TestHandlerHandlePart(MockerTestCase):
self.payload)
self.mocker.replay()
- handlers.run_part(mod_mock, self.data, self.ctype, self.filename,
- self.payload, self.frequency)
+ handlers.run_part(mod_mock, self.data, self.filename,
+ self.payload, self.frequency, self.headers)
def test_no_handle_when_modfreq_once(self):
"""C{handle_part} is not called if frequency is once."""
@@ -163,8 +167,8 @@ class TestHandlerHandlePart(MockerTestCase):
self.mocker.result(settings.PER_ONCE)
self.mocker.replay()
- handlers.run_part(mod_mock, self.data, self.ctype, self.filename,
- self.payload, self.frequency)
+ handlers.run_part(mod_mock, self.data, self.filename,
+ self.payload, self.frequency, self.headers)
def test_exception_is_caught(self):
"""Exceptions within C{handle_part} are caught and logged."""
@@ -178,8 +182,8 @@ class TestHandlerHandlePart(MockerTestCase):
self.mocker.throw(Exception())
self.mocker.replay()
- handlers.run_part(mod_mock, self.data, self.ctype, self.filename,
- self.payload, self.frequency)
+ handlers.run_part(mod_mock, self.data, self.filename,
+ self.payload, self.frequency, self.headers)
class TestCmdlineUrl(MockerTestCase):
diff --git a/tests/unittests/test_builtin_handlers.py b/tests/unittests/test_builtin_handlers.py
index 5f41cb3d..9cf28215 100644
--- a/tests/unittests/test_builtin_handlers.py
+++ b/tests/unittests/test_builtin_handlers.py
@@ -1,8 +1,9 @@
"""Tests of the built-in user data handlers."""
import os
+import unittest
-from mocker import MockerTestCase
+from tests.unittests import helpers as test_helpers
from cloudinit import handlers
from cloudinit import helpers
@@ -13,7 +14,7 @@ from cloudinit.handlers import upstart_job
from cloudinit.settings import (PER_ALWAYS, PER_INSTANCE)
-class TestBuiltins(MockerTestCase):
+class TestBuiltins(test_helpers.FilesystemMockingTestCase):
def test_upstart_frequency_no_out(self):
c_root = self.makeDir()
@@ -34,15 +35,20 @@ class TestBuiltins(MockerTestCase):
None, None, None)
self.assertEquals(0, len(os.listdir(up_root)))
+ @unittest.skip("until LP: #1124384 fixed")
def test_upstart_frequency_single(self):
# files should be written out when frequency is ! per-instance
- c_root = self.makeDir()
- up_root = self.makeDir()
+ new_root = self.makeDir()
+ freq = PER_INSTANCE
+
+ self.patchOS(new_root)
+ self.patchUtils(new_root)
paths = helpers.Paths({
- 'cloud_dir': c_root,
- 'upstart_dir': up_root,
+ 'upstart_dir': "/etc/upstart",
})
- freq = PER_INSTANCE
+
+ util.ensure_dir("/run")
+ util.ensure_dir("/etc/upstart")
mock_subp = self.mocker.replace(util.subp, passthrough=False)
mock_subp(["initctl", "reload-configuration"], capture=False)
@@ -55,4 +61,5 @@ class TestBuiltins(MockerTestCase):
'test.conf', 'blah', freq)
h.handle_part('', handlers.CONTENT_END,
None, None, None)
- self.assertEquals(1, len(os.listdir(up_root)))
+
+ self.assertEquals(1, len(os.listdir('/etc/upstart')))
diff --git a/tests/unittests/test_datasource/test_nocloud.py b/tests/unittests/test_datasource/test_nocloud.py
index 28e0a472..62fc5358 100644
--- a/tests/unittests/test_datasource/test_nocloud.py
+++ b/tests/unittests/test_datasource/test_nocloud.py
@@ -1,7 +1,7 @@
from cloudinit import helpers
-from tests.unittests.helpers import populate_dir
from cloudinit.sources import DataSourceNoCloud
from cloudinit import util
+from tests.unittests.helpers import populate_dir
from mocker import MockerTestCase
import os
diff --git a/tests/unittests/test_handler/test_handler_growpart.py b/tests/unittests/test_handler/test_handler_growpart.py
new file mode 100644
index 00000000..b1b872b0
--- /dev/null
+++ b/tests/unittests/test_handler/test_handler_growpart.py
@@ -0,0 +1,255 @@
+from mocker import MockerTestCase
+
+from cloudinit import cloud
+from cloudinit import util
+
+from cloudinit.config import cc_growpart
+
+import errno
+import logging
+import os
+import re
+
+# growpart:
+# mode: auto # off, on, auto, 'growpart', 'parted'
+# devices: ['root']
+
+HELP_PARTED_NO_RESIZE = """
+Usage: parted [OPTION]... [DEVICE [COMMAND [PARAMETERS]...]...]
+Apply COMMANDs with PARAMETERS to DEVICE. If no COMMAND(s) are given, run in
+interactive mode.
+
+OPTIONs:
+<SNIP>
+
+COMMANDs:
+<SNIP>
+ quit exit program
+ rescue START END rescue a lost partition near START
+ and END
+ resize NUMBER START END resize partition NUMBER and its file
+ system
+ rm NUMBER delete partition NUMBER
+<SNIP>
+Report bugs to bug-parted@gnu.org
+"""
+
+HELP_PARTED_RESIZE = """
+Usage: parted [OPTION]... [DEVICE [COMMAND [PARAMETERS]...]...]
+Apply COMMANDs with PARAMETERS to DEVICE. If no COMMAND(s) are given, run in
+interactive mode.
+
+OPTIONs:
+<SNIP>
+
+COMMANDs:
+<SNIP>
+ quit exit program
+ rescue START END rescue a lost partition near START
+ and END
+ resize NUMBER START END resize partition NUMBER and its file
+ system
+ resizepart NUMBER END resize partition NUMBER
+ rm NUMBER delete partition NUMBER
+<SNIP>
+Report bugs to bug-parted@gnu.org
+"""
+
+HELP_GROWPART_RESIZE = """
+growpart disk partition
+ rewrite partition table so that partition takes up all the space it can
+ options:
+ -h | --help print Usage and exit
+<SNIP>
+ -u | --update R update the the kernel partition table info after growing
+ this requires kernel support and 'partx --update'
+ R is one of:
+ - 'auto' : [default] update partition if possible
+<SNIP>
+ Example:
+ - growpart /dev/sda 1
+ Resize partition 1 on /dev/sda
+"""
+
+HELP_GROWPART_NO_RESIZE = """
+growpart disk partition
+ rewrite partition table so that partition takes up all the space it can
+ options:
+ -h | --help print Usage and exit
+<SNIP>
+ Example:
+ - growpart /dev/sda 1
+ Resize partition 1 on /dev/sda
+"""
+
+
+class TestDisabled(MockerTestCase):
+ def setUp(self):
+ super(TestDisabled, self).setUp()
+ self.name = "growpart"
+ self.cloud_init = None
+ self.log = logging.getLogger("TestDisabled")
+ self.args = []
+
+ self.handle = cc_growpart.handle
+
+ def test_mode_off(self):
+ #Test that nothing is done if mode is off.
+
+ # this really only verifies that resizer_factory isn't called
+ config = {'growpart': {'mode': 'off'}}
+ self.mocker.replace(cc_growpart.resizer_factory,
+ passthrough=False)
+ self.mocker.replay()
+
+ self.handle(self.name, config, self.cloud_init, self.log, self.args)
+
+
+class TestConfig(MockerTestCase):
+ def setUp(self):
+ super(TestConfig, self).setUp()
+ self.name = "growpart"
+ self.paths = None
+ self.cloud = cloud.Cloud(None, self.paths, None, None, None)
+ self.log = logging.getLogger("TestConfig")
+ self.args = []
+ os.environ = {}
+
+ self.cloud_init = None
+ self.handle = cc_growpart.handle
+
+ # Order must be correct
+ self.mocker.order()
+
+ def test_no_resizers_auto_is_fine(self):
+ subp = self.mocker.replace(util.subp, passthrough=False)
+ subp(['parted', '--help'], env={'LANG': 'C'})
+ self.mocker.result((HELP_PARTED_NO_RESIZE, ""))
+ subp(['growpart', '--help'], env={'LANG': 'C'})
+ self.mocker.result((HELP_GROWPART_NO_RESIZE, ""))
+ self.mocker.replay()
+
+ config = {'growpart': {'mode': 'auto'}}
+ self.handle(self.name, config, self.cloud_init, self.log, self.args)
+
+ def test_no_resizers_mode_growpart_is_exception(self):
+ subp = self.mocker.replace(util.subp, passthrough=False)
+ subp(['growpart', '--help'], env={'LANG': 'C'})
+ self.mocker.result((HELP_GROWPART_NO_RESIZE, ""))
+ self.mocker.replay()
+
+ config = {'growpart': {'mode': "growpart"}}
+ self.assertRaises(ValueError, self.handle, self.name, config,
+ self.cloud_init, self.log, self.args)
+
+ def test_mode_auto_prefers_parted(self):
+ subp = self.mocker.replace(util.subp, passthrough=False)
+ subp(['parted', '--help'], env={'LANG': 'C'})
+ self.mocker.result((HELP_PARTED_RESIZE, ""))
+ self.mocker.replay()
+
+ ret = cc_growpart.resizer_factory(mode="auto")
+ self.assertTrue(isinstance(ret, cc_growpart.ResizeParted))
+
+ def test_handle_with_no_growpart_entry(self):
+ #if no 'growpart' entry in config, then mode=auto should be used
+
+ myresizer = object()
+
+ factory = self.mocker.replace(cc_growpart.resizer_factory,
+ passthrough=False)
+ rsdevs = self.mocker.replace(cc_growpart.resize_devices,
+ passthrough=False)
+ factory("auto")
+ self.mocker.result(myresizer)
+ rsdevs(myresizer, ["/"])
+ self.mocker.result((("/", cc_growpart.RESIZE.CHANGED, "my-message",),))
+ self.mocker.replay()
+
+ try:
+ orig_resizers = cc_growpart.RESIZERS
+ cc_growpart.RESIZERS = (('mysizer', object),)
+ self.handle(self.name, {}, self.cloud_init, self.log, self.args)
+ finally:
+ cc_growpart.RESIZERS = orig_resizers
+
+
+class TestResize(MockerTestCase):
+ def setUp(self):
+ super(TestResize, self).setUp()
+ self.name = "growpart"
+ self.log = logging.getLogger("TestResize")
+
+ # Order must be correct
+ self.mocker.order()
+
+ def test_simple_devices(self):
+ #test simple device list
+ # this patches out devent2dev, os.stat, and device_part_info
+ # so in the end, doesn't test a lot
+ devs = ["/dev/XXda1", "/dev/YYda2"]
+ devstat_ret = Bunch(st_mode=25008, st_ino=6078, st_dev=5L,
+ st_nlink=1, st_uid=0, st_gid=6, st_size=0,
+ st_atime=0, st_mtime=0, st_ctime=0)
+ enoent = ["/dev/NOENT"]
+ real_stat = os.stat
+ resize_calls = []
+
+ class myresizer(object):
+ def resize(self, diskdev, partnum, partdev):
+ resize_calls.append((diskdev, partnum, partdev))
+ if partdev == "/dev/YYda2":
+ return (1024, 2048)
+ return (1024, 1024) # old size, new size
+
+ def mystat(path):
+ if path in devs:
+ return devstat_ret
+ if path in enoent:
+ e = OSError("%s: does not exist" % path)
+ e.errno = errno.ENOENT
+ raise e
+ return real_stat(path)
+
+ try:
+ opinfo = cc_growpart.device_part_info
+ cc_growpart.device_part_info = simple_device_part_info
+ os.stat = mystat
+
+ resized = cc_growpart.resize_devices(myresizer(), devs + enoent)
+
+ def find(name, res):
+ for f in res:
+ if f[0] == name:
+ return f
+ return None
+
+ self.assertEqual(cc_growpart.RESIZE.NOCHANGE,
+ find("/dev/XXda1", resized)[1])
+ self.assertEqual(cc_growpart.RESIZE.CHANGED,
+ find("/dev/YYda2", resized)[1])
+ self.assertEqual(cc_growpart.RESIZE.SKIPPED,
+ find(enoent[0], resized)[1])
+ #self.assertEqual(resize_calls,
+ #[("/dev/XXda", "1", "/dev/XXda1"),
+ #("/dev/YYda", "2", "/dev/YYda2")])
+ finally:
+ cc_growpart.device_part_info = opinfo
+ os.stat = real_stat
+
+
+def simple_device_part_info(devpath):
+ # simple stupid return (/dev/vda, 1) for /dev/vda
+ ret = re.search("([^0-9]*)([0-9]*)$", devpath)
+ x = (ret.group(1), ret.group(2))
+ return x
+
+
+class Bunch:
+ st_mode = None # fix pylint complaint
+
+ def __init__(self, **kwds):
+ self.__dict__.update(kwds)
+
+
+# vi: ts=4 expandtab
diff --git a/tests/unittests/test_merging.py b/tests/unittests/test_merging.py
index 0037b966..ad137e85 100644
--- a/tests/unittests/test_merging.py
+++ b/tests/unittests/test_merging.py
@@ -1,62 +1,142 @@
-from mocker import MockerTestCase
-
-from cloudinit import util
-
-
-class TestMergeDict(MockerTestCase):
- def test_simple_merge(self):
- """Test simple non-conflict merge."""
- source = {"key1": "value1"}
- candidate = {"key2": "value2"}
- result = util.mergedict(source, candidate)
- self.assertEqual({"key1": "value1", "key2": "value2"}, result)
-
- def test_nested_merge(self):
- """Test nested merge."""
- source = {"key1": {"key1.1": "value1.1"}}
- candidate = {"key1": {"key1.2": "value1.2"}}
- result = util.mergedict(source, candidate)
- self.assertEqual(
- {"key1": {"key1.1": "value1.1", "key1.2": "value1.2"}}, result)
-
- def test_merge_does_not_override(self):
- """Test that candidate doesn't override source."""
- source = {"key1": "value1", "key2": "value2"}
- candidate = {"key1": "value2", "key2": "NEW VALUE"}
- result = util.mergedict(source, candidate)
- self.assertEqual(source, result)
-
- def test_empty_candidate(self):
- """Test empty candidate doesn't change source."""
- source = {"key": "value"}
- candidate = {}
- result = util.mergedict(source, candidate)
- self.assertEqual(source, result)
-
- def test_empty_source(self):
- """Test empty source is replaced by candidate."""
- source = {}
- candidate = {"key": "value"}
- result = util.mergedict(source, candidate)
- self.assertEqual(candidate, result)
-
- def test_non_dict_candidate(self):
- """Test non-dict candidate is discarded."""
- source = {"key": "value"}
- candidate = "not a dict"
- result = util.mergedict(source, candidate)
- self.assertEqual(source, result)
-
- def test_non_dict_source(self):
- """Test non-dict source is not modified with a dict candidate."""
- source = "not a dict"
- candidate = {"key": "value"}
- result = util.mergedict(source, candidate)
- self.assertEqual(source, result)
-
- def test_neither_dict(self):
- """Test if neither candidate or source is dict source wins."""
- source = "source"
- candidate = "candidate"
- result = util.mergedict(source, candidate)
- self.assertEqual(source, result)
+from tests.unittests import helpers
+
+from cloudinit import mergers
+
+
+class TestSimpleRun(helpers.MockerTestCase):
+ def test_basic_merge(self):
+ source = {
+ 'Blah': ['blah2'],
+ 'Blah3': 'c',
+ }
+ merge_with = {
+ 'Blah2': ['blah3'],
+ 'Blah3': 'b',
+ 'Blah': ['123'],
+ }
+ # Basic merge should not do thing special
+ merge_how = "list()+dict()+str()"
+ merger_set = mergers.string_extract_mergers(merge_how)
+ self.assertEquals(3, len(merger_set))
+ merger = mergers.construct(merger_set)
+ merged = merger.merge(source, merge_with)
+ self.assertEquals(merged['Blah'], ['blah2'])
+ self.assertEquals(merged['Blah2'], ['blah3'])
+ self.assertEquals(merged['Blah3'], 'c')
+
+ def test_dict_overwrite(self):
+ source = {
+ 'Blah': ['blah2'],
+ }
+ merge_with = {
+ 'Blah': ['123'],
+ }
+ # Now lets try a dict overwrite
+ merge_how = "list()+dict(overwrite)+str()"
+ merger_set = mergers.string_extract_mergers(merge_how)
+ self.assertEquals(3, len(merger_set))
+ merger = mergers.construct(merger_set)
+ merged = merger.merge(source, merge_with)
+ self.assertEquals(merged['Blah'], ['123'])
+
+ def test_string_append(self):
+ source = {
+ 'Blah': 'blah2',
+ }
+ merge_with = {
+ 'Blah': '345',
+ }
+ merge_how = "list()+dict()+str(append)"
+ merger_set = mergers.string_extract_mergers(merge_how)
+ self.assertEquals(3, len(merger_set))
+ merger = mergers.construct(merger_set)
+ merged = merger.merge(source, merge_with)
+ self.assertEquals(merged['Blah'], 'blah2345')
+
+ def test_list_extend(self):
+ source = ['abc']
+ merge_with = ['123']
+ merge_how = "list(extend)+dict()+str()"
+ merger_set = mergers.string_extract_mergers(merge_how)
+ self.assertEquals(3, len(merger_set))
+ merger = mergers.construct(merger_set)
+ merged = merger.merge(source, merge_with)
+ self.assertEquals(merged, ['abc', '123'])
+
+ def test_deep_merge(self):
+ source = {
+ 'a': [1, 'b', 2],
+ 'b': 'blahblah',
+ 'c': {
+ 'e': [1, 2, 3],
+ 'f': 'bigblobof',
+ 'iamadict': {
+ 'ok': 'ok',
+ }
+ },
+ 'run': [
+ 'runme',
+ 'runme2',
+ ],
+ 'runmereally': [
+ 'e', ['a'], 'd',
+ ],
+ }
+ merge_with = {
+ 'a': ['e', 'f', 'g'],
+ 'b': 'more',
+ 'c': {
+ 'a': 'b',
+ 'f': 'stuff',
+ },
+ 'run': [
+ 'morecmd',
+ 'moremoremore',
+ ],
+ 'runmereally': [
+ 'blah', ['b'], 'e',
+ ],
+ }
+ merge_how = "list(extend)+dict()+str(append)"
+ merger_set = mergers.string_extract_mergers(merge_how)
+ self.assertEquals(3, len(merger_set))
+ merger = mergers.construct(merger_set)
+ merged = merger.merge(source, merge_with)
+ self.assertEquals(merged['a'], [1, 'b', 2, 'e', 'f', 'g'])
+ self.assertEquals(merged['b'], 'blahblahmore')
+ self.assertEquals(merged['c']['f'], 'bigblobofstuff')
+ self.assertEquals(merged['run'], ['runme', 'runme2', 'morecmd',
+ 'moremoremore'])
+ self.assertEquals(merged['runmereally'], ['e', ['a'], 'd', 'blah',
+ ['b'], 'e'])
+
+ def test_dict_overwrite_layered(self):
+ source = {
+ 'Blah3': {
+ 'f': '3',
+ 'g': {
+ 'a': 'b',
+ }
+ }
+ }
+ merge_with = {
+ 'Blah3': {
+ 'e': '2',
+ 'g': {
+ 'e': 'f',
+ }
+ }
+ }
+ merge_how = "list()+dict()+str()"
+ merger_set = mergers.string_extract_mergers(merge_how)
+ self.assertEquals(3, len(merger_set))
+ merger = mergers.construct(merger_set)
+ merged = merger.merge(source, merge_with)
+ self.assertEquals(merged['Blah3'], {
+ 'e': '2',
+ 'f': '3',
+ 'g': {
+ 'a': 'b',
+ 'e': 'f',
+ }
+ })
diff --git a/tests/unittests/test_sshutil.py b/tests/unittests/test_sshutil.py
new file mode 100644
index 00000000..d8662cac
--- /dev/null
+++ b/tests/unittests/test_sshutil.py
@@ -0,0 +1,101 @@
+from cloudinit import ssh_util
+from unittest import TestCase
+
+
+VALID_CONTENT = {
+ 'dsa': (
+ "AAAAB3NzaC1kc3MAAACBAIrjOQSlSea19bExXBMBKBvcLhBoVvNBjCppNzllipF"
+ "W4jgIOMcNanULRrZGjkOKat6MWJNetSbV1E6IOFDQ16rQgsh/OvYU9XhzM8seLa"
+ "A21VszZuhIV7/2DE3vxu7B54zVzueG1O1Deq6goQCRGWBUnqO2yluJiG4HzrnDa"
+ "jzRAAAAFQDMPO96qXd4F5A+5b2f2MO7SpVomQAAAIBpC3K2zIbDLqBBs1fn7rsv"
+ "KcJvwihdlVjG7UXsDB76P2GNqVG+IlYPpJZ8TO/B/fzTMtrdXp9pSm9OY1+BgN4"
+ "REsZ2WNcvfgY33aWaEM+ieCcQigvxrNAF2FTVcbUIIxAn6SmHuQSWrLSfdHc8H7"
+ "hsrgeUPPdzjBD/cv2ZmqwZ1AAAAIAplIsScrJut5wJMgyK1JG0Kbw9JYQpLe95P"
+ "obB069g8+mYR8U0fysmTEdR44mMu0VNU5E5OhTYoTGfXrVrkR134LqFM2zpVVbE"
+ "JNDnIqDHxTkc6LY2vu8Y2pQ3/bVnllZZOda2oD5HQ7ovygQa6CH+fbaZHbdDUX/"
+ "5z7u2rVAlDw=="
+ ),
+ 'ecdsa': (
+ "AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBITrGBB3cgJ"
+ "J7fPxvtMW9H3oRisNpJ3OAslxZeyP7I0A9BPAW0RQIwHVtVnM7zrp4nI+JLZov/"
+ "Ql7lc2leWL7CY="
+ ),
+ 'rsa': (
+ "AAAAB3NzaC1yc2EAAAABIwAAAQEA3I7VUf2l5gSn5uavROsc5HRDpZdQueUq5oz"
+ "emNSj8T7enqKHOEaFoU2VoPgGEWC9RyzSQVeyD6s7APMcE82EtmW4skVEgEGSbD"
+ "c1pvxzxtchBj78hJP6Cf5TCMFSXw+Fz5rF1dR23QDbN1mkHs7adr8GW4kSWqU7Q"
+ "7NDwfIrJJtO7Hi42GyXtvEONHbiRPOe8stqUly7MvUoN+5kfjBM8Qqpfl2+FNhT"
+ "YWpMfYdPUnE7u536WqzFmsaqJctz3gBxH9Ex7dFtrxR4qiqEr9Qtlu3xGn7Bw07"
+ "/+i1D+ey3ONkZLN+LQ714cgj8fRS4Hj29SCmXp5Kt5/82cD/VN3NtHw=="
+ ),
+}
+
+TEST_OPTIONS = ("no-port-forwarding,no-agent-forwarding,no-X11-forwarding,"
+ 'command="echo \'Please login as the user \"ubuntu\" rather than the'
+ 'user \"root\".\';echo;sleep 10"')
+
+
+class TestAuthKeyLineParser(TestCase):
+ def test_simple_parse(self):
+ # test key line with common 3 fields (keytype, base64, comment)
+ parser = ssh_util.AuthKeyLineParser()
+ for ktype in ['rsa', 'ecdsa', 'dsa']:
+ content = VALID_CONTENT[ktype]
+ comment = 'user-%s@host' % ktype
+ line = ' '.join((ktype, content, comment,))
+ key = parser.parse(line)
+
+ self.assertEqual(key.base64, content)
+ self.assertFalse(key.options)
+ self.assertEqual(key.comment, comment)
+ self.assertEqual(key.keytype, ktype)
+
+ def test_parse_no_comment(self):
+ # test key line with key type and base64 only
+ parser = ssh_util.AuthKeyLineParser()
+ for ktype in ['rsa', 'ecdsa', 'dsa']:
+ content = VALID_CONTENT[ktype]
+ line = ' '.join((ktype, content,))
+ key = parser.parse(line)
+
+ self.assertEqual(key.base64, content)
+ self.assertFalse(key.options)
+ self.assertFalse(key.comment)
+ self.assertEqual(key.keytype, ktype)
+
+ def test_parse_with_keyoptions(self):
+ # test key line with options in it
+ parser = ssh_util.AuthKeyLineParser()
+ options = TEST_OPTIONS
+ for ktype in ['rsa', 'ecdsa', 'dsa']:
+ content = VALID_CONTENT[ktype]
+ comment = 'user-%s@host' % ktype
+ line = ' '.join((options, ktype, content, comment,))
+ key = parser.parse(line)
+
+ self.assertEqual(key.base64, content)
+ self.assertEqual(key.options, options)
+ self.assertEqual(key.comment, comment)
+ self.assertEqual(key.keytype, ktype)
+
+ def test_parse_with_options_passed_in(self):
+ # test key line with key type and base64 only
+ parser = ssh_util.AuthKeyLineParser()
+
+ baseline = ' '.join(("rsa", VALID_CONTENT['rsa'], "user@host"))
+ myopts = "no-port-forwarding,no-agent-forwarding"
+
+ key = parser.parse("allowedopt" + " " + baseline)
+ self.assertEqual(key.options, "allowedopt")
+
+ key = parser.parse("overridden_opt " + baseline, options=myopts)
+ self.assertEqual(key.options, myopts)
+
+ def test_parse_invalid_keytype(self):
+ parser = ssh_util.AuthKeyLineParser()
+ key = parser.parse(' '.join(["badkeytype", VALID_CONTENT['rsa']]))
+
+ self.assertFalse(key.valid())
+
+
+# vi: ts=4 expandtab
diff --git a/tests/unittests/test_userdata.py b/tests/unittests/test_userdata.py
index 82a4c555..fdfe2542 100644
--- a/tests/unittests/test_userdata.py
+++ b/tests/unittests/test_userdata.py
@@ -7,14 +7,17 @@ import os
from email.mime.base import MIMEBase
-from mocker import MockerTestCase
-
+from cloudinit import handlers
+from cloudinit import helpers as c_helpers
from cloudinit import log
from cloudinit import sources
from cloudinit import stages
+from cloudinit import util
INSTANCE_ID = "i-testing"
+from tests.unittests import helpers
+
class FakeDataSource(sources.DataSource):
@@ -26,22 +29,16 @@ class FakeDataSource(sources.DataSource):
# FIXME: these tests shouldn't be checking log output??
# Weirddddd...
-
-
-class TestConsumeUserData(MockerTestCase):
+class TestConsumeUserData(helpers.FilesystemMockingTestCase):
def setUp(self):
- MockerTestCase.setUp(self)
- # Replace the write so no actual files
- # get written out...
- self.mock_write = self.mocker.replace("cloudinit.util.write_file",
- passthrough=False)
+ helpers.FilesystemMockingTestCase.setUp(self)
self._log = None
self._log_file = None
self._log_handler = None
def tearDown(self):
- MockerTestCase.tearDown(self)
+ helpers.FilesystemMockingTestCase.tearDown(self)
if self._log_handler and self._log:
self._log.removeHandler(self._log_handler)
@@ -53,13 +50,77 @@ class TestConsumeUserData(MockerTestCase):
self._log.addHandler(self._log_handler)
return log_file
+ def test_merging_cloud_config(self):
+ blob = '''
+#cloud-config
+a: b
+e: f
+run:
+ - b
+ - c
+'''
+ message1 = MIMEBase("text", "cloud-config")
+ message1['Merge-Type'] = 'dict()+list(extend)+str(append)'
+ message1.set_payload(blob)
+
+ blob2 = '''
+#cloud-config
+a: e
+e: g
+run:
+ - stuff
+ - morestuff
+'''
+ message2 = MIMEBase("text", "cloud-config")
+ message2['X-Merge-Type'] = 'dict()+list(extend)+str()'
+ message2.set_payload(blob2)
+
+ blob3 = '''
+#cloud-config
+e:
+ - 1
+ - 2
+ - 3
+p: 1
+'''
+ message3 = MIMEBase("text", "cloud-config")
+ message3['Merge-Type'] = 'dict()+list()+str()'
+ message3.set_payload(blob3)
+
+ messages = [message1, message2, message3]
+
+ paths = c_helpers.Paths({}, ds=FakeDataSource(''))
+ cloud_cfg = handlers.cloud_config.CloudConfigPartHandler(paths)
+
+ new_root = self.makeDir()
+ self.patchUtils(new_root)
+ self.patchOS(new_root)
+ cloud_cfg.handle_part(None, handlers.CONTENT_START, None, None, None,
+ None)
+ for i, m in enumerate(messages):
+ headers = dict(m)
+ fn = "part-%s" % (i + 1)
+ payload = m.get_payload(decode=True)
+ cloud_cfg.handle_part(None, headers['Content-Type'],
+ fn, payload, None, headers)
+ cloud_cfg.handle_part(None, handlers.CONTENT_END, None, None, None,
+ None)
+ contents = util.load_file(paths.get_ipath('cloud_config'))
+ contents = util.load_yaml(contents)
+ self.assertEquals(contents['run'], ['b', 'c', 'stuff', 'morestuff'])
+ self.assertEquals(contents['a'], 'be')
+ self.assertEquals(contents['e'], 'fg')
+ self.assertEquals(contents['p'], 1)
+
def test_unhandled_type_warning(self):
"""Raw text without magic is ignored but shows warning."""
ci = stages.Init()
data = "arbitrary text\n"
ci.datasource = FakeDataSource(data)
- self.mock_write(ci.paths.get_ipath("cloud_config"), "", 0600)
+ mock_write = self.mocker.replace("cloudinit.util.write_file",
+ passthrough=False)
+ mock_write(ci.paths.get_ipath("cloud_config"), "", 0600)
self.mocker.replay()
log_file = self.capture_log(logging.WARNING)
@@ -76,7 +137,9 @@ class TestConsumeUserData(MockerTestCase):
message.set_payload("Just text")
ci.datasource = FakeDataSource(message.as_string())
- self.mock_write(ci.paths.get_ipath("cloud_config"), "", 0600)
+ mock_write = self.mocker.replace("cloudinit.util.write_file",
+ passthrough=False)
+ mock_write(ci.paths.get_ipath("cloud_config"), "", 0600)
self.mocker.replay()
log_file = self.capture_log(logging.WARNING)
@@ -93,8 +156,10 @@ class TestConsumeUserData(MockerTestCase):
ci.datasource = FakeDataSource(script)
outpath = os.path.join(ci.paths.get_ipath_cur("scripts"), "part-001")
- self.mock_write(ci.paths.get_ipath("cloud_config"), "", 0600)
- self.mock_write(outpath, script, 0700)
+ mock_write = self.mocker.replace("cloudinit.util.write_file",
+ passthrough=False)
+ mock_write(ci.paths.get_ipath("cloud_config"), "", 0600)
+ mock_write(outpath, script, 0700)
self.mocker.replay()
log_file = self.capture_log(logging.WARNING)
@@ -111,8 +176,10 @@ class TestConsumeUserData(MockerTestCase):
ci.datasource = FakeDataSource(message.as_string())
outpath = os.path.join(ci.paths.get_ipath_cur("scripts"), "part-001")
- self.mock_write(ci.paths.get_ipath("cloud_config"), "", 0600)
- self.mock_write(outpath, script, 0700)
+ mock_write = self.mocker.replace("cloudinit.util.write_file",
+ passthrough=False)
+ mock_write(ci.paths.get_ipath("cloud_config"), "", 0600)
+ mock_write(outpath, script, 0700)
self.mocker.replay()
log_file = self.capture_log(logging.WARNING)
@@ -129,8 +196,10 @@ class TestConsumeUserData(MockerTestCase):
ci.datasource = FakeDataSource(message.as_string())
outpath = os.path.join(ci.paths.get_ipath_cur("scripts"), "part-001")
- self.mock_write(outpath, script, 0700)
- self.mock_write(ci.paths.get_ipath("cloud_config"), "", 0600)
+ mock_write = self.mocker.replace("cloudinit.util.write_file",
+ passthrough=False)
+ mock_write(outpath, script, 0700)
+ mock_write(ci.paths.get_ipath("cloud_config"), "", 0600)
self.mocker.replay()
log_file = self.capture_log(logging.WARNING)
diff --git a/tests/unittests/test_util.py b/tests/unittests/test_util.py
index 02611581..7ff9a57f 100644
--- a/tests/unittests/test_util.py
+++ b/tests/unittests/test_util.py
@@ -248,4 +248,99 @@ class TestLoadYaml(TestCase):
myobj)
+class TestMountinfoParsing(TestCase):
+ precise_ext4_mountinfo = \
+"""15 20 0:14 / /sys rw,nosuid,nodev,noexec,relatime - sysfs sysfs rw
+16 20 0:3 / /proc rw,nosuid,nodev,noexec,relatime - proc proc rw
+17 20 0:5 / /dev rw,relatime - devtmpfs udev rw,size=16422216k,nr_inodes=4105554,mode=755
+18 17 0:11 / /dev/pts rw,nosuid,noexec,relatime - devpts devpts rw,gid=5,mode=620,ptmxmode=000
+19 20 0:15 / /run rw,nosuid,relatime - tmpfs tmpfs rw,size=6572812k,mode=755
+20 1 252:1 / / rw,relatime - ext4 /dev/mapper/vg0-root rw,errors=remount-ro,data=ordered
+21 15 0:16 / /sys/fs/cgroup rw,relatime - tmpfs cgroup rw,mode=755
+22 15 0:17 / /sys/fs/fuse/connections rw,relatime - fusectl none rw
+23 15 0:6 / /sys/kernel/debug rw,relatime - debugfs none rw
+25 15 0:10 / /sys/kernel/security rw,relatime - securityfs none rw
+26 19 0:19 / /run/lock rw,nosuid,nodev,noexec,relatime - tmpfs none rw,size=5120k
+27 19 0:20 / /run/shm rw,nosuid,nodev,relatime - tmpfs none rw
+28 19 0:21 / /run/user rw,nosuid,nodev,noexec,relatime - tmpfs none rw,size=102400k,mode=755
+24 21 0:18 / /sys/fs/cgroup/cpuset rw,relatime - cgroup cgroup rw,cpuset
+29 21 0:22 / /sys/fs/cgroup/cpu rw,relatime - cgroup cgroup rw,cpu
+30 21 0:23 / /sys/fs/cgroup/cpuacct rw,relatime - cgroup cgroup rw,cpuacct
+31 21 0:24 / /sys/fs/cgroup/memory rw,relatime - cgroup cgroup rw,memory
+32 21 0:25 / /sys/fs/cgroup/devices rw,relatime - cgroup cgroup rw,devices
+33 21 0:26 / /sys/fs/cgroup/freezer rw,relatime - cgroup cgroup rw,freezer
+34 21 0:27 / /sys/fs/cgroup/blkio rw,relatime - cgroup cgroup rw,blkio
+35 21 0:28 / /sys/fs/cgroup/perf_event rw,relatime - cgroup cgroup rw,perf_event
+36 20 9:0 / /boot rw,relatime - ext4 /dev/md0 rw,data=ordered
+37 16 0:29 / /proc/sys/fs/binfmt_misc rw,nosuid,nodev,noexec,relatime - binfmt_misc binfmt_misc rw
+39 28 0:30 / /run/user/foobar/gvfs rw,nosuid,nodev,relatime - fuse.gvfsd-fuse gvfsd-fuse rw,user_id=1000,group_id=1000"""
+
+ raring_btrfs_mountinfo = \
+"""15 20 0:14 / /sys rw,nosuid,nodev,noexec,relatime - sysfs sysfs rw
+16 20 0:3 / /proc rw,nosuid,nodev,noexec,relatime - proc proc rw
+17 20 0:5 / /dev rw,relatime - devtmpfs udev rw,size=865556k,nr_inodes=216389,mode=755
+18 17 0:11 / /dev/pts rw,nosuid,noexec,relatime - devpts devpts rw,gid=5,mode=620,ptmxmode=000
+19 20 0:15 / /run rw,nosuid,relatime - tmpfs tmpfs rw,size=348196k,mode=755
+20 1 0:16 /@ / rw,relatime - btrfs /dev/vda1 rw,compress=lzo,space_cache
+21 15 0:19 / /sys/fs/fuse/connections rw,relatime - fusectl none rw
+22 15 0:6 / /sys/kernel/debug rw,relatime - debugfs none rw
+23 15 0:10 / /sys/kernel/security rw,relatime - securityfs none rw
+24 19 0:20 / /run/lock rw,nosuid,nodev,noexec,relatime - tmpfs none rw,size=5120k
+25 19 0:21 / /run/shm rw,nosuid,nodev,relatime - tmpfs none rw
+26 19 0:22 / /run/user rw,nosuid,nodev,noexec,relatime - tmpfs none rw,size=102400k,mode=755
+27 20 0:16 /@home /home rw,relatime - btrfs /dev/vda1 rw,compress=lzo,space_cache"""
+
+ def test_invalid_mountinfo(self):
+ line = "20 1 252:1 / / rw,relatime - ext4 /dev/mapper/vg0-root rw,errors=remount-ro,data=ordered"
+ elements = line.split()
+ for i in range(len(elements) + 1):
+ lines = [' '.join(elements[0:i])]
+ if i < 10:
+ expected = None
+ else:
+ expected = ('/dev/mapper/vg0-root', 'ext4', '/')
+ self.assertEqual(expected, util.parse_mount_info('/', lines))
+
+ def test_precise_ext4_root(self):
+ lines = TestMountinfoParsing.precise_ext4_mountinfo.splitlines()
+
+ expected = ('/dev/mapper/vg0-root', 'ext4', '/')
+ self.assertEqual(expected, util.parse_mount_info('/', lines))
+ self.assertEqual(expected, util.parse_mount_info('/usr', lines))
+ self.assertEqual(expected, util.parse_mount_info('/usr/bin', lines))
+
+ expected = ('/dev/md0', 'ext4', '/boot')
+ self.assertEqual(expected, util.parse_mount_info('/boot', lines))
+ self.assertEqual(expected, util.parse_mount_info('/boot/grub', lines))
+
+ expected = ('/dev/mapper/vg0-root', 'ext4', '/')
+ self.assertEqual(expected, util.parse_mount_info('/home', lines))
+ self.assertEqual(expected, util.parse_mount_info('/home/me', lines))
+
+ expected = ('tmpfs', 'tmpfs', '/run')
+ self.assertEqual(expected, util.parse_mount_info('/run', lines))
+
+ expected = ('none', 'tmpfs', '/run/lock')
+ self.assertEqual(expected, util.parse_mount_info('/run/lock', lines))
+
+ def test_raring_btrfs_root(self):
+ lines = TestMountinfoParsing.raring_btrfs_mountinfo.splitlines()
+
+ expected = ('/dev/vda1', 'btrfs', '/')
+ self.assertEqual(expected, util.parse_mount_info('/', lines))
+ self.assertEqual(expected, util.parse_mount_info('/usr', lines))
+ self.assertEqual(expected, util.parse_mount_info('/usr/bin', lines))
+ self.assertEqual(expected, util.parse_mount_info('/boot', lines))
+ self.assertEqual(expected, util.parse_mount_info('/boot/grub', lines))
+
+ expected = ('/dev/vda1', 'btrfs', '/home')
+ self.assertEqual(expected, util.parse_mount_info('/home', lines))
+ self.assertEqual(expected, util.parse_mount_info('/home/me', lines))
+
+ expected = ('tmpfs', 'tmpfs', '/run')
+ self.assertEqual(expected, util.parse_mount_info('/run', lines))
+
+ expected = ('none', 'tmpfs', '/run/lock')
+ self.assertEqual(expected, util.parse_mount_info('/run/lock', lines))
+
# vi: ts=4 expandtab
diff --git a/upstart/cloud-init-nonet.conf b/upstart/cloud-init-nonet.conf
index 118ffc1c..36b99fb5 100644
--- a/upstart/cloud-init-nonet.conf
+++ b/upstart/cloud-init-nonet.conf
@@ -10,19 +10,55 @@ task
console output
script
- # /run/network/static-network-up-emitted is written by
- # upstart (via /etc/network/if-up.d/upstart). its presense would
- # indicate that static-network-up has already fired.
- EMITTED="/run/network/static-network-up-emitted"
- [ -e "$EMITTED" -o -e "/var/$EMITTED" ] && exit 0
+ set +e # you cannot trap TERM reliably with 'set -e'
+ SLEEP_CHILD=""
+ static_network_up() {
+ local emitted="/run/network/static-network-up-emitted"
+ # /run/network/static-network-up-emitted is written by
+ # upstart (via /etc/network/if-up.d/upstart). its presense would
+ # indicate that static-network-up has already fired.
+ [ -e "$emitted" -o -e "/var/$emitted" ]
+ }
+ msg() {
+ local uptime="" idle=""
+ if [ -r /proc/uptime ]; then
+ read uptime idle < /proc/uptime
+ fi
+ echo "$UPSTART_JOB${uptime:+[${uptime}]}:" "$1"
+ }
+
+ handle_sigterm() {
+ # if we received sigterm and static networking is up then it probably
+ # came from upstart as a result of 'stop on static-network-up'
+ [ -z "$SLEEP_CHILD" ] || kill $SLEEP_CHILD
+ if static_network_up; then
+ msg "static networking is now up"
+ exit 0
+ fi
+ msg "recieved SIGTERM, networking not up"
+ exit 2
+ }
+
+ dowait() {
+ msg "waiting $1 seconds for network device"
+ sleep "$1" &
+ SLEEP_CHILD=$!
+ wait $SLEEP_CHILD
+ SLEEP_CHILD=""
+ }
+
+ trap handle_sigterm TERM
+
+ # static_network_up already occurred
+ static_network_up && exit 0
+
+ # obj.pkl comes from cloud-init-local (or previous boot and
+ # manual_cache_clean)
[ -f /var/lib/cloud/instance/obj.pkl ] && exit 0
- short=10; long=120;
- sleep ${short}
- echo $UPSTART_JOB "waiting ${long} seconds for a network device."
- sleep ${long}
- echo $UPSTART_JOB "gave up waiting for a network device."
+ dowait 10
+ dowait 120
+ msg "gave up waiting for a network device."
: > /var/lib/cloud/data/no-net
end script
-# EOF