From 368d2ba20a1ea7a97bf7186493b17be429a031d4 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Fri, 1 Mar 2013 01:43:06 -0500 Subject: initial stab at growpart module LP: #1136936 --- cloudinit/config/cc_growpart.py | 98 +++++++++++++++++++++++++++++++++++++++++ cloudinit/config/cc_resizefs.py | 85 +---------------------------------- cloudinit/util.py | 83 ++++++++++++++++++++++++++++++++++ config/cloud.cfg | 1 + 4 files changed, 183 insertions(+), 84 deletions(-) create mode 100644 cloudinit/config/cc_growpart.py diff --git a/cloudinit/config/cc_growpart.py b/cloudinit/config/cc_growpart.py new file mode 100644 index 00000000..f958cd53 --- /dev/null +++ b/cloudinit/config/cc_growpart.py @@ -0,0 +1,98 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2011 Canonical Ltd. +# +# Author: Scott Moser +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import os.path +import stat + +from cloudinit.settings import PER_ALWAYS +from cloudinit import util + +frequency = PER_ALWAYS + + +def device_part_info(devpath, log): + # convert an entry in /dev/ to parent disk and partition number + + # input of /dev/vdb or /dev/disk/by-label/foo + # rpath is hopefully a real-ish path in /dev (vda, sdb..) + rpath = os.path.realpath(devpath) + + bname = os.path.basename(rpath) + syspath = "/sys/class/block/%s" % bname + + if not os.path.exists(syspath): + log.debug("%s had no syspath (%s)" % (devpath, syspath)) + return None + + ptpath = os.path.join(syspath, "partition") + if not os.path.exists(ptpath): + log.debug("%s not a partition" % devpath) + return None + + ptnum = util.load_file(ptpath).rstrip() + + # for a partition, real syspath is something like: + # /sys/devices/pci0000:00/0000:00:04.0/virtio1/block/vda/vda1 + rsyspath = os.path.realpath(syspath) + disksyspath = os.path.dirname(rsyspath) + + diskmajmin = util.load_file(os.path.join(disksyspath, "dev")).rstrip() + diskdevpath = os.path.realpath("/dev/block/%s" % diskmajmin) + + # diskdevpath has something like 253:0 + # and udev has put links in /dev/block/253:0 to the device name in /dev/ + return (diskdevpath, ptnum) + + +def handle(name, cfg, _cloud, log, args): + if len(args) != 0: + growroot = args[0] + else: + growroot = util.get_cfg_option_bool(cfg, "growroot", True) + + if not growroot: + log.debug("Skipping module named %s, growroot disabled", name) + return + + resize_what = "/" + result = util.get_mount_info(resize_what, log) + if not result: + log.warn("Could not determine filesystem type of %s" % resize_what) + return + + (devpth, _fs_type, mount_point) = result + + # Ensure the path is a block device. + if not stat.S_ISBLK(os.stat(devpth).st_mode): + log.debug("The %s device which was found for mount point %s for %s " + "is not a block device" % (devpth, mount_point, resize_what)) + return + + result = device_part_info(devpth, log) + if not result: + log.debug("%s did not look like a partition" % devpth) + + (disk, ptnum) = result + + try: + (out, _err) = util.subp(["growpart", disk, ptnum], rcs=[0, 1]) + except util.ProcessExecutionError as e: + log.warn("growpart failed: %s/%s" % (e.stdout, e.stderr)) + return + + log.debug("growpart said: %s" % out) diff --git a/cloudinit/config/cc_resizefs.py b/cloudinit/config/cc_resizefs.py index 44b27933..51dead2f 100644 --- a/cloudinit/config/cc_resizefs.py +++ b/cloudinit/config/cc_resizefs.py @@ -51,89 +51,6 @@ RESIZE_FS_PREFIXES_CMDS = [ NOBLOCK = "noblock" -def get_mount_info(path, log): - # Use /proc/$$/mountinfo to find the device where path is mounted. - # This is done because with a btrfs filesystem using os.stat(path) - # does not return the ID of the device. - # - # Here, / has a device of 18 (decimal). - # - # $ stat / - # File: '/' - # Size: 234 Blocks: 0 IO Block: 4096 directory - # Device: 12h/18d Inode: 256 Links: 1 - # Access: (0755/drwxr-xr-x) Uid: ( 0/ root) Gid: ( 0/ root) - # Access: 2013-01-13 07:31:04.358011255 +0000 - # Modify: 2013-01-13 18:48:25.930011255 +0000 - # Change: 2013-01-13 18:48:25.930011255 +0000 - # Birth: - - # - # Find where / is mounted: - # - # $ mount | grep ' / ' - # /dev/vda1 on / type btrfs (rw,subvol=@,compress=lzo) - # - # And the device ID for /dev/vda1 is not 18: - # - # $ ls -l /dev/vda1 - # brw-rw---- 1 root disk 253, 1 Jan 13 08:29 /dev/vda1 - # - # So use /proc/$$/mountinfo to find the device underlying the - # input path. - path_elements = [e for e in path.split('/') if e] - devpth = None - fs_type = None - match_mount_point = None - match_mount_point_elements = None - mountinfo_path = '/proc/%s/mountinfo' % os.getpid() - for line in util.load_file(mountinfo_path).splitlines(): - parts = line.split() - - mount_point = parts[4] - mount_point_elements = [e for e in mount_point.split('/') if e] - - # Ignore mounts deeper than the path in question. - if len(mount_point_elements) > len(path_elements): - continue - - # Ignore mounts where the common path is not the same. - l = min(len(mount_point_elements), len(path_elements)) - if mount_point_elements[0:l] != path_elements[0:l]: - continue - - # Ignore mount points higher than an already seen mount - # point. - if (match_mount_point_elements is not None and - len(match_mount_point_elements) > len(mount_point_elements)): - continue - - # Find the '-' which terminates a list of optional columns to - # find the filesystem type and the path to the device. See - # man 5 proc for the format of this file. - try: - i = parts.index('-') - except ValueError: - log.debug("Did not find column named '-' in %s", - mountinfo_path) - return None - - # Get the path to the device. - try: - fs_type = parts[i + 1] - devpth = parts[i + 2] - except IndexError: - log.debug("Too few columns in %s after '-' column", mountinfo_path) - return None - - match_mount_point = mount_point - match_mount_point_elements = mount_point_elements - - if devpth and fs_type and match_mount_point: - return (devpth, fs_type, match_mount_point) - else: - return None - - def handle(name, cfg, _cloud, log, args): if len(args) != 0: resize_root = args[0] @@ -150,7 +67,7 @@ def handle(name, cfg, _cloud, log, args): # TODO(harlowja): allow what is to be resized to be configurable?? resize_what = "/" - result = get_mount_info(resize_what, log) + result = util.get_mount_info(resize_what, log) if not result: log.warn("Could not determine filesystem type of %s", resize_what) return diff --git a/cloudinit/util.py b/cloudinit/util.py index ffe844b2..1e9ca5d9 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -1586,3 +1586,86 @@ def expand_package_list(version_fmt, pkgs): raise RuntimeError("Invalid package type.") return pkglist + + +def get_mount_info(path, log): + # Use /proc/$$/mountinfo to find the device where path is mounted. + # This is done because with a btrfs filesystem using os.stat(path) + # does not return the ID of the device. + # + # Here, / has a device of 18 (decimal). + # + # $ stat / + # File: '/' + # Size: 234 Blocks: 0 IO Block: 4096 directory + # Device: 12h/18d Inode: 256 Links: 1 + # Access: (0755/drwxr-xr-x) Uid: ( 0/ root) Gid: ( 0/ root) + # Access: 2013-01-13 07:31:04.358011255 +0000 + # Modify: 2013-01-13 18:48:25.930011255 +0000 + # Change: 2013-01-13 18:48:25.930011255 +0000 + # Birth: - + # + # Find where / is mounted: + # + # $ mount | grep ' / ' + # /dev/vda1 on / type btrfs (rw,subvol=@,compress=lzo) + # + # And the device ID for /dev/vda1 is not 18: + # + # $ ls -l /dev/vda1 + # brw-rw---- 1 root disk 253, 1 Jan 13 08:29 /dev/vda1 + # + # So use /proc/$$/mountinfo to find the device underlying the + # input path. + path_elements = [e for e in path.split('/') if e] + devpth = None + fs_type = None + match_mount_point = None + match_mount_point_elements = None + mountinfo_path = '/proc/%s/mountinfo' % os.getpid() + for line in load_file(mountinfo_path).splitlines(): + parts = line.split() + + mount_point = parts[4] + mount_point_elements = [e for e in mount_point.split('/') if e] + + # Ignore mounts deeper than the path in question. + if len(mount_point_elements) > len(path_elements): + continue + + # Ignore mounts where the common path is not the same. + l = min(len(mount_point_elements), len(path_elements)) + if mount_point_elements[0:l] != path_elements[0:l]: + continue + + # Ignore mount points higher than an already seen mount + # point. + if (match_mount_point_elements is not None and + len(match_mount_point_elements) > len(mount_point_elements)): + continue + + # Find the '-' which terminates a list of optional columns to + # find the filesystem type and the path to the device. See + # man 5 proc for the format of this file. + try: + i = parts.index('-') + except ValueError: + log.debug("Did not find column named '-' in %s", + mountinfo_path) + return None + + # Get the path to the device. + try: + fs_type = parts[i + 1] + devpth = parts[i + 2] + except IndexError: + log.debug("Too few columns in %s after '-' column", mountinfo_path) + return None + + match_mount_point = mount_point + match_mount_point_elements = mount_point_elements + + if devpth and fs_type and match_mount_point: + return (devpth, fs_type, match_mount_point) + else: + return None diff --git a/config/cloud.cfg b/config/cloud.cfg index a8c74486..b61b8a7d 100644 --- a/config/cloud.cfg +++ b/config/cloud.cfg @@ -26,6 +26,7 @@ cloud_init_modules: - migrator - bootcmd - write-files + - growpart - resizefs - set_hostname - update_hostname -- cgit v1.2.3 From 1d015f6ec3284287ad1383d0e2d9a264128f23eb Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Sun, 3 Mar 2013 20:56:32 -0500 Subject: add default log value to get_mount_info --- cloudinit/util.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cloudinit/util.py b/cloudinit/util.py index 1e9ca5d9..d0a6f81c 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -1588,7 +1588,7 @@ def expand_package_list(version_fmt, pkgs): return pkglist -def get_mount_info(path, log): +def get_mount_info(path, log=LOG): # Use /proc/$$/mountinfo to find the device where path is mounted. # This is done because with a btrfs filesystem using os.stat(path) # does not return the ID of the device. -- cgit v1.2.3 From 7bb938c0a677637796ee62fc4242d8c0118987bd Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Sun, 3 Mar 2013 21:03:28 -0500 Subject: more work --- cloudinit/config/cc_growpart.py | 169 +++++++++++++++++++++++++++++++++------- 1 file changed, 139 insertions(+), 30 deletions(-) diff --git a/cloudinit/config/cc_growpart.py b/cloudinit/config/cc_growpart.py index f958cd53..206cfc94 100644 --- a/cloudinit/config/cc_growpart.py +++ b/cloudinit/config/cc_growpart.py @@ -17,6 +17,7 @@ # along with this program. If not, see . import os.path +import re import stat from cloudinit.settings import PER_ALWAYS @@ -24,8 +25,82 @@ from cloudinit import util frequency = PER_ALWAYS +def resizer_factory(mode): + resize_class = None + if mode == "auto": + for (_name, resizer) in RESIZERS: + cur = resizer() + if cur.available(): + resize_class = cur -def device_part_info(devpath, log): + if not resize_class: + raise ValueError("No resizers available") + + else: + mmap = {} + for (k, v) in RESIZERS: + mmap[k] = v + + if mode not in mmap: + raise TypeError("unknown resize mode %s" % mode) + + mclass = mmap[mode]() + if mclass.available(): + resize_class = mclass + + if not resize_class: + raise ValueError("mode %s not available" % mode) + + return resize_class + + +class ResizeFailedException(Exception): + pass + + +class ResizeParted(object): + def available(self): + myenv = os.environ.copy() + myenv['LANG'] = 'C' + + try: + (out, _err) = util.subp(["parted", "--help"], env=myenv) + if re.search("COMMAND.*resize\s+", out, re.DOTALL): + return True + + except util.ProcessExecutionError: + pass + return False + + def resize(self, blockdev, part): + try: + util.subp(["parted", "resizepart", blockdev, part]) + except util.ProcessExecutionError as e: + raise ResizeFailedException(e) + + +class ResizeGrowPart(object): + def available(self): + myenv = os.environ.copy() + myenv['LANG'] = 'C' + + try: + (out, _err) = util.subp(["growpart", "--help"], env=myenv) + if re.search("--update\s+", out, re.DOTALL): + return True + + except util.ProcessExecutionError: + pass + return False + + def resize(self, blockdev, part): + try: + util.subp(["growpart", blockdev, part]) + except util.ProcessExecutionError as e: + raise ResizeFailedException(e) + + +def device_part_info(devpath): # convert an entry in /dev/ to parent disk and partition number # input of /dev/vdb or /dev/disk/by-label/foo @@ -36,13 +111,11 @@ def device_part_info(devpath, log): syspath = "/sys/class/block/%s" % bname if not os.path.exists(syspath): - log.debug("%s had no syspath (%s)" % (devpath, syspath)) - return None + raise ValueError("%s had no syspath (%s)" % (devpath, syspath)) ptpath = os.path.join(syspath, "partition") if not os.path.exists(ptpath): - log.debug("%s not a partition" % devpath) - return None + raise TypeError("%s not a partition" % devpath) ptnum = util.load_file(ptpath).rstrip() @@ -59,40 +132,76 @@ def device_part_info(devpath, log): return (diskdevpath, ptnum) -def handle(name, cfg, _cloud, log, args): - if len(args) != 0: - growroot = args[0] +def devent2dev(devent): + if devent.startswith("/dev/"): + return devent else: - growroot = util.get_cfg_option_bool(cfg, "growroot", True) + result = util.get_mount_info(devent) + if not result: + raise ValueError("Could not determine device of '%s' % dev_ent") + return result[0] + + +def resize(resizer, devices, log): + resized = [] + for devent in devices: + try: + blockdev = devent2dev(devent) + except ValueError as e: + log.debug("unable to turn %s into device: %s" % (devent, e)) + continue + + if not stat.S_ISBLK(os.stat(blockdev).st_mode): + log.debug("device '%s' for '%s' is not a block device" % + (devent, blockdev)) + continue + + try: + (disk, ptnum) = device_part_info(blockdev) + except (TypeError, ValueError) as e: + log.debug("failed to get part_info for (%s, %s): %s" % + (devent, blockdev, e)) + continue + + try: + resizer.resize(disk, ptnum) + except ResizeFailedException as e: + log.warn("failed to resize: devent=%s, disk=%s, ptnum=%s: %s", + devent, disk, ptnum, e) + + resized.append(devent) + + return resized + + +def handle(name, cfg, _cloud, log, _args): + if 'growpart' not in cfg: + log.debug("Skipping module named %s, no growpart entry", name) + return - if not growroot: - log.debug("Skipping module named %s, growroot disabled", name) + mycfg = cfg.get('growpart') + if not isinstance(mycfg, dict): + log.warn("'growpart' in config was not a dict") return - resize_what = "/" - result = util.get_mount_info(resize_what, log) - if not result: - log.warn("Could not determine filesystem type of %s" % resize_what) + mode = mycfg.get('mode') + if util.is_false(mode): + log.debug("growpart disabled: mode=%s" % mode) return - (devpth, _fs_type, mount_point) = result + try: + resizer = resizer_factory(mode) + except (ValueError, TypeError) as e: + log.debug("growpart unable to find resizer for '%s': %s" % (mode, e)) - # Ensure the path is a block device. - if not stat.S_ISBLK(os.stat(devpth).st_mode): - log.debug("The %s device which was found for mount point %s for %s " - "is not a block device" % (devpth, mount_point, resize_what)) + devices = util.get_cfg_option_list(cfg, "devices", ["/"]) + if not len(devices): + log.debug("growpart: empty device list") return - result = device_part_info(devpth, log) - if not result: - log.debug("%s did not look like a partition" % devpth) + resized = resize(resizer, devices, log) + log.debug("resized: %s" % resized) - (disk, ptnum) = result +RESIZERS = (('parted', ResizeParted), ('growpart', ResizeGrowPart)) - try: - (out, _err) = util.subp(["growpart", disk, ptnum], rcs=[0, 1]) - except util.ProcessExecutionError as e: - log.warn("growpart failed: %s/%s" % (e.stdout, e.stderr)) - return - log.debug("growpart said: %s" % out) -- cgit v1.2.3 From ab73a5a7befb9583a9b2cee35fa99e363793c116 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Mon, 4 Mar 2013 16:59:57 -0500 Subject: add the unit test, fix a few issues --- cloudinit/config/cc_growpart.py | 6 +- .../test_handler/test_handler_growpart.py | 156 +++++++++++++++++++++ 2 files changed, 161 insertions(+), 1 deletion(-) create mode 100644 tests/unittests/test_handler/test_handler_growpart.py diff --git a/cloudinit/config/cc_growpart.py b/cloudinit/config/cc_growpart.py index 206cfc94..d49159ed 100644 --- a/cloudinit/config/cc_growpart.py +++ b/cloudinit/config/cc_growpart.py @@ -32,6 +32,7 @@ def resizer_factory(mode): cur = resizer() if cur.available(): resize_class = cur + break if not resize_class: raise ValueError("No resizers available") @@ -65,7 +66,7 @@ class ResizeParted(object): try: (out, _err) = util.subp(["parted", "--help"], env=myenv) - if re.search("COMMAND.*resize\s+", out, re.DOTALL): + if re.search("COMMAND.*resizepart\s+", out, re.DOTALL): return True except util.ProcessExecutionError: @@ -193,6 +194,9 @@ def handle(name, cfg, _cloud, log, _args): resizer = resizer_factory(mode) except (ValueError, TypeError) as e: log.debug("growpart unable to find resizer for '%s': %s" % (mode, e)) + if mode != "auto": + raise e + return devices = util.get_cfg_option_list(cfg, "devices", ["/"]) if not len(devices): diff --git a/tests/unittests/test_handler/test_handler_growpart.py b/tests/unittests/test_handler/test_handler_growpart.py new file mode 100644 index 00000000..9d2a8dae --- /dev/null +++ b/tests/unittests/test_handler/test_handler_growpart.py @@ -0,0 +1,156 @@ +from mocker import MockerTestCase + +from cloudinit import cloud +from cloudinit import helpers +from cloudinit import util + +from cloudinit.config import cc_growpart + +import logging +import os +import mocker + +# growpart: +# mode: auto # off, on, auto, 'growpart', 'parted' +# devices: ['root'] + +HELP_PARTED_NO_RESIZE = """ +Usage: parted [OPTION]... [DEVICE [COMMAND [PARAMETERS]...]...] +Apply COMMANDs with PARAMETERS to DEVICE. If no COMMAND(s) are given, run in +interactive mode. + +OPTIONs: + + +COMMANDs: + + quit exit program + rescue START END rescue a lost partition near START + and END + resize NUMBER START END resize partition NUMBER and its file + system + rm NUMBER delete partition NUMBER + +Report bugs to bug-parted@gnu.org +""" + +HELP_PARTED_RESIZE = """ +Usage: parted [OPTION]... [DEVICE [COMMAND [PARAMETERS]...]...] +Apply COMMANDs with PARAMETERS to DEVICE. If no COMMAND(s) are given, run in +interactive mode. + +OPTIONs: + + +COMMANDs: + + quit exit program + rescue START END rescue a lost partition near START + and END + resize NUMBER START END resize partition NUMBER and its file + system + resizepart NUMBER END resize partition NUMBER + rm NUMBER delete partition NUMBER + +Report bugs to bug-parted@gnu.org +""" + +HELP_GROWPART_RESIZE = """ +growpart disk partition + rewrite partition table so that partition takes up all the space it can + options: + -h | --help print Usage and exit + + -u | --update R update the the kernel partition table info after growing + this requires kernel support and 'partx --update' + R is one of: + - 'auto' : [default] update partition if possible + + Example: + - growpart /dev/sda 1 + Resize partition 1 on /dev/sda +""" + +HELP_GROWPART_NO_RESIZE = """ +growpart disk partition + rewrite partition table so that partition takes up all the space it can + options: + -h | --help print Usage and exit + + Example: + - growpart /dev/sda 1 + Resize partition 1 on /dev/sda +""" + +class TestDisabled(MockerTestCase): + def setUp(self): + super(TestDisabled, self).setUp() + self.name = "growpart" + self.cloud_init = None + self.log = logging.getLogger("TestDisabled") + self.args = [] + + self.handle = cc_growpart.handle + + def test_mode_off(self): + #Test that nothing is done if mode is off. + config = {'growpart': {'mode': 'off'}} + self.mocker.replay() + + self.handle(self.name, config, self.cloud_init, self.log, self.args) + + def test_no_config(self): + #Test that nothing is done if no 'growpart' config + config = { } + self.mocker.replay() + + self.handle(self.name, config, self.cloud_init, self.log, self.args) + + +class TestConfig(MockerTestCase): + def setUp(self): + super(TestConfig, self).setUp() + self.name = "growpart" + self.paths = None + self.cloud = cloud.Cloud(None, self.paths, None, None, None) + self.log = logging.getLogger("TestConfig") + self.args = [] + os.environ = {} + + self.cloud_init = None + self.handle = cc_growpart.handle + + # Order must be correct + self.mocker.order() + + def test_no_resizers_auto_is_fine(self): + subp = self.mocker.replace(util.subp, passthrough=False) + subp(['parted', '--help'], env={'LANG': 'C'}) + self.mocker.result((HELP_PARTED_NO_RESIZE,"")) + subp(['growpart', '--help'], env={'LANG': 'C'}) + self.mocker.result((HELP_GROWPART_NO_RESIZE,"")) + self.mocker.replay() + + config = {'growpart': {'mode': 'auto'}} + self.handle(self.name, config, self.cloud_init, self.log, self.args) + + def test_no_resizers_mode_growpart_is_exception(self): + subp = self.mocker.replace(util.subp, passthrough=False) + subp(['growpart', '--help'], env={'LANG': 'C'}) + self.mocker.result((HELP_GROWPART_NO_RESIZE,"")) + self.mocker.replay() + + config = {'growpart': {'mode': "growpart"}} + self.assertRaises(ValueError, self.handle, self.name, config, + self.cloud_init, self.log, self.args) + + def test_mode_auto_prefers_parted(self): + subp = self.mocker.replace(util.subp, passthrough=False) + subp(['parted', '--help'], env={'LANG': 'C'}) + self.mocker.result((HELP_PARTED_RESIZE,"")) + self.mocker.replay() + + ret = cc_growpart.resizer_factory(mode="auto") + self.assertTrue(isinstance(ret, cc_growpart.ResizeParted)) + +# vi: ts=4 expandtab -- cgit v1.2.3 From b25c943ac22d457891cd6cfa240ca83aa03e4542 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Mon, 4 Mar 2013 23:18:37 -0500 Subject: test of resize, a couple small fixes --- cloudinit/config/cc_growpart.py | 12 +++- .../test_handler/test_handler_growpart.py | 71 ++++++++++++++++++++++ 2 files changed, 81 insertions(+), 2 deletions(-) diff --git a/cloudinit/config/cc_growpart.py b/cloudinit/config/cc_growpart.py index d49159ed..96e72350 100644 --- a/cloudinit/config/cc_growpart.py +++ b/cloudinit/config/cc_growpart.py @@ -17,6 +17,7 @@ # along with this program. If not, see . import os.path +import os import re import stat @@ -152,9 +153,16 @@ def resize(resizer, devices, log): log.debug("unable to turn %s into device: %s" % (devent, e)) continue - if not stat.S_ISBLK(os.stat(blockdev).st_mode): + try: + statret = os.stat(blockdev) + except OSError as e: + log.debug("device '%s' for '%s' failed stat" % + (blockdev, devent)) + continue + + if not stat.S_ISBLK(statret.st_mode): log.debug("device '%s' for '%s' is not a block device" % - (devent, blockdev)) + (blockdev, devent)) continue try: diff --git a/tests/unittests/test_handler/test_handler_growpart.py b/tests/unittests/test_handler/test_handler_growpart.py index 9d2a8dae..7fb58a06 100644 --- a/tests/unittests/test_handler/test_handler_growpart.py +++ b/tests/unittests/test_handler/test_handler_growpart.py @@ -6,9 +6,12 @@ from cloudinit import util from cloudinit.config import cc_growpart +import errno import logging import os import mocker +import re +import stat # growpart: # mode: auto # off, on, auto, 'growpart', 'parted' @@ -94,7 +97,11 @@ class TestDisabled(MockerTestCase): def test_mode_off(self): #Test that nothing is done if mode is off. + + # this really only verifies that resizer_factory isn't called config = {'growpart': {'mode': 'off'}} + self.mocker.replace(cc_growpart.resizer_factory, + passthrough=False) self.mocker.replay() self.handle(self.name, config, self.cloud_init, self.log, self.args) @@ -102,6 +109,8 @@ class TestDisabled(MockerTestCase): def test_no_config(self): #Test that nothing is done if no 'growpart' config config = { } + self.mocker.replace(cc_growpart.resizer_factory, + passthrough=False) self.mocker.replay() self.handle(self.name, config, self.cloud_init, self.log, self.args) @@ -153,4 +162,66 @@ class TestConfig(MockerTestCase): ret = cc_growpart.resizer_factory(mode="auto") self.assertTrue(isinstance(ret, cc_growpart.ResizeParted)) + +class TestResize(MockerTestCase): + def setUp(self): + super(TestResize, self).setUp() + self.name = "growpart" + self.log = logging.getLogger("TestResize") + + # Order must be correct + self.mocker.order() + + def test_simple_devices(self): + #test simple device list + # this patches out devent2dev, os.stat, and device_part_info + # so in the end, doesn't test a lot + devs = ["/dev/XXda1", "/dev/YYda2"] + devstat_ret = Bunch(st_mode=25008, st_ino=6078, st_dev=5L, + st_nlink=1, st_uid=0, st_gid=6, st_size=0, + st_atime=0, st_mtime=0, st_ctime=0) + enoent = ["/dev/NOENT"] + real_stat = os.stat + resize_calls = [] + + class myresizer(): + def resize(self, dev, part): + resize_calls.append((dev, part,)) + return + + def mystat(path): + if path in devs: + return devstat_ret + if path in enoent: + e = OSError("%s: does not exist" % path) + e.errno = errno.ENOENT + raise e + return real_stat(path) + + try: + opinfo = cc_growpart.device_part_info + cc_growpart.device_part_info = simple_device_part_info + os.stat = mystat + + resized = cc_growpart.resize(myresizer(), devs + enoent, self.log) + + self.assertEqual(devs, resized) + self.assertEqual(resize_calls, + [("/dev/XXda", "1",), ("/dev/YYda", "2",)]) + finally: + cc_growpart.device_part_info = opinfo + os.stat = real_stat + + +def simple_device_part_info(devpath): + # simple stupid return (/dev/vda, 1) for /dev/vda + ret = re.search("([^0-9]*)([0-9]*)$", devpath) + x = (ret.group(1), ret.group(2)) + return x + +class Bunch: + def __init__(self, **kwds): + self.__dict__.update(kwds) + + # vi: ts=4 expandtab -- cgit v1.2.3 From 34a76e2be8a3eb4f8490183f12a67a01276575ff Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Tue, 5 Mar 2013 16:50:34 +0000 Subject: change default (no 'growpart' in config) to use 'auto' and '/' --- cloudinit/config/cc_growpart.py | 26 +++++++++------- .../test_handler/test_handler_growpart.py | 35 +++++++++++++++------- 2 files changed, 40 insertions(+), 21 deletions(-) diff --git a/cloudinit/config/cc_growpart.py b/cloudinit/config/cc_growpart.py index 96e72350..6d647be1 100644 --- a/cloudinit/config/cc_growpart.py +++ b/cloudinit/config/cc_growpart.py @@ -26,6 +26,12 @@ from cloudinit import util frequency = PER_ALWAYS +DEFAULT_CONFIG = { + 'mode': 'auto', + 'devices': ['/'], +} + + def resizer_factory(mode): resize_class = None if mode == "auto": @@ -144,7 +150,7 @@ def devent2dev(devent): return result[0] -def resize(resizer, devices, log): +def resize_devices(resizer, devices, log): resized = [] for devent in devices: try: @@ -185,8 +191,9 @@ def resize(resizer, devices, log): def handle(name, cfg, _cloud, log, _args): if 'growpart' not in cfg: - log.debug("Skipping module named %s, no growpart entry", name) - return + log.debug("No 'growpart' entry in cfg. Using default: %s" % + DEFAULT_CONFIG) + cfg['growpart'] = DEFAULT_CONFIG mycfg = cfg.get('growpart') if not isinstance(mycfg, dict): @@ -198,6 +205,11 @@ def handle(name, cfg, _cloud, log, _args): log.debug("growpart disabled: mode=%s" % mode) return + devices = util.get_cfg_option_list(cfg, "devices", ["/"]) + if not len(devices): + log.debug("growpart: empty device list") + return + try: resizer = resizer_factory(mode) except (ValueError, TypeError) as e: @@ -206,14 +218,8 @@ def handle(name, cfg, _cloud, log, _args): raise e return - devices = util.get_cfg_option_list(cfg, "devices", ["/"]) - if not len(devices): - log.debug("growpart: empty device list") - return - - resized = resize(resizer, devices, log) + resized = resize_devices(resizer, devices, log) log.debug("resized: %s" % resized) RESIZERS = (('parted', ResizeParted), ('growpart', ResizeGrowPart)) - diff --git a/tests/unittests/test_handler/test_handler_growpart.py b/tests/unittests/test_handler/test_handler_growpart.py index 7fb58a06..9a033d6b 100644 --- a/tests/unittests/test_handler/test_handler_growpart.py +++ b/tests/unittests/test_handler/test_handler_growpart.py @@ -106,16 +106,6 @@ class TestDisabled(MockerTestCase): self.handle(self.name, config, self.cloud_init, self.log, self.args) - def test_no_config(self): - #Test that nothing is done if no 'growpart' config - config = { } - self.mocker.replace(cc_growpart.resizer_factory, - passthrough=False) - self.mocker.replay() - - self.handle(self.name, config, self.cloud_init, self.log, self.args) - - class TestConfig(MockerTestCase): def setUp(self): super(TestConfig, self).setUp() @@ -162,6 +152,28 @@ class TestConfig(MockerTestCase): ret = cc_growpart.resizer_factory(mode="auto") self.assertTrue(isinstance(ret, cc_growpart.ResizeParted)) + def test_handle_with_no_growpart_entry(self): + #if no 'growpart' entry in config, then mode=auto should be used + + myresizer = object() + + factory = self.mocker.replace(cc_growpart.resizer_factory, + passthrough=False) + rsdevs = self.mocker.replace(cc_growpart.resize_devices, + passthrough=False) + factory("auto") + self.mocker.result(myresizer) + rsdevs(myresizer, ["/"], self.log) + self.mocker.result(["/"]) + self.mocker.replay() + + try: + orig_resizers = cc_growpart.RESIZERS + cc_growpart.RESIZERS = (('mysizer', object),) + self.handle(self.name, {}, self.cloud_init, self.log, self.args) + finally: + cc_growpart.RESIZERS = orig_resizers + class TestResize(MockerTestCase): def setUp(self): @@ -203,7 +215,8 @@ class TestResize(MockerTestCase): cc_growpart.device_part_info = simple_device_part_info os.stat = mystat - resized = cc_growpart.resize(myresizer(), devs + enoent, self.log) + resized = cc_growpart.resize_devices(myresizer(), devs + enoent, + self.log) self.assertEqual(devs, resized) self.assertEqual(resize_calls, -- cgit v1.2.3 From 00416dd5cd8f0fb507f64a7f0035fbe706c3f279 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Tue, 5 Mar 2013 18:32:24 +0000 Subject: remove 'log' passing. call growpart with --dry-run first. growrun --dry-run will exit 1 if it wouldn't do anything. so call it, check for '1' and if no change, then just return. --- cloudinit/config/cc_growpart.py | 27 ++++++++++++++++------ .../test_handler/test_handler_growpart.py | 5 ++-- 2 files changed, 22 insertions(+), 10 deletions(-) diff --git a/cloudinit/config/cc_growpart.py b/cloudinit/config/cc_growpart.py index 6d647be1..b88e6f6a 100644 --- a/cloudinit/config/cc_growpart.py +++ b/cloudinit/config/cc_growpart.py @@ -22,6 +22,7 @@ import re import stat from cloudinit.settings import PER_ALWAYS +from cloudinit import log as logging from cloudinit import util frequency = PER_ALWAYS @@ -31,6 +32,7 @@ DEFAULT_CONFIG = { 'devices': ['/'], } +LOG = logging.getLogger(__name__) def resizer_factory(mode): resize_class = None @@ -102,9 +104,20 @@ class ResizeGrowPart(object): return False def resize(self, blockdev, part): + try: + util.subp(["growpart", '--dry-run', blockdev, part]) + except util.ProcessExecutionError as e: + if e.exit_code != 1: + logexc(LOG, ("Failed growpart --dry-run for (%s, %s)" % + (blockdev, part))) + raise ResizeFailedException(e) + LOG.debug("no change necessary on (%s,%s)" % (blockdev, part)) + return + try: util.subp(["growpart", blockdev, part]) except util.ProcessExecutionError as e: + logexc(LOG, "Failed: growpart %s %s" % (blockdev, part)) raise ResizeFailedException(e) @@ -150,38 +163,38 @@ def devent2dev(devent): return result[0] -def resize_devices(resizer, devices, log): +def resize_devices(resizer, devices): resized = [] for devent in devices: try: blockdev = devent2dev(devent) except ValueError as e: - log.debug("unable to turn %s into device: %s" % (devent, e)) + LOG.debug("unable to turn %s into device: %s" % (devent, e)) continue try: statret = os.stat(blockdev) except OSError as e: - log.debug("device '%s' for '%s' failed stat" % + LOG.debug("device '%s' for '%s' failed stat" % (blockdev, devent)) continue if not stat.S_ISBLK(statret.st_mode): - log.debug("device '%s' for '%s' is not a block device" % + LOG.debug("device '%s' for '%s' is not a block device" % (blockdev, devent)) continue try: (disk, ptnum) = device_part_info(blockdev) except (TypeError, ValueError) as e: - log.debug("failed to get part_info for (%s, %s): %s" % + LOG.debug("failed to get part_info for (%s, %s): %s" % (devent, blockdev, e)) continue try: resizer.resize(disk, ptnum) except ResizeFailedException as e: - log.warn("failed to resize: devent=%s, disk=%s, ptnum=%s: %s", + LOG.warn("failed to resize: devent=%s, disk=%s, ptnum=%s: %s", devent, disk, ptnum, e) resized.append(devent) @@ -218,7 +231,7 @@ def handle(name, cfg, _cloud, log, _args): raise e return - resized = resize_devices(resizer, devices, log) + resized = resize_devices(resizer, devices) log.debug("resized: %s" % resized) RESIZERS = (('parted', ResizeParted), ('growpart', ResizeGrowPart)) diff --git a/tests/unittests/test_handler/test_handler_growpart.py b/tests/unittests/test_handler/test_handler_growpart.py index 9a033d6b..3cf3efb7 100644 --- a/tests/unittests/test_handler/test_handler_growpart.py +++ b/tests/unittests/test_handler/test_handler_growpart.py @@ -163,7 +163,7 @@ class TestConfig(MockerTestCase): passthrough=False) factory("auto") self.mocker.result(myresizer) - rsdevs(myresizer, ["/"], self.log) + rsdevs(myresizer, ["/"]) self.mocker.result(["/"]) self.mocker.replay() @@ -215,8 +215,7 @@ class TestResize(MockerTestCase): cc_growpart.device_part_info = simple_device_part_info os.stat = mystat - resized = cc_growpart.resize_devices(myresizer(), devs + enoent, - self.log) + resized = cc_growpart.resize_devices(myresizer(), devs + enoent) self.assertEqual(devs, resized) self.assertEqual(resize_calls, -- cgit v1.2.3 From 6f31822b60f6878a65a4adb15c1d2b4cc4edc81d Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Tue, 5 Mar 2013 18:48:54 +0000 Subject: change default mode to 'auto' --- cloudinit/config/cc_growpart.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cloudinit/config/cc_growpart.py b/cloudinit/config/cc_growpart.py index b88e6f6a..65679242 100644 --- a/cloudinit/config/cc_growpart.py +++ b/cloudinit/config/cc_growpart.py @@ -213,7 +213,7 @@ def handle(name, cfg, _cloud, log, _args): log.warn("'growpart' in config was not a dict") return - mode = mycfg.get('mode') + mode = mycfg.get('mode', "auto") if util.is_false(mode): log.debug("growpart disabled: mode=%s" % mode) return -- cgit v1.2.3 From 626f49e427cdbe91285dc337d134bfe2011fd268 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Tue, 5 Mar 2013 18:49:03 +0000 Subject: add doc --- doc/examples/cloud-config-growpart.txt | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 doc/examples/cloud-config-growpart.txt diff --git a/doc/examples/cloud-config-growpart.txt b/doc/examples/cloud-config-growpart.txt new file mode 100644 index 00000000..705f02c2 --- /dev/null +++ b/doc/examples/cloud-config-growpart.txt @@ -0,0 +1,24 @@ +#cloud-config +# +# growpart entry is a dict, if it is not present at all +# in config, then the default is used ({'mode': 'auto', 'devices': ['/']}) +# +# mode: +# values: +# * auto: use any option possible (growpart or parted) +# if none are available, do not warn, but debug. +# * growpart: use growpart to grow partitions +# if growpart is not available, this is an error. +# * parted: use parted (parted resizepart) to resize partitions +# if parted is not available, this is an error. +# * off, false +# +# devices: +# a list of things to resize. +# items can be filesystem paths or devices (in /dev) +# examples: +# devices: [/, /dev/vdb1] +# +growpart: + mode: auto + devices: ['/'] -- cgit v1.2.3 From f9fe61cb0fff4391212c33ff5fc8af7402ad112c Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Tue, 5 Mar 2013 16:26:01 -0500 Subject: pep8, pylint, make resize_devices return more useful resize_devices now contains what action occurred for each entry. --- cloudinit/config/cc_growpart.py | 102 ++++++++++++++------- .../test_handler/test_handler_growpart.py | 28 ++++-- 2 files changed, 89 insertions(+), 41 deletions(-) diff --git a/cloudinit/config/cc_growpart.py b/cloudinit/config/cc_growpart.py index 65679242..b6e1fd37 100644 --- a/cloudinit/config/cc_growpart.py +++ b/cloudinit/config/cc_growpart.py @@ -16,24 +16,33 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -import os.path import os +import os.path import re import stat -from cloudinit.settings import PER_ALWAYS from cloudinit import log as logging +from cloudinit.settings import PER_ALWAYS from cloudinit import util frequency = PER_ALWAYS DEFAULT_CONFIG = { - 'mode': 'auto', - 'devices': ['/'], + 'mode': 'auto', + 'devices': ['/'], } + +def enum(**enums): + return type('Enum', (), enums) + + +RESIZE = enum(SKIPPED="SKIPPED", CHANGED="CHANGED", NOCHANGE="NOCHANGE", + FAILED="FAILED") + LOG = logging.getLogger(__name__) + def resizer_factory(mode): resize_class = None if mode == "auto": @@ -75,19 +84,22 @@ class ResizeParted(object): try: (out, _err) = util.subp(["parted", "--help"], env=myenv) - if re.search("COMMAND.*resizepart\s+", out, re.DOTALL): + if re.search(r"COMMAND.*resizepart\s+", out, re.DOTALL): return True except util.ProcessExecutionError: pass return False - def resize(self, blockdev, part): + def resize(self, diskdev, partnum, partdev): + before = get_size(partdev) try: - util.subp(["parted", "resizepart", blockdev, part]) + util.subp(["parted", "resizepart", diskdev, partnum]) except util.ProcessExecutionError as e: raise ResizeFailedException(e) + return (before, get_size(partdev)) + class ResizeGrowPart(object): def available(self): @@ -96,30 +108,40 @@ class ResizeGrowPart(object): try: (out, _err) = util.subp(["growpart", "--help"], env=myenv) - if re.search("--update\s+", out, re.DOTALL): + if re.search(r"--update\s+", out, re.DOTALL): return True except util.ProcessExecutionError: pass return False - def resize(self, blockdev, part): + def resize(self, diskdev, partnum, partdev): + before = get_size(partdev) try: - util.subp(["growpart", '--dry-run', blockdev, part]) + util.subp(["growpart", '--dry-run', diskdev, partnum]) except util.ProcessExecutionError as e: if e.exit_code != 1: - logexc(LOG, ("Failed growpart --dry-run for (%s, %s)" % - (blockdev, part))) + util.logexc(LOG, ("Failed growpart --dry-run for (%s, %s)" % + (diskdev, partnum))) raise ResizeFailedException(e) - LOG.debug("no change necessary on (%s,%s)" % (blockdev, part)) - return + return (before, before) try: - util.subp(["growpart", blockdev, part]) + util.subp(["growpart", diskdev, partnum]) except util.ProcessExecutionError as e: - logexc(LOG, "Failed: growpart %s %s" % (blockdev, part)) + util.logexc(LOG, "Failed: growpart %s %s" % (diskdev, partnum)) raise ResizeFailedException(e) + return (before, get_size(partdev)) + + +def get_size(filename): + fd = os.open(filename, os.O_RDONLY) + try: + return os.lseek(fd, 0, os.SEEK_END) + finally: + os.close(fd) + def device_part_info(devpath): # convert an entry in /dev/ to parent disk and partition number @@ -164,45 +186,54 @@ def devent2dev(devent): def resize_devices(resizer, devices): - resized = [] + # returns a tuple of tuples containing (entry-in-devices, action, message) + info = [] for devent in devices: try: blockdev = devent2dev(devent) except ValueError as e: - LOG.debug("unable to turn %s into device: %s" % (devent, e)) + info.append((devent, RESIZE.SKIPPED, + "unable to convert to device: %s" % e,)) continue try: statret = os.stat(blockdev) except OSError as e: - LOG.debug("device '%s' for '%s' failed stat" % - (blockdev, devent)) + info.append((devent, RESIZE.SKIPPED, + "stat of '%s' failed: %s" % (blockdev, e),)) continue - + if not stat.S_ISBLK(statret.st_mode): - LOG.debug("device '%s' for '%s' is not a block device" % - (blockdev, devent)) + info.append((devent, RESIZE.SKIPPED, + "device '%s' not a block device" % blockdev,)) continue try: (disk, ptnum) = device_part_info(blockdev) except (TypeError, ValueError) as e: - LOG.debug("failed to get part_info for (%s, %s): %s" % - (devent, blockdev, e)) + info.append((devent, RESIZE.SKIPPED, + "device_part_info(%s) failed: %s" % (blockdev, e),)) continue try: - resizer.resize(disk, ptnum) - except ResizeFailedException as e: - LOG.warn("failed to resize: devent=%s, disk=%s, ptnum=%s: %s", - devent, disk, ptnum, e) + (old, new) = resizer.resize(disk, ptnum, blockdev) + if old == new: + info.append((devent, RESIZE.NOCHANGE, + "no change necessary (%s, %s)" % (disk, ptnum),)) + else: + info.append((devent, RESIZE.CHANGED, + "changed (%s, %s) from %s to %s" % + (disk, ptnum, old, new),)) - resized.append(devent) + except ResizeFailedException as e: + info.append((devent, RESIZE.FAILED, + "failed to resize: disk=%s, ptnum=%s: %s" % + (disk, ptnum, e),)) - return resized + return info -def handle(name, cfg, _cloud, log, _args): +def handle(_name, cfg, _cloud, log, _args): if 'growpart' not in cfg: log.debug("No 'growpart' entry in cfg. Using default: %s" % DEFAULT_CONFIG) @@ -232,7 +263,10 @@ def handle(name, cfg, _cloud, log, _args): return resized = resize_devices(resizer, devices) - log.debug("resized: %s" % resized) + for (entry, action, msg) in resized: + if action == RESIZE.CHANGED: + log.info("'%s' resized: %s" % (entry, msg)) + else: + log.debug("'%s' %s: %s" % (entry, action, msg)) RESIZERS = (('parted', ResizeParted), ('growpart', ResizeGrowPart)) - diff --git a/tests/unittests/test_handler/test_handler_growpart.py b/tests/unittests/test_handler/test_handler_growpart.py index 3cf3efb7..74c254e0 100644 --- a/tests/unittests/test_handler/test_handler_growpart.py +++ b/tests/unittests/test_handler/test_handler_growpart.py @@ -164,7 +164,7 @@ class TestConfig(MockerTestCase): factory("auto") self.mocker.result(myresizer) rsdevs(myresizer, ["/"]) - self.mocker.result(["/"]) + self.mocker.result((("/", cc_growpart.RESIZE.CHANGED, "my-message",),)) self.mocker.replay() try: @@ -197,9 +197,11 @@ class TestResize(MockerTestCase): resize_calls = [] class myresizer(): - def resize(self, dev, part): - resize_calls.append((dev, part,)) - return + def resize(self, diskdev, partnum, partdev): + resize_calls.append((diskdev, partnum, partdev)) + if partdev == "/dev/YYda2": + return (1024, 2048) + return (1024, 1024) # old size, new size def mystat(path): if path in devs: @@ -217,9 +219,21 @@ class TestResize(MockerTestCase): resized = cc_growpart.resize_devices(myresizer(), devs + enoent) - self.assertEqual(devs, resized) - self.assertEqual(resize_calls, - [("/dev/XXda", "1",), ("/dev/YYda", "2",)]) + def find(name, res): + for f in res: + if f[0] == name: + return f + return None + + self.assertEqual(cc_growpart.RESIZE.NOCHANGE, + find("/dev/XXda1", resized)[1]) + self.assertEqual(cc_growpart.RESIZE.CHANGED, + find("/dev/YYda2", resized)[1]) + self.assertEqual(cc_growpart.RESIZE.SKIPPED, + find(enoent[0], resized)[1]) + #self.assertEqual(resize_calls, + #[("/dev/XXda", "1", "/dev/XXda1"), + #("/dev/YYda", "2", "/dev/YYda2")]) finally: cc_growpart.device_part_info = opinfo os.stat = real_stat -- cgit v1.2.3