From e76fea0e88ecc361eb7e301c0916a89cb4505c24 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Tue, 30 Sep 2014 16:01:00 -0400 Subject: add code for setting up swap file --- cloudinit/config/cc_mounts.py | 100 ++++++++++++++++++++++++++++++++++++++++++ cloudinit/util.py | 21 +++++++++ 2 files changed, 121 insertions(+) diff --git a/cloudinit/config/cc_mounts.py b/cloudinit/config/cc_mounts.py index ba1303d1..7af73c71 100644 --- a/cloudinit/config/cc_mounts.py +++ b/cloudinit/config/cc_mounts.py @@ -75,6 +75,92 @@ def sanitize_devname(startname, transformer, log): return devnode_for_dev_part(blockdev, part) +def suggested_swapsize(memsize=None, maxsize=None, fsys=None): + # make a suggestion on the size of swap for this system. + if memsize is None: + memsize = util.read_meminfo()['total'] + + GB = 2 ** 30 + sugg_max = 8 * GB + + if fsys is None and maxsize is None: + # set max to 8GB default if no filesystem given + maxsize = sugg_max + elif fsys: + statvfs = os.statvfs(fsys) + avail = statvfs.f_frsize * statvfs.f_bfree + + if maxsize is None: + # set to 25% of filesystem space + maxsize = min(int(avail / 4), sugg_max) + elif maxsize > ((avail * .9)): + # set to 90% of available disk space + maxsize = int(avail * .9) + + formulas = [ + # < 1G: swap = double memory + (1 * GB, lambda x: x * 2), + # < 2G: swap = 2G + (2 * GB, lambda x: 2 * GB), + # < 4G: swap = memory + (4 * GB, lambda x: x), + # < 16G: 4G + (16 * GB, lambda x: 4 * GB), + # < 64G: 1/2 M up to max + (64 * GB, lambda x: x / 2), + ] + + size = None + for top, func in formulas: + if memsize <= top: + size = min(func(memsize), maxsize) + # if less than 1/2 memory and not much, return 0 + if size < (memsize / 2) and size < 4 * GB: + return 0 + return size + + return maxsize + + +def setup_swapfile(fname, size=None, maxsize=None): + """ + fname: full path string of filename to setup + size: the size to create. set to "auto" for recommended + maxsize: the maximum size + """ + tdir = os.path.dirname(fname) + if str(size).lower() == "auto": + try: + memsize = util.read_meminfo()['total'] + except IOError as e: + LOG.debug("Not creating swap. failed to read meminfo") + return + + util.ensure_dir(tdir) + size = suggested_swapsize(fsys=tdir, maxsize=maxsize, + memsize=memsize) + + if not size: + LOG.debug("Not creating swap: suggested size was 0") + return + + mbsize = str(int(size / (2 ** 20))) + msg = "creating swap file '%s' of %sMB" % (fname, mbsize) + try: + util.ensure_dir(tdir) + util.log_time(LOG.debug, msg, func=util.subp, + args=[['sh', '-c', + ('rm -f "$1" && umask 0066 && ' + 'dd if=/dev/zero "of=$1" bs=1M "count=$2" && ' + 'mkswap "$1" || { r=$?; rm -f "$1"; exit $r; }'), + 'setup_swap', fname, mbsize]]) + + except Exception as e: + raise IOError("Failed %s: %s" % (msg, e)) + + return fname, size + + def handle(_name, cfg, cloud, log, _args): # fs_spec, fs_file, fs_vfstype, fs_mntops, fs-freq, fs_passno defvals = [None, None, "auto", "defaults,nobootwait", "0", "2"] @@ -162,6 +248,20 @@ def handle(_name, cfg, cloud, log, _args): else: actlist.append(x) + swapcfg = cfg.get('swap', {}) + swapfile = swapcfg.get('filename', '/swap.img') + if swapcfg.get('size') and swapfile: + try: + sret = setup_swapfile(fpath=swapfile, + size=swapcfg.get('size'), + maxsize=swapcfg.get('maxsize')) + if sret is not None: + actlist.append([sret[0], "none", "swap", "sw", "0", "0"]) + except Exception as e: + log.warn("failed to setup swap: %s", e) + else: + log.debug("no swap to setup") + if len(actlist) == 0: log.debug("No modifications to fstab needed.") return diff --git a/cloudinit/util.py b/cloudinit/util.py index 76e91951..349d0155 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -1957,3 +1957,24 @@ def pathprefix2dict(base, required=None, optional=None, delim=os.path.sep): raise ValueError("Missing required files: %s", ','.join(missing)) return ret + + +def read_meminfo(meminfo="/proc/meminfo", raw=False): + # read a /proc/meminfo style file and return + # a dict with 'total', 'free', and 'available' + mpliers = {'kB': 2**10, 'mB': 2 ** 20, 'B': 1, 'gB': 2 ** 30} + kmap = {'MemTotal:': 'total', 'MemFree:': 'free', + 'MemAvailable:': 'available'} + ret = {} + for line in load_file(meminfo).splitlines(): + try: + key, value, unit = line.split() + except ValueError: + key, value = line.split() + unit = 'B' + if raw: + ret[key] = int(value) * mpliers[unit] + elif key in kmap: + ret[kmap[key]] = int(value) * mpliers[unit] + + return ret -- cgit v1.2.3 From 9e645c06677ef146ab5dd36ce6e4be2329a1202b Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Tue, 30 Sep 2014 16:24:38 -0400 Subject: add doc --- doc/examples/cloud-config-mount-points.txt | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/doc/examples/cloud-config-mount-points.txt b/doc/examples/cloud-config-mount-points.txt index 416006db..3b45b47f 100644 --- a/doc/examples/cloud-config-mount-points.txt +++ b/doc/examples/cloud-config-mount-points.txt @@ -37,3 +37,10 @@ mounts: # complete. This must be an array, and must have 7 fields. mount_default_fields: [ None, None, "auto", "defaults,nobootwait", "0", "2" ] + +# swap can also be set up by the 'mounts' module +# default is to not create any swap files, because 'size' is set to 0 +swap: + filename: /swap.img + size: "auto" or size in bytes + maxsize: size in bytes -- cgit v1.2.3 From 9736b260434af860c6ec81776f4278640f1fa9be Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Tue, 30 Sep 2014 16:24:54 -0400 Subject: support human2bytes, separate handling out to method --- cloudinit/config/cc_mounts.py | 43 ++++++++++++++++++++++++++++++------------- cloudinit/util.py | 29 +++++++++++++++++++++++++++++ 2 files changed, 59 insertions(+), 13 deletions(-) diff --git a/cloudinit/config/cc_mounts.py b/cloudinit/config/cc_mounts.py index 7af73c71..b9aa9a12 100644 --- a/cloudinit/config/cc_mounts.py +++ b/cloudinit/config/cc_mounts.py @@ -128,6 +128,7 @@ def setup_swapfile(fname, size=None, maxsize=None): size: the size to create. set to "auto" for recommended maxsize: the maximum size """ + print("fname: %s, size: %s maxsize: %s" % (fname, size, maxsize)) tdir = os.path.dirname(fname) if str(size).lower() == "auto": try: @@ -161,6 +162,32 @@ def setup_swapfile(fname, size=None, maxsize=None): return fname, size +def handle_swapcfg(swapcfg): + """handle the swap config, calling setup_swap if necessary. + return None or (filename, size) + """ + fname = swapcfg.get('filename', '/swap.img') + size = swapcfg.get('size', 0) + maxsize = swapcfg.get('maxsize', 0) + + if not (size and fname): + LOG.debug("no need to setup swap") + return + + try: + if isinstance(size, str) and size != "auto": + size = util.human2bytes(size) + if isinstance(maxsize, str): + maxsize = util.human2bytes(maxsize) + return setup_swapfile(fname=fname, size=size, maxsize=maxsize) + + except Exception as e: + LOG.warn("failed to setup swap: %s", e) + + return None + + + def handle(_name, cfg, cloud, log, _args): # fs_spec, fs_file, fs_vfstype, fs_mntops, fs-freq, fs_passno defvals = [None, None, "auto", "defaults,nobootwait", "0", "2"] @@ -248,19 +275,9 @@ def handle(_name, cfg, cloud, log, _args): else: actlist.append(x) - swapcfg = cfg.get('swap', {}) - swapfile = swapcfg.get('filename', '/swap.img') - if swapcfg.get('size') and swapfile: - try: - sret = setup_swapfile(fpath=swapfile, - size=swapcfg.get('size'), - maxsize=swapcfg.get('maxsize')) - if sret is not None: - actlist.append([sret[0], "none", "swap", "sw", "0", "0"]) - except Exception as e: - log.warn("failed to setup swap: %s", e) - else: - log.debug("no swap to setup") + swapret = handle_swapcfg(cfg.get('swap')) + if swapret: + actlist.append([swapret[0], "none", "swap", "sw", "0", "0"]) if len(actlist) == 0: log.debug("No modifications to fstab needed.") diff --git a/cloudinit/util.py b/cloudinit/util.py index 349d0155..f236d0bf 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -1978,3 +1978,32 @@ def read_meminfo(meminfo="/proc/meminfo", raw=False): ret[kmap[key]] = int(value) * mpliers[unit] return ret + + +def human2bytes(size): + """Convert human string or integer to size in bytes + 10M => 10485760 + .5G => 536870912 + """ + size_in = size + if size.endswith("B"): + size = size[:-1] + + mpliers = {'B': 1, 'K': 2 ** 10, 'M': 2 ** 20, 'G': 2 ** 30, 'T': 2 ** 40} + + num = size + mplier = 'B' + for m in mpliers: + if size.endswith(m): + mplier = m + num = size[0:-len(m)] + + try: + num = float(num) + except ValueError: + raise ValueError("'%s' is not valid input." % size_in) + + if num < 0: + raise ValueError("'%s': cannot be negative" % size_in) + + return int(num * mpliers[mplier]) -- cgit v1.2.3 From 96a5532ed39ace7ec7b80978387e26c9b203200f Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Wed, 1 Oct 2014 13:59:13 -0400 Subject: maxsize must be 'None' otherwise it is zero, and no swap allowed --- cloudinit/config/cc_mounts.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cloudinit/config/cc_mounts.py b/cloudinit/config/cc_mounts.py index b9aa9a12..ded0f413 100644 --- a/cloudinit/config/cc_mounts.py +++ b/cloudinit/config/cc_mounts.py @@ -168,7 +168,7 @@ def handle_swapcfg(swapcfg): """ fname = swapcfg.get('filename', '/swap.img') size = swapcfg.get('size', 0) - maxsize = swapcfg.get('maxsize', 0) + maxsize = swapcfg.get('maxsize', None) if not (size and fname): LOG.debug("no need to setup swap") -- cgit v1.2.3 From 4fb4b31a889f9f6c32290b4fde99edbcd30a3660 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Wed, 1 Oct 2014 14:28:04 -0400 Subject: add debug statement for recommended --- cloudinit/config/cc_mounts.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/cloudinit/config/cc_mounts.py b/cloudinit/config/cc_mounts.py index ded0f413..db09d2fb 100644 --- a/cloudinit/config/cc_mounts.py +++ b/cloudinit/config/cc_mounts.py @@ -116,10 +116,15 @@ def suggested_swapsize(memsize=None, maxsize=None, fsys=None): size = min(func(memsize), maxsize) # if less than 1/2 memory and not much, return 0 if size < (memsize / 2) and size < 4 * GB: - return 0 - return size + size = 0 + break + break - return maxsize + if size is not None: + size = maxsize + LOG.debug("suggest %s for %s memory with %s available disk and max=%s", + size, memsize, avail, maxsize) + return size def setup_swapfile(fname, size=None, maxsize=None): -- cgit v1.2.3 From f3c8ec58cb2a2d1bce73bec08bae08e16f1b1f99 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Wed, 1 Oct 2014 14:58:14 -0400 Subject: nice log message --- cloudinit/config/cc_mounts.py | 24 +++++++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/cloudinit/config/cc_mounts.py b/cloudinit/config/cc_mounts.py index db09d2fb..2503ed98 100644 --- a/cloudinit/config/cc_mounts.py +++ b/cloudinit/config/cc_mounts.py @@ -82,6 +82,9 @@ def suggested_swapsize(memsize=None, maxsize=None, fsys=None): GB = 2 ** 30 sugg_max = 8 * GB + max_in = maxsize + + info = {'avail': 'na', 'max_in': maxsize, 'mem': memsize} if fsys is None and maxsize is None: # set max to 8GB default if no filesystem given @@ -89,6 +92,7 @@ def suggested_swapsize(memsize=None, maxsize=None, fsys=None): elif fsys: statvfs = os.statvfs(fsys) avail = statvfs.f_frsize * statvfs.f_bfree + info['avail'] = avail if maxsize is None: # set to 25% of filesystem space @@ -96,6 +100,10 @@ def suggested_swapsize(memsize=None, maxsize=None, fsys=None): elif maxsize > ((avail * .9)): # set to 90% of available disk space maxsize = int(avail * .9) + elif maxsize is None: + maxsize = sugg_max + + info['max'] = maxsize formulas = [ # < 1G: swap = double memory @@ -122,8 +130,19 @@ def suggested_swapsize(memsize=None, maxsize=None, fsys=None): if size is not None: size = maxsize - LOG.debug("suggest %s for %s memory with %s available disk and max=%s", - size, memsize, avail, maxsize) + + info['size'] = size + + MB = 2 ** 20 + pinfo = {} + for k, v in info.items(): + if isinstance(v, int): + pinfo[k] = "%s MB" % (v / MB) + else: + pinfo[k] = v + + LOG.debug("suggest %(size)s swap for %(mem)s memory with '%(avail)s'" + " disk given max=%(max_in)s [max=%(max)s]'" % pinfo) return size @@ -192,7 +211,6 @@ def handle_swapcfg(swapcfg): return None - def handle(_name, cfg, cloud, log, _args): # fs_spec, fs_file, fs_vfstype, fs_mntops, fs-freq, fs_passno defvals = [None, None, "auto", "defaults,nobootwait", "0", "2"] -- cgit v1.2.3 From 615b484218ef3510f97a5627a996a068b9766f94 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Wed, 1 Oct 2014 15:09:12 -0400 Subject: handle non-dictionary 'swap' input --- cloudinit/config/cc_mounts.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/cloudinit/config/cc_mounts.py b/cloudinit/config/cc_mounts.py index 2503ed98..f0571c52 100644 --- a/cloudinit/config/cc_mounts.py +++ b/cloudinit/config/cc_mounts.py @@ -190,6 +190,10 @@ def handle_swapcfg(swapcfg): """handle the swap config, calling setup_swap if necessary. return None or (filename, size) """ + if not isinstance(swapcfg, dict): + LOG.warn("input for swap config was not a dict.") + return None + fname = swapcfg.get('filename', '/swap.img') size = swapcfg.get('size', 0) maxsize = swapcfg.get('maxsize', None) @@ -298,7 +302,7 @@ def handle(_name, cfg, cloud, log, _args): else: actlist.append(x) - swapret = handle_swapcfg(cfg.get('swap')) + swapret = handle_swapcfg(cfg.get('swap'), {}) if swapret: actlist.append([swapret[0], "none", "swap", "sw", "0", "0"]) -- cgit v1.2.3 From 31f415e3582a912c2172d8c450f14fd35c43f0b6 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Wed, 1 Oct 2014 15:09:36 -0400 Subject: pyflakes --- cloudinit/config/cc_mounts.py | 1 - 1 file changed, 1 deletion(-) diff --git a/cloudinit/config/cc_mounts.py b/cloudinit/config/cc_mounts.py index f0571c52..0c3ce15e 100644 --- a/cloudinit/config/cc_mounts.py +++ b/cloudinit/config/cc_mounts.py @@ -82,7 +82,6 @@ def suggested_swapsize(memsize=None, maxsize=None, fsys=None): GB = 2 ** 30 sugg_max = 8 * GB - max_in = maxsize info = {'avail': 'na', 'max_in': maxsize, 'mem': memsize} -- cgit v1.2.3 From c8b3a21b4e4f5b50767c5df3641a11ad02975e4d Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Wed, 1 Oct 2014 15:29:45 -0400 Subject: check for existing file in /proc/swaps to be safe. --- cloudinit/config/cc_mounts.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/cloudinit/config/cc_mounts.py b/cloudinit/config/cc_mounts.py index 0c3ce15e..33fe4ddc 100644 --- a/cloudinit/config/cc_mounts.py +++ b/cloudinit/config/cc_mounts.py @@ -201,6 +201,20 @@ def handle_swapcfg(swapcfg): LOG.debug("no need to setup swap") return + if os.path.exists(fname): + if not os.path.exists("/proc/swaps"): + LOG.debug("swap file %s existed. no /proc/swaps. Being safe.", + fname) + return + try: + for line in util.load_file("/proc/swaps").splitlines(): + if line.startswith(fname + " "): + LOG.debug("swap file %s already used", fname) + return + except: + LOG.warn("swap file %s existed. Error reading /proc/swaps", fname) + return + try: if isinstance(size, str) and size != "auto": size = util.human2bytes(size) -- cgit v1.2.3 From d5a11b041823ec17e0256a30de580b8f716030cb Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Wed, 1 Oct 2014 15:32:54 -0400 Subject: fix --- cloudinit/config/cc_mounts.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cloudinit/config/cc_mounts.py b/cloudinit/config/cc_mounts.py index 33fe4ddc..186d6848 100644 --- a/cloudinit/config/cc_mounts.py +++ b/cloudinit/config/cc_mounts.py @@ -315,7 +315,7 @@ def handle(_name, cfg, cloud, log, _args): else: actlist.append(x) - swapret = handle_swapcfg(cfg.get('swap'), {}) + swapret = handle_swapcfg(cfg.get('swap', {})) if swapret: actlist.append([swapret[0], "none", "swap", "sw", "0", "0"]) -- cgit v1.2.3 From 93e5b85dd980ba871155ec30f7cadae2723ab128 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Wed, 1 Oct 2014 15:34:18 -0400 Subject: remove debug print --- cloudinit/config/cc_mounts.py | 1 - 1 file changed, 1 deletion(-) diff --git a/cloudinit/config/cc_mounts.py b/cloudinit/config/cc_mounts.py index 186d6848..24c5d24d 100644 --- a/cloudinit/config/cc_mounts.py +++ b/cloudinit/config/cc_mounts.py @@ -151,7 +151,6 @@ def setup_swapfile(fname, size=None, maxsize=None): size: the size to create. set to "auto" for recommended maxsize: the maximum size """ - print("fname: %s, size: %s maxsize: %s" % (fname, size, maxsize)) tdir = os.path.dirname(fname) if str(size).lower() == "auto": try: -- cgit v1.2.3 From a451caaf7128f8415ec33ab9aebcad61243d2dc8 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Wed, 1 Oct 2014 15:37:51 -0400 Subject: message clean up --- cloudinit/config/cc_mounts.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cloudinit/config/cc_mounts.py b/cloudinit/config/cc_mounts.py index 24c5d24d..e0d047d6 100644 --- a/cloudinit/config/cc_mounts.py +++ b/cloudinit/config/cc_mounts.py @@ -208,8 +208,9 @@ def handle_swapcfg(swapcfg): try: for line in util.load_file("/proc/swaps").splitlines(): if line.startswith(fname + " "): - LOG.debug("swap file %s already used", fname) + LOG.debug("swap file %s already in use.", fname) return + LOG.debug("swap file %s existed, but not in /proc/swaps", fname) except: LOG.warn("swap file %s existed. Error reading /proc/swaps", fname) return -- cgit v1.2.3 From 5bb6a49f66c76d9accb912a5612c970a52f5b964 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Wed, 1 Oct 2014 15:58:30 -0400 Subject: return only the filename, as it might be all known --- cloudinit/config/cc_mounts.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/cloudinit/config/cc_mounts.py b/cloudinit/config/cc_mounts.py index e0d047d6..1cb1e839 100644 --- a/cloudinit/config/cc_mounts.py +++ b/cloudinit/config/cc_mounts.py @@ -181,7 +181,7 @@ def setup_swapfile(fname, size=None, maxsize=None): except Exception as e: raise IOError("Failed %s: %s" % (msg, e)) - return fname, size + return fname def handle_swapcfg(swapcfg): @@ -204,16 +204,16 @@ def handle_swapcfg(swapcfg): if not os.path.exists("/proc/swaps"): LOG.debug("swap file %s existed. no /proc/swaps. Being safe.", fname) - return + return fname try: for line in util.load_file("/proc/swaps").splitlines(): if line.startswith(fname + " "): LOG.debug("swap file %s already in use.", fname) - return + return fname LOG.debug("swap file %s existed, but not in /proc/swaps", fname) except: LOG.warn("swap file %s existed. Error reading /proc/swaps", fname) - return + return fname try: if isinstance(size, str) and size != "auto": @@ -317,7 +317,7 @@ def handle(_name, cfg, cloud, log, _args): swapret = handle_swapcfg(cfg.get('swap', {})) if swapret: - actlist.append([swapret[0], "none", "swap", "sw", "0", "0"]) + actlist.append([swapret, "none", "swap", "sw", "0", "0"]) if len(actlist) == 0: log.debug("No modifications to fstab needed.") -- cgit v1.2.3