From acc25d8d7d603313059ac35b4253b504efc560a9 Mon Sep 17 00:00:00 2001 From: "Jason Zions (MSFT)" Date: Wed, 8 May 2019 22:47:07 +0000 Subject: cc_mounts: check if mount -a on no-change fstab path Under some circumstances, cc_disk_setup may reformat volumes which already appear in /etc/fstab (e.g. Azure ephemeral drive is reformatted from NTFS to ext4 after service-heal). Normally, cc_mounts only calls mount -a if it altered /etc/fstab. With this change cc_mounts will read /proc/mounts and verify if configured mounts are already mounted and if not raise flag to request a mount -a. This handles the case where no changes to fstab occur but a mount -a is required due to change in underlying device which prevented the .mount unit from running until after disk was reformatted. LP: #1825596 --- .../unittests/test_handler/test_handler_mounts.py | 30 +++++++++++++++++++++- 1 file changed, 29 insertions(+), 1 deletion(-) (limited to 'tests/unittests/test_handler/test_handler_mounts.py') diff --git a/tests/unittests/test_handler/test_handler_mounts.py b/tests/unittests/test_handler/test_handler_mounts.py index 8fea6c2a..0fb160be 100644 --- a/tests/unittests/test_handler/test_handler_mounts.py +++ b/tests/unittests/test_handler/test_handler_mounts.py @@ -154,7 +154,15 @@ class TestFstabHandling(test_helpers.FilesystemMockingTestCase): return_value=True) self.add_patch('cloudinit.config.cc_mounts.util.subp', - 'mock_util_subp') + 'm_util_subp') + + self.add_patch('cloudinit.config.cc_mounts.util.mounts', + 'mock_util_mounts', + return_value={ + '/dev/sda1': {'fstype': 'ext4', + 'mountpoint': '/', + 'opts': 'rw,relatime,discard' + }}) self.mock_cloud = mock.Mock() self.mock_log = mock.Mock() @@ -230,4 +238,24 @@ class TestFstabHandling(test_helpers.FilesystemMockingTestCase): fstab_new_content = fd.read() self.assertEqual(fstab_expected_content, fstab_new_content) + def test_no_change_fstab_sets_needs_mount_all(self): + '''verify unchanged fstab entries are mounted if not call mount -a''' + fstab_original_content = ( + 'LABEL=cloudimg-rootfs / ext4 defaults 0 0\n' + 'LABEL=UEFI /boot/efi vfat defaults 0 0\n' + '/dev/vdb /mnt auto defaults,noexec,comment=cloudconfig 0 2\n' + ) + fstab_expected_content = fstab_original_content + cc = {'mounts': [ + ['/dev/vdb', '/mnt', 'auto', 'defaults,noexec']]} + with open(cc_mounts.FSTAB_PATH, 'w') as fd: + fd.write(fstab_original_content) + with open(cc_mounts.FSTAB_PATH, 'r') as fd: + fstab_new_content = fd.read() + self.assertEqual(fstab_expected_content, fstab_new_content) + cc_mounts.handle(None, cc, self.mock_cloud, self.mock_log, []) + self.m_util_subp.assert_has_calls([ + mock.call(['mount', '-a']), + mock.call(['systemctl', 'daemon-reload'])]) + # vi: ts=4 expandtab -- cgit v1.2.3 From 6603706eec1c39d9d591c8ffa0ef7171b74d84d6 Mon Sep 17 00:00:00 2001 From: Eduardo Otubo Date: Thu, 23 Jan 2020 17:41:48 +0100 Subject: Do not use fallocate in swap file creation on xfs. (#70) When creating a swap file on an xfs filesystem, fallocate cannot be used. Doing so results in failure of swapon and a message like: swapon: swapfile has holes The solution here is to maintain a list (currently containing only XFS) of filesystems where fallocate cannot be used. The, on those fileystems use the slower but functional 'dd' method. Signed-off-by: Eduardo Otubo Co-authored-by: Adam Dobrawy Co-authored-by: Scott Moser Co-authored-by: Daniel Watkins LP: #1781781 --- cloudinit/config/cc_mounts.py | 67 ++++++++++++++++------ .../unittests/test_handler/test_handler_mounts.py | 12 ++++ 2 files changed, 62 insertions(+), 17 deletions(-) (limited to 'tests/unittests/test_handler/test_handler_mounts.py') diff --git a/cloudinit/config/cc_mounts.py b/cloudinit/config/cc_mounts.py index c741c746..4293844c 100644 --- a/cloudinit/config/cc_mounts.py +++ b/cloudinit/config/cc_mounts.py @@ -223,13 +223,58 @@ def suggested_swapsize(memsize=None, maxsize=None, fsys=None): return size +def create_swapfile(fname, size): + """Size is in MiB.""" + + errmsg = "Failed to create swapfile '%s' of size %dMB via %s: %s" + + def create_swap(fname, size, method): + LOG.debug("Creating swapfile in '%s' on fstype '%s' using '%s'", + fname, fstype, method) + + if method == "fallocate": + cmd = ['fallocate', '-l', '%dM' % size, fname] + elif method == "dd": + cmd = ['dd', 'if=/dev/zero', 'of=%s' % fname, 'bs=1M', + 'count=%d' % size] + + try: + util.subp(cmd, capture=True) + except util.ProcessExecutionError as e: + LOG.warning(errmsg, fname, size, method, e) + util.del_file(fname) + + swap_dir = os.path.dirname(fname) + util.ensure_dir(swap_dir) + + fstype = util.get_mount_info(swap_dir)[1] + + if fstype in ("xfs", "btrfs"): + create_swap(fname, size, "dd") + else: + try: + create_swap(fname, size, "fallocate") + except util.ProcessExecutionError as e: + LOG.warning(errmsg, fname, size, "dd", e) + LOG.warning("Will attempt with dd.") + create_swap(fname, size, "dd") + + util.chmod(fname, 0o600) + try: + util.subp(['mkswap', fname]) + except util.ProcessExecutionError: + util.del_file(fname) + raise + + def setup_swapfile(fname, size=None, maxsize=None): """ fname: full path string of filename to setup size: the size to create. set to "auto" for recommended maxsize: the maximum size """ - tdir = os.path.dirname(fname) + swap_dir = os.path.dirname(fname) + mibsize = str(int(size / (2 ** 20))) if str(size).lower() == "auto": try: memsize = util.read_meminfo()['total'] @@ -237,28 +282,16 @@ def setup_swapfile(fname, size=None, maxsize=None): LOG.debug("Not creating swap: failed to read meminfo") return - util.ensure_dir(tdir) - size = suggested_swapsize(fsys=tdir, maxsize=maxsize, + util.ensure_dir(swap_dir) + size = suggested_swapsize(fsys=swap_dir, maxsize=maxsize, memsize=memsize) if not size: LOG.debug("Not creating swap: suggested size was 0") return - mbsize = str(int(size / (2 ** 20))) - msg = "creating swap file '%s' of %sMB" % (fname, mbsize) - try: - util.ensure_dir(tdir) - util.log_time(LOG.debug, msg, func=util.subp, - args=[['sh', '-c', - ('rm -f "$1" && umask 0066 && ' - '{ fallocate -l "${2}M" "$1" || ' - 'dd if=/dev/zero "of=$1" bs=1M "count=$2"; } && ' - 'mkswap "$1" || { r=$?; rm -f "$1"; exit $r; }'), - 'setup_swap', fname, mbsize]]) - - except Exception as e: - raise IOError("Failed %s: %s" % (msg, e)) + util.log_time(LOG.debug, msg="Setting up swap file", func=create_swapfile, + args=[fname, mibsize]) return fname diff --git a/tests/unittests/test_handler/test_handler_mounts.py b/tests/unittests/test_handler/test_handler_mounts.py index 0fb160be..7bcefa0a 100644 --- a/tests/unittests/test_handler/test_handler_mounts.py +++ b/tests/unittests/test_handler/test_handler_mounts.py @@ -181,6 +181,18 @@ class TestFstabHandling(test_helpers.FilesystemMockingTestCase): return dev + def test_swap_integrity(self): + '''Ensure that the swap file is correctly created and can + swapon successfully. Fixing the corner case of: + kernel: swapon: swapfile has holes''' + + fstab = '/swap.img swap swap defaults 0 0\n' + + with open(cc_mounts.FSTAB_PATH, 'w') as fd: + fd.write(fstab) + cc = {'swap': ['filename: /swap.img', 'size: 512', 'maxsize: 512']} + cc_mounts.handle(None, cc, self.mock_cloud, self.mock_log, []) + def test_fstab_no_swap_device(self): '''Ensure that cloud-init adds a discovered swap partition to /etc/fstab.''' -- cgit v1.2.3 From 6b157aa60a10fa74374287f0ec0313255e0462b4 Mon Sep 17 00:00:00 2001 From: Daniel Watkins Date: Thu, 23 Jan 2020 16:32:01 -0500 Subject: cloudinit: remove ImportError handling for mock imports (#182) We only run on Python 3 now, so we can unambiguously expect unittest.mock to exist. --- tests/unittests/test_data.py | 6 +----- tests/unittests/test_distros/test_generic.py | 6 +----- tests/unittests/test_distros/test_netconfig.py | 6 +----- .../test_handler/test_handler_apt_configure_sources_list_v1.py | 6 +----- .../test_handler/test_handler_apt_configure_sources_list_v3.py | 8 ++------ tests/unittests/test_handler/test_handler_apt_source_v1.py | 8 ++------ tests/unittests/test_handler/test_handler_apt_source_v3.py | 9 ++------- tests/unittests/test_handler/test_handler_ca_certs.py | 5 +---- tests/unittests/test_handler/test_handler_growpart.py | 5 +---- tests/unittests/test_handler/test_handler_lxd.py | 5 +---- tests/unittests/test_handler/test_handler_mounts.py | 6 +----- tests/unittests/test_handler/test_handler_spacewalk.py | 6 +----- tests/unittests/test_util.py | 6 +----- 13 files changed, 16 insertions(+), 66 deletions(-) (limited to 'tests/unittests/test_handler/test_handler_mounts.py') diff --git a/tests/unittests/test_data.py b/tests/unittests/test_data.py index e55feb22..c59db33d 100644 --- a/tests/unittests/test_data.py +++ b/tests/unittests/test_data.py @@ -5,11 +5,7 @@ import gzip import logging import os - -try: - from unittest import mock -except ImportError: - import mock +from unittest import mock from six import BytesIO, StringIO diff --git a/tests/unittests/test_distros/test_generic.py b/tests/unittests/test_distros/test_generic.py index 7e0da4f2..02b334e3 100644 --- a/tests/unittests/test_distros/test_generic.py +++ b/tests/unittests/test_distros/test_generic.py @@ -8,11 +8,7 @@ from cloudinit.tests import helpers import os import shutil import tempfile - -try: - from unittest import mock -except ImportError: - import mock +from unittest import mock unknown_arch_info = { 'arches': ['default'], diff --git a/tests/unittests/test_distros/test_netconfig.py b/tests/unittests/test_distros/test_netconfig.py index aeaadaa0..5ede4d77 100644 --- a/tests/unittests/test_distros/test_netconfig.py +++ b/tests/unittests/test_distros/test_netconfig.py @@ -4,11 +4,7 @@ import copy import os from six import StringIO from textwrap import dedent - -try: - from unittest import mock -except ImportError: - import mock +from unittest import mock from cloudinit import distros from cloudinit.distros.parsers.sys_conf import SysConf diff --git a/tests/unittests/test_handler/test_handler_apt_configure_sources_list_v1.py b/tests/unittests/test_handler/test_handler_apt_configure_sources_list_v1.py index 7c17a264..69009a44 100644 --- a/tests/unittests/test_handler/test_handler_apt_configure_sources_list_v1.py +++ b/tests/unittests/test_handler/test_handler_apt_configure_sources_list_v1.py @@ -7,11 +7,7 @@ import logging import os import shutil import tempfile - -try: - from unittest import mock -except ImportError: - import mock +from unittest import mock from cloudinit import cloud from cloudinit import distros diff --git a/tests/unittests/test_handler/test_handler_apt_configure_sources_list_v3.py b/tests/unittests/test_handler/test_handler_apt_configure_sources_list_v3.py index 0a68cb8f..0aa3d51a 100644 --- a/tests/unittests/test_handler/test_handler_apt_configure_sources_list_v3.py +++ b/tests/unittests/test_handler/test_handler_apt_configure_sources_list_v3.py @@ -7,12 +7,8 @@ import logging import os import shutil import tempfile - -try: - from unittest import mock -except ImportError: - import mock -from mock import call +from unittest import mock +from unittest.mock import call from cloudinit import cloud from cloudinit import distros diff --git a/tests/unittests/test_handler/test_handler_apt_source_v1.py b/tests/unittests/test_handler/test_handler_apt_source_v1.py index 652d97ab..866752ef 100644 --- a/tests/unittests/test_handler/test_handler_apt_source_v1.py +++ b/tests/unittests/test_handler/test_handler_apt_source_v1.py @@ -9,12 +9,8 @@ import os import re import shutil import tempfile - -try: - from unittest import mock -except ImportError: - import mock -from mock import call +from unittest import mock +from unittest.mock import call from cloudinit.config import cc_apt_configure from cloudinit import gpg diff --git a/tests/unittests/test_handler/test_handler_apt_source_v3.py b/tests/unittests/test_handler/test_handler_apt_source_v3.py index c5cf6785..90949b6d 100644 --- a/tests/unittests/test_handler/test_handler_apt_source_v3.py +++ b/tests/unittests/test_handler/test_handler_apt_source_v3.py @@ -11,13 +11,8 @@ import shutil import socket import tempfile -from unittest import TestCase - -try: - from unittest import mock -except ImportError: - import mock -from mock import call +from unittest import TestCase, mock +from unittest.mock import call from cloudinit import cloud from cloudinit import distros diff --git a/tests/unittests/test_handler/test_handler_ca_certs.py b/tests/unittests/test_handler/test_handler_ca_certs.py index 06e14db0..5b4105dd 100644 --- a/tests/unittests/test_handler/test_handler_ca_certs.py +++ b/tests/unittests/test_handler/test_handler_ca_certs.py @@ -11,11 +11,8 @@ import logging import shutil import tempfile import unittest +from unittest import mock -try: - from unittest import mock -except ImportError: - import mock try: from contextlib import ExitStack except ImportError: diff --git a/tests/unittests/test_handler/test_handler_growpart.py b/tests/unittests/test_handler/test_handler_growpart.py index 1f39ebe7..43b53745 100644 --- a/tests/unittests/test_handler/test_handler_growpart.py +++ b/tests/unittests/test_handler/test_handler_growpart.py @@ -11,11 +11,8 @@ import logging import os import re import unittest +from unittest import mock -try: - from unittest import mock -except ImportError: - import mock try: from contextlib import ExitStack except ImportError: diff --git a/tests/unittests/test_handler/test_handler_lxd.py b/tests/unittests/test_handler/test_handler_lxd.py index b63db616..40b521e5 100644 --- a/tests/unittests/test_handler/test_handler_lxd.py +++ b/tests/unittests/test_handler/test_handler_lxd.py @@ -5,10 +5,7 @@ from cloudinit.sources import DataSourceNoCloud from cloudinit import (distros, helpers, cloud) from cloudinit.tests import helpers as t_help -try: - from unittest import mock -except ImportError: - import mock +from unittest import mock class TestLxd(t_help.CiTestCase): diff --git a/tests/unittests/test_handler/test_handler_mounts.py b/tests/unittests/test_handler/test_handler_mounts.py index 7bcefa0a..05ac183e 100644 --- a/tests/unittests/test_handler/test_handler_mounts.py +++ b/tests/unittests/test_handler/test_handler_mounts.py @@ -1,16 +1,12 @@ # This file is part of cloud-init. See LICENSE file for license information. import os.path +from unittest import mock from cloudinit.config import cc_mounts from cloudinit.tests import helpers as test_helpers -try: - from unittest import mock -except ImportError: - import mock - class TestSanitizeDevname(test_helpers.FilesystemMockingTestCase): diff --git a/tests/unittests/test_handler/test_handler_spacewalk.py b/tests/unittests/test_handler/test_handler_spacewalk.py index ddbf4a79..410e6f77 100644 --- a/tests/unittests/test_handler/test_handler_spacewalk.py +++ b/tests/unittests/test_handler/test_handler_spacewalk.py @@ -6,11 +6,7 @@ from cloudinit import util from cloudinit.tests import helpers import logging - -try: - from unittest import mock -except ImportError: - import mock +from unittest import mock LOG = logging.getLogger(__name__) diff --git a/tests/unittests/test_util.py b/tests/unittests/test_util.py index 75a3f0b4..9ff17f52 100644 --- a/tests/unittests/test_util.py +++ b/tests/unittests/test_util.py @@ -12,15 +12,11 @@ import stat import sys import tempfile import yaml +from unittest import mock from cloudinit import importer, util from cloudinit.tests import helpers -try: - from unittest import mock -except ImportError: - import mock - BASH = util.which('bash') BOGUS_COMMAND = 'this-is-not-expected-to-be-a-program-name' -- cgit v1.2.3