merge from master at 18.2

7 files changed, 148 insertions, 70 deletions

diff --git a/tests/unittests/test_handler/test_handler_apt_source_v1.py b/tests/unittests/test_handler/test_handler_apt_source_v1.py
index 3a3f95ca..46ca4ce4 100644
--- a/tests/unittests/test_handler/test_handler_apt_source_v1.py
+++ b/tests/unittests/test_handler/test_handler_apt_source_v1.py
@@ -569,7 +569,8 @@ class TestAptSourceConfig(TestCase):
         newcfg = cc_apt_configure.convert_to_v3_apt_format(cfg_3_only)
         self.assertEqual(newcfg, cfg_3_only)
         # collision (unequal)
-        with self.assertRaises(ValueError):
+        match = "Old and New.*unequal.*apt_proxy"
+        with self.assertRaisesRegex(ValueError, match):
             cc_apt_configure.convert_to_v3_apt_format(cfgconflict)
 
     def test_convert_to_new_format_dict_collision(self):
diff --git a/tests/unittests/test_handler/test_handler_bootcmd.py b/tests/unittests/test_handler/test_handler_bootcmd.py
index dbf43e0d..29fc25e4 100644
--- a/tests/unittests/test_handler/test_handler_bootcmd.py
+++ b/tests/unittests/test_handler/test_handler_bootcmd.py
@@ -3,17 +3,11 @@
 from cloudinit.config import cc_bootcmd
 from cloudinit.sources import DataSourceNone
 from cloudinit import (distros, helpers, cloud, util)
-from cloudinit.tests.helpers import CiTestCase, mock, skipIf
+from cloudinit.tests.helpers import CiTestCase, mock, skipUnlessJsonSchema
 
 import logging
 import tempfile
 
-try:
-    import jsonschema
-    assert jsonschema  # avoid pyflakes error F401: import unused
-    _missing_jsonschema_dep = False
-except ImportError:
-    _missing_jsonschema_dep = True
 
 LOG = logging.getLogger(__name__)
 
@@ -69,10 +63,10 @@ class TestBootcmd(CiTestCase):
             cc_bootcmd.handle('cc_bootcmd', invalid_config, cc, LOG, [])
         self.assertIn('Failed to shellify bootcmd', self.logs.getvalue())
         self.assertEqual(
-            "'int' object is not iterable",
+            "Input to shellify was type 'int'. Expected list or tuple.",
             str(context_manager.exception))
 
-    @skipIf(_missing_jsonschema_dep, "No python-jsonschema dependency")
+    @skipUnlessJsonSchema()
     def test_handler_schema_validation_warns_non_array_type(self):
         """Schema validation warns of non-array type for bootcmd key.
 
@@ -88,7 +82,7 @@ class TestBootcmd(CiTestCase):
             self.logs.getvalue())
         self.assertIn('Failed to shellify', self.logs.getvalue())
 
-    @skipIf(_missing_jsonschema_dep, 'No python-jsonschema dependency')
+    @skipUnlessJsonSchema()
     def test_handler_schema_validation_warns_non_array_item_type(self):
         """Schema validation warns of non-array or string bootcmd items.
 
@@ -98,7 +92,7 @@ class TestBootcmd(CiTestCase):
         invalid_config = {
             'bootcmd': ['ls /', 20, ['wget', 'http://stuff/blah'], {'a': 'n'}]}
         cc = self._get_cloud('ubuntu')
-        with self.assertRaises(RuntimeError) as context_manager:
+        with self.assertRaises(TypeError) as context_manager:
             cc_bootcmd.handle('cc_bootcmd', invalid_config, cc, LOG, [])
         expected_warnings = [
             'bootcmd.1: 20 is not valid under any of the given schemas',
@@ -110,7 +104,8 @@ class TestBootcmd(CiTestCase):
             self.assertIn(warning, logs)
         self.assertIn('Failed to shellify', logs)
         self.assertEqual(
-            'Unable to shellify type int which is not a list or string',
+            ("Unable to shellify type 'int'. Expected list, string, tuple. "
+             "Got: 20"),
             str(context_manager.exception))
 
     def test_handler_creates_and_runs_bootcmd_script_with_instance_id(self):
diff --git a/tests/unittests/test_handler/test_handler_ntp.py b/tests/unittests/test_handler/test_handler_ntp.py
index 28a8455d..695897c0 100644
--- a/tests/unittests/test_handler/test_handler_ntp.py
+++ b/tests/unittests/test_handler/test_handler_ntp.py
@@ -3,7 +3,8 @@
 from cloudinit.config import cc_ntp
 from cloudinit.sources import DataSourceNone
 from cloudinit import (distros, helpers, cloud, util)
-from cloudinit.tests.helpers import FilesystemMockingTestCase, mock, skipIf
+from cloudinit.tests.helpers import (
+    FilesystemMockingTestCase, mock, skipUnlessJsonSchema)
 
 
 import os
@@ -24,13 +25,6 @@ NTP={% for host in servers|list + pools|list %}{{ host }} {% endfor -%}
 {% endif -%}
 """
 
-try:
-    import jsonschema
-    assert jsonschema  # avoid pyflakes error F401: import unused
-    _missing_jsonschema_dep = False
-except ImportError:
-    _missing_jsonschema_dep = True
-
 
 class TestNtp(FilesystemMockingTestCase):
 
@@ -312,7 +306,7 @@ class TestNtp(FilesystemMockingTestCase):
                 content)
         self.assertNotIn('Invalid config:', self.logs.getvalue())
 
-    @skipIf(_missing_jsonschema_dep, "No python-jsonschema dependency")
+    @skipUnlessJsonSchema()
     def test_ntp_handler_schema_validation_warns_non_string_item_type(self):
         """Ntp schema validation warns of non-strings in pools or servers.
 
@@ -333,7 +327,7 @@ class TestNtp(FilesystemMockingTestCase):
             content = stream.read()
         self.assertEqual("servers ['valid', None]\npools [123]\n", content)
 
-    @skipIf(_missing_jsonschema_dep, "No python-jsonschema dependency")
+    @skipUnlessJsonSchema()
     def test_ntp_handler_schema_validation_warns_of_non_array_type(self):
         """Ntp schema validation warns of non-array pools or servers types.
 
@@ -354,7 +348,7 @@ class TestNtp(FilesystemMockingTestCase):
             content = stream.read()
         self.assertEqual("servers non-array\npools 123\n", content)
 
-    @skipIf(_missing_jsonschema_dep, "No python-jsonschema dependency")
+    @skipUnlessJsonSchema()
     def test_ntp_handler_schema_validation_warns_invalid_key_present(self):
         """Ntp schema validation warns of invalid keys present in ntp config.
 
@@ -378,7 +372,7 @@ class TestNtp(FilesystemMockingTestCase):
             "servers []\npools ['0.mycompany.pool.ntp.org']\n",
             content)
 
-    @skipIf(_missing_jsonschema_dep, "No python-jsonschema dependency")
+    @skipUnlessJsonSchema()
     def test_ntp_handler_schema_validation_warns_of_duplicates(self):
         """Ntp schema validation warns of duplicates in servers or pools.
 
diff --git a/tests/unittests/test_handler/test_handler_resizefs.py b/tests/unittests/test_handler/test_handler_resizefs.py
index 5aa3c498..7a7ba1ff 100644
--- a/tests/unittests/test_handler/test_handler_resizefs.py
+++ b/tests/unittests/test_handler/test_handler_resizefs.py
@@ -1,27 +1,20 @@
 # This file is part of cloud-init. See LICENSE file for license information.
 
 from cloudinit.config.cc_resizefs import (
-    can_skip_resize, handle, maybe_get_writable_device_path, _resize_btrfs)
+    can_skip_resize, handle, maybe_get_writable_device_path, _resize_btrfs,
+    _resize_zfs, _resize_xfs, _resize_ext, _resize_ufs)
 
 from collections import namedtuple
 import logging
 import textwrap
 
-from cloudinit.tests.helpers import (CiTestCase, mock, skipIf, util,
-                                     wrap_and_call)
+from cloudinit.tests.helpers import (
+    CiTestCase, mock, skipUnlessJsonSchema, util, wrap_and_call)
 
 
 LOG = logging.getLogger(__name__)
 
 
-try:
-    import jsonschema
-    assert jsonschema  # avoid pyflakes error F401: import unused
-    _missing_jsonschema_dep = False
-except ImportError:
-    _missing_jsonschema_dep = True
-
-
 class TestResizefs(CiTestCase):
     with_logs = True
 
@@ -68,6 +61,9 @@ class TestResizefs(CiTestCase):
         res = can_skip_resize(fs_type, resize_what, devpth)
         self.assertTrue(res)
 
+    def test_can_skip_resize_ext(self):
+        self.assertFalse(can_skip_resize('ext', '/', '/dev/sda1'))
+
     def test_handle_noops_on_disabled(self):
         """The handle function logs when the configuration disables resize."""
         cfg = {'resize_rootfs': False}
@@ -76,7 +72,7 @@ class TestResizefs(CiTestCase):
             'DEBUG: Skipping module named cc_resizefs, resizing disabled\n',
             self.logs.getvalue())
 
-    @skipIf(_missing_jsonschema_dep, "No python-jsonschema dependency")
+    @skipUnlessJsonSchema()
     def test_handle_schema_validation_logs_invalid_resize_rootfs_value(self):
         """The handle reports json schema violations as a warning.
 
@@ -130,6 +126,51 @@ class TestResizefs(CiTestCase):
         logs = self.logs.getvalue()
         self.assertIn("WARNING: Unable to find device '/dev/root'", logs)
 
+    def test_resize_zfs_cmd_return(self):
+        zpool = 'zroot'
+        devpth = 'gpt/system'
+        self.assertEqual(('zpool', 'online', '-e', zpool, devpth),
+                         _resize_zfs(zpool, devpth))
+
+    def test_resize_xfs_cmd_return(self):
+        mount_point = '/mnt/test'
+        devpth = '/dev/sda1'
+        self.assertEqual(('xfs_growfs', mount_point),
+                         _resize_xfs(mount_point, devpth))
+
+    def test_resize_ext_cmd_return(self):
+        mount_point = '/'
+        devpth = '/dev/sdb1'
+        self.assertEqual(('resize2fs', devpth),
+                         _resize_ext(mount_point, devpth))
+
+    def test_resize_ufs_cmd_return(self):
+        mount_point = '/'
+        devpth = '/dev/sda2'
+        self.assertEqual(('growfs', devpth),
+                         _resize_ufs(mount_point, devpth))
+
+    @mock.patch('cloudinit.util.get_mount_info')
+    @mock.patch('cloudinit.util.get_device_info_from_zpool')
+    @mock.patch('cloudinit.util.parse_mount')
+    def test_handle_zfs_root(self, mount_info, zpool_info, parse_mount):
+        devpth = 'vmzroot/ROOT/freebsd'
+        disk = 'gpt/system'
+        fs_type = 'zfs'
+        mount_point = '/'
+
+        mount_info.return_value = (devpth, fs_type, mount_point)
+        zpool_info.return_value = disk
+        parse_mount.return_value = (devpth, fs_type, mount_point)
+
+        cfg = {'resize_rootfs': True}
+
+        with mock.patch('cloudinit.config.cc_resizefs.do_resize') as dresize:
+            handle('cc_resizefs', cfg, _cloud=None, log=LOG, args=[])
+            ret = dresize.call_args[0][0]
+
+        self.assertEqual(('zpool', 'online', '-e', 'vmzroot', disk), ret)
+
 
 class TestRootDevFromCmdline(CiTestCase):
 
@@ -313,5 +354,12 @@ class TestMaybeGetDevicePathAsWritableBlock(CiTestCase):
             ('btrfs', 'filesystem', 'resize', 'max', '/'),
             _resize_btrfs("/", "/dev/sda1"))
 
+    @mock.patch('cloudinit.util.is_FreeBSD')
+    def test_maybe_get_writable_device_path_zfs_freebsd(self, freebsd):
+        freebsd.return_value = True
+        info = 'dev=gpt/system mnt_point=/ path=/'
+        devpth = maybe_get_writable_device_path('gpt/system', info, LOG)
+        self.assertEqual('gpt/system', devpth)
+
 
 # vi: ts=4 expandtab
diff --git a/tests/unittests/test_handler/test_handler_runcmd.py b/tests/unittests/test_handler/test_handler_runcmd.py
index 374c1d31..dbbb2717 100644
--- a/tests/unittests/test_handler/test_handler_runcmd.py
+++ b/tests/unittests/test_handler/test_handler_runcmd.py
@@ -3,19 +3,13 @@
 from cloudinit.config import cc_runcmd
 from cloudinit.sources import DataSourceNone
 from cloudinit import (distros, helpers, cloud, util)
-from cloudinit.tests.helpers import FilesystemMockingTestCase, skipIf
+from cloudinit.tests.helpers import (
+    FilesystemMockingTestCase, skipUnlessJsonSchema)
 
 import logging
 import os
 import stat
 
-try:
-    import jsonschema
-    assert jsonschema  # avoid pyflakes error F401: import unused
-    _missing_jsonschema_dep = False
-except ImportError:
-    _missing_jsonschema_dep = True
-
 LOG = logging.getLogger(__name__)
 
 
@@ -56,7 +50,7 @@ class TestRuncmd(FilesystemMockingTestCase):
             ' /var/lib/cloud/instances/iid-datasource-none/scripts/runcmd',
             self.logs.getvalue())
 
-    @skipIf(_missing_jsonschema_dep, "No python-jsonschema dependency")
+    @skipUnlessJsonSchema()
     def test_handler_schema_validation_warns_non_array_type(self):
         """Schema validation warns of non-array type for runcmd key.
 
@@ -71,7 +65,7 @@ class TestRuncmd(FilesystemMockingTestCase):
             self.logs.getvalue())
         self.assertIn('Failed to shellify', self.logs.getvalue())
 
-    @skipIf(_missing_jsonschema_dep, 'No python-jsonschema dependency')
+    @skipUnlessJsonSchema()
     def test_handler_schema_validation_warns_non_array_item_type(self):
         """Schema validation warns of non-array or string runcmd items.
 
diff --git a/tests/unittests/test_handler/test_handler_set_hostname.py b/tests/unittests/test_handler/test_handler_set_hostname.py
index abdc17e7..d09ec23a 100644
--- a/tests/unittests/test_handler/test_handler_set_hostname.py
+++ b/tests/unittests/test_handler/test_handler_set_hostname.py
@@ -11,6 +11,7 @@ from cloudinit.tests import helpers as t_help
 
 from configobj import ConfigObj
 import logging
+import os
 import shutil
 from six import BytesIO
 import tempfile
@@ -19,14 +20,18 @@ LOG = logging.getLogger(__name__)
 
 
 class TestHostname(t_help.FilesystemMockingTestCase):
+
+    with_logs = True
+
     def setUp(self):
         super(TestHostname, self).setUp()
         self.tmp = tempfile.mkdtemp()
+        util.ensure_dir(os.path.join(self.tmp, 'data'))
         self.addCleanup(shutil.rmtree, self.tmp)
 
     def _fetch_distro(self, kind):
         cls = distros.fetch(kind)
-        paths = helpers.Paths({})
+        paths = helpers.Paths({'cloud_dir': self.tmp})
         return cls(kind, {}, paths)
 
     def test_write_hostname_rhel(self):
@@ -34,7 +39,7 @@ class TestHostname(t_help.FilesystemMockingTestCase):
             'hostname': 'blah.blah.blah.yahoo.com',
         }
         distro = self._fetch_distro('rhel')
-        paths = helpers.Paths({})
+        paths = helpers.Paths({'cloud_dir': self.tmp})
         ds = None
         cc = cloud.Cloud(ds, paths, {}, distro, None)
         self.patchUtils(self.tmp)
@@ -51,7 +56,7 @@ class TestHostname(t_help.FilesystemMockingTestCase):
             'hostname': 'blah.blah.blah.yahoo.com',
         }
         distro = self._fetch_distro('debian')
-        paths = helpers.Paths({})
+        paths = helpers.Paths({'cloud_dir': self.tmp})
         ds = None
         cc = cloud.Cloud(ds, paths, {}, distro, None)
         self.patchUtils(self.tmp)
@@ -65,7 +70,7 @@ class TestHostname(t_help.FilesystemMockingTestCase):
             'hostname': 'blah.blah.blah.suse.com',
         }
         distro = self._fetch_distro('sles')
-        paths = helpers.Paths({})
+        paths = helpers.Paths({'cloud_dir': self.tmp})
         ds = None
         cc = cloud.Cloud(ds, paths, {}, distro, None)
         self.patchUtils(self.tmp)
@@ -74,4 +79,48 @@ class TestHostname(t_help.FilesystemMockingTestCase):
             contents = util.load_file(distro.hostname_conf_fn)
             self.assertEqual('blah', contents.strip())
 
+    def test_multiple_calls_skips_unchanged_hostname(self):
+        """Only new hostname or fqdn values will generate a hostname call."""
+        distro = self._fetch_distro('debian')
+        paths = helpers.Paths({'cloud_dir': self.tmp})
+        ds = None
+        cc = cloud.Cloud(ds, paths, {}, distro, None)
+        self.patchUtils(self.tmp)
+        cc_set_hostname.handle(
+            'cc_set_hostname', {'hostname': 'hostname1.me.com'}, cc, LOG, [])
+        contents = util.load_file("/etc/hostname")
+        self.assertEqual('hostname1', contents.strip())
+        cc_set_hostname.handle(
+            'cc_set_hostname', {'hostname': 'hostname1.me.com'}, cc, LOG, [])
+        self.assertIn(
+            'DEBUG: No hostname changes. Skipping set-hostname\n',
+            self.logs.getvalue())
+        cc_set_hostname.handle(
+            'cc_set_hostname', {'hostname': 'hostname2.me.com'}, cc, LOG, [])
+        contents = util.load_file("/etc/hostname")
+        self.assertEqual('hostname2', contents.strip())
+        self.assertIn(
+            'Non-persistently setting the system hostname to hostname2',
+            self.logs.getvalue())
+
+    def test_error_on_distro_set_hostname_errors(self):
+        """Raise SetHostnameError on exceptions from distro.set_hostname."""
+        distro = self._fetch_distro('debian')
+
+        def set_hostname_error(hostname, fqdn):
+            raise Exception("OOPS on: %s" % fqdn)
+
+        distro.set_hostname = set_hostname_error
+        paths = helpers.Paths({'cloud_dir': self.tmp})
+        ds = None
+        cc = cloud.Cloud(ds, paths, {}, distro, None)
+        self.patchUtils(self.tmp)
+        with self.assertRaises(cc_set_hostname.SetHostnameError) as ctx_mgr:
+            cc_set_hostname.handle(
+                'somename', {'hostname': 'hostname1.me.com'}, cc, LOG, [])
+        self.assertEqual(
+            'Failed to set the hostname to hostname1.me.com (hostname1):'
+            ' OOPS on: hostname1.me.com',
+            str(ctx_mgr.exception))
+
 # vi: ts=4 expandtab
diff --git a/tests/unittests/test_handler/test_schema.py b/tests/unittests/test_handler/test_schema.py
index 648573f6..ac41f124 100644
--- a/tests/unittests/test_handler/test_schema.py
+++ b/tests/unittests/test_handler/test_schema.py
@@ -6,7 +6,7 @@ from cloudinit.config.schema import (
     validate_cloudconfig_schema, main)
 from cloudinit.util import subp, write_file
 
-from cloudinit.tests.helpers import CiTestCase, mock, skipIf
+from cloudinit.tests.helpers import CiTestCase, mock, skipUnlessJsonSchema
 
 from copy import copy
 import os
@@ -14,13 +14,6 @@ from six import StringIO
 from textwrap import dedent
 from yaml import safe_load
 
-try:
-    import jsonschema
-    assert jsonschema  # avoid pyflakes error F401: import unused
-    _missing_jsonschema_dep = False
-except ImportError:
-    _missing_jsonschema_dep = True
-
 
 class GetSchemaTest(CiTestCase):
 
@@ -33,6 +26,8 @@ class GetSchemaTest(CiTestCase):
                 'cc_ntp',
                 'cc_resizefs',
                 'cc_runcmd',
+                'cc_snap',
+                'cc_ubuntu_advantage',
                 'cc_zypper_add_repo'
             ],
             [subschema['id'] for subschema in schema['allOf']])
@@ -73,7 +68,7 @@ class ValidateCloudConfigSchemaTest(CiTestCase):
 
     with_logs = True
 
-    @skipIf(_missing_jsonschema_dep, "No python-jsonschema dependency")
+    @skipUnlessJsonSchema()
     def test_validateconfig_schema_non_strict_emits_warnings(self):
         """When strict is False validate_cloudconfig_schema emits warnings."""
         schema = {'properties': {'p1': {'type': 'string'}}}
@@ -82,7 +77,7 @@ class ValidateCloudConfigSchemaTest(CiTestCase):
             "Invalid config:\np1: -1 is not of type 'string'\n",
             self.logs.getvalue())
 
-    @skipIf(_missing_jsonschema_dep, "No python-jsonschema dependency")
+    @skipUnlessJsonSchema()
     def test_validateconfig_schema_emits_warning_on_missing_jsonschema(self):
         """Warning from validate_cloudconfig_schema when missing jsonschema."""
         schema = {'properties': {'p1': {'type': 'string'}}}
@@ -92,7 +87,7 @@ class ValidateCloudConfigSchemaTest(CiTestCase):
             'Ignoring schema validation. python-jsonschema is not present',
             self.logs.getvalue())
 
-    @skipIf(_missing_jsonschema_dep, "No python-jsonschema dependency")
+    @skipUnlessJsonSchema()
     def test_validateconfig_schema_strict_raises_errors(self):
         """When strict is True validate_cloudconfig_schema raises errors."""
         schema = {'properties': {'p1': {'type': 'string'}}}
@@ -102,7 +97,7 @@ class ValidateCloudConfigSchemaTest(CiTestCase):
             "Cloud config schema errors: p1: -1 is not of type 'string'",
             str(context_mgr.exception))
 
-    @skipIf(_missing_jsonschema_dep, "No python-jsonschema dependency")
+    @skipUnlessJsonSchema()
     def test_validateconfig_schema_honors_formats(self):
         """With strict True, validate_cloudconfig_schema errors on format."""
         schema = {
@@ -153,7 +148,7 @@ class ValidateCloudConfigFileTest(CiTestCase):
                 self.config_file),
             str(context_mgr.exception))
 
-    @skipIf(_missing_jsonschema_dep, "No python-jsonschema dependency")
+    @skipUnlessJsonSchema()
     def test_validateconfig_file_sctricty_validates_schema(self):
         """validate_cloudconfig_file raises errors on invalid schema."""
         schema = {
@@ -336,11 +331,13 @@ class MainTest(CiTestCase):
 
     def test_main_missing_args(self):
         """Main exits non-zero and reports an error on missing parameters."""
-        with mock.patch('sys.argv', ['mycmd']):
-            with mock.patch('sys.stderr', new_callable=StringIO) as m_stderr:
-                with self.assertRaises(SystemExit) as context_manager:
-                    main()
-        self.assertEqual('1', str(context_manager.exception))
+        with mock.patch('sys.exit', side_effect=self.sys_exit):
+            with mock.patch('sys.argv', ['mycmd']):
+                with mock.patch('sys.stderr', new_callable=StringIO) as \
+                        m_stderr:
+                    with self.assertRaises(SystemExit) as context_manager:
+                        main()
+        self.assertEqual(1, context_manager.exception.code)
         self.assertEqual(
             'Expected either --config-file argument or --doc\n',
             m_stderr.getvalue())
@@ -374,7 +371,7 @@ class CloudTestsIntegrationTest(CiTestCase):
     raises Warnings or errors on invalid cloud-config schema.
     """
 
-    @skipIf(_missing_jsonschema_dep, "No python-jsonschema dependency")
+    @skipUnlessJsonSchema()
     def test_all_integration_test_cloud_config_schema(self):
         """Validate schema of cloud_tests yaml files looking for warnings."""
         schema = get_schema()