From 55487d9eb52343bd72271b072734740b52b25c1d Mon Sep 17 00:00:00 2001 From: Daniel Watkins Date: Tue, 21 Jul 2015 13:06:11 +0100 Subject: Refactor cc_mounts.sanitize_devname to make it easier to modify. --- .../unittests/test_handler/test_handler_mounts.py | 133 +++++++++++++++++++++ 1 file changed, 133 insertions(+) create mode 100644 tests/unittests/test_handler/test_handler_mounts.py (limited to 'tests') diff --git a/tests/unittests/test_handler/test_handler_mounts.py b/tests/unittests/test_handler/test_handler_mounts.py new file mode 100644 index 00000000..355674b2 --- /dev/null +++ b/tests/unittests/test_handler/test_handler_mounts.py @@ -0,0 +1,133 @@ +import os.path +import shutil +import tempfile + +from cloudinit.config import cc_mounts + +from .. import helpers as test_helpers + +try: + from unittest import mock +except ImportError: + import mock + + +class TestSanitizeDevname(test_helpers.FilesystemMockingTestCase): + + def setUp(self): + super(TestSanitizeDevname, self).setUp() + self.new_root = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, self.new_root) + self.patchOS(self.new_root) + + def _touch(self, path): + path = os.path.join(self.new_root, path.lstrip('/')) + basedir = os.path.dirname(path) + if not os.path.exists(basedir): + os.makedirs(basedir) + open(path, 'a').close() + + def _makedirs(self, directory): + directory = os.path.join(self.new_root, directory.lstrip('/')) + if not os.path.exists(directory): + os.makedirs(directory) + + def mock_existence_of_disk(self, disk_path): + self._touch(disk_path) + self._makedirs(os.path.join('/sys/block', disk_path.split('/')[-1])) + + def mock_existence_of_partition(self, disk_path, partition_number): + self.mock_existence_of_disk(disk_path) + self._touch(disk_path + str(partition_number)) + disk_name = disk_path.split('/')[-1] + self._makedirs(os.path.join('/sys/block', + disk_name, + disk_name + str(partition_number))) + + def test_existent_full_disk_path_is_returned(self): + disk_path = '/dev/sda' + self.mock_existence_of_disk(disk_path) + self.assertEqual(disk_path, + cc_mounts.sanitize_devname(disk_path, + lambda x: None, + mock.Mock())) + + def test_existent_disk_name_returns_full_path(self): + disk_name = 'sda' + disk_path = '/dev/' + disk_name + self.mock_existence_of_disk(disk_path) + self.assertEqual(disk_path, + cc_mounts.sanitize_devname(disk_name, + lambda x: None, + mock.Mock())) + + def test_existent_meta_disk_is_returned(self): + actual_disk_path = '/dev/sda' + self.mock_existence_of_disk(actual_disk_path) + self.assertEqual( + actual_disk_path, + cc_mounts.sanitize_devname('ephemeral0', + lambda x: actual_disk_path, + mock.Mock())) + + def test_existent_meta_partition_is_returned(self): + disk_name, partition_part = '/dev/sda', '1' + actual_partition_path = disk_name + partition_part + self.mock_existence_of_partition(disk_name, partition_part) + self.assertEqual( + actual_partition_path, + cc_mounts.sanitize_devname('ephemeral0.1', + lambda x: disk_name, + mock.Mock())) + + def test_existent_meta_partition_with_p_is_returned(self): + disk_name, partition_part = '/dev/sda', 'p1' + actual_partition_path = disk_name + partition_part + self.mock_existence_of_partition(disk_name, partition_part) + self.assertEqual( + actual_partition_path, + cc_mounts.sanitize_devname('ephemeral0.1', + lambda x: disk_name, + mock.Mock())) + + def test_first_partition_returned_if_existent_disk_is_partitioned(self): + disk_name, partition_part = '/dev/sda', '1' + actual_partition_path = disk_name + partition_part + self.mock_existence_of_partition(disk_name, partition_part) + self.assertEqual( + actual_partition_path, + cc_mounts.sanitize_devname('ephemeral0', + lambda x: disk_name, + mock.Mock())) + + def test_nth_partition_returned_if_requested(self): + disk_name, partition_part = '/dev/sda', '3' + actual_partition_path = disk_name + partition_part + self.mock_existence_of_partition(disk_name, partition_part) + self.assertEqual( + actual_partition_path, + cc_mounts.sanitize_devname('ephemeral0.3', + lambda x: disk_name, + mock.Mock())) + + def test_transformer_returning_none_returns_none(self): + self.assertIsNone( + cc_mounts.sanitize_devname( + 'ephemeral0', lambda x: None, mock.Mock())) + + def test_missing_device_returns_none(self): + self.assertIsNone( + cc_mounts.sanitize_devname('/dev/sda', None, mock.Mock())) + + def test_missing_sys_returns_none(self): + disk_path = '/dev/sda' + self._makedirs(disk_path) + self.assertIsNone( + cc_mounts.sanitize_devname(disk_path, None, mock.Mock())) + + def test_existent_disk_but_missing_partition_returns_none(self): + disk_path = '/dev/sda' + self.mock_existence_of_disk(disk_path) + self.assertIsNone( + cc_mounts.sanitize_devname( + 'ephemeral0.1', lambda x: disk_path, mock.Mock())) -- cgit v1.2.3 From 9461b1235f7278440ffb84f1e3d95b3f906e444b Mon Sep 17 00:00:00 2001 From: Daniel Watkins Date: Tue, 21 Jul 2015 13:06:11 +0100 Subject: Use /dev/disk devices for Azure ephemeral disk. The ephemeral disk will not necessarily be assigned the same name at each boot (LP: #1411582), so we use some udev rules to ensure we always get the right one. --- cloudinit/sources/DataSourceAzure.py | 39 ++++++++++++++------------- tests/unittests/test_datasource/test_azure.py | 6 +++-- 2 files changed, 25 insertions(+), 20 deletions(-) (limited to 'tests') diff --git a/cloudinit/sources/DataSourceAzure.py b/cloudinit/sources/DataSourceAzure.py index d0a882ca..1193d88b 100644 --- a/cloudinit/sources/DataSourceAzure.py +++ b/cloudinit/sources/DataSourceAzure.py @@ -254,7 +254,7 @@ class DataSourceAzureNet(sources.DataSource): self.metadata.update(fabric_data) - found_ephemeral = find_ephemeral_disk() + found_ephemeral = find_fabric_formatted_ephemeral_disk() if found_ephemeral: self.ds_cfg['disk_aliases']['ephemeral0'] = found_ephemeral LOG.debug("using detected ephemeral0 of %s", found_ephemeral) @@ -276,30 +276,33 @@ def count_files(mp): return len(fnmatch.filter(os.listdir(mp), '*[!cdrom]*')) -def find_ephemeral_part(): +def find_fabric_formatted_ephemeral_part(): """ - Locate the default ephmeral0.1 device. This will be the first device - that has a LABEL of DEF_EPHEMERAL_LABEL and is a NTFS device. If Azure - gets more ephemeral devices, this logic will only identify the first - such device. + Locate the first fabric formatted ephemeral device. """ - c_label_devs = util.find_devs_with("LABEL=%s" % DEF_EPHEMERAL_LABEL) - c_fstype_devs = util.find_devs_with("TYPE=ntfs") - for dev in c_label_devs: - if dev in c_fstype_devs: - return dev + potential_locations = ['/dev/disk/cloud/azure_resource-part1', + '/dev/disk/azure/resource-part1'] + device_location = None + for potential_location in potential_locations: + if os.path.exists(potential_location): + device_location = potential_location + break + if device_location is None: + return None + ntfs_devices = util.find_devs_with("TYPE=ntfs") + real_device = os.path.realpath(device_location) + if real_device in ntfs_devices: + return device_location return None -def find_ephemeral_disk(): +def find_fabric_formatted_ephemeral_disk(): """ Get the ephemeral disk. """ - part_dev = find_ephemeral_part() - if part_dev and str(part_dev[-1]).isdigit(): - return part_dev[:-1] - elif part_dev: - return part_dev + part_dev = find_fabric_formatted_ephemeral_part() + if part_dev: + return part_dev.split('-')[0] return None @@ -313,7 +316,7 @@ def support_new_ephemeral(cfg): new ephemeral device is detected, cloud-init overrides the default frequency for both disk-setup and mounts for the current boot only. """ - device = find_ephemeral_part() + device = find_fabric_formatted_ephemeral_part() if not device: LOG.debug("no default fabric formated ephemeral0.1 found") return None diff --git a/tests/unittests/test_datasource/test_azure.py b/tests/unittests/test_datasource/test_azure.py index 33b971f6..3b7e3293 100644 --- a/tests/unittests/test_datasource/test_azure.py +++ b/tests/unittests/test_datasource/test_azure.py @@ -475,10 +475,12 @@ class TestAzureBounce(TestCase): mock.patch.object(DataSourceAzure, 'list_possible_azure_ds_devs', mock.MagicMock(return_value=[]))) self.patches.enter_context( - mock.patch.object(DataSourceAzure, 'find_ephemeral_disk', + mock.patch.object(DataSourceAzure, + 'find_fabric_formatted_ephemeral_disk', mock.MagicMock(return_value=None))) self.patches.enter_context( - mock.patch.object(DataSourceAzure, 'find_ephemeral_part', + mock.patch.object(DataSourceAzure, + 'find_fabric_formatted_ephemeral_part', mock.MagicMock(return_value=None))) self.patches.enter_context( mock.patch.object(DataSourceAzure, 'get_metadata_from_fabric', -- cgit v1.2.3 From 73c5bbfa31b922a0ba403216c0fc1f63b22a9262 Mon Sep 17 00:00:00 2001 From: Daniel Watkins Date: Wed, 22 Jul 2015 13:06:34 +0100 Subject: Make full data source available to code that handles mirror selection. --- cloudinit/distros/__init__.py | 15 +++++++-------- cloudinit/sources/__init__.py | 3 +-- tests/unittests/test_distros/test_generic.py | 22 +++++++++++++++------- 3 files changed, 23 insertions(+), 17 deletions(-) (limited to 'tests') diff --git a/cloudinit/distros/__init__.py b/cloudinit/distros/__init__.py index 8a947867..47b76c68 100644 --- a/cloudinit/distros/__init__.py +++ b/cloudinit/distros/__init__.py @@ -117,12 +117,11 @@ class Distro(object): arch = self.get_primary_arch() return _get_arch_package_mirror_info(mirror_info, arch) - def get_package_mirror_info(self, arch=None, - availability_zone=None): + def get_package_mirror_info(self, arch=None, data_source=None): # This resolves the package_mirrors config option # down to a single dict of {mirror_name: mirror_url} arch_info = self._get_arch_package_mirror_info(arch) - return _get_package_mirror_info(availability_zone=availability_zone, + return _get_package_mirror_info(data_source=data_source, mirror_info=arch_info) def apply_network(self, settings, bring_up=True): @@ -556,7 +555,7 @@ class Distro(object): LOG.info("Added user '%s' to group '%s'" % (member, name)) -def _get_package_mirror_info(mirror_info, availability_zone=None, +def _get_package_mirror_info(mirror_info, data_source=None, mirror_filter=util.search_for_mirror): # given a arch specific 'mirror_info' entry (from package_mirrors) # search through the 'search' entries, and fallback appropriately @@ -572,11 +571,11 @@ def _get_package_mirror_info(mirror_info, availability_zone=None, ec2_az_re = ("^[a-z][a-z]-(%s)-[1-9][0-9]*[a-z]$" % directions_re) subst = {} - if availability_zone: - subst['availability_zone'] = availability_zone + if data_source and data_source.availability_zone: + subst['availability_zone'] = data_source.availability_zone - if availability_zone and re.match(ec2_az_re, availability_zone): - subst['ec2_region'] = "%s" % availability_zone[0:-1] + if re.match(ec2_az_re, data_source.availability_zone): + subst['ec2_region'] = "%s" % data_source.availability_zone[0:-1] results = {} for (name, mirror) in mirror_info.get('failsafe', {}).items(): diff --git a/cloudinit/sources/__init__.py b/cloudinit/sources/__init__.py index 39eab51b..1a036638 100644 --- a/cloudinit/sources/__init__.py +++ b/cloudinit/sources/__init__.py @@ -210,8 +210,7 @@ class DataSource(object): return hostname def get_package_mirror_info(self): - return self.distro.get_package_mirror_info( - availability_zone=self.availability_zone) + return self.distro.get_package_mirror_info(data_source=self) def normalize_pubkey_data(pubkey_data): diff --git a/tests/unittests/test_distros/test_generic.py b/tests/unittests/test_distros/test_generic.py index 8e3bd78a..6ed1704c 100644 --- a/tests/unittests/test_distros/test_generic.py +++ b/tests/unittests/test_distros/test_generic.py @@ -7,6 +7,11 @@ import os import shutil import tempfile +try: + from unittest import mock +except ImportError: + import mock + unknown_arch_info = { 'arches': ['default'], 'failsafe': {'primary': 'http://fs-primary-default', @@ -144,33 +149,35 @@ class TestGenericDistro(helpers.FilesystemMockingTestCase): def test_get_package_mirror_info_az_ec2(self): arch_mirrors = gapmi(package_mirrors, arch="amd64") + data_source_mock = mock.Mock(availability_zone="us-east-1a") - results = gpmi(arch_mirrors, availability_zone="us-east-1a", + results = gpmi(arch_mirrors, data_source=data_source_mock, mirror_filter=self.return_first) self.assertEqual(results, {'primary': 'http://us-east-1.ec2/', 'security': 'http://security-mirror1-intel'}) - results = gpmi(arch_mirrors, availability_zone="us-east-1a", + results = gpmi(arch_mirrors, data_source=data_source_mock, mirror_filter=self.return_second) self.assertEqual(results, {'primary': 'http://us-east-1a.clouds/', 'security': 'http://security-mirror2-intel'}) - results = gpmi(arch_mirrors, availability_zone="us-east-1a", + results = gpmi(arch_mirrors, data_source=data_source_mock, mirror_filter=self.return_none) self.assertEqual(results, package_mirrors[0]['failsafe']) def test_get_package_mirror_info_az_non_ec2(self): arch_mirrors = gapmi(package_mirrors, arch="amd64") + data_source_mock = mock.Mock(availability_zone="nova.cloudvendor") - results = gpmi(arch_mirrors, availability_zone="nova.cloudvendor", + results = gpmi(arch_mirrors, data_source=data_source_mock, mirror_filter=self.return_first) self.assertEqual(results, {'primary': 'http://nova.cloudvendor.clouds/', 'security': 'http://security-mirror1-intel'}) - results = gpmi(arch_mirrors, availability_zone="nova.cloudvendor", + results = gpmi(arch_mirrors, data_source=data_source_mock, mirror_filter=self.return_last) self.assertEqual(results, {'primary': 'http://nova.cloudvendor.clouds/', @@ -178,17 +185,18 @@ class TestGenericDistro(helpers.FilesystemMockingTestCase): def test_get_package_mirror_info_none(self): arch_mirrors = gapmi(package_mirrors, arch="amd64") + data_source_mock = mock.Mock(availability_zone=None) # because both search entries here replacement based on # availability-zone, the filter will be called with an empty list and # failsafe should be taken. - results = gpmi(arch_mirrors, availability_zone=None, + results = gpmi(arch_mirrors, data_source=data_source_mock, mirror_filter=self.return_first) self.assertEqual(results, {'primary': 'http://fs-primary-intel', 'security': 'http://security-mirror1-intel'}) - results = gpmi(arch_mirrors, availability_zone=None, + results = gpmi(arch_mirrors, data_source=data_source_mock, mirror_filter=self.return_last) self.assertEqual(results, {'primary': 'http://fs-primary-intel', -- cgit v1.2.3