From 55487d9eb52343bd72271b072734740b52b25c1d Mon Sep 17 00:00:00 2001 From: Daniel Watkins Date: Tue, 21 Jul 2015 13:06:11 +0100 Subject: Refactor cc_mounts.sanitize_devname to make it easier to modify. --- cloudinit/config/cc_mounts.py | 104 ++++++++++++++++++------------------------ 1 file changed, 44 insertions(+), 60 deletions(-) (limited to 'cloudinit') diff --git a/cloudinit/config/cc_mounts.py b/cloudinit/config/cc_mounts.py index 1cb1e839..f970c2ca 100644 --- a/cloudinit/config/cc_mounts.py +++ b/cloudinit/config/cc_mounts.py @@ -28,15 +28,15 @@ from cloudinit import type_utils from cloudinit import util # Shortname matches 'sda', 'sda1', 'xvda', 'hda', 'sdb', xvdb, vda, vdd1, sr0 -SHORTNAME_FILTER = r"^([x]{0,1}[shv]d[a-z][0-9]*|sr[0-9]+)$" -SHORTNAME = re.compile(SHORTNAME_FILTER) +DEVICE_NAME_FILTER = r"^([x]{0,1}[shv]d[a-z][0-9]*|sr[0-9]+)$" +DEVICE_NAME_RE = re.compile(DEVICE_NAME_FILTER) WS = re.compile("[%s]+" % (whitespace)) FSTAB_PATH = "/etc/fstab" LOG = logging.getLogger(__name__) -def is_mdname(name): +def is_meta_device_name(name): # return true if this is a metadata service name if name in ["ami", "root", "swap"]: return True @@ -48,6 +48,23 @@ def is_mdname(name): return False +def _get_nth_partition_for_device(device_path, partition_number): + potential_suffixes = [str(partition_number), 'p%s' % (partition_number,)] + for suffix in potential_suffixes: + potential_partition_device = '%s%s' % (device_path, suffix) + if os.path.exists(potential_partition_device): + return potential_partition_device + return None + + +def _is_block_device(device_path, partition_path=None): + device_name = device_path.split('/')[-1] + sys_path = os.path.join('/sys/block/', device_name) + if partition_path is not None: + sys_path = os.path.join(sys_path, partition_path.split('/')[-1]) + return os.path.exists(sys_path) + + def sanitize_devname(startname, transformer, log): log.debug("Attempting to determine the real name of %s", startname) @@ -58,21 +75,34 @@ def sanitize_devname(startname, transformer, log): devname = "ephemeral0" log.debug("Adjusted mount option from ephemeral to ephemeral0") - (blockdev, part) = util.expand_dotted_devname(devname) + device_path, partition_number = util.expand_dotted_devname(devname) - if is_mdname(blockdev): - orig = blockdev - blockdev = transformer(blockdev) - if not blockdev: + if is_meta_device_name(device_path): + orig = device_path + device_path = transformer(device_path) + if not device_path: return None - if not blockdev.startswith("/"): - blockdev = "/dev/%s" % blockdev - log.debug("Mapped metadata name %s to %s", orig, blockdev) + if not device_path.startswith("/"): + device_path = "/dev/%s" % (device_path,) + log.debug("Mapped metadata name %s to %s", orig, device_path) + else: + if DEVICE_NAME_RE.match(startname): + device_path = "/dev/%s" % (device_path,) + + partition_path = None + if partition_number is None: + partition_path = _get_nth_partition_for_device(device_path, 1) else: - if SHORTNAME.match(startname): - blockdev = "/dev/%s" % blockdev + partition_path = _get_nth_partition_for_device(device_path, + partition_number) + if partition_path is None: + return None - return devnode_for_dev_part(blockdev, part) + if _is_block_device(device_path, partition_path): + if partition_path is not None: + return partition_path + return device_path + return None def suggested_swapsize(memsize=None, maxsize=None, fsys=None): @@ -366,49 +396,3 @@ def handle(_name, cfg, cloud, log, _args): util.subp(("mount", "-a")) except: util.logexc(log, "Activating mounts via 'mount -a' failed") - - -def devnode_for_dev_part(device, partition): - """ - Find the name of the partition. While this might seem rather - straight forward, its not since some devices are '' - while others are 'p'. For example, /dev/xvda3 on EC2 - will present as /dev/xvda3p1 for the first partition since /dev/xvda3 is - a block device. - """ - if not os.path.exists(device): - return None - - short_name = os.path.basename(device) - sys_path = "/sys/block/%s" % short_name - - if not os.path.exists(sys_path): - LOG.debug("did not find entry for %s in /sys/block", short_name) - return None - - sys_long_path = sys_path + "/" + short_name - - if partition is not None: - partition = str(partition) - - if partition is None: - valid_mappings = [sys_long_path + "1", sys_long_path + "p1"] - elif partition != "0": - valid_mappings = [sys_long_path + "%s" % partition, - sys_long_path + "p%s" % partition] - else: - valid_mappings = [] - - for cdisk in valid_mappings: - if not os.path.exists(cdisk): - continue - - dev_path = "/dev/%s" % os.path.basename(cdisk) - if os.path.exists(dev_path): - return dev_path - - if partition is None or partition == "0": - return device - - LOG.debug("Did not fine partition %s for device %s", partition, device) - return None -- cgit v1.2.3 From edc46ee7192376af65640a81c39335ebdfd196b6 Mon Sep 17 00:00:00 2001 From: Daniel Watkins Date: Tue, 21 Jul 2015 13:06:11 +0100 Subject: Extend disk_setup and mounts to handle /dev/disk symlinks. --- cloudinit/config/cc_disk_setup.py | 5 +++++ cloudinit/config/cc_mounts.py | 8 +++++--- 2 files changed, 10 insertions(+), 3 deletions(-) (limited to 'cloudinit') diff --git a/cloudinit/config/cc_disk_setup.py b/cloudinit/config/cc_disk_setup.py index e2ce6db4..92fa7a94 100644 --- a/cloudinit/config/cc_disk_setup.py +++ b/cloudinit/config/cc_disk_setup.py @@ -648,6 +648,8 @@ def mkpart(device, definition): table_type: Which partition table to use, defaults to MBR device: the device to work on. """ + LOG.debug('Ensuring that we have a real device, not a symbolic link') + device = os.path.realpath(device) LOG.debug("Checking values for %s definition" % device) overwrite = definition.get('overwrite', False) @@ -745,6 +747,9 @@ def mkfs(fs_cfg): fs_replace = fs_cfg.get('replace_fs', False) overwrite = fs_cfg.get('overwrite', False) + LOG.debug('Ensuring that we have a real device, not a symbolic link') + device = os.path.realpath(device) + # This allows you to define the default ephemeral or swap LOG.debug("Checking %s against default devices", device) diff --git a/cloudinit/config/cc_mounts.py b/cloudinit/config/cc_mounts.py index f970c2ca..73b42f91 100644 --- a/cloudinit/config/cc_mounts.py +++ b/cloudinit/config/cc_mounts.py @@ -49,7 +49,8 @@ def is_meta_device_name(name): def _get_nth_partition_for_device(device_path, partition_number): - potential_suffixes = [str(partition_number), 'p%s' % (partition_number,)] + potential_suffixes = [str(partition_number), 'p%s' % (partition_number,), + '-part%s' % (partition_number,)] for suffix in potential_suffixes: potential_partition_device = '%s%s' % (device_path, suffix) if os.path.exists(potential_partition_device): @@ -58,10 +59,11 @@ def _get_nth_partition_for_device(device_path, partition_number): def _is_block_device(device_path, partition_path=None): - device_name = device_path.split('/')[-1] + device_name = os.path.realpath(device_path).split('/')[-1] sys_path = os.path.join('/sys/block/', device_name) if partition_path is not None: - sys_path = os.path.join(sys_path, partition_path.split('/')[-1]) + sys_path = os.path.join( + sys_path, os.path.realpath(partition_path).split('/')[-1]) return os.path.exists(sys_path) -- cgit v1.2.3 From 9461b1235f7278440ffb84f1e3d95b3f906e444b Mon Sep 17 00:00:00 2001 From: Daniel Watkins Date: Tue, 21 Jul 2015 13:06:11 +0100 Subject: Use /dev/disk devices for Azure ephemeral disk. The ephemeral disk will not necessarily be assigned the same name at each boot (LP: #1411582), so we use some udev rules to ensure we always get the right one. --- cloudinit/sources/DataSourceAzure.py | 39 ++++++++++++++------------- tests/unittests/test_datasource/test_azure.py | 6 +++-- 2 files changed, 25 insertions(+), 20 deletions(-) (limited to 'cloudinit') diff --git a/cloudinit/sources/DataSourceAzure.py b/cloudinit/sources/DataSourceAzure.py index d0a882ca..1193d88b 100644 --- a/cloudinit/sources/DataSourceAzure.py +++ b/cloudinit/sources/DataSourceAzure.py @@ -254,7 +254,7 @@ class DataSourceAzureNet(sources.DataSource): self.metadata.update(fabric_data) - found_ephemeral = find_ephemeral_disk() + found_ephemeral = find_fabric_formatted_ephemeral_disk() if found_ephemeral: self.ds_cfg['disk_aliases']['ephemeral0'] = found_ephemeral LOG.debug("using detected ephemeral0 of %s", found_ephemeral) @@ -276,30 +276,33 @@ def count_files(mp): return len(fnmatch.filter(os.listdir(mp), '*[!cdrom]*')) -def find_ephemeral_part(): +def find_fabric_formatted_ephemeral_part(): """ - Locate the default ephmeral0.1 device. This will be the first device - that has a LABEL of DEF_EPHEMERAL_LABEL and is a NTFS device. If Azure - gets more ephemeral devices, this logic will only identify the first - such device. + Locate the first fabric formatted ephemeral device. """ - c_label_devs = util.find_devs_with("LABEL=%s" % DEF_EPHEMERAL_LABEL) - c_fstype_devs = util.find_devs_with("TYPE=ntfs") - for dev in c_label_devs: - if dev in c_fstype_devs: - return dev + potential_locations = ['/dev/disk/cloud/azure_resource-part1', + '/dev/disk/azure/resource-part1'] + device_location = None + for potential_location in potential_locations: + if os.path.exists(potential_location): + device_location = potential_location + break + if device_location is None: + return None + ntfs_devices = util.find_devs_with("TYPE=ntfs") + real_device = os.path.realpath(device_location) + if real_device in ntfs_devices: + return device_location return None -def find_ephemeral_disk(): +def find_fabric_formatted_ephemeral_disk(): """ Get the ephemeral disk. """ - part_dev = find_ephemeral_part() - if part_dev and str(part_dev[-1]).isdigit(): - return part_dev[:-1] - elif part_dev: - return part_dev + part_dev = find_fabric_formatted_ephemeral_part() + if part_dev: + return part_dev.split('-')[0] return None @@ -313,7 +316,7 @@ def support_new_ephemeral(cfg): new ephemeral device is detected, cloud-init overrides the default frequency for both disk-setup and mounts for the current boot only. """ - device = find_ephemeral_part() + device = find_fabric_formatted_ephemeral_part() if not device: LOG.debug("no default fabric formated ephemeral0.1 found") return None diff --git a/tests/unittests/test_datasource/test_azure.py b/tests/unittests/test_datasource/test_azure.py index 33b971f6..3b7e3293 100644 --- a/tests/unittests/test_datasource/test_azure.py +++ b/tests/unittests/test_datasource/test_azure.py @@ -475,10 +475,12 @@ class TestAzureBounce(TestCase): mock.patch.object(DataSourceAzure, 'list_possible_azure_ds_devs', mock.MagicMock(return_value=[]))) self.patches.enter_context( - mock.patch.object(DataSourceAzure, 'find_ephemeral_disk', + mock.patch.object(DataSourceAzure, + 'find_fabric_formatted_ephemeral_disk', mock.MagicMock(return_value=None))) self.patches.enter_context( - mock.patch.object(DataSourceAzure, 'find_ephemeral_part', + mock.patch.object(DataSourceAzure, + 'find_fabric_formatted_ephemeral_part', mock.MagicMock(return_value=None))) self.patches.enter_context( mock.patch.object(DataSourceAzure, 'get_metadata_from_fabric', -- cgit v1.2.3 From 73c5bbfa31b922a0ba403216c0fc1f63b22a9262 Mon Sep 17 00:00:00 2001 From: Daniel Watkins Date: Wed, 22 Jul 2015 13:06:34 +0100 Subject: Make full data source available to code that handles mirror selection. --- cloudinit/distros/__init__.py | 15 +++++++-------- cloudinit/sources/__init__.py | 3 +-- tests/unittests/test_distros/test_generic.py | 22 +++++++++++++++------- 3 files changed, 23 insertions(+), 17 deletions(-) (limited to 'cloudinit') diff --git a/cloudinit/distros/__init__.py b/cloudinit/distros/__init__.py index 8a947867..47b76c68 100644 --- a/cloudinit/distros/__init__.py +++ b/cloudinit/distros/__init__.py @@ -117,12 +117,11 @@ class Distro(object): arch = self.get_primary_arch() return _get_arch_package_mirror_info(mirror_info, arch) - def get_package_mirror_info(self, arch=None, - availability_zone=None): + def get_package_mirror_info(self, arch=None, data_source=None): # This resolves the package_mirrors config option # down to a single dict of {mirror_name: mirror_url} arch_info = self._get_arch_package_mirror_info(arch) - return _get_package_mirror_info(availability_zone=availability_zone, + return _get_package_mirror_info(data_source=data_source, mirror_info=arch_info) def apply_network(self, settings, bring_up=True): @@ -556,7 +555,7 @@ class Distro(object): LOG.info("Added user '%s' to group '%s'" % (member, name)) -def _get_package_mirror_info(mirror_info, availability_zone=None, +def _get_package_mirror_info(mirror_info, data_source=None, mirror_filter=util.search_for_mirror): # given a arch specific 'mirror_info' entry (from package_mirrors) # search through the 'search' entries, and fallback appropriately @@ -572,11 +571,11 @@ def _get_package_mirror_info(mirror_info, availability_zone=None, ec2_az_re = ("^[a-z][a-z]-(%s)-[1-9][0-9]*[a-z]$" % directions_re) subst = {} - if availability_zone: - subst['availability_zone'] = availability_zone + if data_source and data_source.availability_zone: + subst['availability_zone'] = data_source.availability_zone - if availability_zone and re.match(ec2_az_re, availability_zone): - subst['ec2_region'] = "%s" % availability_zone[0:-1] + if re.match(ec2_az_re, data_source.availability_zone): + subst['ec2_region'] = "%s" % data_source.availability_zone[0:-1] results = {} for (name, mirror) in mirror_info.get('failsafe', {}).items(): diff --git a/cloudinit/sources/__init__.py b/cloudinit/sources/__init__.py index 39eab51b..1a036638 100644 --- a/cloudinit/sources/__init__.py +++ b/cloudinit/sources/__init__.py @@ -210,8 +210,7 @@ class DataSource(object): return hostname def get_package_mirror_info(self): - return self.distro.get_package_mirror_info( - availability_zone=self.availability_zone) + return self.distro.get_package_mirror_info(data_source=self) def normalize_pubkey_data(pubkey_data): diff --git a/tests/unittests/test_distros/test_generic.py b/tests/unittests/test_distros/test_generic.py index 8e3bd78a..6ed1704c 100644 --- a/tests/unittests/test_distros/test_generic.py +++ b/tests/unittests/test_distros/test_generic.py @@ -7,6 +7,11 @@ import os import shutil import tempfile +try: + from unittest import mock +except ImportError: + import mock + unknown_arch_info = { 'arches': ['default'], 'failsafe': {'primary': 'http://fs-primary-default', @@ -144,33 +149,35 @@ class TestGenericDistro(helpers.FilesystemMockingTestCase): def test_get_package_mirror_info_az_ec2(self): arch_mirrors = gapmi(package_mirrors, arch="amd64") + data_source_mock = mock.Mock(availability_zone="us-east-1a") - results = gpmi(arch_mirrors, availability_zone="us-east-1a", + results = gpmi(arch_mirrors, data_source=data_source_mock, mirror_filter=self.return_first) self.assertEqual(results, {'primary': 'http://us-east-1.ec2/', 'security': 'http://security-mirror1-intel'}) - results = gpmi(arch_mirrors, availability_zone="us-east-1a", + results = gpmi(arch_mirrors, data_source=data_source_mock, mirror_filter=self.return_second) self.assertEqual(results, {'primary': 'http://us-east-1a.clouds/', 'security': 'http://security-mirror2-intel'}) - results = gpmi(arch_mirrors, availability_zone="us-east-1a", + results = gpmi(arch_mirrors, data_source=data_source_mock, mirror_filter=self.return_none) self.assertEqual(results, package_mirrors[0]['failsafe']) def test_get_package_mirror_info_az_non_ec2(self): arch_mirrors = gapmi(package_mirrors, arch="amd64") + data_source_mock = mock.Mock(availability_zone="nova.cloudvendor") - results = gpmi(arch_mirrors, availability_zone="nova.cloudvendor", + results = gpmi(arch_mirrors, data_source=data_source_mock, mirror_filter=self.return_first) self.assertEqual(results, {'primary': 'http://nova.cloudvendor.clouds/', 'security': 'http://security-mirror1-intel'}) - results = gpmi(arch_mirrors, availability_zone="nova.cloudvendor", + results = gpmi(arch_mirrors, data_source=data_source_mock, mirror_filter=self.return_last) self.assertEqual(results, {'primary': 'http://nova.cloudvendor.clouds/', @@ -178,17 +185,18 @@ class TestGenericDistro(helpers.FilesystemMockingTestCase): def test_get_package_mirror_info_none(self): arch_mirrors = gapmi(package_mirrors, arch="amd64") + data_source_mock = mock.Mock(availability_zone=None) # because both search entries here replacement based on # availability-zone, the filter will be called with an empty list and # failsafe should be taken. - results = gpmi(arch_mirrors, availability_zone=None, + results = gpmi(arch_mirrors, data_source=data_source_mock, mirror_filter=self.return_first) self.assertEqual(results, {'primary': 'http://fs-primary-intel', 'security': 'http://security-mirror1-intel'}) - results = gpmi(arch_mirrors, availability_zone=None, + results = gpmi(arch_mirrors, data_source=data_source_mock, mirror_filter=self.return_last) self.assertEqual(results, {'primary': 'http://fs-primary-intel', -- cgit v1.2.3 From bc7d57a0ae827978c87919c833bb5e8d2d5143c6 Mon Sep 17 00:00:00 2001 From: Daniel Watkins Date: Wed, 22 Jul 2015 13:06:34 +0100 Subject: Add DataSource.region and use it in mirror selection. Also implement DataSource.region for EC2 and GCE data sources. --- cloudinit/distros/__init__.py | 3 +++ cloudinit/sources/DataSourceEc2.py | 7 +++++++ cloudinit/sources/DataSourceGCE.py | 4 ++++ cloudinit/sources/__init__.py | 4 ++++ config/cloud.cfg | 1 + 5 files changed, 19 insertions(+) (limited to 'cloudinit') diff --git a/cloudinit/distros/__init__.py b/cloudinit/distros/__init__.py index 47b76c68..71884b32 100644 --- a/cloudinit/distros/__init__.py +++ b/cloudinit/distros/__init__.py @@ -577,6 +577,9 @@ def _get_package_mirror_info(mirror_info, data_source=None, if re.match(ec2_az_re, data_source.availability_zone): subst['ec2_region'] = "%s" % data_source.availability_zone[0:-1] + if data_source and data_source.region: + subst['region'] = data_source.region + results = {} for (name, mirror) in mirror_info.get('failsafe', {}).items(): results[name] = mirror diff --git a/cloudinit/sources/DataSourceEc2.py b/cloudinit/sources/DataSourceEc2.py index 798869b7..0032d06c 100644 --- a/cloudinit/sources/DataSourceEc2.py +++ b/cloudinit/sources/DataSourceEc2.py @@ -197,6 +197,13 @@ class DataSourceEc2(sources.DataSource): except KeyError: return None + @property + def region(self): + az = self.availability_zone + if az is not None: + return az[:-1] + return None + # Used to match classes to dependencies datasources = [ (DataSourceEc2, (sources.DEP_FILESYSTEM, sources.DEP_NETWORK)), diff --git a/cloudinit/sources/DataSourceGCE.py b/cloudinit/sources/DataSourceGCE.py index 1b28a68c..7e7fc033 100644 --- a/cloudinit/sources/DataSourceGCE.py +++ b/cloudinit/sources/DataSourceGCE.py @@ -152,6 +152,10 @@ class DataSourceGCE(sources.DataSource): def availability_zone(self): return self.metadata['availability-zone'] + @property + def region(self): + return self.availability_zone.rsplit('-', 1)[0] + # Used to match classes to dependencies datasources = [ (DataSourceGCE, (sources.DEP_FILESYSTEM, sources.DEP_NETWORK)), diff --git a/cloudinit/sources/__init__.py b/cloudinit/sources/__init__.py index 1a036638..a21c08c2 100644 --- a/cloudinit/sources/__init__.py +++ b/cloudinit/sources/__init__.py @@ -157,6 +157,10 @@ class DataSource(object): return self.metadata.get('availability-zone', self.metadata.get('availability_zone')) + @property + def region(self): + return self.metadata.get('region') + def get_instance_id(self): if not self.metadata or 'instance-id' not in self.metadata: # Return a magic not really instance id string diff --git a/config/cloud.cfg b/config/cloud.cfg index e96e1781..2b27f379 100644 --- a/config/cloud.cfg +++ b/config/cloud.cfg @@ -104,6 +104,7 @@ system_info: primary: - http://%(ec2_region)s.ec2.archive.ubuntu.com/ubuntu/ - http://%(availability_zone)s.clouds.archive.ubuntu.com/ubuntu/ + - http://%(region)s.clouds.archive.ubuntu.com/ubuntu/ security: [] - arches: [armhf, armel, default] failsafe: -- cgit v1.2.3 From 452ea086beb8b28b41f5ccc610f4e5433010e35b Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Wed, 22 Jul 2015 15:14:33 -0400 Subject: remove some overly verbose log messages --- cloudinit/config/cc_disk_setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'cloudinit') diff --git a/cloudinit/config/cc_disk_setup.py b/cloudinit/config/cc_disk_setup.py index 92fa7a94..d5b0d1d7 100644 --- a/cloudinit/config/cc_disk_setup.py +++ b/cloudinit/config/cc_disk_setup.py @@ -648,7 +648,7 @@ def mkpart(device, definition): table_type: Which partition table to use, defaults to MBR device: the device to work on. """ - LOG.debug('Ensuring that we have a real device, not a symbolic link') + # ensure that we get a real device rather than a symbolic link device = os.path.realpath(device) LOG.debug("Checking values for %s definition" % device) @@ -747,7 +747,7 @@ def mkfs(fs_cfg): fs_replace = fs_cfg.get('replace_fs', False) overwrite = fs_cfg.get('overwrite', False) - LOG.debug('Ensuring that we have a real device, not a symbolic link') + # ensure that we get a real device rather than a symbolic link device = os.path.realpath(device) # This allows you to define the default ephemeral or swap -- cgit v1.2.3