From 9461b1235f7278440ffb84f1e3d95b3f906e444b Mon Sep 17 00:00:00 2001 From: Daniel Watkins Date: Tue, 21 Jul 2015 13:06:11 +0100 Subject: Use /dev/disk devices for Azure ephemeral disk. The ephemeral disk will not necessarily be assigned the same name at each boot (LP: #1411582), so we use some udev rules to ensure we always get the right one. --- cloudinit/sources/DataSourceAzure.py | 39 +++++++++++++++++++----------------- 1 file changed, 21 insertions(+), 18 deletions(-) (limited to 'cloudinit/sources') diff --git a/cloudinit/sources/DataSourceAzure.py b/cloudinit/sources/DataSourceAzure.py index d0a882ca..1193d88b 100644 --- a/cloudinit/sources/DataSourceAzure.py +++ b/cloudinit/sources/DataSourceAzure.py @@ -254,7 +254,7 @@ class DataSourceAzureNet(sources.DataSource): self.metadata.update(fabric_data) - found_ephemeral = find_ephemeral_disk() + found_ephemeral = find_fabric_formatted_ephemeral_disk() if found_ephemeral: self.ds_cfg['disk_aliases']['ephemeral0'] = found_ephemeral LOG.debug("using detected ephemeral0 of %s", found_ephemeral) @@ -276,30 +276,33 @@ def count_files(mp): return len(fnmatch.filter(os.listdir(mp), '*[!cdrom]*')) -def find_ephemeral_part(): +def find_fabric_formatted_ephemeral_part(): """ - Locate the default ephmeral0.1 device. This will be the first device - that has a LABEL of DEF_EPHEMERAL_LABEL and is a NTFS device. If Azure - gets more ephemeral devices, this logic will only identify the first - such device. + Locate the first fabric formatted ephemeral device. """ - c_label_devs = util.find_devs_with("LABEL=%s" % DEF_EPHEMERAL_LABEL) - c_fstype_devs = util.find_devs_with("TYPE=ntfs") - for dev in c_label_devs: - if dev in c_fstype_devs: - return dev + potential_locations = ['/dev/disk/cloud/azure_resource-part1', + '/dev/disk/azure/resource-part1'] + device_location = None + for potential_location in potential_locations: + if os.path.exists(potential_location): + device_location = potential_location + break + if device_location is None: + return None + ntfs_devices = util.find_devs_with("TYPE=ntfs") + real_device = os.path.realpath(device_location) + if real_device in ntfs_devices: + return device_location return None -def find_ephemeral_disk(): +def find_fabric_formatted_ephemeral_disk(): """ Get the ephemeral disk. """ - part_dev = find_ephemeral_part() - if part_dev and str(part_dev[-1]).isdigit(): - return part_dev[:-1] - elif part_dev: - return part_dev + part_dev = find_fabric_formatted_ephemeral_part() + if part_dev: + return part_dev.split('-')[0] return None @@ -313,7 +316,7 @@ def support_new_ephemeral(cfg): new ephemeral device is detected, cloud-init overrides the default frequency for both disk-setup and mounts for the current boot only. """ - device = find_ephemeral_part() + device = find_fabric_formatted_ephemeral_part() if not device: LOG.debug("no default fabric formated ephemeral0.1 found") return None -- cgit v1.2.3 From 73c5bbfa31b922a0ba403216c0fc1f63b22a9262 Mon Sep 17 00:00:00 2001 From: Daniel Watkins Date: Wed, 22 Jul 2015 13:06:34 +0100 Subject: Make full data source available to code that handles mirror selection. --- cloudinit/distros/__init__.py | 15 +++++++-------- cloudinit/sources/__init__.py | 3 +-- tests/unittests/test_distros/test_generic.py | 22 +++++++++++++++------- 3 files changed, 23 insertions(+), 17 deletions(-) (limited to 'cloudinit/sources') diff --git a/cloudinit/distros/__init__.py b/cloudinit/distros/__init__.py index 8a947867..47b76c68 100644 --- a/cloudinit/distros/__init__.py +++ b/cloudinit/distros/__init__.py @@ -117,12 +117,11 @@ class Distro(object): arch = self.get_primary_arch() return _get_arch_package_mirror_info(mirror_info, arch) - def get_package_mirror_info(self, arch=None, - availability_zone=None): + def get_package_mirror_info(self, arch=None, data_source=None): # This resolves the package_mirrors config option # down to a single dict of {mirror_name: mirror_url} arch_info = self._get_arch_package_mirror_info(arch) - return _get_package_mirror_info(availability_zone=availability_zone, + return _get_package_mirror_info(data_source=data_source, mirror_info=arch_info) def apply_network(self, settings, bring_up=True): @@ -556,7 +555,7 @@ class Distro(object): LOG.info("Added user '%s' to group '%s'" % (member, name)) -def _get_package_mirror_info(mirror_info, availability_zone=None, +def _get_package_mirror_info(mirror_info, data_source=None, mirror_filter=util.search_for_mirror): # given a arch specific 'mirror_info' entry (from package_mirrors) # search through the 'search' entries, and fallback appropriately @@ -572,11 +571,11 @@ def _get_package_mirror_info(mirror_info, availability_zone=None, ec2_az_re = ("^[a-z][a-z]-(%s)-[1-9][0-9]*[a-z]$" % directions_re) subst = {} - if availability_zone: - subst['availability_zone'] = availability_zone + if data_source and data_source.availability_zone: + subst['availability_zone'] = data_source.availability_zone - if availability_zone and re.match(ec2_az_re, availability_zone): - subst['ec2_region'] = "%s" % availability_zone[0:-1] + if re.match(ec2_az_re, data_source.availability_zone): + subst['ec2_region'] = "%s" % data_source.availability_zone[0:-1] results = {} for (name, mirror) in mirror_info.get('failsafe', {}).items(): diff --git a/cloudinit/sources/__init__.py b/cloudinit/sources/__init__.py index 39eab51b..1a036638 100644 --- a/cloudinit/sources/__init__.py +++ b/cloudinit/sources/__init__.py @@ -210,8 +210,7 @@ class DataSource(object): return hostname def get_package_mirror_info(self): - return self.distro.get_package_mirror_info( - availability_zone=self.availability_zone) + return self.distro.get_package_mirror_info(data_source=self) def normalize_pubkey_data(pubkey_data): diff --git a/tests/unittests/test_distros/test_generic.py b/tests/unittests/test_distros/test_generic.py index 8e3bd78a..6ed1704c 100644 --- a/tests/unittests/test_distros/test_generic.py +++ b/tests/unittests/test_distros/test_generic.py @@ -7,6 +7,11 @@ import os import shutil import tempfile +try: + from unittest import mock +except ImportError: + import mock + unknown_arch_info = { 'arches': ['default'], 'failsafe': {'primary': 'http://fs-primary-default', @@ -144,33 +149,35 @@ class TestGenericDistro(helpers.FilesystemMockingTestCase): def test_get_package_mirror_info_az_ec2(self): arch_mirrors = gapmi(package_mirrors, arch="amd64") + data_source_mock = mock.Mock(availability_zone="us-east-1a") - results = gpmi(arch_mirrors, availability_zone="us-east-1a", + results = gpmi(arch_mirrors, data_source=data_source_mock, mirror_filter=self.return_first) self.assertEqual(results, {'primary': 'http://us-east-1.ec2/', 'security': 'http://security-mirror1-intel'}) - results = gpmi(arch_mirrors, availability_zone="us-east-1a", + results = gpmi(arch_mirrors, data_source=data_source_mock, mirror_filter=self.return_second) self.assertEqual(results, {'primary': 'http://us-east-1a.clouds/', 'security': 'http://security-mirror2-intel'}) - results = gpmi(arch_mirrors, availability_zone="us-east-1a", + results = gpmi(arch_mirrors, data_source=data_source_mock, mirror_filter=self.return_none) self.assertEqual(results, package_mirrors[0]['failsafe']) def test_get_package_mirror_info_az_non_ec2(self): arch_mirrors = gapmi(package_mirrors, arch="amd64") + data_source_mock = mock.Mock(availability_zone="nova.cloudvendor") - results = gpmi(arch_mirrors, availability_zone="nova.cloudvendor", + results = gpmi(arch_mirrors, data_source=data_source_mock, mirror_filter=self.return_first) self.assertEqual(results, {'primary': 'http://nova.cloudvendor.clouds/', 'security': 'http://security-mirror1-intel'}) - results = gpmi(arch_mirrors, availability_zone="nova.cloudvendor", + results = gpmi(arch_mirrors, data_source=data_source_mock, mirror_filter=self.return_last) self.assertEqual(results, {'primary': 'http://nova.cloudvendor.clouds/', @@ -178,17 +185,18 @@ class TestGenericDistro(helpers.FilesystemMockingTestCase): def test_get_package_mirror_info_none(self): arch_mirrors = gapmi(package_mirrors, arch="amd64") + data_source_mock = mock.Mock(availability_zone=None) # because both search entries here replacement based on # availability-zone, the filter will be called with an empty list and # failsafe should be taken. - results = gpmi(arch_mirrors, availability_zone=None, + results = gpmi(arch_mirrors, data_source=data_source_mock, mirror_filter=self.return_first) self.assertEqual(results, {'primary': 'http://fs-primary-intel', 'security': 'http://security-mirror1-intel'}) - results = gpmi(arch_mirrors, availability_zone=None, + results = gpmi(arch_mirrors, data_source=data_source_mock, mirror_filter=self.return_last) self.assertEqual(results, {'primary': 'http://fs-primary-intel', -- cgit v1.2.3 From bc7d57a0ae827978c87919c833bb5e8d2d5143c6 Mon Sep 17 00:00:00 2001 From: Daniel Watkins Date: Wed, 22 Jul 2015 13:06:34 +0100 Subject: Add DataSource.region and use it in mirror selection. Also implement DataSource.region for EC2 and GCE data sources. --- cloudinit/distros/__init__.py | 3 +++ cloudinit/sources/DataSourceEc2.py | 7 +++++++ cloudinit/sources/DataSourceGCE.py | 4 ++++ cloudinit/sources/__init__.py | 4 ++++ config/cloud.cfg | 1 + 5 files changed, 19 insertions(+) (limited to 'cloudinit/sources') diff --git a/cloudinit/distros/__init__.py b/cloudinit/distros/__init__.py index 47b76c68..71884b32 100644 --- a/cloudinit/distros/__init__.py +++ b/cloudinit/distros/__init__.py @@ -577,6 +577,9 @@ def _get_package_mirror_info(mirror_info, data_source=None, if re.match(ec2_az_re, data_source.availability_zone): subst['ec2_region'] = "%s" % data_source.availability_zone[0:-1] + if data_source and data_source.region: + subst['region'] = data_source.region + results = {} for (name, mirror) in mirror_info.get('failsafe', {}).items(): results[name] = mirror diff --git a/cloudinit/sources/DataSourceEc2.py b/cloudinit/sources/DataSourceEc2.py index 798869b7..0032d06c 100644 --- a/cloudinit/sources/DataSourceEc2.py +++ b/cloudinit/sources/DataSourceEc2.py @@ -197,6 +197,13 @@ class DataSourceEc2(sources.DataSource): except KeyError: return None + @property + def region(self): + az = self.availability_zone + if az is not None: + return az[:-1] + return None + # Used to match classes to dependencies datasources = [ (DataSourceEc2, (sources.DEP_FILESYSTEM, sources.DEP_NETWORK)), diff --git a/cloudinit/sources/DataSourceGCE.py b/cloudinit/sources/DataSourceGCE.py index 1b28a68c..7e7fc033 100644 --- a/cloudinit/sources/DataSourceGCE.py +++ b/cloudinit/sources/DataSourceGCE.py @@ -152,6 +152,10 @@ class DataSourceGCE(sources.DataSource): def availability_zone(self): return self.metadata['availability-zone'] + @property + def region(self): + return self.availability_zone.rsplit('-', 1)[0] + # Used to match classes to dependencies datasources = [ (DataSourceGCE, (sources.DEP_FILESYSTEM, sources.DEP_NETWORK)), diff --git a/cloudinit/sources/__init__.py b/cloudinit/sources/__init__.py index 1a036638..a21c08c2 100644 --- a/cloudinit/sources/__init__.py +++ b/cloudinit/sources/__init__.py @@ -157,6 +157,10 @@ class DataSource(object): return self.metadata.get('availability-zone', self.metadata.get('availability_zone')) + @property + def region(self): + return self.metadata.get('region') + def get_instance_id(self): if not self.metadata or 'instance-id' not in self.metadata: # Return a magic not really instance id string diff --git a/config/cloud.cfg b/config/cloud.cfg index e96e1781..2b27f379 100644 --- a/config/cloud.cfg +++ b/config/cloud.cfg @@ -104,6 +104,7 @@ system_info: primary: - http://%(ec2_region)s.ec2.archive.ubuntu.com/ubuntu/ - http://%(availability_zone)s.clouds.archive.ubuntu.com/ubuntu/ + - http://%(region)s.clouds.archive.ubuntu.com/ubuntu/ security: [] - arches: [armhf, armel, default] failsafe: -- cgit v1.2.3