provide data source to mirror selection code for region-specific mirrors.

  
This makes the full data source available to the mirror selection code,
which means we can implement region logic on the data source for
everything to use rather than it being mirror-selection-specific.

It also implements that region logic for EC2 and GCE.

LP: #1470890

7 files changed, 44 insertions, 17 deletions

diff --git a/ChangeLog b/ChangeLog
index 77b22d5a..d4ad6caf 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -53,6 +53,8 @@
  - CloudStack: fix password setting on cloudstack > 4.5.1 (LP: #1464253)
  - GCE: fix determination of availability zone (LP: #1470880)
  - ssh: generate ed25519 host keys (LP: #1461242)
+ - distro mirrors: provide datasource to mirror selection code to support
+   GCE regional mirrors. (LP: #1470890)
 0.7.6:
  - open 0.7.6
  - Enable vendordata on CloudSigma datasource (LP: #1303986)
diff --git a/cloudinit/distros/__init__.py b/cloudinit/distros/__init__.py
index 8a947867..71884b32 100644
--- a/cloudinit/distros/__init__.py
+++ b/cloudinit/distros/__init__.py
@@ -117,12 +117,11 @@ class Distro(object):
             arch = self.get_primary_arch()
         return _get_arch_package_mirror_info(mirror_info, arch)
 
-    def get_package_mirror_info(self, arch=None,
-                                availability_zone=None):
+    def get_package_mirror_info(self, arch=None, data_source=None):
         # This resolves the package_mirrors config option
         # down to a single dict of {mirror_name: mirror_url}
         arch_info = self._get_arch_package_mirror_info(arch)
-        return _get_package_mirror_info(availability_zone=availability_zone,
+        return _get_package_mirror_info(data_source=data_source,
                                         mirror_info=arch_info)
 
     def apply_network(self, settings, bring_up=True):
@@ -556,7 +555,7 @@ class Distro(object):
                 LOG.info("Added user '%s' to group '%s'" % (member, name))
 
 
-def _get_package_mirror_info(mirror_info, availability_zone=None,
+def _get_package_mirror_info(mirror_info, data_source=None,
                              mirror_filter=util.search_for_mirror):
     # given a arch specific 'mirror_info' entry (from package_mirrors)
     # search through the 'search' entries, and fallback appropriately
@@ -572,11 +571,14 @@ def _get_package_mirror_info(mirror_info, availability_zone=None,
     ec2_az_re = ("^[a-z][a-z]-(%s)-[1-9][0-9]*[a-z]$" % directions_re)
 
     subst = {}
-    if availability_zone:
-        subst['availability_zone'] = availability_zone
+    if data_source and data_source.availability_zone:
+        subst['availability_zone'] = data_source.availability_zone
 
-    if availability_zone and re.match(ec2_az_re, availability_zone):
-        subst['ec2_region'] = "%s" % availability_zone[0:-1]
+        if re.match(ec2_az_re, data_source.availability_zone):
+            subst['ec2_region'] = "%s" % data_source.availability_zone[0:-1]
+
+    if data_source and data_source.region:
+        subst['region'] = data_source.region
 
     results = {}
     for (name, mirror) in mirror_info.get('failsafe', {}).items():
diff --git a/cloudinit/sources/DataSourceEc2.py b/cloudinit/sources/DataSourceEc2.py
index 798869b7..0032d06c 100644
--- a/cloudinit/sources/DataSourceEc2.py
+++ b/cloudinit/sources/DataSourceEc2.py
@@ -197,6 +197,13 @@ class DataSourceEc2(sources.DataSource):
         except KeyError:
             return None
 
+    @property
+    def region(self):
+        az = self.availability_zone
+        if az is not None:
+            return az[:-1]
+        return None
+
 # Used to match classes to dependencies
 datasources = [
   (DataSourceEc2, (sources.DEP_FILESYSTEM, sources.DEP_NETWORK)),
diff --git a/cloudinit/sources/DataSourceGCE.py b/cloudinit/sources/DataSourceGCE.py
index 1b28a68c..7e7fc033 100644
--- a/cloudinit/sources/DataSourceGCE.py
+++ b/cloudinit/sources/DataSourceGCE.py
@@ -152,6 +152,10 @@ class DataSourceGCE(sources.DataSource):
     def availability_zone(self):
         return self.metadata['availability-zone']
 
+    @property
+    def region(self):
+        return self.availability_zone.rsplit('-', 1)[0]
+
 # Used to match classes to dependencies
 datasources = [
     (DataSourceGCE, (sources.DEP_FILESYSTEM, sources.DEP_NETWORK)),
diff --git a/cloudinit/sources/__init__.py b/cloudinit/sources/__init__.py
index 39eab51b..a21c08c2 100644
--- a/cloudinit/sources/__init__.py
+++ b/cloudinit/sources/__init__.py
@@ -157,6 +157,10 @@ class DataSource(object):
         return self.metadata.get('availability-zone',
                                  self.metadata.get('availability_zone'))
 
+    @property
+    def region(self):
+        return self.metadata.get('region')
+
     def get_instance_id(self):
         if not self.metadata or 'instance-id' not in self.metadata:
             # Return a magic not really instance id string
@@ -210,8 +214,7 @@ class DataSource(object):
             return hostname
 
     def get_package_mirror_info(self):
-        return self.distro.get_package_mirror_info(
-            availability_zone=self.availability_zone)
+        return self.distro.get_package_mirror_info(data_source=self)
 
 
 def normalize_pubkey_data(pubkey_data):
diff --git a/config/cloud.cfg b/config/cloud.cfg
index e96e1781..2b27f379 100644
--- a/config/cloud.cfg
+++ b/config/cloud.cfg
@@ -104,6 +104,7 @@ system_info:
          primary:
            - http://%(ec2_region)s.ec2.archive.ubuntu.com/ubuntu/
            - http://%(availability_zone)s.clouds.archive.ubuntu.com/ubuntu/
+           - http://%(region)s.clouds.archive.ubuntu.com/ubuntu/
          security: []
      - arches: [armhf, armel, default]
        failsafe:
diff --git a/tests/unittests/test_distros/test_generic.py b/tests/unittests/test_distros/test_generic.py
index 8e3bd78a..6ed1704c 100644
--- a/tests/unittests/test_distros/test_generic.py
+++ b/tests/unittests/test_distros/test_generic.py
@@ -7,6 +7,11 @@ import os
 import shutil
 import tempfile
 
+try:
+    from unittest import mock
+except ImportError:
+    import mock
+
 unknown_arch_info = {
     'arches': ['default'],
     'failsafe': {'primary': 'http://fs-primary-default',
@@ -144,33 +149,35 @@ class TestGenericDistro(helpers.FilesystemMockingTestCase):
 
     def test_get_package_mirror_info_az_ec2(self):
         arch_mirrors = gapmi(package_mirrors, arch="amd64")
+        data_source_mock = mock.Mock(availability_zone="us-east-1a")
 
-        results = gpmi(arch_mirrors, availability_zone="us-east-1a",
+        results = gpmi(arch_mirrors, data_source=data_source_mock,
                        mirror_filter=self.return_first)
         self.assertEqual(results,
                          {'primary': 'http://us-east-1.ec2/',
                           'security': 'http://security-mirror1-intel'})
 
-        results = gpmi(arch_mirrors, availability_zone="us-east-1a",
+        results = gpmi(arch_mirrors, data_source=data_source_mock,
                        mirror_filter=self.return_second)
         self.assertEqual(results,
                          {'primary': 'http://us-east-1a.clouds/',
                           'security': 'http://security-mirror2-intel'})
 
-        results = gpmi(arch_mirrors, availability_zone="us-east-1a",
+        results = gpmi(arch_mirrors, data_source=data_source_mock,
                        mirror_filter=self.return_none)
         self.assertEqual(results, package_mirrors[0]['failsafe'])
 
     def test_get_package_mirror_info_az_non_ec2(self):
         arch_mirrors = gapmi(package_mirrors, arch="amd64")
+        data_source_mock = mock.Mock(availability_zone="nova.cloudvendor")
 
-        results = gpmi(arch_mirrors, availability_zone="nova.cloudvendor",
+        results = gpmi(arch_mirrors, data_source=data_source_mock,
                        mirror_filter=self.return_first)
         self.assertEqual(results,
                          {'primary': 'http://nova.cloudvendor.clouds/',
                           'security': 'http://security-mirror1-intel'})
 
-        results = gpmi(arch_mirrors, availability_zone="nova.cloudvendor",
+        results = gpmi(arch_mirrors, data_source=data_source_mock,
                        mirror_filter=self.return_last)
         self.assertEqual(results,
                          {'primary': 'http://nova.cloudvendor.clouds/',
@@ -178,17 +185,18 @@ class TestGenericDistro(helpers.FilesystemMockingTestCase):
 
     def test_get_package_mirror_info_none(self):
         arch_mirrors = gapmi(package_mirrors, arch="amd64")
+        data_source_mock = mock.Mock(availability_zone=None)
 
         # because both search entries here replacement based on
         # availability-zone, the filter will be called with an empty list and
         # failsafe should be taken.
-        results = gpmi(arch_mirrors, availability_zone=None,
+        results = gpmi(arch_mirrors, data_source=data_source_mock,
                        mirror_filter=self.return_first)
         self.assertEqual(results,
                          {'primary': 'http://fs-primary-intel',
                           'security': 'http://security-mirror1-intel'})
 
-        results = gpmi(arch_mirrors, availability_zone=None,
+        results = gpmi(arch_mirrors, data_source=data_source_mock,
                        mirror_filter=self.return_last)
         self.assertEqual(results,
                          {'primary': 'http://fs-primary-intel',