rework package mirror selection

There are several changes here.
 * Datasource now has a 'availability_zone' getter.
 * get_package_mirror_info
   * Datasource convenience 'get_package_mirror_info' that calls
     the configured distro, and passes it the availability-zone
   * distro has a get_package_mirror_info method
   * get_package_mirror_info returns a dict that of name:mirror
     this is to facilitate use of 'security' and 'primary' archive.
   * this supports searching based on templates. Any template
     that references undefined values is skipped. These templates
     can contain 'availability_zone' (LP: #1037727)
   * distro's mirrors can be arch specific (LP: #1028501)
   * 'ec2_region' substitution variable is present only if this
     appears (by availability-zone naming convention) to be in EC2.
 * rename_apt_lists supports the "mirror_info" rather than single mirror
 * generate_sources_list supports mirror_info, and as a result, the
   ubuntu mirrors reference '$security' rather than security (LP: #1006963)
 * remove the DataSourceEc2 specific mirror selection, but instead
   rely on the above filtering, and the fact that 'ec2_region' is only
   defined if the availability_zone looks like a ec2 az.
 * package_mirrors is now required in the system_info config, a dict like:
   package_mirrors:
     - arches: [i386, amd64]
       failsafe:
         primary: http://archive.ubuntu.com/ubuntu
         security: http://security.ubuntu.com/ubuntu
       search:
         primary:
           - http://%(ec2_region)s.ec2.archive.ubuntu.com/ubuntu/
           - http://%(availability_zone)s.clouds.archive.ubuntu.com/ubuntu/
         security: []
     - arches: [armhf, armel, default]
       failsafe:
         primary: http://ports.ubuntu.com/ubuntu
         security: http://ports.ubuntu.com/ubuntu

LP: #1006963, #1028501, #1037727

1 files changed, 121 insertions, 0 deletions

diff --git a/tests/unittests/test_distros/test_generic.py b/tests/unittests/test_distros/test_generic.py
new file mode 100644
index 00000000..2df4c2f0
--- /dev/null
+++ b/tests/unittests/test_distros/test_generic.py
@@ -0,0 +1,121 @@
+from mocker import MockerTestCase
+
+from cloudinit import distros
+
+unknown_arch_info = {
+    'arches': ['default'],
+    'failsafe': {'primary': 'http://fs-primary-default',
+                 'security': 'http://fs-security-default'}
+}
+
+package_mirrors = [
+    {'arches': ['i386', 'amd64'],
+     'failsafe': {'primary': 'http://fs-primary-intel',
+                  'security': 'http://fs-security-intel'},
+     'search': {
+         'primary': ['http://%(ec2_region)s.ec2/',
+                     'http://%(availability_zone)s.clouds/'],
+         'security': ['http://security-mirror1-intel',
+                      'http://security-mirror2-intel']}},
+    {'arches': ['armhf', 'armel'],
+     'failsafe': {'primary': 'http://fs-primary-arm',
+                  'security': 'http://fs-security-arm'}},
+    unknown_arch_info
+]
+
+gpmi = distros._get_package_mirror_info  # pylint: disable=W0212
+gapmi = distros._get_arch_package_mirror_info  # pylint: disable=W0212
+
+
+class TestGenericDistro(MockerTestCase):
+
+    def return_first(self, mlist):
+        if not mlist:
+            return None
+        return mlist[0]
+
+    def return_second(self, mlist):
+        if not mlist:
+            return None
+        return mlist[1]
+
+    def return_none(self, _mlist):
+        return None
+
+    def return_last(self, mlist):
+        if not mlist:
+            return None
+        return(mlist[-1])
+
+    def setUp(self):
+        super(TestGenericDistro, self).setUp()
+        # Make a temp directoy for tests to use.
+        self.tmp = self.makeDir()
+
+    def test_arch_package_mirror_info_unknown(self):
+        """for an unknown arch, we should get back that with arch 'default'."""
+        arch_mirrors = gapmi(package_mirrors, arch="unknown")
+        self.assertEqual(unknown_arch_info, arch_mirrors)
+
+    def test_arch_package_mirror_info_known(self):
+        arch_mirrors = gapmi(package_mirrors, arch="amd64")
+        self.assertEqual(package_mirrors[0], arch_mirrors)
+
+    def test_get_package_mirror_info_az_ec2(self):
+        arch_mirrors = gapmi(package_mirrors, arch="amd64")
+
+        results = gpmi(arch_mirrors, availability_zone="us-east-1a",
+                       mirror_filter=self.return_first)
+        self.assertEqual(results,
+                         {'primary': 'http://us-east-1.ec2/',
+                          'security': 'http://security-mirror1-intel'})
+
+        results = gpmi(arch_mirrors, availability_zone="us-east-1a",
+                       mirror_filter=self.return_second)
+        self.assertEqual(results,
+                         {'primary': 'http://us-east-1a.clouds/',
+                          'security': 'http://security-mirror2-intel'})
+
+        results = gpmi(arch_mirrors, availability_zone="us-east-1a",
+                       mirror_filter=self.return_none)
+        self.assertEqual(results, package_mirrors[0]['failsafe'])
+
+    def test_get_package_mirror_info_az_non_ec2(self):
+        arch_mirrors = gapmi(package_mirrors, arch="amd64")
+
+        results = gpmi(arch_mirrors, availability_zone="nova.cloudvendor",
+                       mirror_filter=self.return_first)
+        self.assertEqual(results,
+                         {'primary': 'http://nova.cloudvendor.clouds/',
+                          'security': 'http://security-mirror1-intel'})
+
+        results = gpmi(arch_mirrors, availability_zone="nova.cloudvendor",
+                       mirror_filter=self.return_last)
+        self.assertEqual(results,
+                         {'primary': 'http://nova.cloudvendor.clouds/',
+                          'security': 'http://security-mirror2-intel'})
+
+    def test_get_package_mirror_info_none(self):
+        arch_mirrors = gapmi(package_mirrors, arch="amd64")
+
+        # because both search entries here replacement based on
+        # availability-zone, the filter will be called with an empty list and
+        # failsafe should be taken.
+        results = gpmi(arch_mirrors, availability_zone=None,
+                       mirror_filter=self.return_first)
+        self.assertEqual(results,
+                         {'primary': 'http://fs-primary-intel',
+                          'security': 'http://security-mirror1-intel'})
+
+        results = gpmi(arch_mirrors, availability_zone=None,
+                       mirror_filter=self.return_last)
+        self.assertEqual(results,
+                         {'primary': 'http://fs-primary-intel',
+                          'security': 'http://security-mirror2-intel'})
+
+
+#def _get_package_mirror_info(mirror_info, availability_zone=None,
+#                             mirror_filter=util.search_for_mirror):
+
+
+# vi: ts=4 expandtab