rework package mirror selection

There are several changes here.
 * Datasource now has a 'availability_zone' getter.
 * get_package_mirror_info
   * Datasource convenience 'get_package_mirror_info' that calls
     the configured distro, and passes it the availability-zone
   * distro has a get_package_mirror_info method
   * get_package_mirror_info returns a dict that of name:mirror
     this is to facilitate use of 'security' and 'primary' archive.
   * this supports searching based on templates. Any template
     that references undefined values is skipped. These templates
     can contain 'availability_zone' (LP: #1037727)
   * distro's mirrors can be arch specific (LP: #1028501)
   * 'ec2_region' substitution variable is present only if this
     appears (by availability-zone naming convention) to be in EC2.
 * rename_apt_lists supports the "mirror_info" rather than single mirror
 * generate_sources_list supports mirror_info, and as a result, the
   ubuntu mirrors reference '$security' rather than security (LP: #1006963)
 * remove the DataSourceEc2 specific mirror selection, but instead
   rely on the above filtering, and the fact that 'ec2_region' is only
   defined if the availability_zone looks like a ec2 az.
 * package_mirrors is now required in the system_info config, a dict like:
   package_mirrors:
     - arches: [i386, amd64]
       failsafe:
         primary: http://archive.ubuntu.com/ubuntu
         security: http://security.ubuntu.com/ubuntu
       search:
         primary:
           - http://%(ec2_region)s.ec2.archive.ubuntu.com/ubuntu/
           - http://%(availability_zone)s.clouds.archive.ubuntu.com/ubuntu/
         security: []
     - arches: [armhf, armel, default]
       failsafe:
         primary: http://ports.ubuntu.com/ubuntu
         security: http://ports.ubuntu.com/ubuntu

LP: #1006963, #1028501, #1037727

1 files changed, 71 insertions, 2 deletions

diff --git a/cloudinit/distros/__init__.py b/cloudinit/distros/__init__.py
index b9609b7a..357209a4 100644
--- a/cloudinit/distros/__init__.py
+++ b/cloudinit/distros/__init__.py
@@ -23,6 +23,8 @@
 from StringIO import StringIO
 
 import abc
+import os
+import re
 
 from cloudinit import importer
 from cloudinit import log as logging
@@ -75,8 +77,26 @@ class Distro(object):
     def update_package_sources(self):
         raise NotImplementedError()
 
-    def get_package_mirror(self):
-        return self.get_option('package_mirror')
+    def get_primary_arch(self):
+        arch = os.uname[4]
+        if arch in ("i386", "i486", "i586", "i686"):
+            return "i386"
+        return arch
+
+    def _get_arch_package_mirror_info(self, arch=None):
+        mirror_info = self.get_option("package_mirrors", None)
+        if arch == None:
+            arch = self.get_primary_arch()
+        return _get_arch_package_mirror_info(mirror_info, arch)
+
+    def get_package_mirror_info(self, arch=None,
+                                availability_zone=None):
+        # this resolves the package_mirrors config option
+        # down to a single dict of {mirror_name: mirror_url}
+        arch_info = self._get_arch_package_mirror_info(arch)
+
+        return _get_package_mirror_info(availability_zone=availability_zone,
+                                        mirror_info=arch_info)
 
     def apply_network(self, settings, bring_up=True):
         # Write it out
@@ -151,6 +171,55 @@ class Distro(object):
             return False
 
 
+def _get_package_mirror_info(mirror_info, availability_zone=None,
+                             mirror_filter=util.search_for_mirror):
+    # given a arch specific 'mirror_info' entry (from package_mirrors)
+    # search through the 'search' entries, and fallback appropriately
+    # return a dict with only {name: mirror} entries.
+
+    ec2_az_re = ("^[a-z][a-z]-(%s)-[1-9][0-9]*[a-z]$" %
+        "north|northeast|east|southeast|south|southwest|west|northwest")
+
+    subst = {}
+    if availability_zone:
+        subst['availability_zone'] = availability_zone
+
+    if availability_zone and re.match(ec2_az_re, availability_zone):
+        subst['ec2_region'] = "%s" % availability_zone[0:-1]
+
+    results = {}
+    for (name, mirror) in mirror_info.get('failsafe', {}).iteritems():
+        results[name] = mirror
+
+    for (name, searchlist) in mirror_info.get('search', {}).iteritems():
+        mirrors = []
+        for tmpl in searchlist:
+            try:
+                mirrors.append(tmpl % subst)
+            except KeyError:
+                pass
+
+        found = mirror_filter(mirrors)
+        if found:
+            results[name] = found
+
+    LOG.debug("filtered distro mirror info: %s" % results)
+
+    return results
+
+
+def _get_arch_package_mirror_info(package_mirrors, arch):
+    # pull out the specific arch from a 'package_mirrors' config option
+    default = None
+    for item in package_mirrors:
+        arches = item.get("arches")
+        if arch in arches:
+            return item
+        if "default" in arches:
+            default = item
+    return default
+
+
 def fetch(name):
     locs = importer.find_module(name,
                                 ['', __name__],