add support for reading configuration of mirror and proxy

This adds support for configuration of a mirror by looking
for dns names like 'ubuntu-mirror', and setting a
'apt_mirror_search' configuration variable to find the first
mirror in the list.

Also, allows configuration of an apt proxy via 'apt_proxy'
in cloud-config.

LP: #897688

6 files changed, 120 insertions, 8 deletions

diff --git a/ChangeLog b/ChangeLog
index 2909069d..39edb705 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -3,6 +3,9 @@
  - add Fedora support [Garrent Holstrom] (LP: #883286)
  - fix bug in netinfo.debug_info if no net devices available (LP: #883367)
  - use python module hashlib rather than md5 to avoid deprecation warnings.
+ - support configuration of mirror based on dns name ubuntu-mirror in
+   local domain.
+ - support setting of Acquire::HTTP::Proxy via 'apt_proxy'
 0.6.2:
  - fix bug where update was not done unless update was explicitly set.
    It would not be run if 'upgrade' or packages were set to be installed
diff --git a/cloudinit/CloudConfig/cc_apt_update_upgrade.py b/cloudinit/CloudConfig/cc_apt_update_upgrade.py
index 495b8522..a60512e1 100644
--- a/cloudinit/CloudConfig/cc_apt_update_upgrade.py
+++ b/cloudinit/CloudConfig/cc_apt_update_upgrade.py
@@ -27,10 +27,10 @@ def handle(name,cfg,cloud,log,args):
     upgrade = util.get_cfg_option_bool(cfg, 'apt_upgrade', False)
 
     release = get_release()
-    if cfg.has_key("apt_mirror"):
-        mirror = cfg["apt_mirror"]
-    else:
-        mirror = cloud.get_mirror()
+
+    mirror = find_apt_mirror(cloud, cfg)
+
+    log.debug("selected mirror at: %s" % mirror)
 
     if not util.get_cfg_option_bool(cfg, \
         'apt_preserve_sources_list', False):
@@ -39,6 +39,20 @@ def handle(name,cfg,cloud,log,args):
             "archive.ubuntu.com/ubuntu")
         rename_apt_lists(old_mir, mirror)
 
+
+    # set up proxy
+    proxy = cfg.get("apt_proxy", None)
+    proxy_filename = "/etc/apt/apt.conf.d/95cloud-init-proxy"
+    if proxy:
+        try:
+            contents = "Acquire::HTTP::Proxy \"%s\";\n"
+            with open(proxy_filename,"w") as fp:
+                fp.write(contents % proxy)
+        except Exception as e:
+            log.warn("Failed to write proxy to %s" % proxy_filename)
+    elif os.path.isfile(proxy_filename):
+        os.unlink(proxy_filename)
+
     # process 'apt_sources'
     if cfg.has_key('apt_sources'):
         errors = add_sources(cfg['apt_sources'],
@@ -162,3 +176,47 @@ def add_sources(srclist, searchList={ }):
     return(elst)
 
 
+def find_apt_mirror(cloud, cfg):
+    """ find an apt_mirror given the cloud and cfg provided """
+
+    # TODO: distro and defaults should be configurable
+    distro = "ubuntu"
+    defaults = {
+        'ubuntu': "http://archive.ubuntu.com/ubuntu",
+        'debian': "http://archive.debian.org/debian",
+    }
+    mirror = None
+
+    cfg_mirror = cfg.get("apt_mirror",None)
+    if cfg_mirror:
+        mirror = cfg["apt_mirror"]
+    elif cfg.has_key("apt_mirror_search"):
+        mirror = util.search_for_mirror(cfg['apt_mirror_search'])
+    else:
+        if cloud:
+            mirror = cloud.get_mirror()
+
+        mydom = ""
+
+        doms = []
+
+        if not mirror and cloud:
+            # if we have a fqdn, then search its domain portion first
+            ( hostname, fqdn ) = util.get_hostname_fqdn(cfg, cloud)
+            mydom = ".".join(fqdn.split(".")[1:])
+            if mydom:
+                doms.append(".%s" % mydom)
+
+        doms.extend((".localdomain", "",))
+
+        mirror_list = []
+        mirrorfmt = "http://%s-mirror%s/%s" % (distro, "%s", distro )
+        for post in doms:
+            mirror_list.append(mirrorfmt % post)
+
+        mirror = util.search_for_mirror(mirror_list)
+
+    if not mirror:
+        mirror = defaults[distro]
+
+    return mirror
diff --git a/cloudinit/DataSource.py b/cloudinit/DataSource.py
index 597760fa..06061a2e 100644
--- a/cloudinit/DataSource.py
+++ b/cloudinit/DataSource.py
@@ -94,7 +94,7 @@ class DataSource:
         return('en_US.UTF-8')
 
     def get_local_mirror(self):
-        return('http://archive.ubuntu.com/ubuntu/')
+        return None
 
     def get_instance_id(self):
         if 'instance-id' not in self.metadata:
diff --git a/cloudinit/DataSourceEc2.py b/cloudinit/DataSourceEc2.py
index 0d8f358f..908a29cd 100644
--- a/cloudinit/DataSourceEc2.py
+++ b/cloudinit/DataSourceEc2.py
@@ -69,7 +69,7 @@ class DataSourceEc2(DataSource.DataSource):
         if availability_zone == None:
             availability_zone = self.get_availability_zone()
 
-        fallback = 'http://archive.ubuntu.com/ubuntu/'
+        fallback = None
 
         if self.is_vpc():
             return fallback
diff --git a/cloudinit/util.py b/cloudinit/util.py
index 0c457128..dc461f6c 100644
--- a/cloudinit/util.py
+++ b/cloudinit/util.py
@@ -24,9 +24,11 @@ from Cheetah.Template import Template
 import urllib2
 import urllib
 import logging
+import re
+import socket
 import time
 import traceback
-import re
+import urlparse
 
 try:
     import selinux
@@ -97,7 +99,7 @@ def get_cfg_by_path(yobj,keyp,default=None):
         cur = cur[tok]
     return(cur)
 
-# merge values from src into cand.
+# merge values from cand into source
 # if src has a key, cand will not override
 def mergedict(src,cand):
     if isinstance(src,dict) and isinstance(cand,dict):
@@ -498,3 +500,26 @@ def get_fqdn_from_hosts(hostname, filename="/etc/hosts"):
             pass
 
     return fqdn
+
+def is_resolvable(name):
+    """ determine if a url is resolvable, return a boolean """
+    try:
+        socket.getaddrinfo(name, None)
+        return True
+    except socket.gaierror as e:
+        return False
+
+def is_resolvable_url(url):
+    """ determine if this url is resolvable (existing or ip) """
+    return(is_resolvable(urlparse.urlparse(url).hostname))
+
+def search_for_mirror(candidates):
+    """ Search through a list of mirror urls for one that works """
+    for cand in candidates:
+        try:
+            if is_resolvable_url(cand):
+                return cand
+        except Exception as e:
+            raise
+
+    return None
diff --git a/doc/examples/cloud-config.txt b/doc/examples/cloud-config.txt
index 73a5e099..e73b8743 100644
--- a/doc/examples/cloud-config.txt
+++ b/doc/examples/cloud-config.txt
@@ -17,7 +17,33 @@ apt_upgrade: true
 #
 # Default: auto select based on cloud metadata
 #  in ec2, the default is <region>.archive.ubuntu.com
+# apt_mirror:
+#   use the provided mirror
+# apt_mirror_search:
+#   search the list for the first mirror.
+#   this is currently very limited, only verifying that
+#   the mirror is dns resolvable or an IP address
+#
+# if neither apt_mirror nor apt_mirror search is set (the default)
+# then use the mirror provided by the DataSource found.
+# In EC2, that means using <region>.ec2.archive.ubuntu.com
+# 
+# if no mirror is provided by the DataSource, then search
+# for dns names '<distro>-mirror' in each of
+# - fqdn of this host per cloud metadata
+# - localdomain
+# - no domain (which would search domains listed in /etc/resolv.conf)
+# That gives the cloud provider the opportunity to set mirrors of a distro
+# up and expose them only by creating dns entries.
+#
+# if none of that is found, then the default distro mirror is used
 apt_mirror: http://us.archive.ubuntu.com/ubuntu/
+apt_mirror_search: 
+ - http://local-mirror.mydomain
+ - http://archive.ubuntu.com
+
+# apt_proxy (configure Acquire::HTTP::Proxy)
+apt_proxy: http://my.apt.proxy:3128
 
 # Preserve existing /etc/apt/sources.list
 # Default: overwrite sources_list with mirror.  If this is true