From e25e758be42c482a0273df6c9e80069bf6ef147c Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Fri, 16 Dec 2011 11:43:35 -0500 Subject: initial mirror configuration/discovery --- cloudinit/CloudConfig/cc_apt_update_upgrade.py | 49 +++++++++++++++++++++++--- cloudinit/DataSource.py | 2 +- cloudinit/DataSourceEc2.py | 2 +- cloudinit/util.py | 29 +++++++++++++-- 4 files changed, 74 insertions(+), 8 deletions(-) diff --git a/cloudinit/CloudConfig/cc_apt_update_upgrade.py b/cloudinit/CloudConfig/cc_apt_update_upgrade.py index 495b8522..10707c77 100644 --- a/cloudinit/CloudConfig/cc_apt_update_upgrade.py +++ b/cloudinit/CloudConfig/cc_apt_update_upgrade.py @@ -27,10 +27,8 @@ def handle(name,cfg,cloud,log,args): upgrade = util.get_cfg_option_bool(cfg, 'apt_upgrade', False) release = get_release() - if cfg.has_key("apt_mirror"): - mirror = cfg["apt_mirror"] - else: - mirror = cloud.get_mirror() + + mirror = find_apt_mirror(cloud, cfg) if not util.get_cfg_option_bool(cfg, \ 'apt_preserve_sources_list', False): @@ -162,3 +160,46 @@ def add_sources(srclist, searchList={ }): return(elst) +def find_apt_mirror(cloud, cfg): + """ find an apt_mirror given the cloud and cfg provided """ + + # TODO: distro and defaults should be configurable + distro = "ubuntu" + defaults = { + 'ubuntu': "http://archive.ubuntu.com/ubuntu", + 'debian': "http://archive.debian.org/debian", + } + mirror = None + + if cfg.has_key("apt_mirror"): + mirror = [cfg["apt_mirror"],] + elif cfg.has_key("apt_mirror_search"): + mirror = util.search_for_mirror(cfg['apt_mirror_search']) + else: + if cloud: + mirror = cloud.get_mirror() + + mydom = "" + + doms = [] + + if not mirror and cloud: + # if we have a fqdn, then search its domain portion first + ( hostname, fqdn ) = util.get_hostname_fqdn(cfg, cloud) + mydom = ".".join(fqdn.split(".")[1:]) + if mydom: + doms.append(".%s" % mydom) + + doms.extend((".localdomain", "",)) + + mirror_list = [] + for post in doms: + mirror_list.append("http://ubuntu-mirror%s/ubuntu" % post) + + print "searching %s" % mirror_list + mirror = util.search_for_mirror(mirror_list) + + if not mirror: + mirror = defaults[distro] + + return mirror diff --git a/cloudinit/DataSource.py b/cloudinit/DataSource.py index 5bb7c3b0..22949b6e 100644 --- a/cloudinit/DataSource.py +++ b/cloudinit/DataSource.py @@ -94,7 +94,7 @@ class DataSource: return('en_US.UTF-8') def get_local_mirror(self): - return('http://archive.ubuntu.com/ubuntu/') + return None def get_instance_id(self): if 'instance-id' not in self.metadata: diff --git a/cloudinit/DataSourceEc2.py b/cloudinit/DataSourceEc2.py index 0d8f358f..908a29cd 100644 --- a/cloudinit/DataSourceEc2.py +++ b/cloudinit/DataSourceEc2.py @@ -69,7 +69,7 @@ class DataSourceEc2(DataSource.DataSource): if availability_zone == None: availability_zone = self.get_availability_zone() - fallback = 'http://archive.ubuntu.com/ubuntu/' + fallback = None if self.is_vpc(): return fallback diff --git a/cloudinit/util.py b/cloudinit/util.py index 0c457128..dc461f6c 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -24,9 +24,11 @@ from Cheetah.Template import Template import urllib2 import urllib import logging +import re +import socket import time import traceback -import re +import urlparse try: import selinux @@ -97,7 +99,7 @@ def get_cfg_by_path(yobj,keyp,default=None): cur = cur[tok] return(cur) -# merge values from src into cand. +# merge values from cand into source # if src has a key, cand will not override def mergedict(src,cand): if isinstance(src,dict) and isinstance(cand,dict): @@ -498,3 +500,26 @@ def get_fqdn_from_hosts(hostname, filename="/etc/hosts"): pass return fqdn + +def is_resolvable(name): + """ determine if a url is resolvable, return a boolean """ + try: + socket.getaddrinfo(name, None) + return True + except socket.gaierror as e: + return False + +def is_resolvable_url(url): + """ determine if this url is resolvable (existing or ip) """ + return(is_resolvable(urlparse.urlparse(url).hostname)) + +def search_for_mirror(candidates): + """ Search through a list of mirror urls for one that works """ + for cand in candidates: + try: + if is_resolvable_url(cand): + return cand + except Exception as e: + raise + + return None -- cgit v1.2.3 From a0b11d8ba00b48214522adff63ef28a2b40e8efa Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Fri, 16 Dec 2011 12:22:53 -0500 Subject: replace a print with a log.debug --- cloudinit/CloudConfig/cc_apt_update_upgrade.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cloudinit/CloudConfig/cc_apt_update_upgrade.py b/cloudinit/CloudConfig/cc_apt_update_upgrade.py index 10707c77..5491f3c8 100644 --- a/cloudinit/CloudConfig/cc_apt_update_upgrade.py +++ b/cloudinit/CloudConfig/cc_apt_update_upgrade.py @@ -30,6 +30,8 @@ def handle(name,cfg,cloud,log,args): mirror = find_apt_mirror(cloud, cfg) + log.debug("selected mirror at: %s" % mirror) + if not util.get_cfg_option_bool(cfg, \ 'apt_preserve_sources_list', False): generate_sources_list(release, mirror) @@ -196,7 +198,6 @@ def find_apt_mirror(cloud, cfg): for post in doms: mirror_list.append("http://ubuntu-mirror%s/ubuntu" % post) - print "searching %s" % mirror_list mirror = util.search_for_mirror(mirror_list) if not mirror: -- cgit v1.2.3 From baa79b1d6dc5969fc55b2ced65af075846ebea7a Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Fri, 16 Dec 2011 12:47:07 -0500 Subject: fix issue if apt_mirror is provided in config --- cloudinit/CloudConfig/cc_apt_update_upgrade.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cloudinit/CloudConfig/cc_apt_update_upgrade.py b/cloudinit/CloudConfig/cc_apt_update_upgrade.py index 5491f3c8..8e95bc33 100644 --- a/cloudinit/CloudConfig/cc_apt_update_upgrade.py +++ b/cloudinit/CloudConfig/cc_apt_update_upgrade.py @@ -174,7 +174,7 @@ def find_apt_mirror(cloud, cfg): mirror = None if cfg.has_key("apt_mirror"): - mirror = [cfg["apt_mirror"],] + mirror = cfg["apt_mirror"] elif cfg.has_key("apt_mirror_search"): mirror = util.search_for_mirror(cfg['apt_mirror_search']) else: -- cgit v1.2.3 From 71e77a29142dcce86350c47b5848c8f0beebfe4d Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Fri, 16 Dec 2011 15:40:21 -0500 Subject: only use apt_mirror if it is non-empty if apt_mirror was set to "" or False in the config, we would have used that. --- cloudinit/CloudConfig/cc_apt_update_upgrade.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cloudinit/CloudConfig/cc_apt_update_upgrade.py b/cloudinit/CloudConfig/cc_apt_update_upgrade.py index 8e95bc33..ea675d51 100644 --- a/cloudinit/CloudConfig/cc_apt_update_upgrade.py +++ b/cloudinit/CloudConfig/cc_apt_update_upgrade.py @@ -173,7 +173,8 @@ def find_apt_mirror(cloud, cfg): } mirror = None - if cfg.has_key("apt_mirror"): + cfg_mirror = cfg.get("apt_mirror",None) + if cfg_mirror: mirror = cfg["apt_mirror"] elif cfg.has_key("apt_mirror_search"): mirror = util.search_for_mirror(cfg['apt_mirror_search']) -- cgit v1.2.3 From fd01c686cf918b8008568430095089f6a59fea70 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Fri, 16 Dec 2011 16:05:45 -0500 Subject: use the distro when searching instead of only searching ubuntu.localdomain, search -mirror.localdomain --- cloudinit/CloudConfig/cc_apt_update_upgrade.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cloudinit/CloudConfig/cc_apt_update_upgrade.py b/cloudinit/CloudConfig/cc_apt_update_upgrade.py index ea675d51..8d98b3a8 100644 --- a/cloudinit/CloudConfig/cc_apt_update_upgrade.py +++ b/cloudinit/CloudConfig/cc_apt_update_upgrade.py @@ -196,8 +196,9 @@ def find_apt_mirror(cloud, cfg): doms.extend((".localdomain", "",)) mirror_list = [] + mirrorfmt = "http://%s-mirror%s/%s" % (distro, "%s", distro ) for post in doms: - mirror_list.append("http://ubuntu-mirror%s/ubuntu" % post) + mirror_list.append(mirrorfmt % post) mirror = util.search_for_mirror(mirror_list) -- cgit v1.2.3 From 9d13bb03c7615291b5a37b3f7afb18f3621df41b Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Fri, 16 Dec 2011 16:06:43 -0500 Subject: add documentation for mirror selection --- doc/examples/cloud-config.txt | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/doc/examples/cloud-config.txt b/doc/examples/cloud-config.txt index 73a5e099..db37fc4b 100644 --- a/doc/examples/cloud-config.txt +++ b/doc/examples/cloud-config.txt @@ -17,7 +17,30 @@ apt_upgrade: true # # Default: auto select based on cloud metadata # in ec2, the default is .archive.ubuntu.com +# apt_mirror: +# use the provided mirror +# apt_mirror_search: +# search the list for the first mirror. +# this is currently very limited, only verifying that +# the mirror is dns resolvable or an IP address +# +# if neither apt_mirror nor apt_mirror search is set (the default) +# then use the mirror provided by the DataSource found. +# In EC2, that means using .ec2.archive.ubuntu.com +# +# if no mirror is provided by the DataSource, then search +# for dns names '-mirror' in each of +# - fqdn of this host per cloud metadata +# - localdomain +# - no domain (which would search domains listed in /etc/resolv.conf) +# That gives the cloud provider the opportunity to set mirrors of a distro +# up and expose them only by creating dns entries. +# +# if none of that is found, then the default distro mirror is used apt_mirror: http://us.archive.ubuntu.com/ubuntu/ +apt_mirror_search: + - http://local-mirror.mydomain + - http://archive.ubuntu.com # Preserve existing /etc/apt/sources.list # Default: overwrite sources_list with mirror. If this is true -- cgit v1.2.3 From cdbbd670524a5d99113026e4caaebe79c886e89c Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Fri, 16 Dec 2011 16:37:01 -0500 Subject: add apt_proxy support --- cloudinit/CloudConfig/cc_apt_update_upgrade.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/cloudinit/CloudConfig/cc_apt_update_upgrade.py b/cloudinit/CloudConfig/cc_apt_update_upgrade.py index 8d98b3a8..1e057b31 100644 --- a/cloudinit/CloudConfig/cc_apt_update_upgrade.py +++ b/cloudinit/CloudConfig/cc_apt_update_upgrade.py @@ -39,6 +39,20 @@ def handle(name,cfg,cloud,log,args): "archive.ubuntu.com/ubuntu") rename_apt_lists(old_mir, mirror) + + # set up proxy + proxy = cfg.get("apt_proxy", None) + proxy_filename = "/etc/apt/apt.conf.d/95cloud-init-proxy" + if proxy: + try: + contents = "Acquire::HTTP::Proxy:: \"%s\";\n" + with open(proxy_filename,"w") as fp: + fp.write(contents % proxy) + except Exception as e: + log.warn("Failed to write proxy to %s" % proxy_filename) + elif os.path.isfile(proxy_filename): + os.unlink(proxy_filename) + # process 'apt_sources' if cfg.has_key('apt_sources'): errors = add_sources(cfg['apt_sources'], -- cgit v1.2.3 From 9dc0f4698e6291a64abf846f6f22a3526758c183 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Fri, 16 Dec 2011 16:39:29 -0500 Subject: remove trailing :: on apt config for proxy --- cloudinit/CloudConfig/cc_apt_update_upgrade.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cloudinit/CloudConfig/cc_apt_update_upgrade.py b/cloudinit/CloudConfig/cc_apt_update_upgrade.py index 1e057b31..a60512e1 100644 --- a/cloudinit/CloudConfig/cc_apt_update_upgrade.py +++ b/cloudinit/CloudConfig/cc_apt_update_upgrade.py @@ -45,7 +45,7 @@ def handle(name,cfg,cloud,log,args): proxy_filename = "/etc/apt/apt.conf.d/95cloud-init-proxy" if proxy: try: - contents = "Acquire::HTTP::Proxy:: \"%s\";\n" + contents = "Acquire::HTTP::Proxy \"%s\";\n" with open(proxy_filename,"w") as fp: fp.write(contents % proxy) except Exception as e: -- cgit v1.2.3 From 32e3e2f3d637051fb79f63f791270eeb05bc721b Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Fri, 16 Dec 2011 16:40:38 -0500 Subject: add apt_proxy doc to cloud-config.txt --- doc/examples/cloud-config.txt | 3 +++ 1 file changed, 3 insertions(+) diff --git a/doc/examples/cloud-config.txt b/doc/examples/cloud-config.txt index db37fc4b..e73b8743 100644 --- a/doc/examples/cloud-config.txt +++ b/doc/examples/cloud-config.txt @@ -42,6 +42,9 @@ apt_mirror_search: - http://local-mirror.mydomain - http://archive.ubuntu.com +# apt_proxy (configure Acquire::HTTP::Proxy) +apt_proxy: http://my.apt.proxy:3128 + # Preserve existing /etc/apt/sources.list # Default: overwrite sources_list with mirror. If this is true # then apt_mirror above will have no effect -- cgit v1.2.3