From e25e758be42c482a0273df6c9e80069bf6ef147c Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Fri, 16 Dec 2011 11:43:35 -0500 Subject: initial mirror configuration/discovery --- cloudinit/CloudConfig/cc_apt_update_upgrade.py | 49 +++++++++++++++++++++++--- cloudinit/DataSource.py | 2 +- cloudinit/DataSourceEc2.py | 2 +- cloudinit/util.py | 29 +++++++++++++-- 4 files changed, 74 insertions(+), 8 deletions(-) (limited to 'cloudinit') diff --git a/cloudinit/CloudConfig/cc_apt_update_upgrade.py b/cloudinit/CloudConfig/cc_apt_update_upgrade.py index 495b8522..10707c77 100644 --- a/cloudinit/CloudConfig/cc_apt_update_upgrade.py +++ b/cloudinit/CloudConfig/cc_apt_update_upgrade.py @@ -27,10 +27,8 @@ def handle(name,cfg,cloud,log,args): upgrade = util.get_cfg_option_bool(cfg, 'apt_upgrade', False) release = get_release() - if cfg.has_key("apt_mirror"): - mirror = cfg["apt_mirror"] - else: - mirror = cloud.get_mirror() + + mirror = find_apt_mirror(cloud, cfg) if not util.get_cfg_option_bool(cfg, \ 'apt_preserve_sources_list', False): @@ -162,3 +160,46 @@ def add_sources(srclist, searchList={ }): return(elst) +def find_apt_mirror(cloud, cfg): + """ find an apt_mirror given the cloud and cfg provided """ + + # TODO: distro and defaults should be configurable + distro = "ubuntu" + defaults = { + 'ubuntu': "http://archive.ubuntu.com/ubuntu", + 'debian': "http://archive.debian.org/debian", + } + mirror = None + + if cfg.has_key("apt_mirror"): + mirror = [cfg["apt_mirror"],] + elif cfg.has_key("apt_mirror_search"): + mirror = util.search_for_mirror(cfg['apt_mirror_search']) + else: + if cloud: + mirror = cloud.get_mirror() + + mydom = "" + + doms = [] + + if not mirror and cloud: + # if we have a fqdn, then search its domain portion first + ( hostname, fqdn ) = util.get_hostname_fqdn(cfg, cloud) + mydom = ".".join(fqdn.split(".")[1:]) + if mydom: + doms.append(".%s" % mydom) + + doms.extend((".localdomain", "",)) + + mirror_list = [] + for post in doms: + mirror_list.append("http://ubuntu-mirror%s/ubuntu" % post) + + print "searching %s" % mirror_list + mirror = util.search_for_mirror(mirror_list) + + if not mirror: + mirror = defaults[distro] + + return mirror diff --git a/cloudinit/DataSource.py b/cloudinit/DataSource.py index 5bb7c3b0..22949b6e 100644 --- a/cloudinit/DataSource.py +++ b/cloudinit/DataSource.py @@ -94,7 +94,7 @@ class DataSource: return('en_US.UTF-8') def get_local_mirror(self): - return('http://archive.ubuntu.com/ubuntu/') + return None def get_instance_id(self): if 'instance-id' not in self.metadata: diff --git a/cloudinit/DataSourceEc2.py b/cloudinit/DataSourceEc2.py index 0d8f358f..908a29cd 100644 --- a/cloudinit/DataSourceEc2.py +++ b/cloudinit/DataSourceEc2.py @@ -69,7 +69,7 @@ class DataSourceEc2(DataSource.DataSource): if availability_zone == None: availability_zone = self.get_availability_zone() - fallback = 'http://archive.ubuntu.com/ubuntu/' + fallback = None if self.is_vpc(): return fallback diff --git a/cloudinit/util.py b/cloudinit/util.py index 0c457128..dc461f6c 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -24,9 +24,11 @@ from Cheetah.Template import Template import urllib2 import urllib import logging +import re +import socket import time import traceback -import re +import urlparse try: import selinux @@ -97,7 +99,7 @@ def get_cfg_by_path(yobj,keyp,default=None): cur = cur[tok] return(cur) -# merge values from src into cand. +# merge values from cand into source # if src has a key, cand will not override def mergedict(src,cand): if isinstance(src,dict) and isinstance(cand,dict): @@ -498,3 +500,26 @@ def get_fqdn_from_hosts(hostname, filename="/etc/hosts"): pass return fqdn + +def is_resolvable(name): + """ determine if a url is resolvable, return a boolean """ + try: + socket.getaddrinfo(name, None) + return True + except socket.gaierror as e: + return False + +def is_resolvable_url(url): + """ determine if this url is resolvable (existing or ip) """ + return(is_resolvable(urlparse.urlparse(url).hostname)) + +def search_for_mirror(candidates): + """ Search through a list of mirror urls for one that works """ + for cand in candidates: + try: + if is_resolvable_url(cand): + return cand + except Exception as e: + raise + + return None -- cgit v1.2.3 From a0b11d8ba00b48214522adff63ef28a2b40e8efa Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Fri, 16 Dec 2011 12:22:53 -0500 Subject: replace a print with a log.debug --- cloudinit/CloudConfig/cc_apt_update_upgrade.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'cloudinit') diff --git a/cloudinit/CloudConfig/cc_apt_update_upgrade.py b/cloudinit/CloudConfig/cc_apt_update_upgrade.py index 10707c77..5491f3c8 100644 --- a/cloudinit/CloudConfig/cc_apt_update_upgrade.py +++ b/cloudinit/CloudConfig/cc_apt_update_upgrade.py @@ -30,6 +30,8 @@ def handle(name,cfg,cloud,log,args): mirror = find_apt_mirror(cloud, cfg) + log.debug("selected mirror at: %s" % mirror) + if not util.get_cfg_option_bool(cfg, \ 'apt_preserve_sources_list', False): generate_sources_list(release, mirror) @@ -196,7 +198,6 @@ def find_apt_mirror(cloud, cfg): for post in doms: mirror_list.append("http://ubuntu-mirror%s/ubuntu" % post) - print "searching %s" % mirror_list mirror = util.search_for_mirror(mirror_list) if not mirror: -- cgit v1.2.3 From baa79b1d6dc5969fc55b2ced65af075846ebea7a Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Fri, 16 Dec 2011 12:47:07 -0500 Subject: fix issue if apt_mirror is provided in config --- cloudinit/CloudConfig/cc_apt_update_upgrade.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'cloudinit') diff --git a/cloudinit/CloudConfig/cc_apt_update_upgrade.py b/cloudinit/CloudConfig/cc_apt_update_upgrade.py index 5491f3c8..8e95bc33 100644 --- a/cloudinit/CloudConfig/cc_apt_update_upgrade.py +++ b/cloudinit/CloudConfig/cc_apt_update_upgrade.py @@ -174,7 +174,7 @@ def find_apt_mirror(cloud, cfg): mirror = None if cfg.has_key("apt_mirror"): - mirror = [cfg["apt_mirror"],] + mirror = cfg["apt_mirror"] elif cfg.has_key("apt_mirror_search"): mirror = util.search_for_mirror(cfg['apt_mirror_search']) else: -- cgit v1.2.3 From 71e77a29142dcce86350c47b5848c8f0beebfe4d Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Fri, 16 Dec 2011 15:40:21 -0500 Subject: only use apt_mirror if it is non-empty if apt_mirror was set to "" or False in the config, we would have used that. --- cloudinit/CloudConfig/cc_apt_update_upgrade.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'cloudinit') diff --git a/cloudinit/CloudConfig/cc_apt_update_upgrade.py b/cloudinit/CloudConfig/cc_apt_update_upgrade.py index 8e95bc33..ea675d51 100644 --- a/cloudinit/CloudConfig/cc_apt_update_upgrade.py +++ b/cloudinit/CloudConfig/cc_apt_update_upgrade.py @@ -173,7 +173,8 @@ def find_apt_mirror(cloud, cfg): } mirror = None - if cfg.has_key("apt_mirror"): + cfg_mirror = cfg.get("apt_mirror",None) + if cfg_mirror: mirror = cfg["apt_mirror"] elif cfg.has_key("apt_mirror_search"): mirror = util.search_for_mirror(cfg['apt_mirror_search']) -- cgit v1.2.3 From fd01c686cf918b8008568430095089f6a59fea70 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Fri, 16 Dec 2011 16:05:45 -0500 Subject: use the distro when searching instead of only searching ubuntu.localdomain, search -mirror.localdomain --- cloudinit/CloudConfig/cc_apt_update_upgrade.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'cloudinit') diff --git a/cloudinit/CloudConfig/cc_apt_update_upgrade.py b/cloudinit/CloudConfig/cc_apt_update_upgrade.py index ea675d51..8d98b3a8 100644 --- a/cloudinit/CloudConfig/cc_apt_update_upgrade.py +++ b/cloudinit/CloudConfig/cc_apt_update_upgrade.py @@ -196,8 +196,9 @@ def find_apt_mirror(cloud, cfg): doms.extend((".localdomain", "",)) mirror_list = [] + mirrorfmt = "http://%s-mirror%s/%s" % (distro, "%s", distro ) for post in doms: - mirror_list.append("http://ubuntu-mirror%s/ubuntu" % post) + mirror_list.append(mirrorfmt % post) mirror = util.search_for_mirror(mirror_list) -- cgit v1.2.3 From cdbbd670524a5d99113026e4caaebe79c886e89c Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Fri, 16 Dec 2011 16:37:01 -0500 Subject: add apt_proxy support --- cloudinit/CloudConfig/cc_apt_update_upgrade.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) (limited to 'cloudinit') diff --git a/cloudinit/CloudConfig/cc_apt_update_upgrade.py b/cloudinit/CloudConfig/cc_apt_update_upgrade.py index 8d98b3a8..1e057b31 100644 --- a/cloudinit/CloudConfig/cc_apt_update_upgrade.py +++ b/cloudinit/CloudConfig/cc_apt_update_upgrade.py @@ -39,6 +39,20 @@ def handle(name,cfg,cloud,log,args): "archive.ubuntu.com/ubuntu") rename_apt_lists(old_mir, mirror) + + # set up proxy + proxy = cfg.get("apt_proxy", None) + proxy_filename = "/etc/apt/apt.conf.d/95cloud-init-proxy" + if proxy: + try: + contents = "Acquire::HTTP::Proxy:: \"%s\";\n" + with open(proxy_filename,"w") as fp: + fp.write(contents % proxy) + except Exception as e: + log.warn("Failed to write proxy to %s" % proxy_filename) + elif os.path.isfile(proxy_filename): + os.unlink(proxy_filename) + # process 'apt_sources' if cfg.has_key('apt_sources'): errors = add_sources(cfg['apt_sources'], -- cgit v1.2.3 From 9dc0f4698e6291a64abf846f6f22a3526758c183 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Fri, 16 Dec 2011 16:39:29 -0500 Subject: remove trailing :: on apt config for proxy --- cloudinit/CloudConfig/cc_apt_update_upgrade.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'cloudinit') diff --git a/cloudinit/CloudConfig/cc_apt_update_upgrade.py b/cloudinit/CloudConfig/cc_apt_update_upgrade.py index 1e057b31..a60512e1 100644 --- a/cloudinit/CloudConfig/cc_apt_update_upgrade.py +++ b/cloudinit/CloudConfig/cc_apt_update_upgrade.py @@ -45,7 +45,7 @@ def handle(name,cfg,cloud,log,args): proxy_filename = "/etc/apt/apt.conf.d/95cloud-init-proxy" if proxy: try: - contents = "Acquire::HTTP::Proxy:: \"%s\";\n" + contents = "Acquire::HTTP::Proxy \"%s\";\n" with open(proxy_filename,"w") as fp: fp.write(contents % proxy) except Exception as e: -- cgit v1.2.3