From d713d7bc5e2b308d11364ccc8701fc2968f9f151 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Wed, 22 Aug 2012 00:28:29 -0400 Subject: rework package mirror selection There are several changes here. * Datasource now has a 'availability_zone' getter. * get_package_mirror_info * Datasource convenience 'get_package_mirror_info' that calls the configured distro, and passes it the availability-zone * distro has a get_package_mirror_info method * get_package_mirror_info returns a dict that of name:mirror this is to facilitate use of 'security' and 'primary' archive. * this supports searching based on templates. Any template that references undefined values is skipped. These templates can contain 'availability_zone' (LP: #1037727) * distro's mirrors can be arch specific (LP: #1028501) * rename_apt_lists supports the "mirror_info" rather than single mirror * generate_sources_list supports mirror_info, and as a result, the ubuntu mirrors reference '$security' rather than security (LP: #1006963) * remove the DataSourceEc2 specific mirror selection, but instead rely on the above filtering, and the fact that 'ec2_region' is only defined if the availability_zone looks like a ec2 az. --- cloudinit/distros/__init__.py | 73 +++++++++++++++++++++++++++++++++++++++++-- cloudinit/distros/debian.py | 4 +++ 2 files changed, 75 insertions(+), 2 deletions(-) (limited to 'cloudinit/distros') diff --git a/cloudinit/distros/__init__.py b/cloudinit/distros/__init__.py index da4d0180..3057ecfc 100644 --- a/cloudinit/distros/__init__.py +++ b/cloudinit/distros/__init__.py @@ -23,6 +23,8 @@ from StringIO import StringIO import abc +import os +import re from cloudinit import importer from cloudinit import log as logging @@ -75,8 +77,26 @@ class Distro(object): def update_package_sources(self): raise NotImplementedError() - def get_package_mirror(self): - return self.get_option('package_mirror') + def get_primary_arch(self): + arch = os.uname[4] + if arch in ("i386", "i486", "i586", "i686"): + return "i386" + return arch + + def _get_arch_package_mirror_info(self, arch=None): + mirror_info = self.get_option("package_mirrors", None) + if arch == None: + arch = self.get_primary_arch() + return _get_arch_package_mirror_info(mirror_info, arch) + + def get_package_mirror_info(self, arch=None, + availability_zone=None): + # this resolves the package_mirrors config option + # down to a single dict of {mirror_name: mirror_url} + arch_info = self._get_arch_package_mirror_info(arch) + + return _get_package_mirror_info(availability_zone=availability_zone, + mirror_info=arch_info) def apply_network(self, settings, bring_up=True): # Write it out @@ -151,6 +171,55 @@ class Distro(object): return False +def _get_package_mirror_info(mirror_info, availability_zone=None, + mirror_filter=util.search_for_mirror): + # given a arch specific 'mirror_info' entry (from package_mirrors) + # search through the 'search' entries, and fallback appropriately + # return a dict with only {name: mirror} entries. + + ec2_az_re = ("^[a-z][a-z]-(%s)-[1-9][0-9]*[a-z]$" % + "north|northeast|east|southeast|south|southwest|west|northwest") + + unset_value = "_UNSET_VALUE_USED_" + azone = availability_zone + + if azone and re.match(ec2_az_re, azone): + ec2_region = "%s" % azone[0:-1] + elif azone: + ec2_region = unset_value + else: + azone = unset_value + ec2_region = unset_value + + results = {} + for (name, mirror) in mirror_info.get('failsafe', {}).iteritems(): + results[name] = mirror + + for (name, searchlist) in mirror_info.get('search', {}).iteritems(): + mirrors = [m % {'ec2_region': ec2_region, 'availability_zone': azone} + for m in searchlist] + # now filter out anything that used the unset availability zone + mirrors = [m for m in mirrors if m.find(unset_value) < 0] + + found = mirror_filter(mirrors) + if found: + results[name] = found + + LOG.debug("filtered distro mirror info: %s" % results) + + return results + +def _get_arch_package_mirror_info(package_mirrors, arch): + # pull out the specific arch from a 'package_mirrors' config option + default = None + for item in package_mirrors: + arches = item.get("arches") + if arch in arches: + return item + if "default" in arches: + default = item + return default + def fetch(name): locs = importer.find_module(name, ['', __name__], diff --git a/cloudinit/distros/debian.py b/cloudinit/distros/debian.py index 3247d7ce..da8c1a5b 100644 --- a/cloudinit/distros/debian.py +++ b/cloudinit/distros/debian.py @@ -147,3 +147,7 @@ class Distro(distros.Distro): def update_package_sources(self): self._runner.run("update-sources", self.package_command, ["update"], freq=PER_INSTANCE) + + def get_primary_arch(self): + (arch, _err) = util.subp(['dpkg', '--print-architecture']) + return str(arch).strip() -- cgit v1.2.3 From cb740b3018f55c21bda3a38b216d4ed61ab30d42 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Wed, 22 Aug 2012 14:36:02 -0400 Subject: fix for pep8 issues and seemingly functional now. --- cloudinit/config/cc_apt_update_upgrade.py | 9 +++++---- cloudinit/distros/__init__.py | 2 ++ 2 files changed, 7 insertions(+), 4 deletions(-) (limited to 'cloudinit/distros') diff --git a/cloudinit/config/cc_apt_update_upgrade.py b/cloudinit/config/cc_apt_update_upgrade.py index 4b5f6a6d..49a93415 100644 --- a/cloudinit/config/cc_apt_update_upgrade.py +++ b/cloudinit/config/cc_apt_update_upgrade.py @@ -66,8 +66,8 @@ def handle(name, cfg, cloud, log, _args): 'apt_preserve_sources_list', False): generate_sources_list(release, mirrors, cloud, log) old_mirrors = cfg.get('apt_old_mirrors', - old_mirrors = {"primary": "archive.ubuntu.com/ubuntu", - "security": "security.ubuntu.com/ubuntu"}) + {"primary": "archive.ubuntu.com/ubuntu", + "security": "security.ubuntu.com/ubuntu"}) rename_apt_lists(old_mirrors, mirrors) # Set up any apt proxy @@ -166,6 +166,7 @@ def rename_apt_lists(old_mirrors, new_mirrors, lists_d="/var/lib/apt/lists"): for filename in glob.glob("%s_*" % oprefix): util.rename(filename, "%s%s" % (nprefix, filename[olen:])) + def get_release(): (stdout, _stderr) = util.subp(['lsb_release', '-cs']) return stdout.strip() @@ -276,8 +277,8 @@ def find_apt_mirror_info(cloud, cfg): mirror_list.append(mirrorfmt % (post)) mirror = util.search_for_mirror(mirror_list) - - mirror_info = cloud.get_package_mirror_info() + + mirror_info = cloud.datasource.get_package_mirror_info() # this is a bit strange. # if mirror is set, then one of the legacy options above set it diff --git a/cloudinit/distros/__init__.py b/cloudinit/distros/__init__.py index 62728a53..dce8fd06 100644 --- a/cloudinit/distros/__init__.py +++ b/cloudinit/distros/__init__.py @@ -209,6 +209,7 @@ def _get_package_mirror_info(mirror_info, availability_zone=None, return results + def _get_arch_package_mirror_info(package_mirrors, arch): # pull out the specific arch from a 'package_mirrors' config option default = None @@ -220,6 +221,7 @@ def _get_arch_package_mirror_info(package_mirrors, arch): default = item return default + def fetch(name): locs = importer.find_module(name, ['', __name__], -- cgit v1.2.3 From 0781b86b414b73c8b5dd9123bda95b7dd74b0243 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Wed, 22 Aug 2012 15:18:01 -0400 Subject: improve the check for "uses unknown key" in mirror templates instead of substituting and then checking for presense of a unlikely to occur string, this only adds to the search list if there is no KeyError raised. --- cloudinit/distros/__init__.py | 24 +++++++++++------------- 1 file changed, 11 insertions(+), 13 deletions(-) (limited to 'cloudinit/distros') diff --git a/cloudinit/distros/__init__.py b/cloudinit/distros/__init__.py index dce8fd06..357209a4 100644 --- a/cloudinit/distros/__init__.py +++ b/cloudinit/distros/__init__.py @@ -180,26 +180,24 @@ def _get_package_mirror_info(mirror_info, availability_zone=None, ec2_az_re = ("^[a-z][a-z]-(%s)-[1-9][0-9]*[a-z]$" % "north|northeast|east|southeast|south|southwest|west|northwest") - unset_value = "_UNSET_VALUE_USED_" - azone = availability_zone + subst = {} + if availability_zone: + subst['availability_zone'] = availability_zone - if azone and re.match(ec2_az_re, azone): - ec2_region = "%s" % azone[0:-1] - elif azone: - ec2_region = unset_value - else: - azone = unset_value - ec2_region = unset_value + if availability_zone and re.match(ec2_az_re, availability_zone): + subst['ec2_region'] = "%s" % availability_zone[0:-1] results = {} for (name, mirror) in mirror_info.get('failsafe', {}).iteritems(): results[name] = mirror for (name, searchlist) in mirror_info.get('search', {}).iteritems(): - mirrors = [m % {'ec2_region': ec2_region, 'availability_zone': azone} - for m in searchlist] - # now filter out anything that used the unset availability zone - mirrors = [m for m in mirrors if m.find(unset_value) < 0] + mirrors = [] + for tmpl in searchlist: + try: + mirrors.append(tmpl % subst) + except KeyError: + pass found = mirror_filter(mirrors) if found: -- cgit v1.2.3