diff options
| -rw-r--r-- | ChangeLog | 2 | ||||
| -rw-r--r-- | cloudinit/config/cc_apt_configure.py | 114 | ||||
| -rw-r--r-- | cloudinit/templater.py | 5 | ||||
| -rw-r--r-- | cloudinit/util.py | 10 | ||||
| -rw-r--r-- | doc/examples/cloud-config.txt | 127 | ||||
| -rw-r--r-- | tests/unittests/test_handler/test_handler_apt_configure_sources_list.py | 165 | ||||
| -rw-r--r-- | tests/unittests/test_handler/test_handler_apt_source.py | 551 |
7 files changed, 932 insertions, 42 deletions
@@ -113,6 +113,8 @@ - settings on the kernel command line (cc:) override all local settings rather than only those in /etc/cloud/cloud.cfg (LP: #1582323) - Improve merging documentation [Daniel Watkins] + - apt sources: support inserting key/key-id only, custom sources.list, + long gpg key fingerprints with spaces, and dictionary format (LP: #1574113) 0.7.6: - open 0.7.6 diff --git a/cloudinit/config/cc_apt_configure.py b/cloudinit/config/cc_apt_configure.py index e3fadc12..7a9777c0 100644 --- a/cloudinit/config/cc_apt_configure.py +++ b/cloudinit/config/cc_apt_configure.py @@ -40,9 +40,9 @@ EXPORT_GPG_KEYID = """ k=${1} ks=${2}; exec 2>/dev/null [ -n "$k" ] || exit 1; - armour=$(gpg --list-keys --armour "${k}") + armour=$(gpg --export --armour "${k}") if [ -z "${armour}" ]; then - gpg --keyserver ${ks} --recv $k >/dev/null && + gpg --keyserver ${ks} --recv "${k}" >/dev/null && armour=$(gpg --export --armour "${k}") && gpg --batch --yes --delete-keys "${k}" fi @@ -70,7 +70,7 @@ def handle(name, cfg, cloud, log, _args): if not util.get_cfg_option_bool(cfg, 'apt_preserve_sources_list', False): - generate_sources_list(release, mirrors, cloud, log) + generate_sources_list(cfg, release, mirrors, cloud, log) old_mirrors = cfg.get('apt_old_mirrors', {"primary": "archive.ubuntu.com/ubuntu", "security": "security.ubuntu.com/ubuntu"}) @@ -149,7 +149,17 @@ def get_release(): return stdout.strip() -def generate_sources_list(codename, mirrors, cloud, log): +def generate_sources_list(cfg, codename, mirrors, cloud, log): + params = {'codename': codename} + for k in mirrors: + params[k] = mirrors[k] + + custtmpl = cfg.get('apt_custom_sources_list', None) + if custtmpl is not None: + templater.render_string_to_file(custtmpl, + '/etc/apt/sources.list', params) + return + template_fn = cloud.get_template_filename('sources.list.%s' % (cloud.distro.name)) if not template_fn: @@ -158,12 +168,60 @@ def generate_sources_list(codename, mirrors, cloud, log): log.warn("No template found, not rendering /etc/apt/sources.list") return - params = {'codename': codename} - for k in mirrors: - params[k] = mirrors[k] templater.render_to_file(template_fn, '/etc/apt/sources.list', params) +def add_key_raw(key): + """ + actual adding of a key as defined in key argument + to the system + """ + try: + util.subp(('apt-key', 'add', '-'), key) + except util.ProcessExecutionError: + raise Exception('failed add key') + + +def add_key(ent): + """ + add key to the system as defined in ent (if any) + supports raw keys or keyid's + The latter will as a first step fetch the raw key from a keyserver + """ + if 'keyid' in ent and 'key' not in ent: + keyserver = "keyserver.ubuntu.com" + if 'keyserver' in ent: + keyserver = ent['keyserver'] + ent['key'] = getkeybyid(ent['keyid'], keyserver) + + if 'key' in ent: + add_key_raw(ent['key']) + + +def convert_to_new_format(srclist): + """convert_to_new_format + convert the old list based format to the new dict based one + """ + srcdict = {} + if isinstance(srclist, list): + for srcent in srclist: + if 'filename' not in srcent: + # file collides for multiple !filename cases for compatibility + # yet we need them all processed, so not same dictionary key + srcent['filename'] = "cloud_config_sources.list" + key = util.rand_dict_key(srcdict, "cloud_config_sources.list") + else: + # all with filename use that as key (matching new format) + key = srcent['filename'] + srcdict[key] = srcent + elif isinstance(srclist, dict): + srcdict = srclist + else: + raise ValueError("unknown apt_sources format") + + return srcdict + + def add_sources(srclist, template_params=None, aa_repo_match=None): """ add entries in /etc/apt/sources.list.d for each abbreviated @@ -178,14 +236,29 @@ def add_sources(srclist, template_params=None, aa_repo_match=None): return False errorlist = [] - for ent in srclist: + srcdict = convert_to_new_format(srclist) + + for filename in srcdict: + ent = srcdict[filename] + if 'filename' not in ent: + ent['filename'] = filename + + # keys can be added without specifying a source + try: + add_key(ent) + except Exception as detail: + errorlist.append([ent, detail]) + if 'source' not in ent: errorlist.append(["", "missing source"]) continue - source = ent['source'] source = templater.render_string(source, template_params) + if not ent['filename'].startswith(os.path.sep): + ent['filename'] = os.path.join("/etc/apt/sources.list.d/", + ent['filename']) + if aa_repo_match(source): try: util.subp(["add-apt-repository", source]) @@ -194,29 +267,6 @@ def add_sources(srclist, template_params=None, aa_repo_match=None): ("add-apt-repository failed. " + str(e))]) continue - if 'filename' not in ent: - ent['filename'] = 'cloud_config_sources.list' - - if not ent['filename'].startswith("/"): - ent['filename'] = os.path.join("/etc/apt/sources.list.d/", - ent['filename']) - - if ('keyid' in ent and 'key' not in ent): - ks = "keyserver.ubuntu.com" - if 'keyserver' in ent: - ks = ent['keyserver'] - try: - ent['key'] = getkeybyid(ent['keyid'], ks) - except Exception: - errorlist.append([source, "failed to get key from %s" % ks]) - continue - - if 'key' in ent: - try: - util.subp(('apt-key', 'add', '-'), ent['key']) - except Exception: - errorlist.append([source, "failed add key"]) - try: contents = "%s\n" % (source) util.write_file(ent['filename'], contents, omode="ab") diff --git a/cloudinit/templater.py b/cloudinit/templater.py index a9231482..8a6ad417 100644 --- a/cloudinit/templater.py +++ b/cloudinit/templater.py @@ -142,6 +142,11 @@ def render_to_file(fn, outfn, params, mode=0o644): util.write_file(outfn, contents, mode=mode) +def render_string_to_file(content, outfn, params, mode=0o644): + contents = render_string(content, params) + util.write_file(outfn, contents, mode=mode) + + def render_string(content, params): if not params: params = {} diff --git a/cloudinit/util.py b/cloudinit/util.py index 8d6cbb4b..d6b80dbe 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -336,6 +336,16 @@ def rand_str(strlen=32, select_from=None): return "".join([random.choice(select_from) for _x in range(0, strlen)]) +def rand_dict_key(dictionary, postfix=None): + if not postfix: + postfix = "" + while True: + newkey = rand_str(strlen=8) + "_" + postfix + if newkey not in dictionary: + break + return newkey + + def read_conf(fname): try: return load_yaml(load_file(fname), default={}) diff --git a/doc/examples/cloud-config.txt b/doc/examples/cloud-config.txt index 1236796c..62b297bc 100644 --- a/doc/examples/cloud-config.txt +++ b/doc/examples/cloud-config.txt @@ -72,14 +72,87 @@ apt_pipelining: False # then apt_mirror above will have no effect apt_preserve_sources_list: true +# Provide a custom template for rendering sources.list +# Default: a default template for Ubuntu/Debain will be used as packaged in +# Ubuntu: /etc/cloud/templates/sources.list.ubuntu.tmpl +# Debian: /etc/cloud/templates/sources.list.debian.tmpl +# Others: n/a +# This will follow the normal mirror/codename replacement rules before +# being written to disk. +apt_custom_sources_list: | + ## template:jinja + ## Note, this file is written by cloud-init on first boot of an instance + ## modifications made here will not survive a re-bundle. + ## if you wish to make changes you can: + ## a.) add 'apt_preserve_sources_list: true' to /etc/cloud/cloud.cfg + ## or do the same in user-data + ## b.) add sources in /etc/apt/sources.list.d + ## c.) make changes to template file /etc/cloud/templates/sources.list.tmpl + deb {{mirror}} {{codename}} main restricted + deb-src {{mirror}} {{codename}} main restricted + + # could drop some of the usually used entries + + # could refer to other mirrors + deb http://ddebs.ubuntu.com {{codename}} main restricted universe multiverse + deb http://ddebs.ubuntu.com {{codename}}-updates main restricted universe multiverse + deb http://ddebs.ubuntu.com {{codename}}-proposed main restricted universe multiverse + + # or even more uncommon examples like local or NFS mounted repos, + # eventually whatever is compatible with sources.list syntax + deb file:/home/apt/debian unstable main contrib non-free + # 'source' entries in apt-sources that match this python regex # expression will be passed to add-apt-repository add_apt_repo_match: '^[\w-]+:\w' +# 'apt_sources' is a dictionary +# The key is the filename and will be prepended by /etc/apt/sources.list.d/ if +# it doesn't start with a '/'. +# There are certain cases - where no content is written into a source.list file +# where the filename will be ignored - yet it can still be used as index for +# merging. +# The value it maps to is a dictionary with the following optional entries: +# source: a sources.list entry (some variable replacements apply) +# keyid: providing a key to import via shortid or fingerprint +# key: providing a raw PGP key +# keyserver: keyserver to fetch keys from, default is keyserver.ubuntu.com +# filename: for compatibility with the older format (now the key to this +# dictionary is the filename). If specified this overwrites the +# filename given as key. + +# the new "filename: {specification-dictionary}, filename2: ..." format allows +# better merging between multiple input files than a list like: +# cloud-config1 +# sources: + s1: {'key': 'key1', 'source': 'source1'} +# cloud-config2 +# sources: + s2: {'key': 'key2'} + s1: {filename: 'foo'} +# this would be merged to +#sources: +# s1: +# filename: foo +# key: key1 +# source: source1 +# s2: +# key: key2 +# Be aware that this style of merging is not the default (for backward +# compatibility reasons). You should specify the following merge_how to get +# this more complete and modern merging behaviour: +# merge_how: "list()+dict()+str()" +# This would then also be equivalent to the config merging used in curtin +# (https://launchpad.net/curtin). + +# for more details see below in the various examples + apt_sources: - - source: "deb http://ppa.launchpad.net/byobu/ppa/ubuntu karmic main" + byobu-ppa.list: + source: "deb http://ppa.launchpad.net/byobu/ppa/ubuntu karmic main" keyid: F430BBA5 # GPG key ID published on a key server - filename: byobu-ppa.list + # adding a source.list line, importing a gpg key for a given key id and + # storing it in the file /etc/apt/sources.list.d/byobu-ppa.list # PPA shortcut: # * Setup correct apt sources.list line @@ -87,7 +160,9 @@ apt_sources: # # See https://help.launchpad.net/Packaging/PPA for more information # this requires 'add-apt-repository' - - source: "ppa:smoser/ppa" # Quote the string + # due to that the filename key is ignored in this case + ignored1: + source: "ppa:smoser/ppa" # Quote the string # Custom apt repository: # * all that is required is 'source' @@ -95,29 +170,60 @@ apt_sources: # * [optional] Import the apt signing key from the keyserver # * Defaults: # + keyserver: keyserver.ubuntu.com - # + filename: cloud_config_sources.list # # See sources.list man page for more information about the format - - source: deb http://archive.ubuntu.com/ubuntu karmic-backports main universe multiverse restricted + my-repo.list: + source: deb http://archive.ubuntu.com/ubuntu karmic-backports main universe multiverse restricted # sources can use $MIRROR and $RELEASE and they will be replaced # with the local mirror for this cloud, and the running release # the entry below would be possibly turned into: - # - source: deb http://us-east-1.ec2.archive.ubuntu.com/ubuntu natty multiverse - - source: deb $MIRROR $RELEASE multiverse + # source: deb http://us-east-1.ec2.archive.ubuntu.com/ubuntu natty multiverse + my-repo.list: + source: deb $MIRROR $RELEASE multiverse # this would have the same end effect as 'ppa:byobu/ppa' - - source: "deb http://ppa.launchpad.net/byobu/ppa/ubuntu karmic main" + my-repo.list: + source: "deb http://ppa.launchpad.net/byobu/ppa/ubuntu karmic main" keyid: F430BBA5 # GPG key ID published on a key server filename: byobu-ppa.list + # this would only import the key without adding a ppa or other source spec + # since this doesn't generate a source.list file the filename key is ignored + ignored2: + keyid: F430BBA5 # GPG key ID published on a key server + + # In general keyid's can also be specified via their long fingerprints + # since this doesn't generate a source.list file the filename key is ignored + ignored3: + keyid: B59D 5F15 97A5 04B7 E230 6DCA 0620 BBCF 0368 3F77 + # Custom apt repository: # * The apt signing key can also be specified # by providing a pgp public key block - # * Providing the PBG key here is the most robust method for + # * Providing the PGP key here is the most robust method for # specifying a key, as it removes dependency on a remote key server + my-repo.list: + source: deb http://ppa.launchpad.net/alestic/ppa/ubuntu karmic main + key: | # The value needs to start with -----BEGIN PGP PUBLIC KEY BLOCK----- + -----BEGIN PGP PUBLIC KEY BLOCK----- + Version: SKS 1.0.10 + + mI0ESpA3UQEEALdZKVIMq0j6qWAXAyxSlF63SvPVIgxHPb9Nk0DZUixn+akqytxG4zKCONz6 + qLjoBBfHnynyVLfT4ihg9an1PqxRnTO+JKQxl8NgKGz6Pon569GtAOdWNKw15XKinJTDLjnj + 9y96ljJqRcpV9t/WsIcdJPcKFR5voHTEoABE2aEXABEBAAG0GUxhdW5jaHBhZCBQUEEgZm9y + IEFsZXN0aWOItgQTAQIAIAUCSpA3UQIbAwYLCQgHAwIEFQIIAwQWAgMBAh4BAheAAAoJEA7H + 5Qi+CcVxWZ8D/1MyYvfj3FJPZUm2Yo1zZsQ657vHI9+pPouqflWOayRR9jbiyUFIn0VdQBrP + t0FwvnOFArUovUWoKAEdqR8hPy3M3APUZjl5K4cMZR/xaMQeQRZ5CHpS4DBKURKAHC0ltS5o + uBJKQOZm5iltJp15cgyIkBkGe8Mx18VFyVglAZey + =Y2oI + -----END PGP PUBLIC KEY BLOCK----- - - source: deb http://ppa.launchpad.net/alestic/ppa/ubuntu karmic main + # Custom gpg key: + # * As with keyid, a key may also be specified without a related source. + # * all other facts mentioned above still apply + # since this doesn't generate a source.list file the filename key is ignored + ignored4: key: | # The value needs to start with -----BEGIN PGP PUBLIC KEY BLOCK----- -----BEGIN PGP PUBLIC KEY BLOCK----- Version: SKS 1.0.10 @@ -132,6 +238,7 @@ apt_sources: =Y2oI -----END PGP PUBLIC KEY BLOCK----- + ## apt config via system_info: # under the 'system_info', you can further customize cloud-init's interaction # with apt. diff --git a/tests/unittests/test_handler/test_handler_apt_configure_sources_list.py b/tests/unittests/test_handler/test_handler_apt_configure_sources_list.py new file mode 100644 index 00000000..5d0417a2 --- /dev/null +++ b/tests/unittests/test_handler/test_handler_apt_configure_sources_list.py @@ -0,0 +1,165 @@ +""" test_handler_apt_configure_sources_list +Test templating of sources list +""" +import logging +import os +import shutil +import tempfile + +try: + from unittest import mock +except ImportError: + import mock + +from cloudinit import cloud +from cloudinit import distros +from cloudinit import helpers +from cloudinit import templater +from cloudinit import util + +from cloudinit.config import cc_apt_configure +from cloudinit.sources import DataSourceNone + +from .. import helpers as t_help + +LOG = logging.getLogger(__name__) + +YAML_TEXT_CUSTOM_SL = """ +apt_mirror: http://archive.ubuntu.com/ubuntu/ +apt_custom_sources_list: | + ## template:jinja + ## Note, this file is written by cloud-init on first boot of an instance + ## modifications made here will not survive a re-bundle. + ## if you wish to make changes you can: + ## a.) add 'apt_preserve_sources_list: true' to /etc/cloud/cloud.cfg + ## or do the same in user-data + ## b.) add sources in /etc/apt/sources.list.d + ## c.) make changes to template file /etc/cloud/templates/sources.list.tmpl + + # See http://help.ubuntu.com/community/UpgradeNotes for how to upgrade to + # newer versions of the distribution. + deb {{mirror}} {{codename}} main restricted + deb-src {{mirror}} {{codename}} main restricted + # FIND_SOMETHING_SPECIAL +""" + +EXPECTED_CONVERTED_CONTENT = ( + """## Note, this file is written by cloud-init on first boot of an instance +## modifications made here will not survive a re-bundle. +## if you wish to make changes you can: +## a.) add 'apt_preserve_sources_list: true' to /etc/cloud/cloud.cfg +## or do the same in user-data +## b.) add sources in /etc/apt/sources.list.d +## c.) make changes to template file /etc/cloud/templates/sources.list.tmpl + +# See http://help.ubuntu.com/community/UpgradeNotes for how to upgrade to +# newer versions of the distribution. +deb http://archive.ubuntu.com/ubuntu/ fakerelease main restricted +deb-src http://archive.ubuntu.com/ubuntu/ fakerelease main restricted +# FIND_SOMETHING_SPECIAL +""") + + +def load_tfile_or_url(*args, **kwargs): + """load_tfile_or_url + load file and return content after decoding + """ + return util.decode_binary(util.read_file_or_url(*args, **kwargs).contents) + + +class TestAptSourceConfigSourceList(t_help.FilesystemMockingTestCase): + """TestAptSourceConfigSourceList + Main Class to test sources list rendering + """ + def setUp(self): + super(TestAptSourceConfigSourceList, self).setUp() + self.subp = util.subp + self.new_root = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, self.new_root) + + def _get_cloud(self, distro, metadata=None): + self.patchUtils(self.new_root) + paths = helpers.Paths({}) + cls = distros.fetch(distro) + mydist = cls(distro, {}, paths) + myds = DataSourceNone.DataSourceNone({}, mydist, paths) + if metadata: + myds.metadata.update(metadata) + return cloud.Cloud(myds, paths, {}, mydist, None) + + def apt_source_list(self, distro, mirror, mirrorcheck=None): + """apt_source_list + Test rendering of a source.list from template for a given distro + """ + if mirrorcheck is None: + mirrorcheck = mirror + + if isinstance(mirror, list): + cfg = {'apt_mirror_search': mirror} + else: + cfg = {'apt_mirror': mirror} + mycloud = self._get_cloud(distro) + + with mock.patch.object(templater, 'render_to_file') as mocktmpl: + with mock.patch.object(os.path, 'isfile', + return_value=True) as mockisfile: + cc_apt_configure.handle("notimportant", cfg, mycloud, + LOG, None) + + mockisfile.assert_any_call( + ('/etc/cloud/templates/sources.list.%s.tmpl' % distro)) + mocktmpl.assert_called_once_with( + ('/etc/cloud/templates/sources.list.%s.tmpl' % distro), + '/etc/apt/sources.list', + {'codename': '', 'primary': mirrorcheck, 'mirror': mirrorcheck}) + + def test_apt_source_list_debian(self): + """test_apt_source_list_debian + Test rendering of a source.list from template for debian + """ + self.apt_source_list('debian', 'http://httpredir.debian.org/debian') + + def test_apt_source_list_ubuntu(self): + """test_apt_source_list_ubuntu + Test rendering of a source.list from template for ubuntu + """ + self.apt_source_list('ubuntu', 'http://archive.ubuntu.com/ubuntu/') + + def test_apt_srcl_debian_mirrorfail(self): + """test_apt_source_list_debian_mirrorfail + Test rendering of a source.list from template for debian + """ + self.apt_source_list('debian', ['http://does.not.exist', + 'http://httpredir.debian.org/debian'], + 'http://httpredir.debian.org/debian') + + def test_apt_srcl_ubuntu_mirrorfail(self): + """test_apt_source_list_ubuntu_mirrorfail + Test rendering of a source.list from template for ubuntu + """ + self.apt_source_list('ubuntu', ['http://does.not.exist', + 'http://archive.ubuntu.com/ubuntu/'], + 'http://archive.ubuntu.com/ubuntu/') + + def test_apt_srcl_custom(self): + """test_apt_srcl_custom + Test rendering from a custom source.list template + """ + cfg = util.load_yaml(YAML_TEXT_CUSTOM_SL) + mycloud = self._get_cloud('ubuntu') + + # the second mock restores the original subp + with mock.patch.object(util, 'write_file') as mockwrite: + with mock.patch.object(util, 'subp', self.subp): + with mock.patch.object(cc_apt_configure, 'get_release', + return_value='fakerelease'): + cc_apt_configure.handle("notimportant", cfg, mycloud, + LOG, None) + + mockwrite.assert_called_once_with( + '/etc/apt/sources.list', + EXPECTED_CONVERTED_CONTENT, + mode=420) + + +# vi: ts=4 expandtab diff --git a/tests/unittests/test_handler/test_handler_apt_source.py b/tests/unittests/test_handler/test_handler_apt_source.py new file mode 100644 index 00000000..4dbe69f0 --- /dev/null +++ b/tests/unittests/test_handler/test_handler_apt_source.py @@ -0,0 +1,551 @@ +""" test_handler_apt_source +Testing various config variations of the apt_source config +""" +import os +import re +import shutil +import tempfile + +try: + from unittest import mock +except ImportError: + import mock +from mock import call + +from cloudinit.config import cc_apt_configure +from cloudinit import util + +from ..helpers import TestCase + +EXPECTEDKEY = """-----BEGIN PGP PUBLIC KEY BLOCK----- +Version: GnuPG v1 + +mI0ESuZLUgEEAKkqq3idtFP7g9hzOu1a8+v8ImawQN4TrvlygfScMU1TIS1eC7UQ +NUA8Qqgr9iUaGnejb0VciqftLrU9D6WYHSKz+EITefgdyJ6SoQxjoJdsCpJ7o9Jy +8PQnpRttiFm4qHu6BVnKnBNxw/z3ST9YMqW5kbMQpfxbGe+obRox59NpABEBAAG0 +HUxhdW5jaHBhZCBQUEEgZm9yIFNjb3R0IE1vc2VyiLYEEwECACAFAkrmS1ICGwMG +CwkIBwMCBBUCCAMEFgIDAQIeAQIXgAAKCRAGILvPA2g/d3aEA/9tVjc10HOZwV29 +OatVuTeERjjrIbxflO586GLA8cp0C9RQCwgod/R+cKYdQcHjbqVcP0HqxveLg0RZ +FJpWLmWKamwkABErwQLGlM/Hwhjfade8VvEQutH5/0JgKHmzRsoqfR+LMO6OS+Sm +S0ORP6HXET3+jC8BMG4tBWCTK/XEZw== +=ACB2 +-----END PGP PUBLIC KEY BLOCK-----""" + + +def load_tfile_or_url(*args, **kwargs): + """load_tfile_or_url + load file and return content after decoding + """ + return util.decode_binary(util.read_file_or_url(*args, **kwargs).contents) + + +class TestAptSourceConfig(TestCase): + """TestAptSourceConfig + Main Class to test apt_source configs + """ + def setUp(self): + super(TestAptSourceConfig, self).setUp() + self.tmp = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, self.tmp) + self.aptlistfile = os.path.join(self.tmp, "single-deb.list") + self.aptlistfile2 = os.path.join(self.tmp, "single-deb2.list") + self.aptlistfile3 = os.path.join(self.tmp, "single-deb3.list") + self.join = os.path.join + # mock fallback filename into writable tmp dir + self.fallbackfn = os.path.join(self.tmp, "etc/apt/sources.list.d/", + "cloud_config_sources.list") + + @staticmethod + def _get_default_params(): + """get_default_params + Get the most basic default mrror and release info to be used in tests + """ + params = {} + params['RELEASE'] = cc_apt_configure.get_release() + params['MIRROR'] = "http://archive.ubuntu.com/ubuntu" + return params + + def myjoin(self, *args, **kwargs): + """myjoin - redir into writable tmpdir""" + if (args[0] == "/etc/apt/sources.list.d/" and + args[1] == "cloud_config_sources.list" and + len(args) == 2): + return self.join(self.tmp, args[0].lstrip("/"), args[1]) + else: + return self.join(*args, **kwargs) + + def apt_src_basic(self, filename, cfg): + """apt_src_basic + Test Fix deb source string, has to overwrite mirror conf in params + """ + params = self._get_default_params() + + cc_apt_configure.add_sources(cfg, params) + + self.assertTrue(os.path.isfile(filename)) + + contents = load_tfile_or_url(filename) + self.assertTrue(re.search(r"%s %s %s %s\n" % + ("deb", "http://archive.ubuntu.com/ubuntu", + "karmic-backports", + "main universe multiverse restricted"), + contents, flags=re.IGNORECASE)) + + def test_apt_src_basic(self): + """test_apt_src_basic + Test Fix deb source string, has to overwrite mirror conf in params. + Test with a filename provided in config. + """ + cfg = {'source': ('deb http://archive.ubuntu.com/ubuntu' + ' karmic-backports' + ' main universe multiverse restricted'), + 'filename': self.aptlistfile} + self.apt_src_basic(self.aptlistfile, [cfg]) + + def test_apt_src_basic_dict(self): + """test_apt_src_basic_dict + Test Fix deb source string, has to overwrite mirror conf in params. + Test with a filename provided in config. + Provided in a dictionary with filename being the key (new format) + """ + cfg = {self.aptlistfile: {'source': + ('deb http://archive.ubuntu.com/ubuntu' + ' karmic-backports' + ' main universe multiverse restricted')}} + self.apt_src_basic(self.aptlistfile, cfg) + + def apt_src_basic_tri(self, cfg): + """apt_src_basic_tri + Test Fix three deb source string, has to overwrite mirror conf in + params. Test with filenames provided in config. + generic part to check three files with different content + """ + self.apt_src_basic(self.aptlistfile, cfg) + + # extra verify on two extra files of this test + contents = load_tfile_or_url(self.aptlistfile2) + self.assertTrue(re.search(r"%s %s %s %s\n" % + ("deb", "http://archive.ubuntu.com/ubuntu", + "precise-backports", + "main universe multiverse restricted"), + contents, flags=re.IGNORECASE)) + contents = load_tfile_or_url(self.aptlistfile3) + self.assertTrue(re.search(r"%s %s %s %s\n" % + ("deb", "http://archive.ubuntu.com/ubuntu", + "lucid-backports", + "main universe multiverse restricted"), + contents, flags=re.IGNORECASE)) + + def test_apt_src_basic_tri(self): + """test_apt_src_basic_tri + Test Fix three deb source string, has to overwrite mirror conf in + params. Test with filenames provided in config. + """ + cfg1 = {'source': ('deb http://archive.ubuntu.com/ubuntu' + ' karmic-backports' + ' main universe multiverse restricted'), + 'filename': self.aptlistfile} + cfg2 = {'source': ('deb http://archive.ubuntu.com/ubuntu' + ' precise-backports' + ' main universe multiverse restricted'), + 'filename': self.aptlistfile2} + cfg3 = {'source': ('deb http://archive.ubuntu.com/ubuntu' + ' lucid-backports' + ' main universe multiverse restricted'), + 'filename': self.aptlistfile3} + self.apt_src_basic_tri([cfg1, cfg2, cfg3]) + + def test_apt_src_basic_dict_tri(self): + """test_apt_src_basic_dict_tri + Test Fix three deb source string, has to overwrite mirror conf in + params. Test with filenames provided in config. + Provided in a dictionary with filename being the key (new format) + """ + cfg = {self.aptlistfile: {'source': + ('deb http://archive.ubuntu.com/ubuntu' + ' karmic-backports' + ' main universe multiverse restricted')}, + self.aptlistfile2: {'source': + ('deb http://archive.ubuntu.com/ubuntu' + ' precise-backports' + ' main universe multiverse restricted')}, + self.aptlistfile3: {'source': + ('deb http://archive.ubuntu.com/ubuntu' + ' lucid-backports' + ' main universe multiverse restricted')}} + self.apt_src_basic_tri(cfg) + + def test_apt_src_basic_nofn(self): + """test_apt_src_basic_nofn + Test Fix deb source string, has to overwrite mirror conf in params. + Test without a filename provided in config and test for known fallback. + """ + cfg = {'source': ('deb http://archive.ubuntu.com/ubuntu' + ' karmic-backports' + ' main universe multiverse restricted')} + with mock.patch.object(os.path, 'join', side_effect=self.myjoin): + self.apt_src_basic(self.fallbackfn, [cfg]) + + def apt_src_replacement(self, filename, cfg): + """apt_src_replace + Test Autoreplacement of MIRROR and RELEASE in source specs + """ + params = self._get_default_params() + cc_apt_configure.add_sources(cfg, params) + + self.assertTrue(os.path.isfile(filename)) + + contents = load_tfile_or_url(filename) + self.assertTrue(re.search(r"%s %s %s %s\n" % + ("deb", params['MIRROR'], params['RELEASE'], + "multiverse"), + contents, flags=re.IGNORECASE)) + + def test_apt_src_replace(self): + """test_apt_src_replace + Test Autoreplacement of MIRROR and RELEASE in source specs with + Filename being set + """ + cfg = {'source': 'deb $MIRROR $RELEASE multiverse', + 'filename': self.aptlistfile} + self.apt_src_replacement(self.aptlistfile, [cfg]) + + def apt_src_replace_tri(self, cfg): + """apt_src_replace_tri + Test three autoreplacements of MIRROR and RELEASE in source specs with + generic part + """ + self.apt_src_replacement(self.aptlistfile, cfg) + + # extra verify on two extra files of this test + params = self._get_default_params() + contents = load_tfile_or_url(self.aptlistfile2) + self.assertTrue(re.search(r"%s %s %s %s\n" % + ("deb", params['MIRROR'], params['RELEASE'], + "main"), + contents, flags=re.IGNORECASE)) + contents = load_tfile_or_url(self.aptlistfile3) + self.assertTrue(re.search(r"%s %s %s %s\n" % + ("deb", params['MIRROR'], params['RELEASE'], + "universe"), + contents, flags=re.IGNORECASE)) + + def test_apt_src_replace_tri(self): + """test_apt_src_replace_tri + Test three autoreplacements of MIRROR and RELEASE in source specs with + Filename being set + """ + cfg1 = {'source': 'deb $MIRROR $RELEASE multiverse', + 'filename': self.aptlistfile} + cfg2 = {'source': 'deb $MIRROR $RELEASE main', + 'filename': self.aptlistfile2} + cfg3 = {'source': 'deb $MIRROR $RELEASE universe', + 'filename': self.aptlistfile3} + self.apt_src_replace_tri([cfg1, cfg2, cfg3]) + + def test_apt_src_replace_dict_tri(self): + """test_apt_src_replace_dict_tri + Test three autoreplacements of MIRROR and RELEASE in source specs with + Filename being set + Provided in a dictionary with filename being the key (new format) + We also test a new special conditions of the new format that allows + filenames to be overwritten inside the directory entry. + """ + cfg = {self.aptlistfile: {'source': 'deb $MIRROR $RELEASE multiverse'}, + 'notused': {'source': 'deb $MIRROR $RELEASE main', + 'filename': self.aptlistfile2}, + self.aptlistfile3: {'source': 'deb $MIRROR $RELEASE universe'}} + self.apt_src_replace_tri(cfg) + + def test_apt_src_replace_nofn(self): + """test_apt_src_replace_nofn + Test Autoreplacement of MIRROR and RELEASE in source specs with + No filename being set + """ + cfg = {'source': 'deb $MIRROR $RELEASE multiverse'} + with mock.patch.object(os.path, 'join', side_effect=self.myjoin): + self.apt_src_replacement(self.fallbackfn, [cfg]) + + def apt_src_keyid(self, filename, cfg, keynum): + """apt_src_keyid + Test specification of a source + keyid + """ + params = self._get_default_params() + + with mock.patch.object(util, 'subp', + return_value=('fakekey 1234', '')) as mockobj: + cc_apt_configure.add_sources(cfg, params) + + # check if it added the right ammount of keys + calls = [] + for _ in range(keynum): + calls.append(call(('apt-key', 'add', '-'), 'fakekey 1234')) + mockobj.assert_has_calls(calls, any_order=True) + + self.assertTrue(os.path.isfile(filename)) + + contents = load_tfile_or_url(filename) + self.assertTrue(re.search(r"%s %s %s %s\n" % + ("deb", + ('http://ppa.launchpad.net/smoser/' + 'cloud-init-test/ubuntu'), + "xenial", "main"), + contents, flags=re.IGNORECASE)) + + def test_apt_src_keyid(self): + """test_apt_src_keyid + Test specification of a source + keyid with filename being set + """ + cfg = {'source': ('deb ' + 'http://ppa.launchpad.net/' + 'smoser/cloud-init-test/ubuntu' + ' xenial main'), + 'keyid': "03683F77", + 'filename': self.aptlistfile} + self.apt_src_keyid(self.aptlistfile, [cfg], 1) + + def test_apt_src_keyid_tri(self): + """test_apt_src_keyid_tri + Test specification of a source + keyid with filename being set + Setting three of such, check for content and keys + """ + cfg1 = {'source': ('deb ' + 'http://ppa.launchpad.net/' + 'smoser/cloud-init-test/ubuntu' + ' xenial main'), + 'keyid': "03683F77", + 'filename': self.aptlistfile} + cfg2 = {'source': ('deb ' + 'http://ppa.launchpad.net/' + 'smoser/cloud-init-test/ubuntu' + ' xenial universe'), + 'keyid': "03683F77", + 'filename': self.aptlistfile2} + cfg3 = {'source': ('deb ' + 'http://ppa.launchpad.net/' + 'smoser/cloud-init-test/ubuntu' + ' xenial multiverse'), + 'keyid': "03683F77", + 'filename': self.aptlistfile3} + + self.apt_src_keyid(self.aptlistfile, [cfg1, cfg2, cfg3], 3) + contents = load_tfile_or_url(self.aptlistfile2) + self.assertTrue(re.search(r"%s %s %s %s\n" % + ("deb", + ('http://ppa.launchpad.net/smoser/' + 'cloud-init-test/ubuntu'), + "xenial", "universe"), + contents, flags=re.IGNORECASE)) + contents = load_tfile_or_url(self.aptlistfile3) + self.assertTrue(re.search(r"%s %s %s %s\n" % + ("deb", + ('http://ppa.launchpad.net/smoser/' + 'cloud-init-test/ubuntu'), + "xenial", "multiverse"), + contents, flags=re.IGNORECASE)) + + def test_apt_src_keyid_nofn(self): + """test_apt_src_keyid_nofn + Test specification of a source + keyid without filename being set + """ + cfg = {'source': ('deb ' + 'http://ppa.launchpad.net/' + 'smoser/cloud-init-test/ubuntu' + ' xenial main'), + 'keyid': "03683F77"} + with mock.patch.object(os.path, 'join', side_effect=self.myjoin): + self.apt_src_keyid(self.fallbackfn, [cfg], 1) + + def apt_src_key(self, filename, cfg): + """apt_src_key + Test specification of a source + key + """ + params = self._get_default_params() + + with mock.patch.object(util, 'subp') as mockobj: + cc_apt_configure.add_sources([cfg], params) + + mockobj.assert_called_with(('apt-key', 'add', '-'), 'fakekey 4321') + + self.assertTrue(os.path.isfile(filename)) + + contents = load_tfile_or_url(filename) + self.assertTrue(re.search(r"%s %s %s %s\n" % + ("deb", + ('http://ppa.launchpad.net/smoser/' + 'cloud-init-test/ubuntu'), + "xenial", "main"), + contents, flags=re.IGNORECASE)) + + def test_apt_src_key(self): + """test_apt_src_key + Test specification of a source + key with filename being set + """ + cfg = {'source': ('deb ' + 'http://ppa.launchpad.net/' + 'smoser/cloud-init-test/ubuntu' + ' xenial main'), + 'key': "fakekey 4321", + 'filename': self.aptlistfile} + self.apt_src_key(self.aptlistfile, cfg) + + def test_apt_src_key_nofn(self): + """test_apt_src_key_nofn + Test specification of a source + key without filename being set + """ + cfg = {'source': ('deb ' + 'http://ppa.launchpad.net/' + 'smoser/cloud-init-test/ubuntu' + ' xenial main'), + 'key': "fakekey 4321"} + with mock.patch.object(os.path, 'join', side_effect=self.myjoin): + self.apt_src_key(self.fallbackfn, cfg) + + def test_apt_src_keyonly(self): + """test_apt_src_keyonly + Test specification key without source + """ + params = self._get_default_params() + cfg = {'key': "fakekey 4242", + 'filename': self.aptlistfile} + + with mock.patch.object(util, 'subp') as mockobj: + cc_apt_configure.add_sources([cfg], params) + + mockobj.assert_called_once_with(('apt-key', 'add', '-'), + 'fakekey 4242') + + # filename should be ignored on key only + self.assertFalse(os.path.isfile(self.aptlistfile)) + + def test_apt_src_keyidonly(self): + """test_apt_src_keyidonly + Test specification of a keyid without source + """ + params = self._get_default_params() + cfg = {'keyid': "03683F77", + 'filename': self.aptlistfile} + + with mock.patch.object(util, 'subp', + return_value=('fakekey 1212', '')) as mockobj: + cc_apt_configure.add_sources([cfg], params) + + mockobj.assert_called_with(('apt-key', 'add', '-'), 'fakekey 1212') + + # filename should be ignored on key only + self.assertFalse(os.path.isfile(self.aptlistfile)) + + def test_apt_src_keyid_real(self): + """test_apt_src_keyid_real + Test specification of a keyid without source incl + up to addition of the key (nothing but add_key_raw mocked) + """ + keyid = "03683F77" + params = self._get_default_params() + cfg = {'keyid': keyid, + 'filename': self.aptlistfile} + + with mock.patch.object(cc_apt_configure, 'add_key_raw') as mockobj: + cc_apt_configure.add_sources([cfg], params) + + mockobj.assert_called_with(EXPECTEDKEY) + + # filename should be ignored on key only + self.assertFalse(os.path.isfile(self.aptlistfile)) + + def test_apt_src_longkeyid_real(self): + """test_apt_src_longkeyid_real + Test specification of a long key fingerprint without source incl + up to addition of the key (nothing but add_key_raw mocked) + """ + keyid = "B59D 5F15 97A5 04B7 E230 6DCA 0620 BBCF 0368 3F77" + params = self._get_default_params() + cfg = {'keyid': keyid, + 'filename': self.aptlistfile} + + with mock.patch.object(cc_apt_configure, 'add_key_raw') as mockobj: + cc_apt_configure.add_sources([cfg], params) + + mockobj.assert_called_with(EXPECTEDKEY) + + # filename should be ignored on key only + self.assertFalse(os.path.isfile(self.aptlistfile)) + + def test_apt_src_ppa(self): + """test_apt_src_ppa + Test specification of a ppa + """ + params = self._get_default_params() + cfg = {'source': 'ppa:smoser/cloud-init-test', + 'filename': self.aptlistfile} + + # default matcher needed for ppa + matcher = re.compile(r'^[\w-]+:\w').search + + with mock.patch.object(util, 'subp') as mockobj: + cc_apt_configure.add_sources([cfg], params, aa_repo_match=matcher) + mockobj.assert_called_once_with(['add-apt-repository', + 'ppa:smoser/cloud-init-test']) + + # adding ppa should ignore filename (uses add-apt-repository) + self.assertFalse(os.path.isfile(self.aptlistfile)) + + def test_apt_src_ppa_tri(self): + """test_apt_src_ppa_tri + Test specification of a ppa + """ + params = self._get_default_params() + cfg1 = {'source': 'ppa:smoser/cloud-init-test', + 'filename': self.aptlistfile} + cfg2 = {'source': 'ppa:smoser/cloud-init-test2', + 'filename': self.aptlistfile2} + cfg3 = {'source': 'ppa:smoser/cloud-init-test3', + 'filename': self.aptlistfile3} + + # default matcher needed for ppa + matcher = re.compile(r'^[\w-]+:\w').search + + with mock.patch.object(util, 'subp') as mockobj: + cc_apt_configure.add_sources([cfg1, cfg2, cfg3], params, + aa_repo_match=matcher) + calls = [call(['add-apt-repository', 'ppa:smoser/cloud-init-test']), + call(['add-apt-repository', 'ppa:smoser/cloud-init-test2']), + call(['add-apt-repository', 'ppa:smoser/cloud-init-test3'])] + mockobj.assert_has_calls(calls, any_order=True) + + # adding ppa should ignore all filenames (uses add-apt-repository) + self.assertFalse(os.path.isfile(self.aptlistfile)) + self.assertFalse(os.path.isfile(self.aptlistfile2)) + self.assertFalse(os.path.isfile(self.aptlistfile3)) + + def test_convert_to_new_format(self): + """test_convert_to_new_format + Test the conversion of old to new format + And the noop conversion of new to new format as well + """ + cfg1 = {'source': 'deb $MIRROR $RELEASE multiverse', + 'filename': self.aptlistfile} + cfg2 = {'source': 'deb $MIRROR $RELEASE main', + 'filename': self.aptlistfile2} + cfg3 = {'source': 'deb $MIRROR $RELEASE universe', + 'filename': self.aptlistfile3} + checkcfg = {self.aptlistfile: {'filename': self.aptlistfile, + 'source': 'deb $MIRROR $RELEASE ' + 'multiverse'}, + self.aptlistfile2: {'filename': self.aptlistfile2, + 'source': 'deb $MIRROR $RELEASE main'}, + self.aptlistfile3: {'filename': self.aptlistfile3, + 'source': 'deb $MIRROR $RELEASE ' + 'universe'}} + + newcfg = cc_apt_configure.convert_to_new_format([cfg1, cfg2, cfg3]) + self.assertEqual(newcfg, checkcfg) + + newcfg2 = cc_apt_configure.convert_to_new_format(newcfg) + self.assertEqual(newcfg2, checkcfg) + + with self.assertRaises(ValueError): + cc_apt_configure.convert_to_new_format(5) + + +# vi: ts=4 expandtab |