From eeabe2ed7480dc653c9d65d218738a8ed5a21579 Mon Sep 17 00:00:00 2001 From: Christian Ehrhardt Date: Tue, 10 May 2016 14:42:19 +0200 Subject: basic apt_source test --- .../test_handler/test_handler_apt_source.py | 87 ++++++++++++++++++++++ 1 file changed, 87 insertions(+) create mode 100644 tests/unittests/test_handler/test_handler_apt_source.py diff --git a/tests/unittests/test_handler/test_handler_apt_source.py b/tests/unittests/test_handler/test_handler_apt_source.py new file mode 100644 index 00000000..b45fc1d1 --- /dev/null +++ b/tests/unittests/test_handler/test_handler_apt_source.py @@ -0,0 +1,87 @@ +""" test_handler_apt_source +Testing various config variations of the apt_source config +""" +import os +import shutil +import tempfile +import re + +from cloudinit import distros +from cloudinit import util +from cloudinit.config import cc_apt_configure + +from ..helpers import TestCase + +UNKNOWN_ARCH_INFO = { + 'arches': ['default'], + 'failsafe': {'primary': 'http://fs-primary-default', + 'security': 'http://fs-security-default'} +} + +PACKAGE_MIRRORS = [ + {'arches': ['i386', 'amd64'], + 'failsafe': {'primary': 'http://fs-primary-intel', + 'security': 'http://fs-security-intel'}, + 'search': { + 'primary': ['http://%(ec2_region)s.ec2/', + 'http://%(availability_zone)s.clouds/'], + 'security': ['http://security-mirror1-intel', + 'http://security-mirror2-intel']}}, + {'arches': ['armhf', 'armel'], + 'failsafe': {'primary': 'http://fs-primary-arm', + 'security': 'http://fs-security-arm'}}, + UNKNOWN_ARCH_INFO +] + +GAPMI = distros._get_arch_package_mirror_info + +def load_tfile_or_url(*args, **kwargs): + """ load_tfile_or_url + load file and return content after decoding + """ + return util.decode_binary(util.read_file_or_url(*args, **kwargs).contents) + +class TestAptSourceConfig(TestCase): + """ TestAptSourceConfig + Main Class to test apt_source configs + """ + def setUp(self): + super(TestAptSourceConfig, self).setUp() + self.tmp = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, self.tmp) + self.aptlistfile = os.path.join(self.tmp, "single-deb.list") + + + @staticmethod + def _get_default_params(): + """ get_default_params + Get the most basic default mrror and release info to be used in tests + """ + params = {} + params['RELEASE'] = cc_apt_configure.get_release() + params['MIRROR'] = "http://archive.ubuntu.com/ubuntu" + return params + + @staticmethod + def _search_apt_source(contents, params, pre, post): + return re.search(r"%s %s %s %s\n" % + (pre, params['MIRROR'], params['RELEASE'], post), + contents, flags=re.IGNORECASE) + + def test_apt_source_release(self): + """ test_apt_source_release + Test Autoreplacement of MIRROR and RELEASE in source specs + """ + params = self._get_default_params() + cfg = {'source': 'deb $MIRROR $RELEASE multiverse', + 'filename': self.aptlistfile} + + cc_apt_configure.add_sources([cfg], params) + + self.assertTrue(os.path.isfile(self.aptlistfile)) + + contents = load_tfile_or_url(self.aptlistfile) + self.assertTrue(self._search_apt_source(contents, params, + "deb", "multiverse")) + +# vi: ts=4 expandtab -- cgit v1.2.3 From b0494addabfe1d07947427ade99a00d2c7588f12 Mon Sep 17 00:00:00 2001 From: Christian Ehrhardt Date: Tue, 10 May 2016 14:59:57 +0200 Subject: split into basic and replacement test --- .../test_handler/test_handler_apt_source.py | 37 ++++++++++++++++------ 1 file changed, 28 insertions(+), 9 deletions(-) diff --git a/tests/unittests/test_handler/test_handler_apt_source.py b/tests/unittests/test_handler/test_handler_apt_source.py index b45fc1d1..a9647156 100644 --- a/tests/unittests/test_handler/test_handler_apt_source.py +++ b/tests/unittests/test_handler/test_handler_apt_source.py @@ -62,14 +62,31 @@ class TestAptSourceConfig(TestCase): params['MIRROR'] = "http://archive.ubuntu.com/ubuntu" return params - @staticmethod - def _search_apt_source(contents, params, pre, post): - return re.search(r"%s %s %s %s\n" % - (pre, params['MIRROR'], params['RELEASE'], post), - contents, flags=re.IGNORECASE) - def test_apt_source_release(self): - """ test_apt_source_release + def test_apt_source_basic(self): + """ test_apt_source_basic + Test Fix deb source string, has to overwrite mirror conf in params + """ + params = self._get_default_params() + cfg = {'source': ('deb http://archive.ubuntu.com/ubuntu' + ' karmic-backports' + ' main universe multiverse restricted'), + 'filename': self.aptlistfile} + + cc_apt_configure.add_sources([cfg], params) + + self.assertTrue(os.path.isfile(self.aptlistfile)) + + contents = load_tfile_or_url(self.aptlistfile) + self.assertTrue(re.search(r"%s %s %s %s\n" % + ("deb", "http://archive.ubuntu.com/ubuntu", + "karmic-backports", + "main universe multiverse restricted"), + contents, flags=re.IGNORECASE)) + + + def test_apt_source_replacement(self): + """ test_apt_source_replace Test Autoreplacement of MIRROR and RELEASE in source specs """ params = self._get_default_params() @@ -81,7 +98,9 @@ class TestAptSourceConfig(TestCase): self.assertTrue(os.path.isfile(self.aptlistfile)) contents = load_tfile_or_url(self.aptlistfile) - self.assertTrue(self._search_apt_source(contents, params, - "deb", "multiverse")) + self.assertTrue(re.search(r"%s %s %s %s\n" % + ("deb", params['MIRROR'], params['RELEASE'], + "multiverse"), + contents, flags=re.IGNORECASE)) # vi: ts=4 expandtab -- cgit v1.2.3 From 8cb8502cc1c99ec787e23504cf4e9f60c01bf0fe Mon Sep 17 00:00:00 2001 From: Christian Ehrhardt Date: Tue, 10 May 2016 15:30:49 +0200 Subject: add test_apt_source_ppa (failing for now) --- .../test_handler/test_handler_apt_source.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/tests/unittests/test_handler/test_handler_apt_source.py b/tests/unittests/test_handler/test_handler_apt_source.py index a9647156..22a4accf 100644 --- a/tests/unittests/test_handler/test_handler_apt_source.py +++ b/tests/unittests/test_handler/test_handler_apt_source.py @@ -103,4 +103,25 @@ class TestAptSourceConfig(TestCase): "multiverse"), contents, flags=re.IGNORECASE)) + + def test_apt_source_ppa(self): + """ test_apt_source_ppa + Test specification of a ppa + """ + params = self._get_default_params() + cfg = {'source': 'ppa:smoser/cloud-init-test', + 'filename': self.aptlistfile} + + cc_apt_configure.add_sources([cfg], params) + + self.assertTrue(os.path.isfile(self.aptlistfile)) + + # report content before making regex + # FAIL ? goes in "untranslated" + # should become e.g. deb http://ppa.launchpad.net/smoser/cloud-init-test/ubuntu xenial main + contents = load_tfile_or_url(self.aptlistfile) + print(contents) + self.assertTrue(1 == 2) + + # vi: ts=4 expandtab -- cgit v1.2.3 From 66f847f4cba2215490986ffede8e03299dbd5b90 Mon Sep 17 00:00:00 2001 From: Christian Ehrhardt Date: Tue, 10 May 2016 17:14:26 +0200 Subject: add test_apt_source_key for sources with a keyid to import --- .../test_handler/test_handler_apt_source.py | 32 ++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/tests/unittests/test_handler/test_handler_apt_source.py b/tests/unittests/test_handler/test_handler_apt_source.py index 22a4accf..d9942901 100644 --- a/tests/unittests/test_handler/test_handler_apt_source.py +++ b/tests/unittests/test_handler/test_handler_apt_source.py @@ -104,6 +104,38 @@ class TestAptSourceConfig(TestCase): contents, flags=re.IGNORECASE)) + def test_apt_source_key(self): + """ test_apt_source_key + Test specification of a source + key + """ + params = self._get_default_params() + cfg = {'source': ('deb ' + 'http://ppa.launchpad.net/' + 'smoser/cloud-init-test/ubuntu' + ' xenial main'), + 'keyid:': "03683F77", + 'filename': self.aptlistfile} + + cc_apt_configure.add_sources([cfg], params) + + self.assertTrue(os.path.isfile(self.aptlistfile)) + + # report content before making regex + contents = load_tfile_or_url(self.aptlistfile) + self.assertTrue(re.search(r"%s %s %s %s\n" % + ("deb", + ('http://ppa.launchpad.net/smoser/' + 'cloud-init-test/ubuntu'), + "xenial", "main"), + contents, flags=re.IGNORECASE)) + # check if key was imported + try: + util.subp(('apt-key', 'list', '03683F77')) + except util.ProcessExecutionError as err: + print("apt-key failed. " + str(err)) + self.assertTrue(1 == 2) + + def test_apt_source_ppa(self): """ test_apt_source_ppa Test specification of a ppa -- cgit v1.2.3 From 904aeedf343af17ef88bbbaef9896d425eefa778 Mon Sep 17 00:00:00 2001 From: Christian Ehrhardt Date: Tue, 10 May 2016 20:57:30 +0200 Subject: provide valid matcher for ppa so that apt-add-repository is triggered --- .../unittests/test_handler/test_handler_apt_source.py | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/tests/unittests/test_handler/test_handler_apt_source.py b/tests/unittests/test_handler/test_handler_apt_source.py index d9942901..10a03a8d 100644 --- a/tests/unittests/test_handler/test_handler_apt_source.py +++ b/tests/unittests/test_handler/test_handler_apt_source.py @@ -144,14 +144,21 @@ class TestAptSourceConfig(TestCase): cfg = {'source': 'ppa:smoser/cloud-init-test', 'filename': self.aptlistfile} - cc_apt_configure.add_sources([cfg], params) + # default matcher needed for ppa + matcher = re.compile(r'^[\w-]+:\w').search - self.assertTrue(os.path.isfile(self.aptlistfile)) + cc_apt_configure.add_sources([cfg], params, aa_repo_match=matcher) - # report content before making regex - # FAIL ? goes in "untranslated" - # should become e.g. deb http://ppa.launchpad.net/smoser/cloud-init-test/ubuntu xenial main - contents = load_tfile_or_url(self.aptlistfile) + # adding ppa should ignore filename (uses add-apt-repository) + self.assertFalse(os.path.isfile(self.aptlistfile)) + expected_sources_fn=('/etc/apt/sources.list.d/' + 'smoser-ubuntu-cloud-init-test-%s.list' + % params['RELEASE']) + print("filename: %s" % expected_sources_fn) + self.assertTrue(os.path.isfile(expected_sources_fn)) + + # file gets not created, might be permission or env detail + contents = load_tfile_or_url(expected_sources_fn) print(contents) self.assertTrue(1 == 2) -- cgit v1.2.3 From 86c59ffa50a74a1d0001c5ef6ccc78bd6f656fdc Mon Sep 17 00:00:00 2001 From: Christian Ehrhardt Date: Tue, 10 May 2016 21:03:07 +0200 Subject: use proper asserRaises for try catch --- tests/unittests/test_handler/test_handler_apt_source.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/unittests/test_handler/test_handler_apt_source.py b/tests/unittests/test_handler/test_handler_apt_source.py index 10a03a8d..a4d359a5 100644 --- a/tests/unittests/test_handler/test_handler_apt_source.py +++ b/tests/unittests/test_handler/test_handler_apt_source.py @@ -132,8 +132,7 @@ class TestAptSourceConfig(TestCase): try: util.subp(('apt-key', 'list', '03683F77')) except util.ProcessExecutionError as err: - print("apt-key failed. " + str(err)) - self.assertTrue(1 == 2) + self.assertRaises(err, "apt-key failed failed") def test_apt_source_ppa(self): @@ -160,7 +159,8 @@ class TestAptSourceConfig(TestCase): # file gets not created, might be permission or env detail contents = load_tfile_or_url(expected_sources_fn) print(contents) - self.assertTrue(1 == 2) + # intentional debug exit + self.assertRaises(ValueError) # vi: ts=4 expandtab -- cgit v1.2.3 From 4c5d1966ead445dc6d110e9677902b95dfef2dc5 Mon Sep 17 00:00:00 2001 From: Christian Ehrhardt Date: Wed, 11 May 2016 10:57:14 +0200 Subject: test test_apt_source_key with mocked util.subp --- tests/unittests/test_handler/test_handler_apt_source.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/tests/unittests/test_handler/test_handler_apt_source.py b/tests/unittests/test_handler/test_handler_apt_source.py index a4d359a5..29535bee 100644 --- a/tests/unittests/test_handler/test_handler_apt_source.py +++ b/tests/unittests/test_handler/test_handler_apt_source.py @@ -6,6 +6,11 @@ import shutil import tempfile import re +try: + from unittest import mock +except ImportError: + import mock + from cloudinit import distros from cloudinit import util from cloudinit.config import cc_apt_configure @@ -116,11 +121,13 @@ class TestAptSourceConfig(TestCase): 'keyid:': "03683F77", 'filename': self.aptlistfile} - cc_apt_configure.add_sources([cfg], params) + with mock.patch.object(util, 'subp', return_value=('fakekey 1234', '')) as mockobj: + cc_apt_configure.add_sources([cfg], params) + + mockobj.assert_called_with(('apt-key', 'add', '-'), 'fakekey 1234') self.assertTrue(os.path.isfile(self.aptlistfile)) - # report content before making regex contents = load_tfile_or_url(self.aptlistfile) self.assertTrue(re.search(r"%s %s %s %s\n" % ("deb", @@ -128,11 +135,6 @@ class TestAptSourceConfig(TestCase): 'cloud-init-test/ubuntu'), "xenial", "main"), contents, flags=re.IGNORECASE)) - # check if key was imported - try: - util.subp(('apt-key', 'list', '03683F77')) - except util.ProcessExecutionError as err: - self.assertRaises(err, "apt-key failed failed") def test_apt_source_ppa(self): -- cgit v1.2.3 From fef11c6a98ea74774aa84b3b14007b246a1c615e Mon Sep 17 00:00:00 2001 From: Christian Ehrhardt Date: Wed, 11 May 2016 10:57:33 +0200 Subject: fix specification of keyid --- tests/unittests/test_handler/test_handler_apt_source.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unittests/test_handler/test_handler_apt_source.py b/tests/unittests/test_handler/test_handler_apt_source.py index 29535bee..601504bd 100644 --- a/tests/unittests/test_handler/test_handler_apt_source.py +++ b/tests/unittests/test_handler/test_handler_apt_source.py @@ -118,7 +118,7 @@ class TestAptSourceConfig(TestCase): 'http://ppa.launchpad.net/' 'smoser/cloud-init-test/ubuntu' ' xenial main'), - 'keyid:': "03683F77", + 'keyid': "03683F77", 'filename': self.aptlistfile} with mock.patch.object(util, 'subp', return_value=('fakekey 1234', '')) as mockobj: -- cgit v1.2.3 From 0e299c89e10a6ccb583588ff31ac783421e57501 Mon Sep 17 00:00:00 2001 From: Christian Ehrhardt Date: Wed, 11 May 2016 11:03:51 +0200 Subject: convert test_apt_source_ppa to use a mocked util.subp --- tests/unittests/test_handler/test_handler_apt_source.py | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/tests/unittests/test_handler/test_handler_apt_source.py b/tests/unittests/test_handler/test_handler_apt_source.py index 601504bd..e73a72c6 100644 --- a/tests/unittests/test_handler/test_handler_apt_source.py +++ b/tests/unittests/test_handler/test_handler_apt_source.py @@ -148,21 +148,13 @@ class TestAptSourceConfig(TestCase): # default matcher needed for ppa matcher = re.compile(r'^[\w-]+:\w').search - cc_apt_configure.add_sources([cfg], params, aa_repo_match=matcher) + with mock.patch.object(util, 'subp') as mockobj: + cc_apt_configure.add_sources([cfg], params, aa_repo_match=matcher) + mockobj.assert_called_once_with(['add-apt-repository', + 'ppa:smoser/cloud-init-test']) # adding ppa should ignore filename (uses add-apt-repository) self.assertFalse(os.path.isfile(self.aptlistfile)) - expected_sources_fn=('/etc/apt/sources.list.d/' - 'smoser-ubuntu-cloud-init-test-%s.list' - % params['RELEASE']) - print("filename: %s" % expected_sources_fn) - self.assertTrue(os.path.isfile(expected_sources_fn)) - - # file gets not created, might be permission or env detail - contents = load_tfile_or_url(expected_sources_fn) - print(contents) - # intentional debug exit - self.assertRaises(ValueError) # vi: ts=4 expandtab -- cgit v1.2.3 From 0a35ba5b8d85077a865a137887aa8cfd76405eb2 Mon Sep 17 00:00:00 2001 From: Christian Ehrhardt Date: Wed, 11 May 2016 11:06:08 +0200 Subject: fix a few forgotten pep8 warnings in test_handler_apt_source.py --- tests/unittests/test_handler/test_handler_apt_source.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/tests/unittests/test_handler/test_handler_apt_source.py b/tests/unittests/test_handler/test_handler_apt_source.py index e73a72c6..849f23c9 100644 --- a/tests/unittests/test_handler/test_handler_apt_source.py +++ b/tests/unittests/test_handler/test_handler_apt_source.py @@ -40,12 +40,14 @@ PACKAGE_MIRRORS = [ GAPMI = distros._get_arch_package_mirror_info + def load_tfile_or_url(*args, **kwargs): """ load_tfile_or_url load file and return content after decoding """ return util.decode_binary(util.read_file_or_url(*args, **kwargs).contents) + class TestAptSourceConfig(TestCase): """ TestAptSourceConfig Main Class to test apt_source configs @@ -56,7 +58,6 @@ class TestAptSourceConfig(TestCase): self.addCleanup(shutil.rmtree, self.tmp) self.aptlistfile = os.path.join(self.tmp, "single-deb.list") - @staticmethod def _get_default_params(): """ get_default_params @@ -67,7 +68,6 @@ class TestAptSourceConfig(TestCase): params['MIRROR'] = "http://archive.ubuntu.com/ubuntu" return params - def test_apt_source_basic(self): """ test_apt_source_basic Test Fix deb source string, has to overwrite mirror conf in params @@ -89,7 +89,6 @@ class TestAptSourceConfig(TestCase): "main universe multiverse restricted"), contents, flags=re.IGNORECASE)) - def test_apt_source_replacement(self): """ test_apt_source_replace Test Autoreplacement of MIRROR and RELEASE in source specs @@ -108,7 +107,6 @@ class TestAptSourceConfig(TestCase): "multiverse"), contents, flags=re.IGNORECASE)) - def test_apt_source_key(self): """ test_apt_source_key Test specification of a source + key @@ -121,7 +119,8 @@ class TestAptSourceConfig(TestCase): 'keyid': "03683F77", 'filename': self.aptlistfile} - with mock.patch.object(util, 'subp', return_value=('fakekey 1234', '')) as mockobj: + with mock.patch.object(util, 'subp', + return_value=('fakekey 1234', '')) as mockobj: cc_apt_configure.add_sources([cfg], params) mockobj.assert_called_with(('apt-key', 'add', '-'), 'fakekey 1234') @@ -136,7 +135,6 @@ class TestAptSourceConfig(TestCase): "xenial", "main"), contents, flags=re.IGNORECASE)) - def test_apt_source_ppa(self): """ test_apt_source_ppa Test specification of a ppa @@ -151,7 +149,7 @@ class TestAptSourceConfig(TestCase): with mock.patch.object(util, 'subp') as mockobj: cc_apt_configure.add_sources([cfg], params, aa_repo_match=matcher) mockobj.assert_called_once_with(['add-apt-repository', - 'ppa:smoser/cloud-init-test']) + 'ppa:smoser/cloud-init-test']) # adding ppa should ignore filename (uses add-apt-repository) self.assertFalse(os.path.isfile(self.aptlistfile)) -- cgit v1.2.3 From a9cd544d7de1af90f6c5cf9df43135b530acb308 Mon Sep 17 00:00:00 2001 From: Christian Ehrhardt Date: Wed, 11 May 2016 16:52:17 +0200 Subject: split test_apt_source_key into one for key and one for keyid --- .../test_handler/test_handler_apt_source.py | 32 ++++++++++++++++++++-- 1 file changed, 30 insertions(+), 2 deletions(-) diff --git a/tests/unittests/test_handler/test_handler_apt_source.py b/tests/unittests/test_handler/test_handler_apt_source.py index 849f23c9..a7db0fa6 100644 --- a/tests/unittests/test_handler/test_handler_apt_source.py +++ b/tests/unittests/test_handler/test_handler_apt_source.py @@ -107,6 +107,34 @@ class TestAptSourceConfig(TestCase): "multiverse"), contents, flags=re.IGNORECASE)) + def test_apt_source_keyid(self): + """ test_apt_source_keyid + Test specification of a source + keyid + """ + params = self._get_default_params() + cfg = {'source': ('deb ' + 'http://ppa.launchpad.net/' + 'smoser/cloud-init-test/ubuntu' + ' xenial main'), + 'keyid': "03683F77", + 'filename': self.aptlistfile} + + with mock.patch.object(util, 'subp', + return_value=('fakekey 1234', '')) as mockobj: + cc_apt_configure.add_sources([cfg], params) + + mockobj.assert_called_with(('apt-key', 'add', '-'), 'fakekey 1234') + + self.assertTrue(os.path.isfile(self.aptlistfile)) + + contents = load_tfile_or_url(self.aptlistfile) + self.assertTrue(re.search(r"%s %s %s %s\n" % + ("deb", + ('http://ppa.launchpad.net/smoser/' + 'cloud-init-test/ubuntu'), + "xenial", "main"), + contents, flags=re.IGNORECASE)) + def test_apt_source_key(self): """ test_apt_source_key Test specification of a source + key @@ -116,14 +144,14 @@ class TestAptSourceConfig(TestCase): 'http://ppa.launchpad.net/' 'smoser/cloud-init-test/ubuntu' ' xenial main'), - 'keyid': "03683F77", + 'key': "fakekey 4321", 'filename': self.aptlistfile} with mock.patch.object(util, 'subp', return_value=('fakekey 1234', '')) as mockobj: cc_apt_configure.add_sources([cfg], params) - mockobj.assert_called_with(('apt-key', 'add', '-'), 'fakekey 1234') + mockobj.assert_called_with(('apt-key', 'add', '-'), 'fakekey 4321') self.assertTrue(os.path.isfile(self.aptlistfile)) -- cgit v1.2.3 From d82f6c1ecd255ed3a76bd8ef7b76163408f0b398 Mon Sep 17 00:00:00 2001 From: Christian Ehrhardt Date: Wed, 11 May 2016 16:55:37 +0200 Subject: add test_apt_source_keyonly (not yet supported) --- tests/unittests/test_handler/test_handler_apt_source.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/tests/unittests/test_handler/test_handler_apt_source.py b/tests/unittests/test_handler/test_handler_apt_source.py index a7db0fa6..92a92406 100644 --- a/tests/unittests/test_handler/test_handler_apt_source.py +++ b/tests/unittests/test_handler/test_handler_apt_source.py @@ -163,6 +163,22 @@ class TestAptSourceConfig(TestCase): "xenial", "main"), contents, flags=re.IGNORECASE)) + def test_apt_source_keyonly(self): + """ test_apt_source_keyonly + Test specification key without source (not yet supported) + """ + params = self._get_default_params() + cfg = {'key': "fakekey 4242", + 'filename': self.aptlistfile} + + with mock.patch.object(util, 'subp') as mockobj: + cc_apt_configure.add_sources([cfg], params) + + mockobj.assert_called_once_with(('apt-key', 'add', '-'), 'fakekey 4242') + + # filename should be ignored on key only + self.assertFalse(os.path.isfile(self.aptlistfile)) + def test_apt_source_ppa(self): """ test_apt_source_ppa Test specification of a ppa -- cgit v1.2.3 From b4f76a0a855d792acc05807a3a62cc8c72d80792 Mon Sep 17 00:00:00 2001 From: Christian Ehrhardt Date: Wed, 11 May 2016 16:56:12 +0200 Subject: apt_apt_source_key doesn't need a mocked retval --- tests/unittests/test_handler/test_handler_apt_source.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/unittests/test_handler/test_handler_apt_source.py b/tests/unittests/test_handler/test_handler_apt_source.py index 92a92406..091b07da 100644 --- a/tests/unittests/test_handler/test_handler_apt_source.py +++ b/tests/unittests/test_handler/test_handler_apt_source.py @@ -147,8 +147,7 @@ class TestAptSourceConfig(TestCase): 'key': "fakekey 4321", 'filename': self.aptlistfile} - with mock.patch.object(util, 'subp', - return_value=('fakekey 1234', '')) as mockobj: + with mock.patch.object(util, 'subp') as mockobj: cc_apt_configure.add_sources([cfg], params) mockobj.assert_called_with(('apt-key', 'add', '-'), 'fakekey 4321') -- cgit v1.2.3 From d5bc051cdc418efa0a910f8b61790523b89e8f91 Mon Sep 17 00:00:00 2001 From: Christian Ehrhardt Date: Wed, 11 May 2016 16:59:15 +0200 Subject: add test_apt_source_keyidonly (not yet supported) --- tests/unittests/test_handler/test_handler_apt_source.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/tests/unittests/test_handler/test_handler_apt_source.py b/tests/unittests/test_handler/test_handler_apt_source.py index 091b07da..de009174 100644 --- a/tests/unittests/test_handler/test_handler_apt_source.py +++ b/tests/unittests/test_handler/test_handler_apt_source.py @@ -178,6 +178,23 @@ class TestAptSourceConfig(TestCase): # filename should be ignored on key only self.assertFalse(os.path.isfile(self.aptlistfile)) + def test_apt_source_keyidonly(self): + """ test_apt_source_keyidonly + Test specification of a keyid without source (not yet supported) + """ + params = self._get_default_params() + cfg = {'keyid': "03683F77", + 'filename': self.aptlistfile} + + with mock.patch.object(util, 'subp', + return_value=('fakekey 1212', '')) as mockobj: + cc_apt_configure.add_sources([cfg], params) + + mockobj.assert_called_with(('apt-key', 'add', '-'), 'fakekey 1212') + + # filename should be ignored on key only + self.assertFalse(os.path.isfile(self.aptlistfile)) + def test_apt_source_ppa(self): """ test_apt_source_ppa Test specification of a ppa -- cgit v1.2.3 From da2640951d2d87f38dd539e53115af98f12c11ac Mon Sep 17 00:00:00 2001 From: Christian Ehrhardt Date: Wed, 11 May 2016 18:44:52 +0200 Subject: fix pep8 warning --- tests/unittests/test_handler/test_handler_apt_source.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/unittests/test_handler/test_handler_apt_source.py b/tests/unittests/test_handler/test_handler_apt_source.py index de009174..01d56559 100644 --- a/tests/unittests/test_handler/test_handler_apt_source.py +++ b/tests/unittests/test_handler/test_handler_apt_source.py @@ -173,7 +173,8 @@ class TestAptSourceConfig(TestCase): with mock.patch.object(util, 'subp') as mockobj: cc_apt_configure.add_sources([cfg], params) - mockobj.assert_called_once_with(('apt-key', 'add', '-'), 'fakekey 4242') + mockobj.assert_called_once_with(('apt-key', 'add', '-'), + 'fakekey 4242') # filename should be ignored on key only self.assertFalse(os.path.isfile(self.aptlistfile)) -- cgit v1.2.3 From cca640d332eb8a6b068033a28b0b319873c7fbf6 Mon Sep 17 00:00:00 2001 From: Christian Ehrhardt Date: Thu, 12 May 2016 09:21:50 +0200 Subject: allow to add keys without specifying a source --- cloudinit/config/cc_apt_configure.py | 42 ++++++++++++++++++++++-------------- 1 file changed, 26 insertions(+), 16 deletions(-) diff --git a/cloudinit/config/cc_apt_configure.py b/cloudinit/config/cc_apt_configure.py index 702977cb..1d3eddff 100644 --- a/cloudinit/config/cc_apt_configure.py +++ b/cloudinit/config/cc_apt_configure.py @@ -164,6 +164,29 @@ def generate_sources_list(codename, mirrors, cloud, log): templater.render_to_file(template_fn, '/etc/apt/sources.list', params) +def add_key(ent, errorlist): + """ + add key to the system as defiend in entry (if any) + suppords raw keys or keyid's + The latter will as a first step fetched to get the raw key + """ + if ('keyid' in ent and 'key' not in ent): + keyserver = "keyserver.ubuntu.com" + if 'keyserver' in ent: + keyserver = ent['keyserver'] + try: + ent['key'] = getkeybyid(ent['keyid'], keyserver) + except: + errorlist.append([ent, "failed to get key from %s" % keyserver]) + return + + if 'key' in ent: + try: + util.subp(('apt-key', 'add', '-'), ent['key']) + except: + errorlist.append([ent, "failed add key"]) + + def add_sources(srclist, template_params=None, aa_repo_match=None): """ add entries in /etc/apt/sources.list.d for each abbreviated @@ -179,6 +202,9 @@ def add_sources(srclist, template_params=None, aa_repo_match=None): errorlist = [] for ent in srclist: + # keys can be added without specifying a source + add_key(ent, errorlist) + if 'source' not in ent: errorlist.append(["", "missing source"]) continue @@ -201,22 +227,6 @@ def add_sources(srclist, template_params=None, aa_repo_match=None): ent['filename'] = os.path.join("/etc/apt/sources.list.d/", ent['filename']) - if ('keyid' in ent and 'key' not in ent): - ks = "keyserver.ubuntu.com" - if 'keyserver' in ent: - ks = ent['keyserver'] - try: - ent['key'] = getkeybyid(ent['keyid'], ks) - except: - errorlist.append([source, "failed to get key from %s" % ks]) - continue - - if 'key' in ent: - try: - util.subp(('apt-key', 'add', '-'), ent['key']) - except: - errorlist.append([source, "failed add key"]) - try: contents = "%s\n" % (source) util.write_file(ent['filename'], contents, omode="ab") -- cgit v1.2.3 From a6282380814750851c0dc2ac2d4d3386eb6fcce4 Mon Sep 17 00:00:00 2001 From: Christian Ehrhardt Date: Thu, 12 May 2016 09:26:27 +0200 Subject: update doc/examples/cloud-config.txt with new key-without-source cases --- doc/examples/cloud-config.txt | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/doc/examples/cloud-config.txt b/doc/examples/cloud-config.txt index 1236796c..f84d526d 100644 --- a/doc/examples/cloud-config.txt +++ b/doc/examples/cloud-config.txt @@ -111,12 +111,14 @@ apt_sources: keyid: F430BBA5 # GPG key ID published on a key server filename: byobu-ppa.list + # this would only import the key without adding a ppa or other source spec + - keyid: F430BBA5 # GPG key ID published on a key server + # Custom apt repository: # * The apt signing key can also be specified # by providing a pgp public key block # * Providing the PBG key here is the most robust method for # specifying a key, as it removes dependency on a remote key server - - source: deb http://ppa.launchpad.net/alestic/ppa/ubuntu karmic main key: | # The value needs to start with -----BEGIN PGP PUBLIC KEY BLOCK----- -----BEGIN PGP PUBLIC KEY BLOCK----- @@ -132,6 +134,24 @@ apt_sources: =Y2oI -----END PGP PUBLIC KEY BLOCK----- + # Custom gpg key: + # * As the keyid also a key can be specified withut a related source + # * all other facts mentioned above still apply + - key: | # The value needs to start with -----BEGIN PGP PUBLIC KEY BLOCK----- + -----BEGIN PGP PUBLIC KEY BLOCK----- + Version: SKS 1.0.10 + + mI0ESpA3UQEEALdZKVIMq0j6qWAXAyxSlF63SvPVIgxHPb9Nk0DZUixn+akqytxG4zKCONz6 + qLjoBBfHnynyVLfT4ihg9an1PqxRnTO+JKQxl8NgKGz6Pon569GtAOdWNKw15XKinJTDLjnj + 9y96ljJqRcpV9t/WsIcdJPcKFR5voHTEoABE2aEXABEBAAG0GUxhdW5jaHBhZCBQUEEgZm9y + IEFsZXN0aWOItgQTAQIAIAUCSpA3UQIbAwYLCQgHAwIEFQIIAwQWAgMBAh4BAheAAAoJEA7H + 5Qi+CcVxWZ8D/1MyYvfj3FJPZUm2Yo1zZsQ657vHI9+pPouqflWOayRR9jbiyUFIn0VdQBrP + t0FwvnOFArUovUWoKAEdqR8hPy3M3APUZjl5K4cMZR/xaMQeQRZ5CHpS4DBKURKAHC0ltS5o + uBJKQOZm5iltJp15cgyIkBkGe8Mx18VFyVglAZey + =Y2oI + -----END PGP PUBLIC KEY BLOCK----- + + ## apt config via system_info: # under the 'system_info', you can further customize cloud-init's interaction # with apt. -- cgit v1.2.3 From 48a50dabf482a8ae029775a94be2d1b84763f98d Mon Sep 17 00:00:00 2001 From: Christian Ehrhardt Date: Thu, 12 May 2016 09:52:02 +0200 Subject: remove unused test parts in test_handler_apt_source --- .../test_handler/test_handler_apt_source.py | 23 ---------------------- 1 file changed, 23 deletions(-) diff --git a/tests/unittests/test_handler/test_handler_apt_source.py b/tests/unittests/test_handler/test_handler_apt_source.py index 01d56559..38c93e0e 100644 --- a/tests/unittests/test_handler/test_handler_apt_source.py +++ b/tests/unittests/test_handler/test_handler_apt_source.py @@ -17,29 +17,6 @@ from cloudinit.config import cc_apt_configure from ..helpers import TestCase -UNKNOWN_ARCH_INFO = { - 'arches': ['default'], - 'failsafe': {'primary': 'http://fs-primary-default', - 'security': 'http://fs-security-default'} -} - -PACKAGE_MIRRORS = [ - {'arches': ['i386', 'amd64'], - 'failsafe': {'primary': 'http://fs-primary-intel', - 'security': 'http://fs-security-intel'}, - 'search': { - 'primary': ['http://%(ec2_region)s.ec2/', - 'http://%(availability_zone)s.clouds/'], - 'security': ['http://security-mirror1-intel', - 'http://security-mirror2-intel']}}, - {'arches': ['armhf', 'armel'], - 'failsafe': {'primary': 'http://fs-primary-arm', - 'security': 'http://fs-security-arm'}}, - UNKNOWN_ARCH_INFO -] - -GAPMI = distros._get_arch_package_mirror_info - def load_tfile_or_url(*args, **kwargs): """ load_tfile_or_url -- cgit v1.2.3 From 1b4a06cd7933f876b16576567bbc0dab46b83b6b Mon Sep 17 00:00:00 2001 From: Christian Ehrhardt Date: Thu, 12 May 2016 13:21:29 +0200 Subject: add testcase for rendering of templates into source.list --- .../test_handler_apt_configure_sources_list.py | 89 ++++++++++++++++++++++ 1 file changed, 89 insertions(+) create mode 100644 tests/unittests/test_handler/test_handler_apt_configure_sources_list.py diff --git a/tests/unittests/test_handler/test_handler_apt_configure_sources_list.py b/tests/unittests/test_handler/test_handler_apt_configure_sources_list.py new file mode 100644 index 00000000..46edb628 --- /dev/null +++ b/tests/unittests/test_handler/test_handler_apt_configure_sources_list.py @@ -0,0 +1,89 @@ +""" test_handler_apt_configure_sources_list +Test templating of sources list +""" +import os +import shutil +import tempfile +import re + +import logging + +try: + from unittest import mock +except ImportError: + import mock + +from cloudinit import cloud +from cloudinit import distros +from cloudinit import util +from cloudinit import helpers +from cloudinit import templater + +from cloudinit.sources import DataSourceNone +from cloudinit.config import cc_apt_configure + +from .. import helpers as t_help + +LOG = logging.getLogger(__name__) + + +def load_tfile_or_url(*args, **kwargs): + """ load_tfile_or_url + load file and return content after decoding + """ + return util.decode_binary(util.read_file_or_url(*args, **kwargs).contents) + + +class TestAptSourceConfigSourceList(t_help.FilesystemMockingTestCase): + """ TestAptSourceConfigSourceList + Main Class to test sources list rendering + """ + def setUp(self): + super(TestAptSourceConfigSourceList, self).setUp() + self.new_root = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, self.new_root) + + def _get_cloud(self, distro, metadata=None): + self.patchUtils(self.new_root) + paths = helpers.Paths({}) + cls = distros.fetch(distro) + mydist = cls(distro, {}, paths) + myds = DataSourceNone.DataSourceNone({}, mydist, paths) + if metadata: + myds.metadata.update(metadata) + return cloud.Cloud(myds, paths, {}, mydist, None) + +# TODO - Ubuntu template +# TODO - Debian template +# TODO Later - custom template filename +# TODO Later - custom template raw + + def test_apt_source_list_ubuntu(self): + """ test_apt_source_list + Test rendering of a source.list from template for ubuntu + """ + self.patchOS(self.new_root) + self.patchUtils(self.new_root) + + cfg = {'apt_mirror': 'http://archive.ubuntu.com/ubuntu/'} + mycloud = self._get_cloud('ubuntu') + + with mock.patch.object(templater, 'render_to_file') as mocktmpl: + with mock.patch.object(os.path, 'isfile', + return_value=True) as mockisfile: + cc_apt_configure.handle("notimportant", cfg, mycloud, + LOG, None) + + mockisfile.assert_any_call(('/etc/cloud/templates/' + 'sources.list.ubuntu.tmpl')) + mocktmpl.assert_called_once_with(('/etc/cloud/templates/' + 'sources.list.ubuntu.tmpl'), + '/etc/apt/sources.list', + {'codename': '', + 'primary': + 'http://archive.ubuntu.com/ubuntu/', + 'mirror': + 'http://archive.ubuntu.com/ubuntu/'}) + + +# vi: ts=4 expandtab -- cgit v1.2.3 From 83f2351bba273273d04759ebacd2a25df9045090 Mon Sep 17 00:00:00 2001 From: Christian Ehrhardt Date: Thu, 12 May 2016 13:28:36 +0200 Subject: test debian and ubuntu source.list templating --- .../test_handler_apt_configure_sources_list.py | 32 ++++++++++++++++------ 1 file changed, 23 insertions(+), 9 deletions(-) diff --git a/tests/unittests/test_handler/test_handler_apt_configure_sources_list.py b/tests/unittests/test_handler/test_handler_apt_configure_sources_list.py index 46edb628..aff272a3 100644 --- a/tests/unittests/test_handler/test_handler_apt_configure_sources_list.py +++ b/tests/unittests/test_handler/test_handler_apt_configure_sources_list.py @@ -58,15 +58,15 @@ class TestAptSourceConfigSourceList(t_help.FilesystemMockingTestCase): # TODO Later - custom template filename # TODO Later - custom template raw - def test_apt_source_list_ubuntu(self): - """ test_apt_source_list - Test rendering of a source.list from template for ubuntu + def apt_source_list(self, distro, mirror): + """ apt_source_list + Test rendering of a source.list from template for a given distro """ self.patchOS(self.new_root) self.patchUtils(self.new_root) - cfg = {'apt_mirror': 'http://archive.ubuntu.com/ubuntu/'} - mycloud = self._get_cloud('ubuntu') + cfg = {'apt_mirror': mirror} + mycloud = self._get_cloud(distro) with mock.patch.object(templater, 'render_to_file') as mocktmpl: with mock.patch.object(os.path, 'isfile', @@ -75,15 +75,29 @@ class TestAptSourceConfigSourceList(t_help.FilesystemMockingTestCase): LOG, None) mockisfile.assert_any_call(('/etc/cloud/templates/' - 'sources.list.ubuntu.tmpl')) + 'sources.list.%s.tmpl' % distro)) mocktmpl.assert_called_once_with(('/etc/cloud/templates/' - 'sources.list.ubuntu.tmpl'), + 'sources.list.%s.tmpl' % distro), '/etc/apt/sources.list', {'codename': '', 'primary': - 'http://archive.ubuntu.com/ubuntu/', + mirror, 'mirror': - 'http://archive.ubuntu.com/ubuntu/'}) + mirror}) + + + def test_apt_source_list_ubuntu(self): + """ test_apt_source_list_ubuntu + Test rendering of a source.list from template for ubuntu + """ + self.apt_source_list('ubuntu', 'http://archive.ubuntu.com/ubuntu/') + + + def test_apt_source_list_debian(self): + """ test_apt_source_list_debian + Test rendering of a source.list from template for debian + """ + self.apt_source_list('debian', 'ftp.us.debian.org') # vi: ts=4 expandtab -- cgit v1.2.3 From 6fc583a4bcb49f7dbecdac095bc63e90dd6edaf3 Mon Sep 17 00:00:00 2001 From: Christian Ehrhardt Date: Thu, 12 May 2016 13:43:59 +0200 Subject: test mirror list with failing mirror --- cloudinit/config/cc_apt_configure.py | 1 + .../test_handler_apt_configure_sources_list.py | 23 ++++++++++++++++++---- 2 files changed, 20 insertions(+), 4 deletions(-) diff --git a/cloudinit/config/cc_apt_configure.py b/cloudinit/config/cc_apt_configure.py index 1d3eddff..ccbdcbc1 100644 --- a/cloudinit/config/cc_apt_configure.py +++ b/cloudinit/config/cc_apt_configure.py @@ -57,6 +57,7 @@ def handle(name, cfg, cloud, log, _args): release = get_release() mirrors = find_apt_mirror_info(cloud, cfg) + print(mirrors) if not mirrors or "primary" not in mirrors: log.debug(("Skipping module named %s," " no package 'mirror' located"), name) diff --git a/tests/unittests/test_handler/test_handler_apt_configure_sources_list.py b/tests/unittests/test_handler/test_handler_apt_configure_sources_list.py index aff272a3..bac2da24 100644 --- a/tests/unittests/test_handler/test_handler_apt_configure_sources_list.py +++ b/tests/unittests/test_handler/test_handler_apt_configure_sources_list.py @@ -58,14 +58,20 @@ class TestAptSourceConfigSourceList(t_help.FilesystemMockingTestCase): # TODO Later - custom template filename # TODO Later - custom template raw - def apt_source_list(self, distro, mirror): + def apt_source_list(self, distro, mirror, mirrorcheck=None): """ apt_source_list Test rendering of a source.list from template for a given distro """ self.patchOS(self.new_root) self.patchUtils(self.new_root) - cfg = {'apt_mirror': mirror} + if mirrorcheck is None: + mirrorcheck = mirror + + if isinstance(mirror, list): + cfg = {'apt_mirror_search': mirror} + else: + cfg = {'apt_mirror': mirror} mycloud = self._get_cloud(distro) with mock.patch.object(templater, 'render_to_file') as mocktmpl: @@ -81,9 +87,9 @@ class TestAptSourceConfigSourceList(t_help.FilesystemMockingTestCase): '/etc/apt/sources.list', {'codename': '', 'primary': - mirror, + mirrorcheck, 'mirror': - mirror}) + mirrorcheck}) def test_apt_source_list_ubuntu(self): @@ -100,4 +106,13 @@ class TestAptSourceConfigSourceList(t_help.FilesystemMockingTestCase): self.apt_source_list('debian', 'ftp.us.debian.org') + def test_apt_srcl_ubuntu_mirrorfail(self): + """ test_apt_source_list_ubuntu_mirrorfail + Test rendering of a source.list from template for ubuntu + """ + self.apt_source_list('ubuntu', ['http://does.not.exist', + 'http://archive.ubuntu.com/ubuntu/'], + 'http://archive.ubuntu.com/ubuntu/') + + # vi: ts=4 expandtab -- cgit v1.2.3 From d2af10ab5d3fef34934c04d4590ea611204f44c5 Mon Sep 17 00:00:00 2001 From: Christian Ehrhardt Date: Thu, 12 May 2016 13:44:29 +0200 Subject: order code in test order --- .../test_handler_apt_configure_sources_list.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/tests/unittests/test_handler/test_handler_apt_configure_sources_list.py b/tests/unittests/test_handler/test_handler_apt_configure_sources_list.py index bac2da24..d48167c9 100644 --- a/tests/unittests/test_handler/test_handler_apt_configure_sources_list.py +++ b/tests/unittests/test_handler/test_handler_apt_configure_sources_list.py @@ -92,13 +92,6 @@ class TestAptSourceConfigSourceList(t_help.FilesystemMockingTestCase): mirrorcheck}) - def test_apt_source_list_ubuntu(self): - """ test_apt_source_list_ubuntu - Test rendering of a source.list from template for ubuntu - """ - self.apt_source_list('ubuntu', 'http://archive.ubuntu.com/ubuntu/') - - def test_apt_source_list_debian(self): """ test_apt_source_list_debian Test rendering of a source.list from template for debian @@ -106,6 +99,13 @@ class TestAptSourceConfigSourceList(t_help.FilesystemMockingTestCase): self.apt_source_list('debian', 'ftp.us.debian.org') + def test_apt_source_list_ubuntu(self): + """ test_apt_source_list_ubuntu + Test rendering of a source.list from template for ubuntu + """ + self.apt_source_list('ubuntu', 'http://archive.ubuntu.com/ubuntu/') + + def test_apt_srcl_ubuntu_mirrorfail(self): """ test_apt_source_list_ubuntu_mirrorfail Test rendering of a source.list from template for ubuntu -- cgit v1.2.3 From 18864f8e7331da359399decb1b080e36fa343f5a Mon Sep 17 00:00:00 2001 From: Christian Ehrhardt Date: Thu, 12 May 2016 13:45:15 +0200 Subject: remove missed test print --- cloudinit/config/cc_apt_configure.py | 1 - .../test_handler/test_handler_apt_configure_sources_list.py | 9 +++++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/cloudinit/config/cc_apt_configure.py b/cloudinit/config/cc_apt_configure.py index ccbdcbc1..1d3eddff 100644 --- a/cloudinit/config/cc_apt_configure.py +++ b/cloudinit/config/cc_apt_configure.py @@ -57,7 +57,6 @@ def handle(name, cfg, cloud, log, _args): release = get_release() mirrors = find_apt_mirror_info(cloud, cfg) - print(mirrors) if not mirrors or "primary" not in mirrors: log.debug(("Skipping module named %s," " no package 'mirror' located"), name) diff --git a/tests/unittests/test_handler/test_handler_apt_configure_sources_list.py b/tests/unittests/test_handler/test_handler_apt_configure_sources_list.py index d48167c9..b8fe03ae 100644 --- a/tests/unittests/test_handler/test_handler_apt_configure_sources_list.py +++ b/tests/unittests/test_handler/test_handler_apt_configure_sources_list.py @@ -106,6 +106,15 @@ class TestAptSourceConfigSourceList(t_help.FilesystemMockingTestCase): self.apt_source_list('ubuntu', 'http://archive.ubuntu.com/ubuntu/') + def test_apt_srcl_debian_mirrorfail(self): + """ test_apt_source_list_debian_mirrorfail + Test rendering of a source.list from template for debian + """ + self.apt_source_list('debian', ['http://does.not.exist', + 'ftp.us.debian.org'], + 'ftp.us.debian.org') + + def test_apt_srcl_ubuntu_mirrorfail(self): """ test_apt_source_list_ubuntu_mirrorfail Test rendering of a source.list from template for ubuntu -- cgit v1.2.3 From dc2b81f3f6922c678806ce9120e9ce4c590243dd Mon Sep 17 00:00:00 2001 From: Christian Ehrhardt Date: Thu, 12 May 2016 14:09:17 +0200 Subject: use recommended http mirror redirection for debian --- .../test_handler/test_handler_apt_configure_sources_list.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/unittests/test_handler/test_handler_apt_configure_sources_list.py b/tests/unittests/test_handler/test_handler_apt_configure_sources_list.py index b8fe03ae..5255c5b9 100644 --- a/tests/unittests/test_handler/test_handler_apt_configure_sources_list.py +++ b/tests/unittests/test_handler/test_handler_apt_configure_sources_list.py @@ -96,7 +96,7 @@ class TestAptSourceConfigSourceList(t_help.FilesystemMockingTestCase): """ test_apt_source_list_debian Test rendering of a source.list from template for debian """ - self.apt_source_list('debian', 'ftp.us.debian.org') + self.apt_source_list('debian', 'http://httpredir.debian.org/debian') def test_apt_source_list_ubuntu(self): @@ -111,8 +111,8 @@ class TestAptSourceConfigSourceList(t_help.FilesystemMockingTestCase): Test rendering of a source.list from template for debian """ self.apt_source_list('debian', ['http://does.not.exist', - 'ftp.us.debian.org'], - 'ftp.us.debian.org') + 'http://httpredir.debian.org/debian'], + 'http://httpredir.debian.org/debian') def test_apt_srcl_ubuntu_mirrorfail(self): -- cgit v1.2.3 From 9a75e200402410e6f0ae841cdadebb20840c6a8e Mon Sep 17 00:00:00 2001 From: Christian Ehrhardt Date: Thu, 12 May 2016 14:30:11 +0200 Subject: initial version of a test for a custom source.list template --- .../test_handler_apt_configure_sources_list.py | 49 +++++++++++++++++++--- 1 file changed, 44 insertions(+), 5 deletions(-) diff --git a/tests/unittests/test_handler/test_handler_apt_configure_sources_list.py b/tests/unittests/test_handler/test_handler_apt_configure_sources_list.py index 5255c5b9..8777d2a9 100644 --- a/tests/unittests/test_handler/test_handler_apt_configure_sources_list.py +++ b/tests/unittests/test_handler/test_handler_apt_configure_sources_list.py @@ -26,6 +26,24 @@ from .. import helpers as t_help LOG = logging.getLogger(__name__) +YAML_TEXT_CUSTOM_SL = """ +apt_mirror: http://archive.ubuntu.com/ubuntu/ +apt_custom_sources_list: | + ## template:jinja + ## Note, this file is written by cloud-init on first boot of an instance + ## modifications made here will not survive a re-bundle. + ## if you wish to make changes you can: + ## a.) add 'apt_preserve_sources_list: true' to /etc/cloud/cloud.cfg + ## or do the same in user-data + ## b.) add sources in /etc/apt/sources.list.d + ## c.) make changes to template file /etc/cloud/templates/sources.list.tmpl + + # See http://help.ubuntu.com/community/UpgradeNotes for how to upgrade to + # newer versions of the distribution. + deb {{mirror}} {{codename}} main restricted + deb-src {{mirror}} {{codename}} main restricted + # FIND_SOMETHING_SPECIAL +""" def load_tfile_or_url(*args, **kwargs): """ load_tfile_or_url @@ -53,11 +71,6 @@ class TestAptSourceConfigSourceList(t_help.FilesystemMockingTestCase): myds.metadata.update(metadata) return cloud.Cloud(myds, paths, {}, mydist, None) -# TODO - Ubuntu template -# TODO - Debian template -# TODO Later - custom template filename -# TODO Later - custom template raw - def apt_source_list(self, distro, mirror, mirrorcheck=None): """ apt_source_list Test rendering of a source.list from template for a given distro @@ -124,4 +137,30 @@ class TestAptSourceConfigSourceList(t_help.FilesystemMockingTestCase): 'http://archive.ubuntu.com/ubuntu/') + def test_apt_srcl_custom(self): + """ test_apt_srcl_custom + Test rendering from a custom source.list template + """ + self.patchOS(self.new_root) + self.patchUtils(self.new_root) + + cfg = util.load_yaml(YAML_TEXT_CUSTOM_SL) + mycloud = self._get_cloud('ubuntu') + mirrorcheck = 'http://archive.ubuntu.com/ubuntu/' + + with mock.patch.object(templater, 'render_to_file') as mocktmpl: + with mock.patch.object(os.path, 'isfile', + return_value=True) as mockisfile: + cc_apt_configure.handle("notimportant", cfg, mycloud, + LOG, None) + + mockisfile.assert_any_call(('/etc/cloud/templates/sources.list.ubuntu.tmpl')) + mocktmpl.assert_called_once_with(('/etc/cloud/templates/sources.list.ubuntu.tmpl'), + '/etc/apt/sources.list', + {'codename': '', + 'primary': + mirrorcheck, + 'mirror': + mirrorcheck}) + # vi: ts=4 expandtab -- cgit v1.2.3 From c6cbba7184f0864f34b90532a4b0d3d61fdd6bc9 Mon Sep 17 00:00:00 2001 From: Christian Ehrhardt Date: Thu, 12 May 2016 15:42:31 +0200 Subject: Finalize test_apt_srcl_custom Adding: - known content after full templating - restore subp from mocking for proper execution - drop all kind of superfluous mocking of util and OS --- .../test_handler_apt_configure_sources_list.py | 41 +++++++++++----------- 1 file changed, 21 insertions(+), 20 deletions(-) diff --git a/tests/unittests/test_handler/test_handler_apt_configure_sources_list.py b/tests/unittests/test_handler/test_handler_apt_configure_sources_list.py index 8777d2a9..c1ca71af 100644 --- a/tests/unittests/test_handler/test_handler_apt_configure_sources_list.py +++ b/tests/unittests/test_handler/test_handler_apt_configure_sources_list.py @@ -58,6 +58,7 @@ class TestAptSourceConfigSourceList(t_help.FilesystemMockingTestCase): """ def setUp(self): super(TestAptSourceConfigSourceList, self).setUp() + self.subp = util.subp self.new_root = tempfile.mkdtemp() self.addCleanup(shutil.rmtree, self.new_root) @@ -75,9 +76,6 @@ class TestAptSourceConfigSourceList(t_help.FilesystemMockingTestCase): """ apt_source_list Test rendering of a source.list from template for a given distro """ - self.patchOS(self.new_root) - self.patchUtils(self.new_root) - if mirrorcheck is None: mirrorcheck = mirror @@ -141,26 +139,29 @@ class TestAptSourceConfigSourceList(t_help.FilesystemMockingTestCase): """ test_apt_srcl_custom Test rendering from a custom source.list template """ - self.patchOS(self.new_root) - self.patchUtils(self.new_root) - cfg = util.load_yaml(YAML_TEXT_CUSTOM_SL) mycloud = self._get_cloud('ubuntu') - mirrorcheck = 'http://archive.ubuntu.com/ubuntu/' - with mock.patch.object(templater, 'render_to_file') as mocktmpl: - with mock.patch.object(os.path, 'isfile', - return_value=True) as mockisfile: - cc_apt_configure.handle("notimportant", cfg, mycloud, - LOG, None) + # the second mock restores the original subp + with mock.patch.object(util, 'write_file') as mockwrite, \ + mock.patch.object(util, 'subp', self.subp) as mocksubp: + cc_apt_configure.handle("notimportant", cfg, mycloud, + LOG, None) + + mockwrite.assert_called_once_with( + '/etc/apt/sources.list', + ("## Note, this file is written by cloud-init on first boot of an" + " instance\n## modifications made here will not survive a re-bun" + "dle.\n## if you wish to make changes you can:\n## a.) add 'apt_" + "preserve_sources_list: true' to /etc/cloud/cloud.cfg\n## or" + " do the same in user-data\n## b.) add sources in /etc/apt/sourc" + "es.list.d\n## c.) make changes to template file /etc/cloud/temp" + "lates/sources.list.tmpl\n\n# See http://help.ubuntu.com/communi" + "ty/UpgradeNotes for how to upgrade to\n# newer versions of the " + "distribution.\ndeb http://archive.ubuntu.com/ubuntu/ xenial mai" + "n restricted\ndeb-src http://archive.ubuntu.com/ubuntu/ xenial " + "main restricted\n# FIND_SOMETHING_SPECIAL\n"), + mode=420) - mockisfile.assert_any_call(('/etc/cloud/templates/sources.list.ubuntu.tmpl')) - mocktmpl.assert_called_once_with(('/etc/cloud/templates/sources.list.ubuntu.tmpl'), - '/etc/apt/sources.list', - {'codename': '', - 'primary': - mirrorcheck, - 'mirror': - mirrorcheck}) # vi: ts=4 expandtab -- cgit v1.2.3 From cdee96acdde84071d54764e2554e6ee27702166c Mon Sep 17 00:00:00 2001 From: Christian Ehrhardt Date: Thu, 12 May 2016 15:51:00 +0200 Subject: use old style nested context to make pep8 happy --- .../test_handler_apt_configure_sources_list.py | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/tests/unittests/test_handler/test_handler_apt_configure_sources_list.py b/tests/unittests/test_handler/test_handler_apt_configure_sources_list.py index c1ca71af..16d6a5d1 100644 --- a/tests/unittests/test_handler/test_handler_apt_configure_sources_list.py +++ b/tests/unittests/test_handler/test_handler_apt_configure_sources_list.py @@ -45,6 +45,7 @@ apt_custom_sources_list: | # FIND_SOMETHING_SPECIAL """ + def load_tfile_or_url(*args, **kwargs): """ load_tfile_or_url load file and return content after decoding @@ -102,21 +103,18 @@ class TestAptSourceConfigSourceList(t_help.FilesystemMockingTestCase): 'mirror': mirrorcheck}) - def test_apt_source_list_debian(self): """ test_apt_source_list_debian Test rendering of a source.list from template for debian """ self.apt_source_list('debian', 'http://httpredir.debian.org/debian') - def test_apt_source_list_ubuntu(self): """ test_apt_source_list_ubuntu Test rendering of a source.list from template for ubuntu """ self.apt_source_list('ubuntu', 'http://archive.ubuntu.com/ubuntu/') - def test_apt_srcl_debian_mirrorfail(self): """ test_apt_source_list_debian_mirrorfail Test rendering of a source.list from template for debian @@ -125,7 +123,6 @@ class TestAptSourceConfigSourceList(t_help.FilesystemMockingTestCase): 'http://httpredir.debian.org/debian'], 'http://httpredir.debian.org/debian') - def test_apt_srcl_ubuntu_mirrorfail(self): """ test_apt_source_list_ubuntu_mirrorfail Test rendering of a source.list from template for ubuntu @@ -134,7 +131,6 @@ class TestAptSourceConfigSourceList(t_help.FilesystemMockingTestCase): 'http://archive.ubuntu.com/ubuntu/'], 'http://archive.ubuntu.com/ubuntu/') - def test_apt_srcl_custom(self): """ test_apt_srcl_custom Test rendering from a custom source.list template @@ -143,10 +139,10 @@ class TestAptSourceConfigSourceList(t_help.FilesystemMockingTestCase): mycloud = self._get_cloud('ubuntu') # the second mock restores the original subp - with mock.patch.object(util, 'write_file') as mockwrite, \ - mock.patch.object(util, 'subp', self.subp) as mocksubp: - cc_apt_configure.handle("notimportant", cfg, mycloud, - LOG, None) + with mock.patch.object(util, 'write_file') as mockwrite: + with mock.patch.object(util, 'subp', self.subp) as mocksubp: + cc_apt_configure.handle("notimportant", cfg, mycloud, + LOG, None) mockwrite.assert_called_once_with( '/etc/apt/sources.list', -- cgit v1.2.3 From 0d8bf6f2c1464b5f4dab735841a50d02016d2caf Mon Sep 17 00:00:00 2001 From: Christian Ehrhardt Date: Thu, 12 May 2016 15:51:21 +0200 Subject: add feature to allow a custom template for source list --- cloudinit/config/cc_apt_configure.py | 18 +++++++++++++----- cloudinit/templater.py | 5 +++++ 2 files changed, 18 insertions(+), 5 deletions(-) diff --git a/cloudinit/config/cc_apt_configure.py b/cloudinit/config/cc_apt_configure.py index 1d3eddff..2ab5e86c 100644 --- a/cloudinit/config/cc_apt_configure.py +++ b/cloudinit/config/cc_apt_configure.py @@ -21,6 +21,7 @@ import glob import os import re +import tempfile from cloudinit import templater from cloudinit import util @@ -70,7 +71,7 @@ def handle(name, cfg, cloud, log, _args): if not util.get_cfg_option_bool(cfg, 'apt_preserve_sources_list', False): - generate_sources_list(release, mirrors, cloud, log) + generate_sources_list(cfg, release, mirrors, cloud, log) old_mirrors = cfg.get('apt_old_mirrors', {"primary": "archive.ubuntu.com/ubuntu", "security": "security.ubuntu.com/ubuntu"}) @@ -149,7 +150,17 @@ def get_release(): return stdout.strip() -def generate_sources_list(codename, mirrors, cloud, log): +def generate_sources_list(cfg, codename, mirrors, cloud, log): + params = {'codename': codename} + for k in mirrors: + params[k] = mirrors[k] + + custtmpl = cfg.get('apt_custom_sources_list', None) + if custtmpl is not None: + templater.render_string_to_file(custtmpl, + '/etc/apt/sources.list', params) + return + template_fn = cloud.get_template_filename('sources.list.%s' % (cloud.distro.name)) if not template_fn: @@ -158,9 +169,6 @@ def generate_sources_list(codename, mirrors, cloud, log): log.warn("No template found, not rendering /etc/apt/sources.list") return - params = {'codename': codename} - for k in mirrors: - params[k] = mirrors[k] templater.render_to_file(template_fn, '/etc/apt/sources.list', params) diff --git a/cloudinit/templater.py b/cloudinit/templater.py index a9231482..8a6ad417 100644 --- a/cloudinit/templater.py +++ b/cloudinit/templater.py @@ -142,6 +142,11 @@ def render_to_file(fn, outfn, params, mode=0o644): util.write_file(outfn, contents, mode=mode) +def render_string_to_file(content, outfn, params, mode=0o644): + contents = render_string(content, params) + util.write_file(outfn, contents, mode=mode) + + def render_string(content, params): if not params: params = {} -- cgit v1.2.3 From 3be3e7452410d97ef9f9d4b525fa828de1f57bc0 Mon Sep 17 00:00:00 2001 From: Christian Ehrhardt Date: Thu, 12 May 2016 16:27:50 +0200 Subject: Document apt_custom_sources_list in examples --- doc/examples/cloud-config.txt | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/doc/examples/cloud-config.txt b/doc/examples/cloud-config.txt index f84d526d..75a4b6d4 100644 --- a/doc/examples/cloud-config.txt +++ b/doc/examples/cloud-config.txt @@ -72,6 +72,36 @@ apt_pipelining: False # then apt_mirror above will have no effect apt_preserve_sources_list: true +# Provide a custom template for rednering sources.list +# Default: a default template for Ubuntu/Debain will be used as packaged in +# Ubuntu: /etc/cloud/templates/sources.list.ubuntu.tmpl +# Debian: /etc/cloud/templates/sources.list.debian.tmpl +# Others: n/a +# This will follow the normal mirror/codename replacement rules before +# being written to disk. +apt_custom_sources_list: | + ## template:jinja + ## Note, this file is written by cloud-init on first boot of an instance + ## modifications made here will not survive a re-bundle. + ## if you wish to make changes you can: + ## a.) add 'apt_preserve_sources_list: true' to /etc/cloud/cloud.cfg + ## or do the same in user-data + ## b.) add sources in /etc/apt/sources.list.d + ## c.) make changes to template file /etc/cloud/templates/sources.list.tmpl + deb {{mirror}} {{codename}} main restricted + deb-src {{mirror}} {{codename}} main restricted + + # could drop some of the usually used entries + + # could refer to other mirrors + deb http://ddebs.ubuntu.com {{codename}} main restricted universe multiverse + deb http://ddebs.ubuntu.com {{codename}}-updates main restricted universe multiverse + deb http://ddebs.ubuntu.com {{codename}}-proposed main restricted universe multiverse + + # or even more uncommon examples like local or NFS mounted repos, + # eventually whatever is compatible with sources.list syntax + deb file:/home/apt/debian unstable main contrib non-free + # 'source' entries in apt-sources that match this python regex # expression will be passed to add-apt-repository add_apt_repo_match: '^[\w-]+:\w' -- cgit v1.2.3 From 4f6b14ea0f21f015ce73a28d985ecded981b931d Mon Sep 17 00:00:00 2001 From: Christian Ehrhardt Date: Thu, 12 May 2016 16:54:20 +0200 Subject: remove no more applicable "not supported" statements --- tests/unittests/test_handler/test_handler_apt_source.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/unittests/test_handler/test_handler_apt_source.py b/tests/unittests/test_handler/test_handler_apt_source.py index 38c93e0e..23f458eb 100644 --- a/tests/unittests/test_handler/test_handler_apt_source.py +++ b/tests/unittests/test_handler/test_handler_apt_source.py @@ -141,7 +141,7 @@ class TestAptSourceConfig(TestCase): def test_apt_source_keyonly(self): """ test_apt_source_keyonly - Test specification key without source (not yet supported) + Test specification key without source """ params = self._get_default_params() cfg = {'key': "fakekey 4242", @@ -158,7 +158,7 @@ class TestAptSourceConfig(TestCase): def test_apt_source_keyidonly(self): """ test_apt_source_keyidonly - Test specification of a keyid without source (not yet supported) + Test specification of a keyid without source """ params = self._get_default_params() cfg = {'keyid': "03683F77", -- cgit v1.2.3 From 9c6b5f54ad5b83131de6d997930bd9f4031e6a83 Mon Sep 17 00:00:00 2001 From: Christian Ehrhardt Date: Thu, 12 May 2016 19:50:01 +0200 Subject: move errorlist.append out of add_key --- cloudinit/config/cc_apt_configure.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/cloudinit/config/cc_apt_configure.py b/cloudinit/config/cc_apt_configure.py index 2ab5e86c..91d02815 100644 --- a/cloudinit/config/cc_apt_configure.py +++ b/cloudinit/config/cc_apt_configure.py @@ -172,7 +172,7 @@ def generate_sources_list(cfg, codename, mirrors, cloud, log): templater.render_to_file(template_fn, '/etc/apt/sources.list', params) -def add_key(ent, errorlist): +def add_key(ent): """ add key to the system as defiend in entry (if any) suppords raw keys or keyid's @@ -185,14 +185,13 @@ def add_key(ent, errorlist): try: ent['key'] = getkeybyid(ent['keyid'], keyserver) except: - errorlist.append([ent, "failed to get key from %s" % keyserver]) - return + raise Exception('failed to get key from %s' % keyserver) if 'key' in ent: try: util.subp(('apt-key', 'add', '-'), ent['key']) except: - errorlist.append([ent, "failed add key"]) + raise Exception('failed add key') def add_sources(srclist, template_params=None, aa_repo_match=None): @@ -211,7 +210,10 @@ def add_sources(srclist, template_params=None, aa_repo_match=None): errorlist = [] for ent in srclist: # keys can be added without specifying a source - add_key(ent, errorlist) + try: + add_key(ent) + except Exception as detail: + errorlist.append([ent, detail]) if 'source' not in ent: errorlist.append(["", "missing source"]) -- cgit v1.2.3 From e55ccfa5670e16aa7431a193d0838aa7d04db4d5 Mon Sep 17 00:00:00 2001 From: Christian Ehrhardt Date: Thu, 12 May 2016 19:50:34 +0200 Subject: remove Unnecessary parens in add_key --- cloudinit/config/cc_apt_configure.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cloudinit/config/cc_apt_configure.py b/cloudinit/config/cc_apt_configure.py index 91d02815..492c3c00 100644 --- a/cloudinit/config/cc_apt_configure.py +++ b/cloudinit/config/cc_apt_configure.py @@ -178,7 +178,7 @@ def add_key(ent): suppords raw keys or keyid's The latter will as a first step fetched to get the raw key """ - if ('keyid' in ent and 'key' not in ent): + if 'keyid' in ent and 'key' not in ent: keyserver = "keyserver.ubuntu.com" if 'keyserver' in ent: keyserver = ent['keyserver'] -- cgit v1.2.3 From 53834934e4c520b2fb8b5acffca641213ddd688a Mon Sep 17 00:00:00 2001 From: Christian Ehrhardt Date: Thu, 12 May 2016 20:29:33 +0200 Subject: fix EXPORT_GPG_KEYID for long key fingerprints --- cloudinit/config/cc_apt_configure.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cloudinit/config/cc_apt_configure.py b/cloudinit/config/cc_apt_configure.py index 492c3c00..28f20939 100644 --- a/cloudinit/config/cc_apt_configure.py +++ b/cloudinit/config/cc_apt_configure.py @@ -43,7 +43,7 @@ EXPORT_GPG_KEYID = """ [ -n "$k" ] || exit 1; armour=$(gpg --list-keys --armour "${k}") if [ -z "${armour}" ]; then - gpg --keyserver ${ks} --recv $k >/dev/null && + gpg --keyserver ${ks} --recv "${k}" >/dev/null && armour=$(gpg --export --armour "${k}") && gpg --batch --yes --delete-keys "${k}" fi -- cgit v1.2.3 From 2202494b72cae19cbf9d34a8f3176d7021becb13 Mon Sep 17 00:00:00 2001 From: Christian Ehrhardt Date: Thu, 12 May 2016 20:53:12 +0200 Subject: split add_key and add_key_raw fior better testability --- cloudinit/config/cc_apt_configure.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/cloudinit/config/cc_apt_configure.py b/cloudinit/config/cc_apt_configure.py index 28f20939..e7b8a9b3 100644 --- a/cloudinit/config/cc_apt_configure.py +++ b/cloudinit/config/cc_apt_configure.py @@ -171,10 +171,19 @@ def generate_sources_list(cfg, codename, mirrors, cloud, log): templater.render_to_file(template_fn, '/etc/apt/sources.list', params) +def add_key_raw(key): + """ + actual adding of a key as defined in key argument + to the system + """ + try: + util.subp(('apt-key', 'add', '-'), key) + except: + raise Exception('failed add key') def add_key(ent): """ - add key to the system as defiend in entry (if any) + add key to the system as defiend in ent (if any) suppords raw keys or keyid's The latter will as a first step fetched to get the raw key """ @@ -188,10 +197,7 @@ def add_key(ent): raise Exception('failed to get key from %s' % keyserver) if 'key' in ent: - try: - util.subp(('apt-key', 'add', '-'), ent['key']) - except: - raise Exception('failed add key') + add_key_raw(ent['key']) def add_sources(srclist, template_params=None, aa_repo_match=None): -- cgit v1.2.3 From 46f930a718a89c514ba7caa22096201e7c6ca0a8 Mon Sep 17 00:00:00 2001 From: Christian Ehrhardt Date: Thu, 12 May 2016 21:00:30 +0200 Subject: Adding test_apt_source_keyid_real and test_apt_source_longkeyid_real This now ensures that the stack of fetching IDs from keyservers and adding them really works by comparing against known good keys that are expected. --- .../test_handler/test_handler_apt_source.py | 49 ++++++++++++++++++++++ 1 file changed, 49 insertions(+) diff --git a/tests/unittests/test_handler/test_handler_apt_source.py b/tests/unittests/test_handler/test_handler_apt_source.py index 23f458eb..e50c7468 100644 --- a/tests/unittests/test_handler/test_handler_apt_source.py +++ b/tests/unittests/test_handler/test_handler_apt_source.py @@ -17,6 +17,19 @@ from cloudinit.config import cc_apt_configure from ..helpers import TestCase +EXPECTEDKEY = """-----BEGIN PGP PUBLIC KEY BLOCK----- +Version: GnuPG v1 + +mI0ESuZLUgEEAKkqq3idtFP7g9hzOu1a8+v8ImawQN4TrvlygfScMU1TIS1eC7UQ +NUA8Qqgr9iUaGnejb0VciqftLrU9D6WYHSKz+EITefgdyJ6SoQxjoJdsCpJ7o9Jy +8PQnpRttiFm4qHu6BVnKnBNxw/z3ST9YMqW5kbMQpfxbGe+obRox59NpABEBAAG0 +HUxhdW5jaHBhZCBQUEEgZm9yIFNjb3R0IE1vc2VyiLYEEwECACAFAkrmS1ICGwMG +CwkIBwMCBBUCCAMEFgIDAQIeAQIXgAAKCRAGILvPA2g/d3aEA/9tVjc10HOZwV29 +OatVuTeERjjrIbxflO586GLA8cp0C9RQCwgod/R+cKYdQcHjbqVcP0HqxveLg0RZ +FJpWLmWKamwkABErwQLGlM/Hwhjfade8VvEQutH5/0JgKHmzRsoqfR+LMO6OS+Sm +S0ORP6HXET3+jC8BMG4tBWCTK/XEZw== +=ACB2 +-----END PGP PUBLIC KEY BLOCK-----""" def load_tfile_or_url(*args, **kwargs): """ load_tfile_or_url @@ -173,6 +186,42 @@ class TestAptSourceConfig(TestCase): # filename should be ignored on key only self.assertFalse(os.path.isfile(self.aptlistfile)) + def test_apt_source_keyid_real(self): + """ test_apt_source_keyid_real + Test specification of a keyid without source incl + up to addition of the key (nothing but add_key_raw mocked) + """ + keyid = "03683F77" + params = self._get_default_params() + cfg = {'keyid': keyid, + 'filename': self.aptlistfile} + + with mock.patch.object(cc_apt_configure, 'add_key_raw') as mockobj: + cc_apt_configure.add_sources([cfg], params) + + mockobj.assert_called_with(EXPECTEDKEY) + + # filename should be ignored on key only + self.assertFalse(os.path.isfile(self.aptlistfile)) + + def test_apt_source_longkeyid_real(self): + """ test_apt_source_keyid_real + Test specification of a long key fingerprint without source incl + up to addition of the key (nothing but add_key_raw mocked) + """ + keyid = "B59D 5F15 97A5 04B7 E230 6DCA 0620 BBCF 0368 3F77" + params = self._get_default_params() + cfg = {'keyid': keyid, + 'filename': self.aptlistfile} + + with mock.patch.object(cc_apt_configure, 'add_key_raw') as mockobj: + cc_apt_configure.add_sources([cfg], params) + + mockobj.assert_called_with(EXPECTEDKEY) + + # filename should be ignored on key only + self.assertFalse(os.path.isfile(self.aptlistfile)) + def test_apt_source_ppa(self): """ test_apt_source_ppa Test specification of a ppa -- cgit v1.2.3 From 9c098751b8065da609566572a7badd0d66c2b6ba Mon Sep 17 00:00:00 2001 From: Christian Ehrhardt Date: Thu, 12 May 2016 21:01:30 +0200 Subject: remove superfluous import --- tests/unittests/test_handler/test_handler_apt_source.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/unittests/test_handler/test_handler_apt_source.py b/tests/unittests/test_handler/test_handler_apt_source.py index e50c7468..88b4ccc6 100644 --- a/tests/unittests/test_handler/test_handler_apt_source.py +++ b/tests/unittests/test_handler/test_handler_apt_source.py @@ -11,7 +11,6 @@ try: except ImportError: import mock -from cloudinit import distros from cloudinit import util from cloudinit.config import cc_apt_configure -- cgit v1.2.3 From 3ece03a2df95bdb40851b9d629b39c43a233868b Mon Sep 17 00:00:00 2001 From: Christian Ehrhardt Date: Thu, 12 May 2016 21:03:51 +0200 Subject: alphabetical import order --- doc/examples/cloud-config.txt | 3 +++ tests/unittests/test_handler/test_handler_apt_source.py | 4 ++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/doc/examples/cloud-config.txt b/doc/examples/cloud-config.txt index 75a4b6d4..8adc5a96 100644 --- a/doc/examples/cloud-config.txt +++ b/doc/examples/cloud-config.txt @@ -144,6 +144,9 @@ apt_sources: # this would only import the key without adding a ppa or other source spec - keyid: F430BBA5 # GPG key ID published on a key server + # In general keyid's can also be specified via their long fingerprints + - keyid: B59D 5F15 97A5 04B7 E230 6DCA 0620 BBCF 0368 3F77 + # Custom apt repository: # * The apt signing key can also be specified # by providing a pgp public key block diff --git a/tests/unittests/test_handler/test_handler_apt_source.py b/tests/unittests/test_handler/test_handler_apt_source.py index 88b4ccc6..439bd038 100644 --- a/tests/unittests/test_handler/test_handler_apt_source.py +++ b/tests/unittests/test_handler/test_handler_apt_source.py @@ -2,17 +2,17 @@ Testing various config variations of the apt_source config """ import os +import re import shutil import tempfile -import re try: from unittest import mock except ImportError: import mock -from cloudinit import util from cloudinit.config import cc_apt_configure +from cloudinit import util from ..helpers import TestCase -- cgit v1.2.3 From 338198c5a802207cabd7c13422fcf0e03c667ce7 Mon Sep 17 00:00:00 2001 From: Christian Ehrhardt Date: Thu, 12 May 2016 21:04:34 +0200 Subject: alphabetical order on imports --- .../test_handler/test_handler_apt_configure_sources_list.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/tests/unittests/test_handler/test_handler_apt_configure_sources_list.py b/tests/unittests/test_handler/test_handler_apt_configure_sources_list.py index 16d6a5d1..d4fdee21 100644 --- a/tests/unittests/test_handler/test_handler_apt_configure_sources_list.py +++ b/tests/unittests/test_handler/test_handler_apt_configure_sources_list.py @@ -1,12 +1,11 @@ """ test_handler_apt_configure_sources_list Test templating of sources list """ +import logging import os +import re import shutil import tempfile -import re - -import logging try: from unittest import mock @@ -15,12 +14,12 @@ except ImportError: from cloudinit import cloud from cloudinit import distros -from cloudinit import util from cloudinit import helpers from cloudinit import templater +from cloudinit import util -from cloudinit.sources import DataSourceNone from cloudinit.config import cc_apt_configure +from cloudinit.sources import DataSourceNone from .. import helpers as t_help -- cgit v1.2.3 From 3c2c85a7c152c5026ff3122a7e134c5db4cfdce7 Mon Sep 17 00:00:00 2001 From: Christian Ehrhardt Date: Thu, 12 May 2016 21:09:34 +0200 Subject: fix old typo in example --- doc/examples/cloud-config.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/examples/cloud-config.txt b/doc/examples/cloud-config.txt index 8adc5a96..31b791b6 100644 --- a/doc/examples/cloud-config.txt +++ b/doc/examples/cloud-config.txt @@ -150,7 +150,7 @@ apt_sources: # Custom apt repository: # * The apt signing key can also be specified # by providing a pgp public key block - # * Providing the PBG key here is the most robust method for + # * Providing the PGP key here is the most robust method for # specifying a key, as it removes dependency on a remote key server - source: deb http://ppa.launchpad.net/alestic/ppa/ubuntu karmic main key: | # The value needs to start with -----BEGIN PGP PUBLIC KEY BLOCK----- -- cgit v1.2.3 From 2a54898f8009f1511862a75467e84a77bef1943a Mon Sep 17 00:00:00 2001 From: Christian Ehrhardt Date: Thu, 12 May 2016 21:13:57 +0200 Subject: improve spacing in apt_source_list test --- .../test_handler_apt_configure_sources_list.py | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/tests/unittests/test_handler/test_handler_apt_configure_sources_list.py b/tests/unittests/test_handler/test_handler_apt_configure_sources_list.py index d4fdee21..3bcd93cf 100644 --- a/tests/unittests/test_handler/test_handler_apt_configure_sources_list.py +++ b/tests/unittests/test_handler/test_handler_apt_configure_sources_list.py @@ -91,16 +91,12 @@ class TestAptSourceConfigSourceList(t_help.FilesystemMockingTestCase): cc_apt_configure.handle("notimportant", cfg, mycloud, LOG, None) - mockisfile.assert_any_call(('/etc/cloud/templates/' - 'sources.list.%s.tmpl' % distro)) - mocktmpl.assert_called_once_with(('/etc/cloud/templates/' - 'sources.list.%s.tmpl' % distro), - '/etc/apt/sources.list', - {'codename': '', - 'primary': - mirrorcheck, - 'mirror': - mirrorcheck}) + mockisfile.assert_any_call( + ('/etc/cloud/templates/sources.list.%s.tmpl' % distro)) + mocktmpl.assert_called_once_with( + ('/etc/cloud/templates/sources.list.%s.tmpl' % distro), + '/etc/apt/sources.list', + {'codename': '', 'primary': mirrorcheck, 'mirror': mirrorcheck}) def test_apt_source_list_debian(self): """ test_apt_source_list_debian -- cgit v1.2.3 From 0a407b63f61f48d310d205f833c1a2792273ec80 Mon Sep 17 00:00:00 2001 From: Christian Ehrhardt Date: Thu, 12 May 2016 21:20:29 +0200 Subject: streamline code and sanitize expected result string definition --- .../test_handler_apt_configure_sources_list.py | 28 +++++++++++++--------- 1 file changed, 17 insertions(+), 11 deletions(-) diff --git a/tests/unittests/test_handler/test_handler_apt_configure_sources_list.py b/tests/unittests/test_handler/test_handler_apt_configure_sources_list.py index 3bcd93cf..1aa4f553 100644 --- a/tests/unittests/test_handler/test_handler_apt_configure_sources_list.py +++ b/tests/unittests/test_handler/test_handler_apt_configure_sources_list.py @@ -44,6 +44,22 @@ apt_custom_sources_list: | # FIND_SOMETHING_SPECIAL """ +EXPECTED_CONVERTED_CONTENT = ( + """## Note, this file is written by cloud-init on first boot of an instance +## modifications made here will not survive a re-bundle. +## if you wish to make changes you can: +## a.) add 'apt_preserve_sources_list: true' to /etc/cloud/cloud.cfg +## or do the same in user-data +## b.) add sources in /etc/apt/sources.list.d +## c.) make changes to template file /etc/cloud/templates/sources.list.tmpl + +# See http://help.ubuntu.com/community/UpgradeNotes for how to upgrade to +# newer versions of the distribution. +deb http://archive.ubuntu.com/ubuntu/ xenial main restricted +deb-src http://archive.ubuntu.com/ubuntu/ xenial main restricted +# FIND_SOMETHING_SPECIAL +""") + def load_tfile_or_url(*args, **kwargs): """ load_tfile_or_url @@ -141,17 +157,7 @@ class TestAptSourceConfigSourceList(t_help.FilesystemMockingTestCase): mockwrite.assert_called_once_with( '/etc/apt/sources.list', - ("## Note, this file is written by cloud-init on first boot of an" - " instance\n## modifications made here will not survive a re-bun" - "dle.\n## if you wish to make changes you can:\n## a.) add 'apt_" - "preserve_sources_list: true' to /etc/cloud/cloud.cfg\n## or" - " do the same in user-data\n## b.) add sources in /etc/apt/sourc" - "es.list.d\n## c.) make changes to template file /etc/cloud/temp" - "lates/sources.list.tmpl\n\n# See http://help.ubuntu.com/communi" - "ty/UpgradeNotes for how to upgrade to\n# newer versions of the " - "distribution.\ndeb http://archive.ubuntu.com/ubuntu/ xenial mai" - "n restricted\ndeb-src http://archive.ubuntu.com/ubuntu/ xenial " - "main restricted\n# FIND_SOMETHING_SPECIAL\n"), + EXPECTED_CONVERTED_CONTENT, mode=420) -- cgit v1.2.3 From 454de24c7d457b980c91849b128efe4faee62032 Mon Sep 17 00:00:00 2001 From: Christian Ehrhardt Date: Thu, 12 May 2016 21:21:36 +0200 Subject: make pep8 happy with a few spaces --- cloudinit/config/cc_apt_configure.py | 2 ++ tests/unittests/test_handler/test_handler_apt_source.py | 1 + 2 files changed, 3 insertions(+) diff --git a/cloudinit/config/cc_apt_configure.py b/cloudinit/config/cc_apt_configure.py index e7b8a9b3..e5a962ac 100644 --- a/cloudinit/config/cc_apt_configure.py +++ b/cloudinit/config/cc_apt_configure.py @@ -171,6 +171,7 @@ def generate_sources_list(cfg, codename, mirrors, cloud, log): templater.render_to_file(template_fn, '/etc/apt/sources.list', params) + def add_key_raw(key): """ actual adding of a key as defined in key argument @@ -181,6 +182,7 @@ def add_key_raw(key): except: raise Exception('failed add key') + def add_key(ent): """ add key to the system as defiend in ent (if any) diff --git a/tests/unittests/test_handler/test_handler_apt_source.py b/tests/unittests/test_handler/test_handler_apt_source.py index 439bd038..e130392c 100644 --- a/tests/unittests/test_handler/test_handler_apt_source.py +++ b/tests/unittests/test_handler/test_handler_apt_source.py @@ -30,6 +30,7 @@ S0ORP6HXET3+jC8BMG4tBWCTK/XEZw== =ACB2 -----END PGP PUBLIC KEY BLOCK-----""" + def load_tfile_or_url(*args, **kwargs): """ load_tfile_or_url load file and return content after decoding -- cgit v1.2.3 From 8575c4f7045c7074370bba73198ea36571224ece Mon Sep 17 00:00:00 2001 From: Christian Ehrhardt Date: Mon, 23 May 2016 08:21:59 +0200 Subject: generalize test_apt_source_basic to be reusable across more testcases --- .../test_handler/test_handler_apt_source.py | 23 ++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/tests/unittests/test_handler/test_handler_apt_source.py b/tests/unittests/test_handler/test_handler_apt_source.py index e130392c..1b294431 100644 --- a/tests/unittests/test_handler/test_handler_apt_source.py +++ b/tests/unittests/test_handler/test_handler_apt_source.py @@ -58,27 +58,34 @@ class TestAptSourceConfig(TestCase): params['MIRROR'] = "http://archive.ubuntu.com/ubuntu" return params - def test_apt_source_basic(self): - """ test_apt_source_basic + def apt_source_basic(self, filename, cfg): + """ apt_source_basic Test Fix deb source string, has to overwrite mirror conf in params """ params = self._get_default_params() - cfg = {'source': ('deb http://archive.ubuntu.com/ubuntu' - ' karmic-backports' - ' main universe multiverse restricted'), - 'filename': self.aptlistfile} cc_apt_configure.add_sources([cfg], params) - self.assertTrue(os.path.isfile(self.aptlistfile)) + self.assertTrue(os.path.isfile(filename)) - contents = load_tfile_or_url(self.aptlistfile) + contents = load_tfile_or_url(filename) self.assertTrue(re.search(r"%s %s %s %s\n" % ("deb", "http://archive.ubuntu.com/ubuntu", "karmic-backports", "main universe multiverse restricted"), contents, flags=re.IGNORECASE)) + def test_apt_source_basic(self): + """ test_apt_source_basic + Test Fix deb source string, has to overwrite mirror conf in params. + Test with a filename provided in config. + """ + cfg = {'source': ('deb http://archive.ubuntu.com/ubuntu' + ' karmic-backports' + ' main universe multiverse restricted'), + 'filename': self.aptlistfile} + self.apt_source_basic(self.aptlistfile, cfg) + def test_apt_source_replacement(self): """ test_apt_source_replace Test Autoreplacement of MIRROR and RELEASE in source specs -- cgit v1.2.3 From 29065e4a52a747b4f4cf30092ddcc3744e6aa350 Mon Sep 17 00:00:00 2001 From: Christian Ehrhardt Date: Mon, 23 May 2016 09:24:44 +0200 Subject: test_apt_source_basic_nofn check for non-specified filename Cloud-inint uses a default fallback, we want to ensure no code change modfies this behaviour. --- .../test_handler/test_handler_apt_source.py | 25 ++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/tests/unittests/test_handler/test_handler_apt_source.py b/tests/unittests/test_handler/test_handler_apt_source.py index 1b294431..ce356fc6 100644 --- a/tests/unittests/test_handler/test_handler_apt_source.py +++ b/tests/unittests/test_handler/test_handler_apt_source.py @@ -47,6 +47,7 @@ class TestAptSourceConfig(TestCase): self.tmp = tempfile.mkdtemp() self.addCleanup(shutil.rmtree, self.tmp) self.aptlistfile = os.path.join(self.tmp, "single-deb.list") + self.join = os.path.join @staticmethod def _get_default_params(): @@ -86,6 +87,30 @@ class TestAptSourceConfig(TestCase): 'filename': self.aptlistfile} self.apt_source_basic(self.aptlistfile, cfg) + def test_apt_source_basic_nofn(self): + """ test_apt_source_basic_nofn + Test Fix deb source string, has to overwrite mirror conf in params. + Test without a filename provided in config and test for known fallback. + """ + cfg = {'source': ('deb http://archive.ubuntu.com/ubuntu' + ' karmic-backports' + ' main universe multiverse restricted')} + # mock into writable tmp dir and check path/content there + filename = os.path.join(self.tmp, "etc/apt/sources.list.d/", + "cloud_config_sources.list") + + def myjoin(*args, **kwargs): + """ myjoin - redir into writable tmpdir""" + if (args[0] == "/etc/apt/sources.list.d/" + and args[1] == "cloud_config_sources.list" + and len(args) == 2): + return self.join(self.tmp, args[0].lstrip("/"), args[1]) + else: + return self.join(*args, **kwargs) + + with mock.patch.object(os.path, 'join', side_effect=myjoin): + self.apt_source_basic(filename, cfg) + def test_apt_source_replacement(self): """ test_apt_source_replace Test Autoreplacement of MIRROR and RELEASE in source specs -- cgit v1.2.3 From f06dd57907caa648743a73566b2b6e62b96be2fb Mon Sep 17 00:00:00 2001 From: Christian Ehrhardt Date: Mon, 23 May 2016 09:34:23 +0200 Subject: drop unused mockappsubp --- tests/unittests/test_handler/test_handler_apt_configure_sources_list.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unittests/test_handler/test_handler_apt_configure_sources_list.py b/tests/unittests/test_handler/test_handler_apt_configure_sources_list.py index 1aa4f553..353422a2 100644 --- a/tests/unittests/test_handler/test_handler_apt_configure_sources_list.py +++ b/tests/unittests/test_handler/test_handler_apt_configure_sources_list.py @@ -151,7 +151,7 @@ class TestAptSourceConfigSourceList(t_help.FilesystemMockingTestCase): # the second mock restores the original subp with mock.patch.object(util, 'write_file') as mockwrite: - with mock.patch.object(util, 'subp', self.subp) as mocksubp: + with mock.patch.object(util, 'subp', self.subp): cc_apt_configure.handle("notimportant", cfg, mycloud, LOG, None) -- cgit v1.2.3 From cf37b78e9bada97a7cc223bd824fb0e8cd8c4c7c Mon Sep 17 00:00:00 2001 From: Christian Ehrhardt Date: Mon, 23 May 2016 10:48:03 +0200 Subject: extend test_apt_source_replace by a no-filename case --- .../test_handler/test_handler_apt_source.py | 53 +++++++++++++++------- 1 file changed, 36 insertions(+), 17 deletions(-) diff --git a/tests/unittests/test_handler/test_handler_apt_source.py b/tests/unittests/test_handler/test_handler_apt_source.py index ce356fc6..050ee78f 100644 --- a/tests/unittests/test_handler/test_handler_apt_source.py +++ b/tests/unittests/test_handler/test_handler_apt_source.py @@ -59,6 +59,15 @@ class TestAptSourceConfig(TestCase): params['MIRROR'] = "http://archive.ubuntu.com/ubuntu" return params + def myjoin(self, *args, **kwargs): + """ myjoin - redir into writable tmpdir""" + if (args[0] == "/etc/apt/sources.list.d/" + and args[1] == "cloud_config_sources.list" + and len(args) == 2): + return self.join(self.tmp, args[0].lstrip("/"), args[1]) + else: + return self.join(*args, **kwargs) + def apt_source_basic(self, filename, cfg): """ apt_source_basic Test Fix deb source string, has to overwrite mirror conf in params @@ -99,36 +108,46 @@ class TestAptSourceConfig(TestCase): filename = os.path.join(self.tmp, "etc/apt/sources.list.d/", "cloud_config_sources.list") - def myjoin(*args, **kwargs): - """ myjoin - redir into writable tmpdir""" - if (args[0] == "/etc/apt/sources.list.d/" - and args[1] == "cloud_config_sources.list" - and len(args) == 2): - return self.join(self.tmp, args[0].lstrip("/"), args[1]) - else: - return self.join(*args, **kwargs) - - with mock.patch.object(os.path, 'join', side_effect=myjoin): + with mock.patch.object(os.path, 'join', side_effect=self.myjoin): self.apt_source_basic(filename, cfg) - def test_apt_source_replacement(self): - """ test_apt_source_replace + def apt_source_replacement(self, filename, cfg): + """ apt_source_replace Test Autoreplacement of MIRROR and RELEASE in source specs """ params = self._get_default_params() - cfg = {'source': 'deb $MIRROR $RELEASE multiverse', - 'filename': self.aptlistfile} - cc_apt_configure.add_sources([cfg], params) - self.assertTrue(os.path.isfile(self.aptlistfile)) + self.assertTrue(os.path.isfile(filename)) - contents = load_tfile_or_url(self.aptlistfile) + contents = load_tfile_or_url(filename) self.assertTrue(re.search(r"%s %s %s %s\n" % ("deb", params['MIRROR'], params['RELEASE'], "multiverse"), contents, flags=re.IGNORECASE)) + def test_apt_source_replace(self): + """ test_apt_source_replace + Test Autoreplacement of MIRROR and RELEASE in source specs with + Filename being set + """ + cfg = {'source': 'deb $MIRROR $RELEASE multiverse', + 'filename': self.aptlistfile} + self.apt_source_replacement(self.aptlistfile, cfg) + + def test_apt_source_replace_nofn(self): + """ test_apt_source_replace_nofn + Test Autoreplacement of MIRROR and RELEASE in source specs with + No filename being set + """ + cfg = {'source': 'deb $MIRROR $RELEASE multiverse'} + # mock into writable tmp dir and check path/content there + filename = os.path.join(self.tmp, "etc/apt/sources.list.d/", + "cloud_config_sources.list") + + with mock.patch.object(os.path, 'join', side_effect=self.myjoin): + self.apt_source_replacement(filename, cfg) + def test_apt_source_keyid(self): """ test_apt_source_keyid Test specification of a source + keyid -- cgit v1.2.3 From 6b7711dbb75cd3fd31a638f3b530e863eb907708 Mon Sep 17 00:00:00 2001 From: Christian Ehrhardt Date: Mon, 23 May 2016 10:53:16 +0200 Subject: extend test_apt_source_keyid by no filename case --- .../test_handler/test_handler_apt_source.py | 41 ++++++++++++++++------ 1 file changed, 31 insertions(+), 10 deletions(-) diff --git a/tests/unittests/test_handler/test_handler_apt_source.py b/tests/unittests/test_handler/test_handler_apt_source.py index 050ee78f..c4ba6d00 100644 --- a/tests/unittests/test_handler/test_handler_apt_source.py +++ b/tests/unittests/test_handler/test_handler_apt_source.py @@ -148,17 +148,11 @@ class TestAptSourceConfig(TestCase): with mock.patch.object(os.path, 'join', side_effect=self.myjoin): self.apt_source_replacement(filename, cfg) - def test_apt_source_keyid(self): - """ test_apt_source_keyid + def apt_source_keyid(self, filename, cfg): + """ apt_source_keyid Test specification of a source + keyid """ params = self._get_default_params() - cfg = {'source': ('deb ' - 'http://ppa.launchpad.net/' - 'smoser/cloud-init-test/ubuntu' - ' xenial main'), - 'keyid': "03683F77", - 'filename': self.aptlistfile} with mock.patch.object(util, 'subp', return_value=('fakekey 1234', '')) as mockobj: @@ -166,9 +160,9 @@ class TestAptSourceConfig(TestCase): mockobj.assert_called_with(('apt-key', 'add', '-'), 'fakekey 1234') - self.assertTrue(os.path.isfile(self.aptlistfile)) + self.assertTrue(os.path.isfile(filename)) - contents = load_tfile_or_url(self.aptlistfile) + contents = load_tfile_or_url(filename) self.assertTrue(re.search(r"%s %s %s %s\n" % ("deb", ('http://ppa.launchpad.net/smoser/' @@ -176,6 +170,33 @@ class TestAptSourceConfig(TestCase): "xenial", "main"), contents, flags=re.IGNORECASE)) + def test_apt_source_keyid(self): + """ test_apt_source_keyid + Test specification of a source + keyid with filename being set + """ + cfg = {'source': ('deb ' + 'http://ppa.launchpad.net/' + 'smoser/cloud-init-test/ubuntu' + ' xenial main'), + 'keyid': "03683F77", + 'filename': self.aptlistfile} + self.apt_source_keyid(self.aptlistfile, cfg) + + def test_apt_source_keyid_nofn(self): + """ test_apt_source_keyid + Test specification of a source + keyid without filename being set + """ + cfg = {'source': ('deb ' + 'http://ppa.launchpad.net/' + 'smoser/cloud-init-test/ubuntu' + ' xenial main'), + 'keyid': "03683F77"} + # mock into writable tmp dir and check path/content there + filename = os.path.join(self.tmp, "etc/apt/sources.list.d/", + "cloud_config_sources.list") + with mock.patch.object(os.path, 'join', side_effect=self.myjoin): + self.apt_source_keyid(filename, cfg) + def test_apt_source_key(self): """ test_apt_source_key Test specification of a source + key -- cgit v1.2.3 From 0b2cbda06a5622bc2b6dcc4f2d4190824528f1f6 Mon Sep 17 00:00:00 2001 From: Christian Ehrhardt Date: Mon, 23 May 2016 10:55:37 +0200 Subject: put fallbackfn to init This was now used by multiple methods, no need to duplicate code. --- .../test_handler/test_handler_apt_source.py | 21 +++++++-------------- 1 file changed, 7 insertions(+), 14 deletions(-) diff --git a/tests/unittests/test_handler/test_handler_apt_source.py b/tests/unittests/test_handler/test_handler_apt_source.py index c4ba6d00..79c33b4e 100644 --- a/tests/unittests/test_handler/test_handler_apt_source.py +++ b/tests/unittests/test_handler/test_handler_apt_source.py @@ -48,6 +48,10 @@ class TestAptSourceConfig(TestCase): self.addCleanup(shutil.rmtree, self.tmp) self.aptlistfile = os.path.join(self.tmp, "single-deb.list") self.join = os.path.join + # mock fallback filename into writable tmp dir + self.fallbackfn = os.path.join(self.tmp, "etc/apt/sources.list.d/", + "cloud_config_sources.list") + @staticmethod def _get_default_params(): @@ -104,12 +108,8 @@ class TestAptSourceConfig(TestCase): cfg = {'source': ('deb http://archive.ubuntu.com/ubuntu' ' karmic-backports' ' main universe multiverse restricted')} - # mock into writable tmp dir and check path/content there - filename = os.path.join(self.tmp, "etc/apt/sources.list.d/", - "cloud_config_sources.list") - with mock.patch.object(os.path, 'join', side_effect=self.myjoin): - self.apt_source_basic(filename, cfg) + self.apt_source_basic(self.fallbackfn, cfg) def apt_source_replacement(self, filename, cfg): """ apt_source_replace @@ -141,12 +141,8 @@ class TestAptSourceConfig(TestCase): No filename being set """ cfg = {'source': 'deb $MIRROR $RELEASE multiverse'} - # mock into writable tmp dir and check path/content there - filename = os.path.join(self.tmp, "etc/apt/sources.list.d/", - "cloud_config_sources.list") - with mock.patch.object(os.path, 'join', side_effect=self.myjoin): - self.apt_source_replacement(filename, cfg) + self.apt_source_replacement(self.fallbackfn, cfg) def apt_source_keyid(self, filename, cfg): """ apt_source_keyid @@ -191,11 +187,8 @@ class TestAptSourceConfig(TestCase): 'smoser/cloud-init-test/ubuntu' ' xenial main'), 'keyid': "03683F77"} - # mock into writable tmp dir and check path/content there - filename = os.path.join(self.tmp, "etc/apt/sources.list.d/", - "cloud_config_sources.list") with mock.patch.object(os.path, 'join', side_effect=self.myjoin): - self.apt_source_keyid(filename, cfg) + self.apt_source_keyid(self.fallbackfn, cfg) def test_apt_source_key(self): """ test_apt_source_key -- cgit v1.2.3 From 1926eb2476f9e1fda3356c7828479231dccc309b Mon Sep 17 00:00:00 2001 From: Christian Ehrhardt Date: Mon, 23 May 2016 10:59:45 +0200 Subject: extend test_apt_source_key by nofn case --- .../test_handler/test_handler_apt_source.py | 38 ++++++++++++++++------ 1 file changed, 28 insertions(+), 10 deletions(-) diff --git a/tests/unittests/test_handler/test_handler_apt_source.py b/tests/unittests/test_handler/test_handler_apt_source.py index 79c33b4e..516dc694 100644 --- a/tests/unittests/test_handler/test_handler_apt_source.py +++ b/tests/unittests/test_handler/test_handler_apt_source.py @@ -190,26 +190,20 @@ class TestAptSourceConfig(TestCase): with mock.patch.object(os.path, 'join', side_effect=self.myjoin): self.apt_source_keyid(self.fallbackfn, cfg) - def test_apt_source_key(self): - """ test_apt_source_key + def apt_source_key(self, filename, cfg): + """ apt_source_key Test specification of a source + key """ params = self._get_default_params() - cfg = {'source': ('deb ' - 'http://ppa.launchpad.net/' - 'smoser/cloud-init-test/ubuntu' - ' xenial main'), - 'key': "fakekey 4321", - 'filename': self.aptlistfile} with mock.patch.object(util, 'subp') as mockobj: cc_apt_configure.add_sources([cfg], params) mockobj.assert_called_with(('apt-key', 'add', '-'), 'fakekey 4321') - self.assertTrue(os.path.isfile(self.aptlistfile)) + self.assertTrue(os.path.isfile(filename)) - contents = load_tfile_or_url(self.aptlistfile) + contents = load_tfile_or_url(filename) self.assertTrue(re.search(r"%s %s %s %s\n" % ("deb", ('http://ppa.launchpad.net/smoser/' @@ -217,6 +211,30 @@ class TestAptSourceConfig(TestCase): "xenial", "main"), contents, flags=re.IGNORECASE)) + def test_apt_source_key(self): + """ test_apt_source_key + Test specification of a source + key with filename being set + """ + cfg = {'source': ('deb ' + 'http://ppa.launchpad.net/' + 'smoser/cloud-init-test/ubuntu' + ' xenial main'), + 'key': "fakekey 4321", + 'filename': self.aptlistfile} + self.apt_source_key(self.aptlistfile, cfg) + + def test_apt_source_key_nofn(self): + """ test_apt_source_key + Test specification of a source + key without filename being set + """ + cfg = {'source': ('deb ' + 'http://ppa.launchpad.net/' + 'smoser/cloud-init-test/ubuntu' + ' xenial main'), + 'key': "fakekey 4321"} + with mock.patch.object(os.path, 'join', side_effect=self.myjoin): + self.apt_source_key(self.fallbackfn, cfg) + def test_apt_source_keyonly(self): """ test_apt_source_keyonly Test specification key without source -- cgit v1.2.3 From ee239517c5342cbd62c9fdeaf735d78d6fd1fbb8 Mon Sep 17 00:00:00 2001 From: Christian Ehrhardt Date: Mon, 23 May 2016 11:59:14 +0200 Subject: support apt_sources to be a dictionary key is the filename, and "old" input shall be handled as it was all the time. For compatibility this will (continue to) overwrite the file of multiple options that did not specify an output file (they all get the same default). Yet it will process them all - as it always did - e.g. to add the keys of all of them. Any users of the new format won't have these issues, as they will always have a key. --- cloudinit/config/cc_apt_configure.py | 34 +++++++++++++++++++++++++--------- cloudinit/util.py | 8 ++++++++ 2 files changed, 33 insertions(+), 9 deletions(-) diff --git a/cloudinit/config/cc_apt_configure.py b/cloudinit/config/cc_apt_configure.py index e5a962ac..a46ebb3e 100644 --- a/cloudinit/config/cc_apt_configure.py +++ b/cloudinit/config/cc_apt_configure.py @@ -215,8 +215,28 @@ def add_sources(srclist, template_params=None, aa_repo_match=None): def aa_repo_match(x): return False + # convert old list format to new dict based format + if isinstance(srclist, list): + srcdict = {} + for srcent in srclist: + if 'filename' not in srcent: + # file collides for multiple !filename cases for compatibility + # yet we need them all processed, so not same dictionary key + srcent['filename'] = "cloud_config_sources.list" + key = util.rand_dict_key(srcdict, "cloud_config_sources.list") + else: + # all with filename use that as key (matching new format) + key = srcent['filename'] + srcdict[key] = srcent + else: + srcdict = srclist + errorlist = [] - for ent in srclist: + for filename in srcdict: + ent = srcdict[filename] + if 'filename' not in ent: + ent[filename] = filename + # keys can be added without specifying a source try: add_key(ent) @@ -226,10 +246,13 @@ def add_sources(srclist, template_params=None, aa_repo_match=None): if 'source' not in ent: errorlist.append(["", "missing source"]) continue - source = ent['source'] source = templater.render_string(source, template_params) + if not ent['filename'].startswith("/"): + ent['filename'] = os.path.join("/etc/apt/sources.list.d/", + ent['filename']) + if aa_repo_match(source): try: util.subp(["add-apt-repository", source]) @@ -238,13 +261,6 @@ def add_sources(srclist, template_params=None, aa_repo_match=None): ("add-apt-repository failed. " + str(e))]) continue - if 'filename' not in ent: - ent['filename'] = 'cloud_config_sources.list' - - if not ent['filename'].startswith("/"): - ent['filename'] = os.path.join("/etc/apt/sources.list.d/", - ent['filename']) - try: contents = "%s\n" % (source) util.write_file(ent['filename'], contents, omode="ab") diff --git a/cloudinit/util.py b/cloudinit/util.py index 0d21e11b..2931efbd 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -335,6 +335,14 @@ def rand_str(strlen=32, select_from=None): select_from = string.ascii_letters + string.digits return "".join([random.choice(select_from) for _x in range(0, strlen)]) +def rand_dict_key(dictionary, postfix=None): + if not postfix: + postfix = "" + while True: + newkey = rand_str(strlen=8) + "_" + postfix + if newkey not in dictionary: + break + return newkey def read_conf(fname): try: -- cgit v1.2.3 From a33f8c09863381006f708a1e9d49997ed9f7befa Mon Sep 17 00:00:00 2001 From: Christian Ehrhardt Date: Mon, 23 May 2016 12:12:26 +0200 Subject: warn about multiple colliding apt_source without filenames --- cloudinit/config/cc_apt_configure.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/cloudinit/config/cc_apt_configure.py b/cloudinit/config/cc_apt_configure.py index a46ebb3e..327b543e 100644 --- a/cloudinit/config/cc_apt_configure.py +++ b/cloudinit/config/cc_apt_configure.py @@ -215,15 +215,22 @@ def add_sources(srclist, template_params=None, aa_repo_match=None): def aa_repo_match(x): return False + errorlist = [] # convert old list format to new dict based format if isinstance(srclist, list): srcdict = {} + fnfallbackused = None for srcent in srclist: if 'filename' not in srcent: # file collides for multiple !filename cases for compatibility # yet we need them all processed, so not same dictionary key srcent['filename'] = "cloud_config_sources.list" key = util.rand_dict_key(srcdict, "cloud_config_sources.list") + if fnfallbackused is not None: + errorlist.append(["multiple apt_source entries without", + "filename will conflict: %s vs %s" % + (srcent, fnfallbackused)]) + fnfallbackused = srcent else: # all with filename use that as key (matching new format) key = srcent['filename'] @@ -231,7 +238,6 @@ def add_sources(srclist, template_params=None, aa_repo_match=None): else: srcdict = srclist - errorlist = [] for filename in srcdict: ent = srcdict[filename] if 'filename' not in ent: -- cgit v1.2.3 From 3560afc50569169ceddf95ff755d231fd5858143 Mon Sep 17 00:00:00 2001 From: Christian Ehrhardt Date: Mon, 23 May 2016 13:34:27 +0200 Subject: fix function names in inline doc --- tests/unittests/test_handler/test_handler_apt_source.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/unittests/test_handler/test_handler_apt_source.py b/tests/unittests/test_handler/test_handler_apt_source.py index 516dc694..6441374d 100644 --- a/tests/unittests/test_handler/test_handler_apt_source.py +++ b/tests/unittests/test_handler/test_handler_apt_source.py @@ -179,7 +179,7 @@ class TestAptSourceConfig(TestCase): self.apt_source_keyid(self.aptlistfile, cfg) def test_apt_source_keyid_nofn(self): - """ test_apt_source_keyid + """ test_apt_source_keyid_nofn Test specification of a source + keyid without filename being set """ cfg = {'source': ('deb ' @@ -224,7 +224,7 @@ class TestAptSourceConfig(TestCase): self.apt_source_key(self.aptlistfile, cfg) def test_apt_source_key_nofn(self): - """ test_apt_source_key + """ test_apt_source_key_nofn Test specification of a source + key without filename being set """ cfg = {'source': ('deb ' -- cgit v1.2.3 From db5188ca3b9eac1097735ce42807881266d29543 Mon Sep 17 00:00:00 2001 From: Christian Ehrhardt Date: Mon, 23 May 2016 13:57:17 +0200 Subject: testcases with multiple source list entries --- .../test_handler/test_handler_apt_source.py | 74 ++++++++++++++++++++-- 1 file changed, 68 insertions(+), 6 deletions(-) diff --git a/tests/unittests/test_handler/test_handler_apt_source.py b/tests/unittests/test_handler/test_handler_apt_source.py index 6441374d..dcbd51e6 100644 --- a/tests/unittests/test_handler/test_handler_apt_source.py +++ b/tests/unittests/test_handler/test_handler_apt_source.py @@ -47,6 +47,8 @@ class TestAptSourceConfig(TestCase): self.tmp = tempfile.mkdtemp() self.addCleanup(shutil.rmtree, self.tmp) self.aptlistfile = os.path.join(self.tmp, "single-deb.list") + self.aptlistfile2 = os.path.join(self.tmp, "single-deb2.list") + self.aptlistfile3 = os.path.join(self.tmp, "single-deb3.list") self.join = os.path.join # mock fallback filename into writable tmp dir self.fallbackfn = os.path.join(self.tmp, "etc/apt/sources.list.d/", @@ -78,7 +80,7 @@ class TestAptSourceConfig(TestCase): """ params = self._get_default_params() - cc_apt_configure.add_sources([cfg], params) + cc_apt_configure.add_sources(cfg, params) self.assertTrue(os.path.isfile(filename)) @@ -98,7 +100,40 @@ class TestAptSourceConfig(TestCase): ' karmic-backports' ' main universe multiverse restricted'), 'filename': self.aptlistfile} - self.apt_source_basic(self.aptlistfile, cfg) + self.apt_source_basic(self.aptlistfile, [cfg]) + + def test_apt_source_basic_triple(self): + """ test_apt_source_basic_triple + Test Fix three deb source string, has to overwrite mirror conf in + params. Test with filenames provided in config. + """ + cfg1 = {'source': ('deb http://archive.ubuntu.com/ubuntu' + ' karmic-backports' + ' main universe multiverse restricted'), + 'filename': self.aptlistfile} + cfg2 = {'source': ('deb http://archive.ubuntu.com/ubuntu' + ' precise-backports' + ' main universe multiverse restricted'), + 'filename': self.aptlistfile2} + cfg3 = {'source': ('deb http://archive.ubuntu.com/ubuntu' + ' lucid-backports' + ' main universe multiverse restricted'), + 'filename': self.aptlistfile3} + self.apt_source_basic(self.aptlistfile, [cfg1, cfg2, cfg3]) + + # extra verify on two extra files of this test + contents = load_tfile_or_url(self.aptlistfile2) + self.assertTrue(re.search(r"%s %s %s %s\n" % + ("deb", "http://archive.ubuntu.com/ubuntu", + "precise-backports", + "main universe multiverse restricted"), + contents, flags=re.IGNORECASE)) + contents = load_tfile_or_url(self.aptlistfile3) + self.assertTrue(re.search(r"%s %s %s %s\n" % + ("deb", "http://archive.ubuntu.com/ubuntu", + "lucid-backports", + "main universe multiverse restricted"), + contents, flags=re.IGNORECASE)) def test_apt_source_basic_nofn(self): """ test_apt_source_basic_nofn @@ -109,14 +144,14 @@ class TestAptSourceConfig(TestCase): ' karmic-backports' ' main universe multiverse restricted')} with mock.patch.object(os.path, 'join', side_effect=self.myjoin): - self.apt_source_basic(self.fallbackfn, cfg) + self.apt_source_basic(self.fallbackfn, [cfg]) def apt_source_replacement(self, filename, cfg): """ apt_source_replace Test Autoreplacement of MIRROR and RELEASE in source specs """ params = self._get_default_params() - cc_apt_configure.add_sources([cfg], params) + cc_apt_configure.add_sources(cfg, params) self.assertTrue(os.path.isfile(filename)) @@ -133,7 +168,34 @@ class TestAptSourceConfig(TestCase): """ cfg = {'source': 'deb $MIRROR $RELEASE multiverse', 'filename': self.aptlistfile} - self.apt_source_replacement(self.aptlistfile, cfg) + self.apt_source_replacement(self.aptlistfile, [cfg]) + + def test_apt_source_replace_triple(self): + """ test_apt_source_replace_triple + Test three autoreplacements of MIRROR and RELEASE in source specs with + Filename being set + """ + cfg1 = {'source': 'deb $MIRROR $RELEASE multiverse', + 'filename': self.aptlistfile} + cfg2 = {'source': 'deb $MIRROR $RELEASE main', + 'filename': self.aptlistfile2} + cfg3 = {'source': 'deb $MIRROR $RELEASE universe', + 'filename': self.aptlistfile3} + self.apt_source_replacement(self.aptlistfile, [cfg1, cfg2, cfg3]) + + # extra verify on two extra files of this test + params = self._get_default_params() + contents = load_tfile_or_url(self.aptlistfile2) + self.assertTrue(re.search(r"%s %s %s %s\n" % + ("deb", params['MIRROR'], params['RELEASE'], + "main"), + contents, flags=re.IGNORECASE)) + contents = load_tfile_or_url(self.aptlistfile3) + self.assertTrue(re.search(r"%s %s %s %s\n" % + ("deb", params['MIRROR'], params['RELEASE'], + "universe"), + contents, flags=re.IGNORECASE)) + def test_apt_source_replace_nofn(self): """ test_apt_source_replace_nofn @@ -142,7 +204,7 @@ class TestAptSourceConfig(TestCase): """ cfg = {'source': 'deb $MIRROR $RELEASE multiverse'} with mock.patch.object(os.path, 'join', side_effect=self.myjoin): - self.apt_source_replacement(self.fallbackfn, cfg) + self.apt_source_replacement(self.fallbackfn, [cfg]) def apt_source_keyid(self, filename, cfg): """ apt_source_keyid -- cgit v1.2.3 From a62d3ddcf6ec06ccf0f3c2dd0fd969c83304d89b Mon Sep 17 00:00:00 2001 From: Christian Ehrhardt Date: Mon, 23 May 2016 14:13:18 +0200 Subject: add triple case for test_apt_source_keyid_triple incl triple key check --- .../test_handler/test_handler_apt_source.py | 55 ++++++++++++++++++++-- 1 file changed, 50 insertions(+), 5 deletions(-) diff --git a/tests/unittests/test_handler/test_handler_apt_source.py b/tests/unittests/test_handler/test_handler_apt_source.py index dcbd51e6..9edd3df5 100644 --- a/tests/unittests/test_handler/test_handler_apt_source.py +++ b/tests/unittests/test_handler/test_handler_apt_source.py @@ -10,6 +10,7 @@ try: from unittest import mock except ImportError: import mock +from mock import call from cloudinit.config import cc_apt_configure from cloudinit import util @@ -206,7 +207,7 @@ class TestAptSourceConfig(TestCase): with mock.patch.object(os.path, 'join', side_effect=self.myjoin): self.apt_source_replacement(self.fallbackfn, [cfg]) - def apt_source_keyid(self, filename, cfg): + def apt_source_keyid(self, filename, cfg, keynum): """ apt_source_keyid Test specification of a source + keyid """ @@ -214,9 +215,13 @@ class TestAptSourceConfig(TestCase): with mock.patch.object(util, 'subp', return_value=('fakekey 1234', '')) as mockobj: - cc_apt_configure.add_sources([cfg], params) + cc_apt_configure.add_sources(cfg, params) - mockobj.assert_called_with(('apt-key', 'add', '-'), 'fakekey 1234') + # check if it added the right ammount of keys + calls = [] + for i in range(keynum): + calls.append(call(('apt-key', 'add', '-'), 'fakekey 1234')) + mockobj.assert_has_calls(calls, any_order=True) self.assertTrue(os.path.isfile(filename)) @@ -238,7 +243,47 @@ class TestAptSourceConfig(TestCase): ' xenial main'), 'keyid': "03683F77", 'filename': self.aptlistfile} - self.apt_source_keyid(self.aptlistfile, cfg) + self.apt_source_keyid(self.aptlistfile, [cfg], 1) + + def test_apt_source_keyid_triple(self): + """ test_apt_source_keyid_triple + Test specification of a source + keyid with filename being set + Setting three of such, check for content and keys + """ + cfg1 = {'source': ('deb ' + 'http://ppa.launchpad.net/' + 'smoser/cloud-init-test/ubuntu' + ' xenial main'), + 'keyid': "03683F77", + 'filename': self.aptlistfile} + cfg2 = {'source': ('deb ' + 'http://ppa.launchpad.net/' + 'smoser/cloud-init-test/ubuntu' + ' xenial universe'), + 'keyid': "03683F77", + 'filename': self.aptlistfile2} + cfg3 = {'source': ('deb ' + 'http://ppa.launchpad.net/' + 'smoser/cloud-init-test/ubuntu' + ' xenial multiverse'), + 'keyid': "03683F77", + 'filename': self.aptlistfile3} + + self.apt_source_keyid(self.aptlistfile, [cfg1, cfg2, cfg3], 3) + contents = load_tfile_or_url(self.aptlistfile2) + self.assertTrue(re.search(r"%s %s %s %s\n" % + ("deb", + ('http://ppa.launchpad.net/smoser/' + 'cloud-init-test/ubuntu'), + "xenial", "universe"), + contents, flags=re.IGNORECASE)) + contents = load_tfile_or_url(self.aptlistfile3) + self.assertTrue(re.search(r"%s %s %s %s\n" % + ("deb", + ('http://ppa.launchpad.net/smoser/' + 'cloud-init-test/ubuntu'), + "xenial", "multiverse"), + contents, flags=re.IGNORECASE)) def test_apt_source_keyid_nofn(self): """ test_apt_source_keyid_nofn @@ -250,7 +295,7 @@ class TestAptSourceConfig(TestCase): ' xenial main'), 'keyid': "03683F77"} with mock.patch.object(os.path, 'join', side_effect=self.myjoin): - self.apt_source_keyid(self.fallbackfn, cfg) + self.apt_source_keyid(self.fallbackfn, [cfg], 1) def apt_source_key(self, filename, cfg): """ apt_source_key -- cgit v1.2.3 From 7a141721c5bb0ba2e65191c514a15ff01220ebca Mon Sep 17 00:00:00 2001 From: Christian Ehrhardt Date: Mon, 23 May 2016 14:14:44 +0200 Subject: make checkers happy about unused loop index --- tests/unittests/test_handler/test_handler_apt_source.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unittests/test_handler/test_handler_apt_source.py b/tests/unittests/test_handler/test_handler_apt_source.py index 9edd3df5..3056e9d0 100644 --- a/tests/unittests/test_handler/test_handler_apt_source.py +++ b/tests/unittests/test_handler/test_handler_apt_source.py @@ -219,7 +219,7 @@ class TestAptSourceConfig(TestCase): # check if it added the right ammount of keys calls = [] - for i in range(keynum): + for _ in range(keynum): calls.append(call(('apt-key', 'add', '-'), 'fakekey 1234')) mockobj.assert_has_calls(calls, any_order=True) -- cgit v1.2.3 From 72ff44b4517eacb4f525e1bf7db6787607ff306a Mon Sep 17 00:00:00 2001 From: Christian Ehrhardt Date: Mon, 23 May 2016 14:21:23 +0200 Subject: add triple test for ppa adding --- .../test_handler/test_handler_apt_source.py | 27 ++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/tests/unittests/test_handler/test_handler_apt_source.py b/tests/unittests/test_handler/test_handler_apt_source.py index 3056e9d0..76099035 100644 --- a/tests/unittests/test_handler/test_handler_apt_source.py +++ b/tests/unittests/test_handler/test_handler_apt_source.py @@ -431,5 +431,32 @@ class TestAptSourceConfig(TestCase): # adding ppa should ignore filename (uses add-apt-repository) self.assertFalse(os.path.isfile(self.aptlistfile)) + def test_apt_source_ppa_triple(self): + """ test_apt_source_ppa_triple + Test specification of a ppa + """ + params = self._get_default_params() + cfg1 = {'source': 'ppa:smoser/cloud-init-test', + 'filename': self.aptlistfile} + cfg2 = {'source': 'ppa:smoser/cloud-init-test2', + 'filename': self.aptlistfile2} + cfg3 = {'source': 'ppa:smoser/cloud-init-test3', + 'filename': self.aptlistfile3} + + # default matcher needed for ppa + matcher = re.compile(r'^[\w-]+:\w').search + + with mock.patch.object(util, 'subp') as mockobj: + cc_apt_configure.add_sources([cfg1, cfg2, cfg3], params, + aa_repo_match=matcher) + calls = [call(['add-apt-repository', 'ppa:smoser/cloud-init-test']), + call(['add-apt-repository', 'ppa:smoser/cloud-init-test2']), + call(['add-apt-repository', 'ppa:smoser/cloud-init-test3'])] + mockobj.assert_has_calls(calls, any_order=True) + + # adding ppa should ignore all filenames (uses add-apt-repository) + self.assertFalse(os.path.isfile(self.aptlistfile)) + self.assertFalse(os.path.isfile(self.aptlistfile2)) + self.assertFalse(os.path.isfile(self.aptlistfile3)) # vi: ts=4 expandtab -- cgit v1.2.3 From 4ed5251d17ee7a44ce12d38d9b3d4fa554279419 Mon Sep 17 00:00:00 2001 From: Christian Ehrhardt Date: Mon, 23 May 2016 15:06:48 +0200 Subject: fix issue with dictionary style apt_sources handling filenames --- cloudinit/config/cc_apt_configure.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cloudinit/config/cc_apt_configure.py b/cloudinit/config/cc_apt_configure.py index 327b543e..a25d6af1 100644 --- a/cloudinit/config/cc_apt_configure.py +++ b/cloudinit/config/cc_apt_configure.py @@ -241,7 +241,7 @@ def add_sources(srclist, template_params=None, aa_repo_match=None): for filename in srcdict: ent = srcdict[filename] if 'filename' not in ent: - ent[filename] = filename + ent['filename'] = filename # keys can be added without specifying a source try: -- cgit v1.2.3 From d1887b6fb38d378bb35298fadd6bea729952f3a0 Mon Sep 17 00:00:00 2001 From: Christian Ehrhardt Date: Mon, 23 May 2016 15:07:37 +0200 Subject: add test_apt_source_basic_dict This is the basic testcase but in the new dictionary format --- tests/unittests/test_handler/test_handler_apt_source.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/tests/unittests/test_handler/test_handler_apt_source.py b/tests/unittests/test_handler/test_handler_apt_source.py index 76099035..d2370e93 100644 --- a/tests/unittests/test_handler/test_handler_apt_source.py +++ b/tests/unittests/test_handler/test_handler_apt_source.py @@ -103,6 +103,18 @@ class TestAptSourceConfig(TestCase): 'filename': self.aptlistfile} self.apt_source_basic(self.aptlistfile, [cfg]) + def test_apt_source_basic_dict(self): + """ test_apt_source_basic_dict + Test Fix deb source string, has to overwrite mirror conf in params. + Test with a filename provided in config. + Provided in a dictionary with filename being the key (new format) + """ + cfg = {self.aptlistfile: {'source': + ('deb http://archive.ubuntu.com/ubuntu' + ' karmic-backports' + ' main universe multiverse restricted')}} + self.apt_source_basic(self.aptlistfile, cfg) + def test_apt_source_basic_triple(self): """ test_apt_source_basic_triple Test Fix three deb source string, has to overwrite mirror conf in -- cgit v1.2.3 From e4968621c802062910d964c2d9c897d7124861b0 Mon Sep 17 00:00:00 2001 From: Christian Ehrhardt Date: Mon, 23 May 2016 15:21:55 +0200 Subject: unify basic triple check and add test_apt_src_basic_dict_triple based on it --- .../test_handler/test_handler_apt_source.py | 57 ++++++++++++++++------ 1 file changed, 43 insertions(+), 14 deletions(-) diff --git a/tests/unittests/test_handler/test_handler_apt_source.py b/tests/unittests/test_handler/test_handler_apt_source.py index d2370e93..f19a78b1 100644 --- a/tests/unittests/test_handler/test_handler_apt_source.py +++ b/tests/unittests/test_handler/test_handler_apt_source.py @@ -115,6 +115,29 @@ class TestAptSourceConfig(TestCase): ' main universe multiverse restricted')}} self.apt_source_basic(self.aptlistfile, cfg) + def apt_source_basic_triple(self, cfg): + """ apt_source_basic_triple + Test Fix three deb source string, has to overwrite mirror conf in + params. Test with filenames provided in config. + generic part to check three files with different content + """ + self.apt_source_basic(self.aptlistfile, cfg) + + # extra verify on two extra files of this test + contents = load_tfile_or_url(self.aptlistfile2) + self.assertTrue(re.search(r"%s %s %s %s\n" % + ("deb", "http://archive.ubuntu.com/ubuntu", + "precise-backports", + "main universe multiverse restricted"), + contents, flags=re.IGNORECASE)) + contents = load_tfile_or_url(self.aptlistfile3) + self.assertTrue(re.search(r"%s %s %s %s\n" % + ("deb", "http://archive.ubuntu.com/ubuntu", + "lucid-backports", + "main universe multiverse restricted"), + contents, flags=re.IGNORECASE)) + + def test_apt_source_basic_triple(self): """ test_apt_source_basic_triple Test Fix three deb source string, has to overwrite mirror conf in @@ -132,21 +155,27 @@ class TestAptSourceConfig(TestCase): ' lucid-backports' ' main universe multiverse restricted'), 'filename': self.aptlistfile3} - self.apt_source_basic(self.aptlistfile, [cfg1, cfg2, cfg3]) + self.apt_source_basic_triple([cfg1, cfg2, cfg3]) - # extra verify on two extra files of this test - contents = load_tfile_or_url(self.aptlistfile2) - self.assertTrue(re.search(r"%s %s %s %s\n" % - ("deb", "http://archive.ubuntu.com/ubuntu", - "precise-backports", - "main universe multiverse restricted"), - contents, flags=re.IGNORECASE)) - contents = load_tfile_or_url(self.aptlistfile3) - self.assertTrue(re.search(r"%s %s %s %s\n" % - ("deb", "http://archive.ubuntu.com/ubuntu", - "lucid-backports", - "main universe multiverse restricted"), - contents, flags=re.IGNORECASE)) + def test_apt_src_basic_dict_triple(self): + """ test_apt_src_basic_dict_triple + Test Fix three deb source string, has to overwrite mirror conf in + params. Test with filenames provided in config. + Provided in a dictionary with filename being the key (new format) + """ + cfg = {self.aptlistfile: {'source': + ('deb http://archive.ubuntu.com/ubuntu' + ' karmic-backports' + ' main universe multiverse restricted')}, + self.aptlistfile2: {'source': + ('deb http://archive.ubuntu.com/ubuntu' + ' precise-backports' + ' main universe multiverse restricted')}, + self.aptlistfile3: {'source': + ('deb http://archive.ubuntu.com/ubuntu' + ' lucid-backports' + ' main universe multiverse restricted')}} + self.apt_source_basic_triple(cfg) def test_apt_source_basic_nofn(self): """ test_apt_source_basic_nofn -- cgit v1.2.3 From 2945e028477ddb031d9a51ada16d5b992380242a Mon Sep 17 00:00:00 2001 From: Christian Ehrhardt Date: Mon, 23 May 2016 16:12:18 +0200 Subject: make sure we only handle list or dict apt_sources and bail out for others --- cloudinit/config/cc_apt_configure.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/cloudinit/config/cc_apt_configure.py b/cloudinit/config/cc_apt_configure.py index a25d6af1..dd199471 100644 --- a/cloudinit/config/cc_apt_configure.py +++ b/cloudinit/config/cc_apt_configure.py @@ -217,8 +217,8 @@ def add_sources(srclist, template_params=None, aa_repo_match=None): errorlist = [] # convert old list format to new dict based format + srcdict = {} if isinstance(srclist, list): - srcdict = {} fnfallbackused = None for srcent in srclist: if 'filename' not in srcent: @@ -235,8 +235,10 @@ def add_sources(srclist, template_params=None, aa_repo_match=None): # all with filename use that as key (matching new format) key = srcent['filename'] srcdict[key] = srcent - else: + elif isinstance(srclist, dict): srcdict = srclist + else: + errorlist.append(["srclist", "unknown apt_sources format"]) for filename in srcdict: ent = srcdict[filename] -- cgit v1.2.3 From 299e3dd9b3769101fa694d7e96895bb89e0375ca Mon Sep 17 00:00:00 2001 From: Christian Ehrhardt Date: Mon, 23 May 2016 16:13:45 +0200 Subject: shorten method names to follow python rules --- .../test_handler/test_handler_apt_source.py | 145 +++++++++++---------- 1 file changed, 76 insertions(+), 69 deletions(-) diff --git a/tests/unittests/test_handler/test_handler_apt_source.py b/tests/unittests/test_handler/test_handler_apt_source.py index f19a78b1..9a2cc268 100644 --- a/tests/unittests/test_handler/test_handler_apt_source.py +++ b/tests/unittests/test_handler/test_handler_apt_source.py @@ -75,8 +75,8 @@ class TestAptSourceConfig(TestCase): else: return self.join(*args, **kwargs) - def apt_source_basic(self, filename, cfg): - """ apt_source_basic + def apt_src_basic(self, filename, cfg): + """ apt_src_basic Test Fix deb source string, has to overwrite mirror conf in params """ params = self._get_default_params() @@ -92,8 +92,8 @@ class TestAptSourceConfig(TestCase): "main universe multiverse restricted"), contents, flags=re.IGNORECASE)) - def test_apt_source_basic(self): - """ test_apt_source_basic + def test_apt_src_basic(self): + """ test_apt_src_basic Test Fix deb source string, has to overwrite mirror conf in params. Test with a filename provided in config. """ @@ -101,10 +101,10 @@ class TestAptSourceConfig(TestCase): ' karmic-backports' ' main universe multiverse restricted'), 'filename': self.aptlistfile} - self.apt_source_basic(self.aptlistfile, [cfg]) + self.apt_src_basic(self.aptlistfile, [cfg]) - def test_apt_source_basic_dict(self): - """ test_apt_source_basic_dict + def test_apt_src_basic_dict(self): + """ test_apt_src_basic_dict Test Fix deb source string, has to overwrite mirror conf in params. Test with a filename provided in config. Provided in a dictionary with filename being the key (new format) @@ -113,15 +113,15 @@ class TestAptSourceConfig(TestCase): ('deb http://archive.ubuntu.com/ubuntu' ' karmic-backports' ' main universe multiverse restricted')}} - self.apt_source_basic(self.aptlistfile, cfg) + self.apt_src_basic(self.aptlistfile, cfg) - def apt_source_basic_triple(self, cfg): - """ apt_source_basic_triple + def apt_src_basic_tri(self, cfg): + """ apt_src_basic_tri Test Fix three deb source string, has to overwrite mirror conf in params. Test with filenames provided in config. generic part to check three files with different content """ - self.apt_source_basic(self.aptlistfile, cfg) + self.apt_src_basic(self.aptlistfile, cfg) # extra verify on two extra files of this test contents = load_tfile_or_url(self.aptlistfile2) @@ -138,8 +138,8 @@ class TestAptSourceConfig(TestCase): contents, flags=re.IGNORECASE)) - def test_apt_source_basic_triple(self): - """ test_apt_source_basic_triple + def test_apt_src_basic_tri(self): + """ test_apt_src_basic_tri Test Fix three deb source string, has to overwrite mirror conf in params. Test with filenames provided in config. """ @@ -155,10 +155,10 @@ class TestAptSourceConfig(TestCase): ' lucid-backports' ' main universe multiverse restricted'), 'filename': self.aptlistfile3} - self.apt_source_basic_triple([cfg1, cfg2, cfg3]) + self.apt_src_basic_tri([cfg1, cfg2, cfg3]) - def test_apt_src_basic_dict_triple(self): - """ test_apt_src_basic_dict_triple + def test_apt_src_basic_dict_tri(self): + """ test_apt_src_basic_dict_tri Test Fix three deb source string, has to overwrite mirror conf in params. Test with filenames provided in config. Provided in a dictionary with filename being the key (new format) @@ -175,10 +175,10 @@ class TestAptSourceConfig(TestCase): ('deb http://archive.ubuntu.com/ubuntu' ' lucid-backports' ' main universe multiverse restricted')}} - self.apt_source_basic_triple(cfg) + self.apt_src_basic_tri(cfg) - def test_apt_source_basic_nofn(self): - """ test_apt_source_basic_nofn + def test_apt_src_basic_nofn(self): + """ test_apt_src_basic_nofn Test Fix deb source string, has to overwrite mirror conf in params. Test without a filename provided in config and test for known fallback. """ @@ -186,10 +186,10 @@ class TestAptSourceConfig(TestCase): ' karmic-backports' ' main universe multiverse restricted')} with mock.patch.object(os.path, 'join', side_effect=self.myjoin): - self.apt_source_basic(self.fallbackfn, [cfg]) + self.apt_src_basic(self.fallbackfn, [cfg]) - def apt_source_replacement(self, filename, cfg): - """ apt_source_replace + def apt_src_replacement(self, filename, cfg): + """ apt_src_replace Test Autoreplacement of MIRROR and RELEASE in source specs """ params = self._get_default_params() @@ -203,27 +203,21 @@ class TestAptSourceConfig(TestCase): "multiverse"), contents, flags=re.IGNORECASE)) - def test_apt_source_replace(self): - """ test_apt_source_replace + def test_apt_src_replace(self): + """ test_apt_src_replace Test Autoreplacement of MIRROR and RELEASE in source specs with Filename being set """ cfg = {'source': 'deb $MIRROR $RELEASE multiverse', 'filename': self.aptlistfile} - self.apt_source_replacement(self.aptlistfile, [cfg]) + self.apt_src_replacement(self.aptlistfile, [cfg]) - def test_apt_source_replace_triple(self): - """ test_apt_source_replace_triple + def apt_src_replace_tri(self, cfg): + """ apt_src_replace_tri Test three autoreplacements of MIRROR and RELEASE in source specs with - Filename being set + generic part """ - cfg1 = {'source': 'deb $MIRROR $RELEASE multiverse', - 'filename': self.aptlistfile} - cfg2 = {'source': 'deb $MIRROR $RELEASE main', - 'filename': self.aptlistfile2} - cfg3 = {'source': 'deb $MIRROR $RELEASE universe', - 'filename': self.aptlistfile3} - self.apt_source_replacement(self.aptlistfile, [cfg1, cfg2, cfg3]) + self.apt_src_replacement(self.aptlistfile, cfg) # extra verify on two extra files of this test params = self._get_default_params() @@ -239,17 +233,30 @@ class TestAptSourceConfig(TestCase): contents, flags=re.IGNORECASE)) - def test_apt_source_replace_nofn(self): - """ test_apt_source_replace_nofn + def test_apt_src_replace_tri(self): + """ test_apt_src_replace_tri + Test three autoreplacements of MIRROR and RELEASE in source specs with + Filename being set + """ + cfg1 = {'source': 'deb $MIRROR $RELEASE multiverse', + 'filename': self.aptlistfile} + cfg2 = {'source': 'deb $MIRROR $RELEASE main', + 'filename': self.aptlistfile2} + cfg3 = {'source': 'deb $MIRROR $RELEASE universe', + 'filename': self.aptlistfile3} + self.apt_src_replace_tri([cfg1, cfg2, cfg3]) + + def test_apt_src_replace_nofn(self): + """ test_apt_src_replace_nofn Test Autoreplacement of MIRROR and RELEASE in source specs with No filename being set """ cfg = {'source': 'deb $MIRROR $RELEASE multiverse'} with mock.patch.object(os.path, 'join', side_effect=self.myjoin): - self.apt_source_replacement(self.fallbackfn, [cfg]) + self.apt_src_replacement(self.fallbackfn, [cfg]) - def apt_source_keyid(self, filename, cfg, keynum): - """ apt_source_keyid + def apt_src_keyid(self, filename, cfg, keynum): + """ apt_src_keyid Test specification of a source + keyid """ params = self._get_default_params() @@ -274,8 +281,8 @@ class TestAptSourceConfig(TestCase): "xenial", "main"), contents, flags=re.IGNORECASE)) - def test_apt_source_keyid(self): - """ test_apt_source_keyid + def test_apt_src_keyid(self): + """ test_apt_src_keyid Test specification of a source + keyid with filename being set """ cfg = {'source': ('deb ' @@ -284,10 +291,10 @@ class TestAptSourceConfig(TestCase): ' xenial main'), 'keyid': "03683F77", 'filename': self.aptlistfile} - self.apt_source_keyid(self.aptlistfile, [cfg], 1) + self.apt_src_keyid(self.aptlistfile, [cfg], 1) - def test_apt_source_keyid_triple(self): - """ test_apt_source_keyid_triple + def test_apt_src_keyid_tri(self): + """ test_apt_src_keyid_tri Test specification of a source + keyid with filename being set Setting three of such, check for content and keys """ @@ -310,7 +317,7 @@ class TestAptSourceConfig(TestCase): 'keyid': "03683F77", 'filename': self.aptlistfile3} - self.apt_source_keyid(self.aptlistfile, [cfg1, cfg2, cfg3], 3) + self.apt_src_keyid(self.aptlistfile, [cfg1, cfg2, cfg3], 3) contents = load_tfile_or_url(self.aptlistfile2) self.assertTrue(re.search(r"%s %s %s %s\n" % ("deb", @@ -326,8 +333,8 @@ class TestAptSourceConfig(TestCase): "xenial", "multiverse"), contents, flags=re.IGNORECASE)) - def test_apt_source_keyid_nofn(self): - """ test_apt_source_keyid_nofn + def test_apt_src_keyid_nofn(self): + """ test_apt_src_keyid_nofn Test specification of a source + keyid without filename being set """ cfg = {'source': ('deb ' @@ -336,10 +343,10 @@ class TestAptSourceConfig(TestCase): ' xenial main'), 'keyid': "03683F77"} with mock.patch.object(os.path, 'join', side_effect=self.myjoin): - self.apt_source_keyid(self.fallbackfn, [cfg], 1) + self.apt_src_keyid(self.fallbackfn, [cfg], 1) - def apt_source_key(self, filename, cfg): - """ apt_source_key + def apt_src_key(self, filename, cfg): + """ apt_src_key Test specification of a source + key """ params = self._get_default_params() @@ -359,8 +366,8 @@ class TestAptSourceConfig(TestCase): "xenial", "main"), contents, flags=re.IGNORECASE)) - def test_apt_source_key(self): - """ test_apt_source_key + def test_apt_src_key(self): + """ test_apt_src_key Test specification of a source + key with filename being set """ cfg = {'source': ('deb ' @@ -369,10 +376,10 @@ class TestAptSourceConfig(TestCase): ' xenial main'), 'key': "fakekey 4321", 'filename': self.aptlistfile} - self.apt_source_key(self.aptlistfile, cfg) + self.apt_src_key(self.aptlistfile, cfg) - def test_apt_source_key_nofn(self): - """ test_apt_source_key_nofn + def test_apt_src_key_nofn(self): + """ test_apt_src_key_nofn Test specification of a source + key without filename being set """ cfg = {'source': ('deb ' @@ -381,10 +388,10 @@ class TestAptSourceConfig(TestCase): ' xenial main'), 'key': "fakekey 4321"} with mock.patch.object(os.path, 'join', side_effect=self.myjoin): - self.apt_source_key(self.fallbackfn, cfg) + self.apt_src_key(self.fallbackfn, cfg) - def test_apt_source_keyonly(self): - """ test_apt_source_keyonly + def test_apt_src_keyonly(self): + """ test_apt_src_keyonly Test specification key without source """ params = self._get_default_params() @@ -400,8 +407,8 @@ class TestAptSourceConfig(TestCase): # filename should be ignored on key only self.assertFalse(os.path.isfile(self.aptlistfile)) - def test_apt_source_keyidonly(self): - """ test_apt_source_keyidonly + def test_apt_src_keyidonly(self): + """ test_apt_src_keyidonly Test specification of a keyid without source """ params = self._get_default_params() @@ -417,8 +424,8 @@ class TestAptSourceConfig(TestCase): # filename should be ignored on key only self.assertFalse(os.path.isfile(self.aptlistfile)) - def test_apt_source_keyid_real(self): - """ test_apt_source_keyid_real + def test_apt_src_keyid_real(self): + """ test_apt_src_keyid_real Test specification of a keyid without source incl up to addition of the key (nothing but add_key_raw mocked) """ @@ -435,8 +442,8 @@ class TestAptSourceConfig(TestCase): # filename should be ignored on key only self.assertFalse(os.path.isfile(self.aptlistfile)) - def test_apt_source_longkeyid_real(self): - """ test_apt_source_keyid_real + def test_apt_src_longkeyid_real(self): + """ test_apt_src_keyid_real Test specification of a long key fingerprint without source incl up to addition of the key (nothing but add_key_raw mocked) """ @@ -453,8 +460,8 @@ class TestAptSourceConfig(TestCase): # filename should be ignored on key only self.assertFalse(os.path.isfile(self.aptlistfile)) - def test_apt_source_ppa(self): - """ test_apt_source_ppa + def test_apt_src_ppa(self): + """ test_apt_src_ppa Test specification of a ppa """ params = self._get_default_params() @@ -472,8 +479,8 @@ class TestAptSourceConfig(TestCase): # adding ppa should ignore filename (uses add-apt-repository) self.assertFalse(os.path.isfile(self.aptlistfile)) - def test_apt_source_ppa_triple(self): - """ test_apt_source_ppa_triple + def test_apt_src_ppa_tri(self): + """ test_apt_src_ppa_tri Test specification of a ppa """ params = self._get_default_params() -- cgit v1.2.3 From 23a19bde96d895e97c5fb6dbbe50620fb1130553 Mon Sep 17 00:00:00 2001 From: Christian Ehrhardt Date: Mon, 23 May 2016 16:14:04 +0200 Subject: add test_apt_src_replace_dict_tri This includes a test for the weird but valid case in the new dictionary syntax that one sets a key (which is the filename) but overwrites the filename value inside of it. --- tests/unittests/test_handler/test_handler_apt_source.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/tests/unittests/test_handler/test_handler_apt_source.py b/tests/unittests/test_handler/test_handler_apt_source.py index 9a2cc268..237cf14d 100644 --- a/tests/unittests/test_handler/test_handler_apt_source.py +++ b/tests/unittests/test_handler/test_handler_apt_source.py @@ -246,6 +246,20 @@ class TestAptSourceConfig(TestCase): 'filename': self.aptlistfile3} self.apt_src_replace_tri([cfg1, cfg2, cfg3]) + def test_apt_src_replace_dict_tri(self): + """ test_apt_src_replace_dict_tri + Test three autoreplacements of MIRROR and RELEASE in source specs with + Filename being set + Provided in a dictionary with filename being the key (new format) + We also test a new special conditions of the new format that allows + filenames to be overwritten inside the directory entry. + """ + cfg = {self.aptlistfile: {'source': 'deb $MIRROR $RELEASE multiverse'}, + 'notused': {'source': 'deb $MIRROR $RELEASE main', + 'filename': self.aptlistfile2}, + self.aptlistfile3: {'source': 'deb $MIRROR $RELEASE universe'}} + self.apt_src_replace_tri(cfg) + def test_apt_src_replace_nofn(self): """ test_apt_src_replace_nofn Test Autoreplacement of MIRROR and RELEASE in source specs with -- cgit v1.2.3 From 65ad82bec66ea3379a20785b1932ed1dc3c17b67 Mon Sep 17 00:00:00 2001 From: Christian Ehrhardt Date: Mon, 23 May 2016 16:46:30 +0200 Subject: modify cloud-config examples to match the new apt_source format --- doc/examples/cloud-config.txt | 76 ++++++++++++++++++++++++++++++++++++------- 1 file changed, 65 insertions(+), 11 deletions(-) diff --git a/doc/examples/cloud-config.txt b/doc/examples/cloud-config.txt index 31b791b6..50c6d282 100644 --- a/doc/examples/cloud-config.txt +++ b/doc/examples/cloud-config.txt @@ -106,10 +106,52 @@ apt_custom_sources_list: | # expression will be passed to add-apt-repository add_apt_repo_match: '^[\w-]+:\w' +# 'apt_sources' is a dictionary +# The key is the filename and will be prepended by /etc/apt/sources.list.d/ if +# it doesn't start with a '/'. +# There are certain cases - where no content is written into a source.list file +# where the filename will be ignored - yet it can still be used as index for +# merging. +# The value it maps to is a dictionary with the following optional entries: +# 'source': a sources.list entry (some variable replacements apply) +# 'keyid': providing a key to import via shortid or fingerprint +# 'key': providing a raw PGP key +# 'filename': for compatibility with the older format (now the key to this +# dictionary is the filename). If specified this overwrites the +# filename given as key. + +# the new "filename: {specification-dictionary}, filename2: ..." format allows +# better merging between multiple input files than a list like: +# cloud-config1 +# sources: + s1: {'key': 'key1', 'source': 'source1'} +# cloud-config2 +# sources: + s2: {'key': 'key2'} + s1: {filename: 'foo'} +# this would be merged to +#sources: +# s1: +# filename: foo +# key: key1 +# source: source1 +# s2: +# key: key2 +# Be aware that this style of merging is not the default (for backward +# compatibility reasons). You should specify the following merge_how to get +# this more complete and modern merging behaviour: +# merge_how: "list()+dict()+str()" +# This would then also be equivalent to the config merging used in curtin +# (https://launchpad.net/curtin). + +# for more details see below in the various examples + apt_sources: - - source: "deb http://ppa.launchpad.net/byobu/ppa/ubuntu karmic main" + byobu-ppa.list: + source: "deb http://ppa.launchpad.net/byobu/ppa/ubuntu karmic main" keyid: F430BBA5 # GPG key ID published on a key server - filename: byobu-ppa.list + # adding a source.list line, importing a gpg key for a given key id and + # storing it in the file /etc/apt/sources.list.d/byobu-ppa.list # PPA shortcut: # * Setup correct apt sources.list line @@ -117,7 +159,9 @@ apt_sources: # # See https://help.launchpad.net/Packaging/PPA for more information # this requires 'add-apt-repository' - - source: "ppa:smoser/ppa" # Quote the string + # due to that the filename key is ignored in this case + ignored1: + source: "ppa:smoser/ppa" # Quote the string # Custom apt repository: # * all that is required is 'source' @@ -128,31 +172,39 @@ apt_sources: # + filename: cloud_config_sources.list # # See sources.list man page for more information about the format - - source: deb http://archive.ubuntu.com/ubuntu karmic-backports main universe multiverse restricted + my-repo.list: + source: deb http://archive.ubuntu.com/ubuntu karmic-backports main universe multiverse restricted # sources can use $MIRROR and $RELEASE and they will be replaced # with the local mirror for this cloud, and the running release # the entry below would be possibly turned into: - # - source: deb http://us-east-1.ec2.archive.ubuntu.com/ubuntu natty multiverse - - source: deb $MIRROR $RELEASE multiverse + # source: deb http://us-east-1.ec2.archive.ubuntu.com/ubuntu natty multiverse + my-repo.list: + source: deb $MIRROR $RELEASE multiverse # this would have the same end effect as 'ppa:byobu/ppa' - - source: "deb http://ppa.launchpad.net/byobu/ppa/ubuntu karmic main" + my-repo.list: + source: "deb http://ppa.launchpad.net/byobu/ppa/ubuntu karmic main" keyid: F430BBA5 # GPG key ID published on a key server filename: byobu-ppa.list # this would only import the key without adding a ppa or other source spec - - keyid: F430BBA5 # GPG key ID published on a key server + # since this doesn't generate a source.list file the filename key is ignored + ignored2: + keyid: F430BBA5 # GPG key ID published on a key server # In general keyid's can also be specified via their long fingerprints - - keyid: B59D 5F15 97A5 04B7 E230 6DCA 0620 BBCF 0368 3F77 + # since this doesn't generate a source.list file the filename key is ignored + ignored3: + keyid: B59D 5F15 97A5 04B7 E230 6DCA 0620 BBCF 0368 3F77 # Custom apt repository: # * The apt signing key can also be specified # by providing a pgp public key block # * Providing the PGP key here is the most robust method for # specifying a key, as it removes dependency on a remote key server - - source: deb http://ppa.launchpad.net/alestic/ppa/ubuntu karmic main + my-repo.list: + source: deb http://ppa.launchpad.net/alestic/ppa/ubuntu karmic main key: | # The value needs to start with -----BEGIN PGP PUBLIC KEY BLOCK----- -----BEGIN PGP PUBLIC KEY BLOCK----- Version: SKS 1.0.10 @@ -170,7 +222,9 @@ apt_sources: # Custom gpg key: # * As the keyid also a key can be specified withut a related source # * all other facts mentioned above still apply - - key: | # The value needs to start with -----BEGIN PGP PUBLIC KEY BLOCK----- + # since this doesn't generate a source.list file the filename key is ignored + ignored4: + key: | # The value needs to start with -----BEGIN PGP PUBLIC KEY BLOCK----- -----BEGIN PGP PUBLIC KEY BLOCK----- Version: SKS 1.0.10 -- cgit v1.2.3 From a63a64a70def97730d2ab544b0df9f87f3484333 Mon Sep 17 00:00:00 2001 From: Christian Ehrhardt Date: Mon, 23 May 2016 16:53:03 +0200 Subject: final pep8 check fixups --- cloudinit/util.py | 2 ++ tests/unittests/test_handler/test_handler_apt_source.py | 9 +++------ 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/cloudinit/util.py b/cloudinit/util.py index 2931efbd..0773af69 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -335,6 +335,7 @@ def rand_str(strlen=32, select_from=None): select_from = string.ascii_letters + string.digits return "".join([random.choice(select_from) for _x in range(0, strlen)]) + def rand_dict_key(dictionary, postfix=None): if not postfix: postfix = "" @@ -344,6 +345,7 @@ def rand_dict_key(dictionary, postfix=None): break return newkey + def read_conf(fname): try: return load_yaml(load_file(fname), default={}) diff --git a/tests/unittests/test_handler/test_handler_apt_source.py b/tests/unittests/test_handler/test_handler_apt_source.py index 237cf14d..c19904fb 100644 --- a/tests/unittests/test_handler/test_handler_apt_source.py +++ b/tests/unittests/test_handler/test_handler_apt_source.py @@ -55,7 +55,6 @@ class TestAptSourceConfig(TestCase): self.fallbackfn = os.path.join(self.tmp, "etc/apt/sources.list.d/", "cloud_config_sources.list") - @staticmethod def _get_default_params(): """ get_default_params @@ -68,9 +67,9 @@ class TestAptSourceConfig(TestCase): def myjoin(self, *args, **kwargs): """ myjoin - redir into writable tmpdir""" - if (args[0] == "/etc/apt/sources.list.d/" - and args[1] == "cloud_config_sources.list" - and len(args) == 2): + if (args[0] == "/etc/apt/sources.list.d/" and + args[1] == "cloud_config_sources.list" and + len(args) == 2): return self.join(self.tmp, args[0].lstrip("/"), args[1]) else: return self.join(*args, **kwargs) @@ -137,7 +136,6 @@ class TestAptSourceConfig(TestCase): "main universe multiverse restricted"), contents, flags=re.IGNORECASE)) - def test_apt_src_basic_tri(self): """ test_apt_src_basic_tri Test Fix three deb source string, has to overwrite mirror conf in @@ -232,7 +230,6 @@ class TestAptSourceConfig(TestCase): "universe"), contents, flags=re.IGNORECASE)) - def test_apt_src_replace_tri(self): """ test_apt_src_replace_tri Test three autoreplacements of MIRROR and RELEASE in source specs with -- cgit v1.2.3 From 1b418ef2db337e9f8bea7462f18f58ee41dea3b2 Mon Sep 17 00:00:00 2001 From: Christian Ehrhardt Date: Tue, 24 May 2016 16:27:24 +0200 Subject: fix typo in examples doc --- doc/examples/cloud-config.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/examples/cloud-config.txt b/doc/examples/cloud-config.txt index 50c6d282..60457093 100644 --- a/doc/examples/cloud-config.txt +++ b/doc/examples/cloud-config.txt @@ -72,7 +72,7 @@ apt_pipelining: False # then apt_mirror above will have no effect apt_preserve_sources_list: true -# Provide a custom template for rednering sources.list +# Provide a custom template for rendering sources.list # Default: a default template for Ubuntu/Debain will be used as packaged in # Ubuntu: /etc/cloud/templates/sources.list.ubuntu.tmpl # Debian: /etc/cloud/templates/sources.list.debian.tmpl -- cgit v1.2.3 From 14040a9c8df6e8406acb79fd653873bb05cb4d40 Mon Sep 17 00:00:00 2001 From: Christian Ehrhardt Date: Tue, 24 May 2016 16:29:31 +0200 Subject: improve examples of ap_source --- doc/examples/cloud-config.txt | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/doc/examples/cloud-config.txt b/doc/examples/cloud-config.txt index 60457093..df59ff57 100644 --- a/doc/examples/cloud-config.txt +++ b/doc/examples/cloud-config.txt @@ -113,10 +113,11 @@ add_apt_repo_match: '^[\w-]+:\w' # where the filename will be ignored - yet it can still be used as index for # merging. # The value it maps to is a dictionary with the following optional entries: -# 'source': a sources.list entry (some variable replacements apply) -# 'keyid': providing a key to import via shortid or fingerprint -# 'key': providing a raw PGP key -# 'filename': for compatibility with the older format (now the key to this +# source: a sources.list entry (some variable replacements apply) +# keyid: providing a key to import via shortid or fingerprint +# key: providing a raw PGP key +# keyserver: keyserver to fetch keys from, default is keyserver.ubuntu.com +# filename: for compatibility with the older format (now the key to this # dictionary is the filename). If specified this overwrites the # filename given as key. @@ -169,7 +170,6 @@ apt_sources: # * [optional] Import the apt signing key from the keyserver # * Defaults: # + keyserver: keyserver.ubuntu.com - # + filename: cloud_config_sources.list # # See sources.list man page for more information about the format my-repo.list: -- cgit v1.2.3 From 2a20382249c35f368f931dc2bcd8225ea1fc4e84 Mon Sep 17 00:00:00 2001 From: Christian Ehrhardt Date: Tue, 24 May 2016 18:55:35 +0200 Subject: integrate further smaller review feedback --- cloudinit/config/cc_apt_configure.py | 52 +++++++++++++++++++----------------- 1 file changed, 27 insertions(+), 25 deletions(-) diff --git a/cloudinit/config/cc_apt_configure.py b/cloudinit/config/cc_apt_configure.py index 8f22c446..02b16336 100644 --- a/cloudinit/config/cc_apt_configure.py +++ b/cloudinit/config/cc_apt_configure.py @@ -21,7 +21,6 @@ import glob import os import re -import tempfile from cloudinit import templater from cloudinit import util @@ -179,44 +178,29 @@ def add_key_raw(key): """ try: util.subp(('apt-key', 'add', '-'), key) - except: + except util.ProcessExecutionError: raise Exception('failed add key') def add_key(ent): """ - add key to the system as defiend in ent (if any) - suppords raw keys or keyid's - The latter will as a first step fetched to get the raw key + add key to the system as defined in ent (if any) + supports raw keys or keyid's + The latter will as a first step fetch the raw key from a keyserver """ if 'keyid' in ent and 'key' not in ent: keyserver = "keyserver.ubuntu.com" if 'keyserver' in ent: keyserver = ent['keyserver'] - try: - ent['key'] = getkeybyid(ent['keyid'], keyserver) - except: - raise Exception('failed to get key from %s' % keyserver) + ent['key'] = getkeybyid(ent['keyid'], keyserver) if 'key' in ent: add_key_raw(ent['key']) - -def add_sources(srclist, template_params=None, aa_repo_match=None): - """ - add entries in /etc/apt/sources.list.d for each abbreviated - sources.list entry in 'srclist'. When rendering template, also - include the values in dictionary searchList +def convert_to_new_format(srclist, errorlist): + """ convert_to_new_format + convert the old list based format to the new dict based one """ - if template_params is None: - template_params = {} - - if aa_repo_match is None: - def aa_repo_match(x): - return False - - errorlist = [] - # convert old list format to new dict based format srcdict = {} if isinstance(srclist, list): fnfallbackused = None @@ -240,6 +224,24 @@ def add_sources(srclist, template_params=None, aa_repo_match=None): else: errorlist.append(["srclist", "unknown apt_sources format"]) + return srcdict + +def add_sources(srclist, template_params=None, aa_repo_match=None): + """ + add entries in /etc/apt/sources.list.d for each abbreviated + sources.list entry in 'srclist'. When rendering template, also + include the values in dictionary searchList + """ + if template_params is None: + template_params = {} + + if aa_repo_match is None: + def aa_repo_match(x): + return False + + errorlist = [] + srcdict = convert_to_new_format(srclist, errorlist) + for filename in srcdict: ent = srcdict[filename] if 'filename' not in ent: @@ -257,7 +259,7 @@ def add_sources(srclist, template_params=None, aa_repo_match=None): source = ent['source'] source = templater.render_string(source, template_params) - if not ent['filename'].startswith("/"): + if not ent['filename'].startswith(os.path.sep): ent['filename'] = os.path.join("/etc/apt/sources.list.d/", ent['filename']) -- cgit v1.2.3 From c2cfbd6831912e6dfd6b16ed21afb929592ce51a Mon Sep 17 00:00:00 2001 From: Christian Ehrhardt Date: Tue, 24 May 2016 19:06:55 +0200 Subject: pacify pep8 regarding the new changes --- cloudinit/config/cc_apt_configure.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cloudinit/config/cc_apt_configure.py b/cloudinit/config/cc_apt_configure.py index 02b16336..ffbf7513 100644 --- a/cloudinit/config/cc_apt_configure.py +++ b/cloudinit/config/cc_apt_configure.py @@ -197,6 +197,7 @@ def add_key(ent): if 'key' in ent: add_key_raw(ent['key']) + def convert_to_new_format(srclist, errorlist): """ convert_to_new_format convert the old list based format to the new dict based one @@ -226,6 +227,7 @@ def convert_to_new_format(srclist, errorlist): return srcdict + def add_sources(srclist, template_params=None, aa_repo_match=None): """ add entries in /etc/apt/sources.list.d for each abbreviated -- cgit v1.2.3 From 150abceed3ec8d1d5099e855ebc4fce0361be65d Mon Sep 17 00:00:00 2001 From: Christian Ehrhardt Date: Tue, 24 May 2016 19:07:15 +0200 Subject: add test for the now isolated convert_to_new_format function --- .../test_handler/test_handler_apt_source.py | 29 ++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/tests/unittests/test_handler/test_handler_apt_source.py b/tests/unittests/test_handler/test_handler_apt_source.py index c19904fb..1b2bc58f 100644 --- a/tests/unittests/test_handler/test_handler_apt_source.py +++ b/tests/unittests/test_handler/test_handler_apt_source.py @@ -518,4 +518,33 @@ class TestAptSourceConfig(TestCase): self.assertFalse(os.path.isfile(self.aptlistfile2)) self.assertFalse(os.path.isfile(self.aptlistfile3)) + def test_convert_to_new_format(self): + """ test_convert_to_new_format + Test the conversion of old to new format + And the noop conversion of new to new format as well + """ + cfg1 = {'source': 'deb $MIRROR $RELEASE multiverse', + 'filename': self.aptlistfile} + cfg2 = {'source': 'deb $MIRROR $RELEASE main', + 'filename': self.aptlistfile2} + cfg3 = {'source': 'deb $MIRROR $RELEASE universe', + 'filename': self.aptlistfile3} + errorlist = [] + checkcfg = {self.aptlistfile: {'filename': self.aptlistfile, + 'source': 'deb $MIRROR $RELEASE ' + 'multiverse'}, + self.aptlistfile2: {'filename': self.aptlistfile2, + 'source': 'deb $MIRROR $RELEASE main'}, + self.aptlistfile3: {'filename': self.aptlistfile3, + 'source': 'deb $MIRROR $RELEASE ' + 'universe'}} + + newcfg = cc_apt_configure.convert_to_new_format([cfg1, cfg2, cfg3], + errorlist) + self.assertEqual(newcfg, checkcfg) + + newcfg2 = cc_apt_configure.convert_to_new_format(newcfg, errorlist) + self.assertEqual(newcfg2, checkcfg) + + # vi: ts=4 expandtab -- cgit v1.2.3 From a84747a8b7a6b527e8ee294e6fc04d70a7090bc5 Mon Sep 17 00:00:00 2001 From: Christian Ehrhardt Date: Wed, 25 May 2016 09:32:39 +0200 Subject: make test_apt_srcl_custom independent to where it is executed --- .../test_handler/test_handler_apt_configure_sources_list.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/tests/unittests/test_handler/test_handler_apt_configure_sources_list.py b/tests/unittests/test_handler/test_handler_apt_configure_sources_list.py index 353422a2..5cf386f8 100644 --- a/tests/unittests/test_handler/test_handler_apt_configure_sources_list.py +++ b/tests/unittests/test_handler/test_handler_apt_configure_sources_list.py @@ -55,8 +55,8 @@ EXPECTED_CONVERTED_CONTENT = ( # See http://help.ubuntu.com/community/UpgradeNotes for how to upgrade to # newer versions of the distribution. -deb http://archive.ubuntu.com/ubuntu/ xenial main restricted -deb-src http://archive.ubuntu.com/ubuntu/ xenial main restricted +deb http://archive.ubuntu.com/ubuntu/ fakerelease main restricted +deb-src http://archive.ubuntu.com/ubuntu/ fakerelease main restricted # FIND_SOMETHING_SPECIAL """) @@ -152,8 +152,10 @@ class TestAptSourceConfigSourceList(t_help.FilesystemMockingTestCase): # the second mock restores the original subp with mock.patch.object(util, 'write_file') as mockwrite: with mock.patch.object(util, 'subp', self.subp): - cc_apt_configure.handle("notimportant", cfg, mycloud, - LOG, None) + with mock.patch.object(cc_apt_configure, 'get_release', + return_value='fakerelease'): + cc_apt_configure.handle("notimportant", cfg, mycloud, + LOG, None) mockwrite.assert_called_once_with( '/etc/apt/sources.list', -- cgit v1.2.3 From 12cf5ce64f0252a67c6a9dcbd9d3c93a1a07ccad Mon Sep 17 00:00:00 2001 From: Christian Ehrhardt Date: Wed, 25 May 2016 13:18:50 +0200 Subject: fix inline doc of test_apt_src_longkeyid_real --- tests/unittests/test_handler/test_handler_apt_source.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unittests/test_handler/test_handler_apt_source.py b/tests/unittests/test_handler/test_handler_apt_source.py index 1b2bc58f..753e86a8 100644 --- a/tests/unittests/test_handler/test_handler_apt_source.py +++ b/tests/unittests/test_handler/test_handler_apt_source.py @@ -454,7 +454,7 @@ class TestAptSourceConfig(TestCase): self.assertFalse(os.path.isfile(self.aptlistfile)) def test_apt_src_longkeyid_real(self): - """ test_apt_src_keyid_real + """ test_apt_src_longkeyid_real Test specification of a long key fingerprint without source incl up to addition of the key (nothing but add_key_raw mocked) """ -- cgit v1.2.3 From aa236033b159b691f5ec31885750a8167c63b2a1 Mon Sep 17 00:00:00 2001 From: Christian Ehrhardt Date: Mon, 30 May 2016 12:27:40 +0200 Subject: drop errorlist from convert_to_new_format --- cloudinit/config/cc_apt_configure.py | 12 +++--------- tests/unittests/test_handler/test_handler_apt_source.py | 6 ++---- 2 files changed, 5 insertions(+), 13 deletions(-) diff --git a/cloudinit/config/cc_apt_configure.py b/cloudinit/config/cc_apt_configure.py index ffbf7513..2dd48844 100644 --- a/cloudinit/config/cc_apt_configure.py +++ b/cloudinit/config/cc_apt_configure.py @@ -198,24 +198,18 @@ def add_key(ent): add_key_raw(ent['key']) -def convert_to_new_format(srclist, errorlist): +def convert_to_new_format(srclist): """ convert_to_new_format convert the old list based format to the new dict based one """ srcdict = {} if isinstance(srclist, list): - fnfallbackused = None for srcent in srclist: if 'filename' not in srcent: # file collides for multiple !filename cases for compatibility # yet we need them all processed, so not same dictionary key srcent['filename'] = "cloud_config_sources.list" key = util.rand_dict_key(srcdict, "cloud_config_sources.list") - if fnfallbackused is not None: - errorlist.append(["multiple apt_source entries without", - "filename will conflict: %s vs %s" % - (srcent, fnfallbackused)]) - fnfallbackused = srcent else: # all with filename use that as key (matching new format) key = srcent['filename'] @@ -223,7 +217,7 @@ def convert_to_new_format(srclist, errorlist): elif isinstance(srclist, dict): srcdict = srclist else: - errorlist.append(["srclist", "unknown apt_sources format"]) + raise ValueError("unknown apt_sources format") return srcdict @@ -242,7 +236,7 @@ def add_sources(srclist, template_params=None, aa_repo_match=None): return False errorlist = [] - srcdict = convert_to_new_format(srclist, errorlist) + srcdict = convert_to_new_format(srclist) for filename in srcdict: ent = srcdict[filename] diff --git a/tests/unittests/test_handler/test_handler_apt_source.py b/tests/unittests/test_handler/test_handler_apt_source.py index 753e86a8..4536c5b2 100644 --- a/tests/unittests/test_handler/test_handler_apt_source.py +++ b/tests/unittests/test_handler/test_handler_apt_source.py @@ -529,7 +529,6 @@ class TestAptSourceConfig(TestCase): 'filename': self.aptlistfile2} cfg3 = {'source': 'deb $MIRROR $RELEASE universe', 'filename': self.aptlistfile3} - errorlist = [] checkcfg = {self.aptlistfile: {'filename': self.aptlistfile, 'source': 'deb $MIRROR $RELEASE ' 'multiverse'}, @@ -539,11 +538,10 @@ class TestAptSourceConfig(TestCase): 'source': 'deb $MIRROR $RELEASE ' 'universe'}} - newcfg = cc_apt_configure.convert_to_new_format([cfg1, cfg2, cfg3], - errorlist) + newcfg = cc_apt_configure.convert_to_new_format([cfg1, cfg2, cfg3]) self.assertEqual(newcfg, checkcfg) - newcfg2 = cc_apt_configure.convert_to_new_format(newcfg, errorlist) + newcfg2 = cc_apt_configure.convert_to_new_format(newcfg) self.assertEqual(newcfg2, checkcfg) -- cgit v1.2.3 From c7b9b2b04490fda948631eecdba1bb678f0e4db9 Mon Sep 17 00:00:00 2001 From: Christian Ehrhardt Date: Mon, 30 May 2016 12:48:00 +0200 Subject: add test for wrong apt_source format --- tests/unittests/test_handler/test_handler_apt_source.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/unittests/test_handler/test_handler_apt_source.py b/tests/unittests/test_handler/test_handler_apt_source.py index 4536c5b2..fe2ffae5 100644 --- a/tests/unittests/test_handler/test_handler_apt_source.py +++ b/tests/unittests/test_handler/test_handler_apt_source.py @@ -544,5 +544,8 @@ class TestAptSourceConfig(TestCase): newcfg2 = cc_apt_configure.convert_to_new_format(newcfg) self.assertEqual(newcfg2, checkcfg) + with self.assertRaises(ValueError): + cc_apt_configure.convert_to_new_format(5) + # vi: ts=4 expandtab -- cgit v1.2.3 From 964ec3ae45c27cf55e0c1349138294ff11debab8 Mon Sep 17 00:00:00 2001 From: Christian Ehrhardt Date: Mon, 30 May 2016 12:53:29 +0200 Subject: improve wording in the examples --- doc/examples/cloud-config.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/examples/cloud-config.txt b/doc/examples/cloud-config.txt index df59ff57..62b297bc 100644 --- a/doc/examples/cloud-config.txt +++ b/doc/examples/cloud-config.txt @@ -220,7 +220,7 @@ apt_sources: -----END PGP PUBLIC KEY BLOCK----- # Custom gpg key: - # * As the keyid also a key can be specified withut a related source + # * As with keyid, a key may also be specified without a related source. # * all other facts mentioned above still apply # since this doesn't generate a source.list file the filename key is ignored ignored4: -- cgit v1.2.3 From 3c85315373306729443ef79fd8e54af46a7bc849 Mon Sep 17 00:00:00 2001 From: Christian Ehrhardt Date: Mon, 30 May 2016 13:07:22 +0200 Subject: fix EXPORT_GPG_KEYID for existing keys This was broken for keys already existing in the local keyring. There instead of the keycontent it reported the header like: pub 1024R/03683F77 2009-10-27 uid Launchpad PPA for Scott Moser --- cloudinit/config/cc_apt_configure.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cloudinit/config/cc_apt_configure.py b/cloudinit/config/cc_apt_configure.py index 2dd48844..d603f417 100644 --- a/cloudinit/config/cc_apt_configure.py +++ b/cloudinit/config/cc_apt_configure.py @@ -40,7 +40,7 @@ EXPORT_GPG_KEYID = """ k=${1} ks=${2}; exec 2>/dev/null [ -n "$k" ] || exit 1; - armour=$(gpg --list-keys --armour "${k}") + armour=$(gpg --export --armour "${k}") if [ -z "${armour}" ]; then gpg --keyserver ${ks} --recv "${k}" >/dev/null && armour=$(gpg --export --armour "${k}") && -- cgit v1.2.3 From 80931f7008971c9a7705c054fabc29fec7a133e2 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Fri, 3 Jun 2016 15:27:32 -0400 Subject: fix tox -e flake8 --- cloudinit/config/cc_apt_configure.py | 4 +- .../test_handler_apt_configure_sources_list.py | 17 +++--- .../test_handler/test_handler_apt_source.py | 66 +++++++++++----------- 3 files changed, 43 insertions(+), 44 deletions(-) diff --git a/cloudinit/config/cc_apt_configure.py b/cloudinit/config/cc_apt_configure.py index d603f417..7a9777c0 100644 --- a/cloudinit/config/cc_apt_configure.py +++ b/cloudinit/config/cc_apt_configure.py @@ -199,8 +199,8 @@ def add_key(ent): def convert_to_new_format(srclist): - """ convert_to_new_format - convert the old list based format to the new dict based one + """convert_to_new_format + convert the old list based format to the new dict based one """ srcdict = {} if isinstance(srclist, list): diff --git a/tests/unittests/test_handler/test_handler_apt_configure_sources_list.py b/tests/unittests/test_handler/test_handler_apt_configure_sources_list.py index 5cf386f8..5d0417a2 100644 --- a/tests/unittests/test_handler/test_handler_apt_configure_sources_list.py +++ b/tests/unittests/test_handler/test_handler_apt_configure_sources_list.py @@ -3,7 +3,6 @@ Test templating of sources list """ import logging import os -import re import shutil import tempfile @@ -62,14 +61,14 @@ deb-src http://archive.ubuntu.com/ubuntu/ fakerelease main restricted def load_tfile_or_url(*args, **kwargs): - """ load_tfile_or_url + """load_tfile_or_url load file and return content after decoding """ return util.decode_binary(util.read_file_or_url(*args, **kwargs).contents) class TestAptSourceConfigSourceList(t_help.FilesystemMockingTestCase): - """ TestAptSourceConfigSourceList + """TestAptSourceConfigSourceList Main Class to test sources list rendering """ def setUp(self): @@ -89,7 +88,7 @@ class TestAptSourceConfigSourceList(t_help.FilesystemMockingTestCase): return cloud.Cloud(myds, paths, {}, mydist, None) def apt_source_list(self, distro, mirror, mirrorcheck=None): - """ apt_source_list + """apt_source_list Test rendering of a source.list from template for a given distro """ if mirrorcheck is None: @@ -115,19 +114,19 @@ class TestAptSourceConfigSourceList(t_help.FilesystemMockingTestCase): {'codename': '', 'primary': mirrorcheck, 'mirror': mirrorcheck}) def test_apt_source_list_debian(self): - """ test_apt_source_list_debian + """test_apt_source_list_debian Test rendering of a source.list from template for debian """ self.apt_source_list('debian', 'http://httpredir.debian.org/debian') def test_apt_source_list_ubuntu(self): - """ test_apt_source_list_ubuntu + """test_apt_source_list_ubuntu Test rendering of a source.list from template for ubuntu """ self.apt_source_list('ubuntu', 'http://archive.ubuntu.com/ubuntu/') def test_apt_srcl_debian_mirrorfail(self): - """ test_apt_source_list_debian_mirrorfail + """test_apt_source_list_debian_mirrorfail Test rendering of a source.list from template for debian """ self.apt_source_list('debian', ['http://does.not.exist', @@ -135,7 +134,7 @@ class TestAptSourceConfigSourceList(t_help.FilesystemMockingTestCase): 'http://httpredir.debian.org/debian') def test_apt_srcl_ubuntu_mirrorfail(self): - """ test_apt_source_list_ubuntu_mirrorfail + """test_apt_source_list_ubuntu_mirrorfail Test rendering of a source.list from template for ubuntu """ self.apt_source_list('ubuntu', ['http://does.not.exist', @@ -143,7 +142,7 @@ class TestAptSourceConfigSourceList(t_help.FilesystemMockingTestCase): 'http://archive.ubuntu.com/ubuntu/') def test_apt_srcl_custom(self): - """ test_apt_srcl_custom + """test_apt_srcl_custom Test rendering from a custom source.list template """ cfg = util.load_yaml(YAML_TEXT_CUSTOM_SL) diff --git a/tests/unittests/test_handler/test_handler_apt_source.py b/tests/unittests/test_handler/test_handler_apt_source.py index fe2ffae5..4dbe69f0 100644 --- a/tests/unittests/test_handler/test_handler_apt_source.py +++ b/tests/unittests/test_handler/test_handler_apt_source.py @@ -33,14 +33,14 @@ S0ORP6HXET3+jC8BMG4tBWCTK/XEZw== def load_tfile_or_url(*args, **kwargs): - """ load_tfile_or_url + """load_tfile_or_url load file and return content after decoding """ return util.decode_binary(util.read_file_or_url(*args, **kwargs).contents) class TestAptSourceConfig(TestCase): - """ TestAptSourceConfig + """TestAptSourceConfig Main Class to test apt_source configs """ def setUp(self): @@ -57,7 +57,7 @@ class TestAptSourceConfig(TestCase): @staticmethod def _get_default_params(): - """ get_default_params + """get_default_params Get the most basic default mrror and release info to be used in tests """ params = {} @@ -66,7 +66,7 @@ class TestAptSourceConfig(TestCase): return params def myjoin(self, *args, **kwargs): - """ myjoin - redir into writable tmpdir""" + """myjoin - redir into writable tmpdir""" if (args[0] == "/etc/apt/sources.list.d/" and args[1] == "cloud_config_sources.list" and len(args) == 2): @@ -75,7 +75,7 @@ class TestAptSourceConfig(TestCase): return self.join(*args, **kwargs) def apt_src_basic(self, filename, cfg): - """ apt_src_basic + """apt_src_basic Test Fix deb source string, has to overwrite mirror conf in params """ params = self._get_default_params() @@ -92,7 +92,7 @@ class TestAptSourceConfig(TestCase): contents, flags=re.IGNORECASE)) def test_apt_src_basic(self): - """ test_apt_src_basic + """test_apt_src_basic Test Fix deb source string, has to overwrite mirror conf in params. Test with a filename provided in config. """ @@ -103,7 +103,7 @@ class TestAptSourceConfig(TestCase): self.apt_src_basic(self.aptlistfile, [cfg]) def test_apt_src_basic_dict(self): - """ test_apt_src_basic_dict + """test_apt_src_basic_dict Test Fix deb source string, has to overwrite mirror conf in params. Test with a filename provided in config. Provided in a dictionary with filename being the key (new format) @@ -115,7 +115,7 @@ class TestAptSourceConfig(TestCase): self.apt_src_basic(self.aptlistfile, cfg) def apt_src_basic_tri(self, cfg): - """ apt_src_basic_tri + """apt_src_basic_tri Test Fix three deb source string, has to overwrite mirror conf in params. Test with filenames provided in config. generic part to check three files with different content @@ -137,7 +137,7 @@ class TestAptSourceConfig(TestCase): contents, flags=re.IGNORECASE)) def test_apt_src_basic_tri(self): - """ test_apt_src_basic_tri + """test_apt_src_basic_tri Test Fix three deb source string, has to overwrite mirror conf in params. Test with filenames provided in config. """ @@ -156,7 +156,7 @@ class TestAptSourceConfig(TestCase): self.apt_src_basic_tri([cfg1, cfg2, cfg3]) def test_apt_src_basic_dict_tri(self): - """ test_apt_src_basic_dict_tri + """test_apt_src_basic_dict_tri Test Fix three deb source string, has to overwrite mirror conf in params. Test with filenames provided in config. Provided in a dictionary with filename being the key (new format) @@ -176,7 +176,7 @@ class TestAptSourceConfig(TestCase): self.apt_src_basic_tri(cfg) def test_apt_src_basic_nofn(self): - """ test_apt_src_basic_nofn + """test_apt_src_basic_nofn Test Fix deb source string, has to overwrite mirror conf in params. Test without a filename provided in config and test for known fallback. """ @@ -187,7 +187,7 @@ class TestAptSourceConfig(TestCase): self.apt_src_basic(self.fallbackfn, [cfg]) def apt_src_replacement(self, filename, cfg): - """ apt_src_replace + """apt_src_replace Test Autoreplacement of MIRROR and RELEASE in source specs """ params = self._get_default_params() @@ -202,7 +202,7 @@ class TestAptSourceConfig(TestCase): contents, flags=re.IGNORECASE)) def test_apt_src_replace(self): - """ test_apt_src_replace + """test_apt_src_replace Test Autoreplacement of MIRROR and RELEASE in source specs with Filename being set """ @@ -211,7 +211,7 @@ class TestAptSourceConfig(TestCase): self.apt_src_replacement(self.aptlistfile, [cfg]) def apt_src_replace_tri(self, cfg): - """ apt_src_replace_tri + """apt_src_replace_tri Test three autoreplacements of MIRROR and RELEASE in source specs with generic part """ @@ -231,7 +231,7 @@ class TestAptSourceConfig(TestCase): contents, flags=re.IGNORECASE)) def test_apt_src_replace_tri(self): - """ test_apt_src_replace_tri + """test_apt_src_replace_tri Test three autoreplacements of MIRROR and RELEASE in source specs with Filename being set """ @@ -244,7 +244,7 @@ class TestAptSourceConfig(TestCase): self.apt_src_replace_tri([cfg1, cfg2, cfg3]) def test_apt_src_replace_dict_tri(self): - """ test_apt_src_replace_dict_tri + """test_apt_src_replace_dict_tri Test three autoreplacements of MIRROR and RELEASE in source specs with Filename being set Provided in a dictionary with filename being the key (new format) @@ -252,13 +252,13 @@ class TestAptSourceConfig(TestCase): filenames to be overwritten inside the directory entry. """ cfg = {self.aptlistfile: {'source': 'deb $MIRROR $RELEASE multiverse'}, - 'notused': {'source': 'deb $MIRROR $RELEASE main', - 'filename': self.aptlistfile2}, + 'notused': {'source': 'deb $MIRROR $RELEASE main', + 'filename': self.aptlistfile2}, self.aptlistfile3: {'source': 'deb $MIRROR $RELEASE universe'}} self.apt_src_replace_tri(cfg) def test_apt_src_replace_nofn(self): - """ test_apt_src_replace_nofn + """test_apt_src_replace_nofn Test Autoreplacement of MIRROR and RELEASE in source specs with No filename being set """ @@ -267,7 +267,7 @@ class TestAptSourceConfig(TestCase): self.apt_src_replacement(self.fallbackfn, [cfg]) def apt_src_keyid(self, filename, cfg, keynum): - """ apt_src_keyid + """apt_src_keyid Test specification of a source + keyid """ params = self._get_default_params() @@ -293,7 +293,7 @@ class TestAptSourceConfig(TestCase): contents, flags=re.IGNORECASE)) def test_apt_src_keyid(self): - """ test_apt_src_keyid + """test_apt_src_keyid Test specification of a source + keyid with filename being set """ cfg = {'source': ('deb ' @@ -305,7 +305,7 @@ class TestAptSourceConfig(TestCase): self.apt_src_keyid(self.aptlistfile, [cfg], 1) def test_apt_src_keyid_tri(self): - """ test_apt_src_keyid_tri + """test_apt_src_keyid_tri Test specification of a source + keyid with filename being set Setting three of such, check for content and keys """ @@ -345,7 +345,7 @@ class TestAptSourceConfig(TestCase): contents, flags=re.IGNORECASE)) def test_apt_src_keyid_nofn(self): - """ test_apt_src_keyid_nofn + """test_apt_src_keyid_nofn Test specification of a source + keyid without filename being set """ cfg = {'source': ('deb ' @@ -357,7 +357,7 @@ class TestAptSourceConfig(TestCase): self.apt_src_keyid(self.fallbackfn, [cfg], 1) def apt_src_key(self, filename, cfg): - """ apt_src_key + """apt_src_key Test specification of a source + key """ params = self._get_default_params() @@ -378,7 +378,7 @@ class TestAptSourceConfig(TestCase): contents, flags=re.IGNORECASE)) def test_apt_src_key(self): - """ test_apt_src_key + """test_apt_src_key Test specification of a source + key with filename being set """ cfg = {'source': ('deb ' @@ -390,7 +390,7 @@ class TestAptSourceConfig(TestCase): self.apt_src_key(self.aptlistfile, cfg) def test_apt_src_key_nofn(self): - """ test_apt_src_key_nofn + """test_apt_src_key_nofn Test specification of a source + key without filename being set """ cfg = {'source': ('deb ' @@ -402,7 +402,7 @@ class TestAptSourceConfig(TestCase): self.apt_src_key(self.fallbackfn, cfg) def test_apt_src_keyonly(self): - """ test_apt_src_keyonly + """test_apt_src_keyonly Test specification key without source """ params = self._get_default_params() @@ -419,7 +419,7 @@ class TestAptSourceConfig(TestCase): self.assertFalse(os.path.isfile(self.aptlistfile)) def test_apt_src_keyidonly(self): - """ test_apt_src_keyidonly + """test_apt_src_keyidonly Test specification of a keyid without source """ params = self._get_default_params() @@ -436,7 +436,7 @@ class TestAptSourceConfig(TestCase): self.assertFalse(os.path.isfile(self.aptlistfile)) def test_apt_src_keyid_real(self): - """ test_apt_src_keyid_real + """test_apt_src_keyid_real Test specification of a keyid without source incl up to addition of the key (nothing but add_key_raw mocked) """ @@ -454,7 +454,7 @@ class TestAptSourceConfig(TestCase): self.assertFalse(os.path.isfile(self.aptlistfile)) def test_apt_src_longkeyid_real(self): - """ test_apt_src_longkeyid_real + """test_apt_src_longkeyid_real Test specification of a long key fingerprint without source incl up to addition of the key (nothing but add_key_raw mocked) """ @@ -472,7 +472,7 @@ class TestAptSourceConfig(TestCase): self.assertFalse(os.path.isfile(self.aptlistfile)) def test_apt_src_ppa(self): - """ test_apt_src_ppa + """test_apt_src_ppa Test specification of a ppa """ params = self._get_default_params() @@ -491,7 +491,7 @@ class TestAptSourceConfig(TestCase): self.assertFalse(os.path.isfile(self.aptlistfile)) def test_apt_src_ppa_tri(self): - """ test_apt_src_ppa_tri + """test_apt_src_ppa_tri Test specification of a ppa """ params = self._get_default_params() @@ -519,7 +519,7 @@ class TestAptSourceConfig(TestCase): self.assertFalse(os.path.isfile(self.aptlistfile3)) def test_convert_to_new_format(self): - """ test_convert_to_new_format + """test_convert_to_new_format Test the conversion of old to new format And the noop conversion of new to new format as well """ -- cgit v1.2.3