From cca640d332eb8a6b068033a28b0b319873c7fbf6 Mon Sep 17 00:00:00 2001
From: Christian Ehrhardt <christian.ehrhardt@canonical.com>
Date: Thu, 12 May 2016 09:21:50 +0200
Subject: allow to add keys without specifying a source

---
 cloudinit/config/cc_apt_configure.py | 42 ++++++++++++++++++++++--------------
 1 file changed, 26 insertions(+), 16 deletions(-)

(limited to 'cloudinit/config')

diff --git a/cloudinit/config/cc_apt_configure.py b/cloudinit/config/cc_apt_configure.py
index 702977cb..1d3eddff 100644
--- a/cloudinit/config/cc_apt_configure.py
+++ b/cloudinit/config/cc_apt_configure.py
@@ -164,6 +164,29 @@ def generate_sources_list(codename, mirrors, cloud, log):
     templater.render_to_file(template_fn, '/etc/apt/sources.list', params)
 
 
+def add_key(ent, errorlist):
+    """
+    add key to the system as defiend in entry (if any)
+    suppords raw keys or keyid's
+    The latter will as a first step fetched to get the raw key
+    """
+    if ('keyid' in ent and 'key' not in ent):
+        keyserver = "keyserver.ubuntu.com"
+        if 'keyserver' in ent:
+            keyserver = ent['keyserver']
+        try:
+            ent['key'] = getkeybyid(ent['keyid'], keyserver)
+        except:
+            errorlist.append([ent, "failed to get key from %s" % keyserver])
+            return
+
+    if 'key' in ent:
+        try:
+            util.subp(('apt-key', 'add', '-'), ent['key'])
+        except:
+            errorlist.append([ent, "failed add key"])
+
+
 def add_sources(srclist, template_params=None, aa_repo_match=None):
     """
     add entries in /etc/apt/sources.list.d for each abbreviated
@@ -179,6 +202,9 @@ def add_sources(srclist, template_params=None, aa_repo_match=None):
 
     errorlist = []
     for ent in srclist:
+        # keys can be added without specifying a source
+        add_key(ent, errorlist)
+
         if 'source' not in ent:
             errorlist.append(["", "missing source"])
             continue
@@ -201,22 +227,6 @@ def add_sources(srclist, template_params=None, aa_repo_match=None):
             ent['filename'] = os.path.join("/etc/apt/sources.list.d/",
                                            ent['filename'])
 
-        if ('keyid' in ent and 'key' not in ent):
-            ks = "keyserver.ubuntu.com"
-            if 'keyserver' in ent:
-                ks = ent['keyserver']
-            try:
-                ent['key'] = getkeybyid(ent['keyid'], ks)
-            except:
-                errorlist.append([source, "failed to get key from %s" % ks])
-                continue
-
-        if 'key' in ent:
-            try:
-                util.subp(('apt-key', 'add', '-'), ent['key'])
-            except:
-                errorlist.append([source, "failed add key"])
-
         try:
             contents = "%s\n" % (source)
             util.write_file(ent['filename'], contents, omode="ab")
-- 
cgit v1.2.3


From 6fc583a4bcb49f7dbecdac095bc63e90dd6edaf3 Mon Sep 17 00:00:00 2001
From: Christian Ehrhardt <christian.ehrhardt@canonical.com>
Date: Thu, 12 May 2016 13:43:59 +0200
Subject: test mirror list with failing mirror

---
 cloudinit/config/cc_apt_configure.py               |  1 +
 .../test_handler_apt_configure_sources_list.py     | 23 ++++++++++++++++++----
 2 files changed, 20 insertions(+), 4 deletions(-)

(limited to 'cloudinit/config')

diff --git a/cloudinit/config/cc_apt_configure.py b/cloudinit/config/cc_apt_configure.py
index 1d3eddff..ccbdcbc1 100644
--- a/cloudinit/config/cc_apt_configure.py
+++ b/cloudinit/config/cc_apt_configure.py
@@ -57,6 +57,7 @@ def handle(name, cfg, cloud, log, _args):
 
     release = get_release()
     mirrors = find_apt_mirror_info(cloud, cfg)
+    print(mirrors)
     if not mirrors or "primary" not in mirrors:
         log.debug(("Skipping module named %s,"
                    " no package 'mirror' located"), name)
diff --git a/tests/unittests/test_handler/test_handler_apt_configure_sources_list.py b/tests/unittests/test_handler/test_handler_apt_configure_sources_list.py
index aff272a3..bac2da24 100644
--- a/tests/unittests/test_handler/test_handler_apt_configure_sources_list.py
+++ b/tests/unittests/test_handler/test_handler_apt_configure_sources_list.py
@@ -58,14 +58,20 @@ class TestAptSourceConfigSourceList(t_help.FilesystemMockingTestCase):
 # TODO Later - custom template filename
 # TODO Later - custom template raw
 
-    def apt_source_list(self, distro, mirror):
+    def apt_source_list(self, distro, mirror, mirrorcheck=None):
         """ apt_source_list
         Test rendering of a source.list from template for a given distro
         """
         self.patchOS(self.new_root)
         self.patchUtils(self.new_root)
 
-        cfg = {'apt_mirror': mirror}
+        if mirrorcheck is None:
+            mirrorcheck = mirror
+
+        if isinstance(mirror, list):
+            cfg = {'apt_mirror_search': mirror}
+        else:
+            cfg = {'apt_mirror': mirror}
         mycloud = self._get_cloud(distro)
 
         with mock.patch.object(templater, 'render_to_file') as mocktmpl:
@@ -81,9 +87,9 @@ class TestAptSourceConfigSourceList(t_help.FilesystemMockingTestCase):
                                          '/etc/apt/sources.list',
                                          {'codename': '',
                                           'primary':
-                                          mirror,
+                                          mirrorcheck,
                                           'mirror':
-                                          mirror})
+                                          mirrorcheck})
 
 
     def test_apt_source_list_ubuntu(self):
@@ -100,4 +106,13 @@ class TestAptSourceConfigSourceList(t_help.FilesystemMockingTestCase):
         self.apt_source_list('debian', 'ftp.us.debian.org')
 
 
+    def test_apt_srcl_ubuntu_mirrorfail(self):
+        """ test_apt_source_list_ubuntu_mirrorfail
+        Test rendering of a source.list from template for ubuntu
+        """
+        self.apt_source_list('ubuntu', ['http://does.not.exist',
+                                        'http://archive.ubuntu.com/ubuntu/'],
+                             'http://archive.ubuntu.com/ubuntu/')
+
+
 # vi: ts=4 expandtab
-- 
cgit v1.2.3


From 18864f8e7331da359399decb1b080e36fa343f5a Mon Sep 17 00:00:00 2001
From: Christian Ehrhardt <christian.ehrhardt@canonical.com>
Date: Thu, 12 May 2016 13:45:15 +0200
Subject: remove missed test print

---
 cloudinit/config/cc_apt_configure.py                             | 1 -
 .../test_handler/test_handler_apt_configure_sources_list.py      | 9 +++++++++
 2 files changed, 9 insertions(+), 1 deletion(-)

(limited to 'cloudinit/config')

diff --git a/cloudinit/config/cc_apt_configure.py b/cloudinit/config/cc_apt_configure.py
index ccbdcbc1..1d3eddff 100644
--- a/cloudinit/config/cc_apt_configure.py
+++ b/cloudinit/config/cc_apt_configure.py
@@ -57,7 +57,6 @@ def handle(name, cfg, cloud, log, _args):
 
     release = get_release()
     mirrors = find_apt_mirror_info(cloud, cfg)
-    print(mirrors)
     if not mirrors or "primary" not in mirrors:
         log.debug(("Skipping module named %s,"
                    " no package 'mirror' located"), name)
diff --git a/tests/unittests/test_handler/test_handler_apt_configure_sources_list.py b/tests/unittests/test_handler/test_handler_apt_configure_sources_list.py
index d48167c9..b8fe03ae 100644
--- a/tests/unittests/test_handler/test_handler_apt_configure_sources_list.py
+++ b/tests/unittests/test_handler/test_handler_apt_configure_sources_list.py
@@ -106,6 +106,15 @@ class TestAptSourceConfigSourceList(t_help.FilesystemMockingTestCase):
         self.apt_source_list('ubuntu', 'http://archive.ubuntu.com/ubuntu/')
 
 
+    def test_apt_srcl_debian_mirrorfail(self):
+        """ test_apt_source_list_debian_mirrorfail
+        Test rendering of a source.list from template for debian
+        """
+        self.apt_source_list('debian', ['http://does.not.exist',
+                                        'ftp.us.debian.org'],
+                             'ftp.us.debian.org')
+
+
     def test_apt_srcl_ubuntu_mirrorfail(self):
         """ test_apt_source_list_ubuntu_mirrorfail
         Test rendering of a source.list from template for ubuntu
-- 
cgit v1.2.3


From 0d8bf6f2c1464b5f4dab735841a50d02016d2caf Mon Sep 17 00:00:00 2001
From: Christian Ehrhardt <christian.ehrhardt@canonical.com>
Date: Thu, 12 May 2016 15:51:21 +0200
Subject: add feature to allow a custom template for source list

---
 cloudinit/config/cc_apt_configure.py | 18 +++++++++++++-----
 cloudinit/templater.py               |  5 +++++
 2 files changed, 18 insertions(+), 5 deletions(-)

(limited to 'cloudinit/config')

diff --git a/cloudinit/config/cc_apt_configure.py b/cloudinit/config/cc_apt_configure.py
index 1d3eddff..2ab5e86c 100644
--- a/cloudinit/config/cc_apt_configure.py
+++ b/cloudinit/config/cc_apt_configure.py
@@ -21,6 +21,7 @@
 import glob
 import os
 import re
+import tempfile
 
 from cloudinit import templater
 from cloudinit import util
@@ -70,7 +71,7 @@ def handle(name, cfg, cloud, log, _args):
 
     if not util.get_cfg_option_bool(cfg,
                                     'apt_preserve_sources_list', False):
-        generate_sources_list(release, mirrors, cloud, log)
+        generate_sources_list(cfg, release, mirrors, cloud, log)
         old_mirrors = cfg.get('apt_old_mirrors',
                               {"primary": "archive.ubuntu.com/ubuntu",
                                "security": "security.ubuntu.com/ubuntu"})
@@ -149,7 +150,17 @@ def get_release():
     return stdout.strip()
 
 
-def generate_sources_list(codename, mirrors, cloud, log):
+def generate_sources_list(cfg, codename, mirrors, cloud, log):
+    params = {'codename': codename}
+    for k in mirrors:
+        params[k] = mirrors[k]
+
+    custtmpl = cfg.get('apt_custom_sources_list', None)
+    if custtmpl is not None:
+        templater.render_string_to_file(custtmpl,
+                                        '/etc/apt/sources.list', params)
+        return
+
     template_fn = cloud.get_template_filename('sources.list.%s' %
                                               (cloud.distro.name))
     if not template_fn:
@@ -158,9 +169,6 @@ def generate_sources_list(codename, mirrors, cloud, log):
             log.warn("No template found, not rendering /etc/apt/sources.list")
             return
 
-    params = {'codename': codename}
-    for k in mirrors:
-        params[k] = mirrors[k]
     templater.render_to_file(template_fn, '/etc/apt/sources.list', params)
 
 
diff --git a/cloudinit/templater.py b/cloudinit/templater.py
index a9231482..8a6ad417 100644
--- a/cloudinit/templater.py
+++ b/cloudinit/templater.py
@@ -142,6 +142,11 @@ def render_to_file(fn, outfn, params, mode=0o644):
     util.write_file(outfn, contents, mode=mode)
 
 
+def render_string_to_file(content, outfn, params, mode=0o644):
+    contents = render_string(content, params)
+    util.write_file(outfn, contents, mode=mode)
+
+
 def render_string(content, params):
     if not params:
         params = {}
-- 
cgit v1.2.3


From 9c6b5f54ad5b83131de6d997930bd9f4031e6a83 Mon Sep 17 00:00:00 2001
From: Christian Ehrhardt <christian.ehrhardt@canonical.com>
Date: Thu, 12 May 2016 19:50:01 +0200
Subject: move errorlist.append out of add_key

---
 cloudinit/config/cc_apt_configure.py | 12 +++++++-----
 1 file changed, 7 insertions(+), 5 deletions(-)

(limited to 'cloudinit/config')

diff --git a/cloudinit/config/cc_apt_configure.py b/cloudinit/config/cc_apt_configure.py
index 2ab5e86c..91d02815 100644
--- a/cloudinit/config/cc_apt_configure.py
+++ b/cloudinit/config/cc_apt_configure.py
@@ -172,7 +172,7 @@ def generate_sources_list(cfg, codename, mirrors, cloud, log):
     templater.render_to_file(template_fn, '/etc/apt/sources.list', params)
 
 
-def add_key(ent, errorlist):
+def add_key(ent):
     """
     add key to the system as defiend in entry (if any)
     suppords raw keys or keyid's
@@ -185,14 +185,13 @@ def add_key(ent, errorlist):
         try:
             ent['key'] = getkeybyid(ent['keyid'], keyserver)
         except:
-            errorlist.append([ent, "failed to get key from %s" % keyserver])
-            return
+            raise Exception('failed to get key from %s' % keyserver)
 
     if 'key' in ent:
         try:
             util.subp(('apt-key', 'add', '-'), ent['key'])
         except:
-            errorlist.append([ent, "failed add key"])
+            raise Exception('failed add key')
 
 
 def add_sources(srclist, template_params=None, aa_repo_match=None):
@@ -211,7 +210,10 @@ def add_sources(srclist, template_params=None, aa_repo_match=None):
     errorlist = []
     for ent in srclist:
         # keys can be added without specifying a source
-        add_key(ent, errorlist)
+        try:
+            add_key(ent)
+        except Exception as detail:
+            errorlist.append([ent, detail])
 
         if 'source' not in ent:
             errorlist.append(["", "missing source"])
-- 
cgit v1.2.3


From e55ccfa5670e16aa7431a193d0838aa7d04db4d5 Mon Sep 17 00:00:00 2001
From: Christian Ehrhardt <christian.ehrhardt@canonical.com>
Date: Thu, 12 May 2016 19:50:34 +0200
Subject: remove Unnecessary parens in add_key

---
 cloudinit/config/cc_apt_configure.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'cloudinit/config')

diff --git a/cloudinit/config/cc_apt_configure.py b/cloudinit/config/cc_apt_configure.py
index 91d02815..492c3c00 100644
--- a/cloudinit/config/cc_apt_configure.py
+++ b/cloudinit/config/cc_apt_configure.py
@@ -178,7 +178,7 @@ def add_key(ent):
     suppords raw keys or keyid's
     The latter will as a first step fetched to get the raw key
     """
-    if ('keyid' in ent and 'key' not in ent):
+    if 'keyid' in ent and 'key' not in ent:
         keyserver = "keyserver.ubuntu.com"
         if 'keyserver' in ent:
             keyserver = ent['keyserver']
-- 
cgit v1.2.3


From 53834934e4c520b2fb8b5acffca641213ddd688a Mon Sep 17 00:00:00 2001
From: Christian Ehrhardt <christian.ehrhardt@canonical.com>
Date: Thu, 12 May 2016 20:29:33 +0200
Subject: fix EXPORT_GPG_KEYID for long key fingerprints

---
 cloudinit/config/cc_apt_configure.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'cloudinit/config')

diff --git a/cloudinit/config/cc_apt_configure.py b/cloudinit/config/cc_apt_configure.py
index 492c3c00..28f20939 100644
--- a/cloudinit/config/cc_apt_configure.py
+++ b/cloudinit/config/cc_apt_configure.py
@@ -43,7 +43,7 @@ EXPORT_GPG_KEYID = """
     [ -n "$k" ] || exit 1;
     armour=$(gpg --list-keys --armour "${k}")
     if [ -z "${armour}" ]; then
-       gpg --keyserver ${ks} --recv $k >/dev/null &&
+       gpg --keyserver ${ks} --recv "${k}" >/dev/null &&
           armour=$(gpg --export --armour "${k}") &&
           gpg --batch --yes --delete-keys "${k}"
     fi
-- 
cgit v1.2.3


From 2202494b72cae19cbf9d34a8f3176d7021becb13 Mon Sep 17 00:00:00 2001
From: Christian Ehrhardt <christian.ehrhardt@canonical.com>
Date: Thu, 12 May 2016 20:53:12 +0200
Subject: split add_key and add_key_raw fior better testability

---
 cloudinit/config/cc_apt_configure.py | 16 +++++++++++-----
 1 file changed, 11 insertions(+), 5 deletions(-)

(limited to 'cloudinit/config')

diff --git a/cloudinit/config/cc_apt_configure.py b/cloudinit/config/cc_apt_configure.py
index 28f20939..e7b8a9b3 100644
--- a/cloudinit/config/cc_apt_configure.py
+++ b/cloudinit/config/cc_apt_configure.py
@@ -171,10 +171,19 @@ def generate_sources_list(cfg, codename, mirrors, cloud, log):
 
     templater.render_to_file(template_fn, '/etc/apt/sources.list', params)
 
+def add_key_raw(key):
+    """
+    actual adding of a key as defined in key argument
+    to the system
+    """
+    try:
+        util.subp(('apt-key', 'add', '-'), key)
+    except:
+        raise Exception('failed add key')
 
 def add_key(ent):
     """
-    add key to the system as defiend in entry (if any)
+    add key to the system as defiend in ent (if any)
     suppords raw keys or keyid's
     The latter will as a first step fetched to get the raw key
     """
@@ -188,10 +197,7 @@ def add_key(ent):
             raise Exception('failed to get key from %s' % keyserver)
 
     if 'key' in ent:
-        try:
-            util.subp(('apt-key', 'add', '-'), ent['key'])
-        except:
-            raise Exception('failed add key')
+        add_key_raw(ent['key'])
 
 
 def add_sources(srclist, template_params=None, aa_repo_match=None):
-- 
cgit v1.2.3


From 454de24c7d457b980c91849b128efe4faee62032 Mon Sep 17 00:00:00 2001
From: Christian Ehrhardt <christian.ehrhardt@canonical.com>
Date: Thu, 12 May 2016 21:21:36 +0200
Subject: make pep8 happy with a few spaces

---
 cloudinit/config/cc_apt_configure.py                    | 2 ++
 tests/unittests/test_handler/test_handler_apt_source.py | 1 +
 2 files changed, 3 insertions(+)

(limited to 'cloudinit/config')

diff --git a/cloudinit/config/cc_apt_configure.py b/cloudinit/config/cc_apt_configure.py
index e7b8a9b3..e5a962ac 100644
--- a/cloudinit/config/cc_apt_configure.py
+++ b/cloudinit/config/cc_apt_configure.py
@@ -171,6 +171,7 @@ def generate_sources_list(cfg, codename, mirrors, cloud, log):
 
     templater.render_to_file(template_fn, '/etc/apt/sources.list', params)
 
+
 def add_key_raw(key):
     """
     actual adding of a key as defined in key argument
@@ -181,6 +182,7 @@ def add_key_raw(key):
     except:
         raise Exception('failed add key')
 
+
 def add_key(ent):
     """
     add key to the system as defiend in ent (if any)
diff --git a/tests/unittests/test_handler/test_handler_apt_source.py b/tests/unittests/test_handler/test_handler_apt_source.py
index 439bd038..e130392c 100644
--- a/tests/unittests/test_handler/test_handler_apt_source.py
+++ b/tests/unittests/test_handler/test_handler_apt_source.py
@@ -30,6 +30,7 @@ S0ORP6HXET3+jC8BMG4tBWCTK/XEZw==
 =ACB2
 -----END PGP PUBLIC KEY BLOCK-----"""
 
+
 def load_tfile_or_url(*args, **kwargs):
     """ load_tfile_or_url
     load file and return content after decoding
-- 
cgit v1.2.3


From ee239517c5342cbd62c9fdeaf735d78d6fd1fbb8 Mon Sep 17 00:00:00 2001
From: Christian Ehrhardt <christian.ehrhardt@canonical.com>
Date: Mon, 23 May 2016 11:59:14 +0200
Subject: support apt_sources to be a dictionary

key is the filename, and "old" input shall be handled as it was all the time.
For compatibility this will (continue to) overwrite the file of multiple
options that did not specify an output file (they all get the same default).
Yet it will process them all - as it always did - e.g. to add the keys of all
of them.

Any users of the new format won't have these issues, as they will always have
a key.
---
 cloudinit/config/cc_apt_configure.py | 34 +++++++++++++++++++++++++---------
 cloudinit/util.py                    |  8 ++++++++
 2 files changed, 33 insertions(+), 9 deletions(-)

(limited to 'cloudinit/config')

diff --git a/cloudinit/config/cc_apt_configure.py b/cloudinit/config/cc_apt_configure.py
index e5a962ac..a46ebb3e 100644
--- a/cloudinit/config/cc_apt_configure.py
+++ b/cloudinit/config/cc_apt_configure.py
@@ -215,8 +215,28 @@ def add_sources(srclist, template_params=None, aa_repo_match=None):
         def aa_repo_match(x):
             return False
 
+    # convert old list format to new dict based format
+    if isinstance(srclist, list):
+        srcdict = {}
+        for srcent in srclist:
+            if 'filename' not in srcent:
+                # file collides for multiple !filename cases for compatibility
+                # yet we need them all processed, so not same dictionary key
+                srcent['filename'] = "cloud_config_sources.list"
+                key = util.rand_dict_key(srcdict, "cloud_config_sources.list")
+            else:
+                # all with filename use that as key (matching new format)
+                key = srcent['filename']
+            srcdict[key] = srcent
+    else:
+        srcdict = srclist
+
     errorlist = []
-    for ent in srclist:
+    for filename in srcdict:
+        ent = srcdict[filename]
+        if 'filename' not in ent:
+            ent[filename] = filename
+
         # keys can be added without specifying a source
         try:
             add_key(ent)
@@ -226,10 +246,13 @@ def add_sources(srclist, template_params=None, aa_repo_match=None):
         if 'source' not in ent:
             errorlist.append(["", "missing source"])
             continue
-
         source = ent['source']
         source = templater.render_string(source, template_params)
 
+        if not ent['filename'].startswith("/"):
+            ent['filename'] = os.path.join("/etc/apt/sources.list.d/",
+                                           ent['filename'])
+
         if aa_repo_match(source):
             try:
                 util.subp(["add-apt-repository", source])
@@ -238,13 +261,6 @@ def add_sources(srclist, template_params=None, aa_repo_match=None):
                                   ("add-apt-repository failed. " + str(e))])
             continue
 
-        if 'filename' not in ent:
-            ent['filename'] = 'cloud_config_sources.list'
-
-        if not ent['filename'].startswith("/"):
-            ent['filename'] = os.path.join("/etc/apt/sources.list.d/",
-                                           ent['filename'])
-
         try:
             contents = "%s\n" % (source)
             util.write_file(ent['filename'], contents, omode="ab")
diff --git a/cloudinit/util.py b/cloudinit/util.py
index 0d21e11b..2931efbd 100644
--- a/cloudinit/util.py
+++ b/cloudinit/util.py
@@ -335,6 +335,14 @@ def rand_str(strlen=32, select_from=None):
         select_from = string.ascii_letters + string.digits
     return "".join([random.choice(select_from) for _x in range(0, strlen)])
 
+def rand_dict_key(dictionary, postfix=None):
+    if not postfix:
+        postfix = ""
+    while True:
+        newkey = rand_str(strlen=8) + "_" + postfix
+        if newkey not in dictionary:
+            break
+    return newkey
 
 def read_conf(fname):
     try:
-- 
cgit v1.2.3


From a33f8c09863381006f708a1e9d49997ed9f7befa Mon Sep 17 00:00:00 2001
From: Christian Ehrhardt <christian.ehrhardt@canonical.com>
Date: Mon, 23 May 2016 12:12:26 +0200
Subject: warn about multiple colliding apt_source without filenames

---
 cloudinit/config/cc_apt_configure.py | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

(limited to 'cloudinit/config')

diff --git a/cloudinit/config/cc_apt_configure.py b/cloudinit/config/cc_apt_configure.py
index a46ebb3e..327b543e 100644
--- a/cloudinit/config/cc_apt_configure.py
+++ b/cloudinit/config/cc_apt_configure.py
@@ -215,15 +215,22 @@ def add_sources(srclist, template_params=None, aa_repo_match=None):
         def aa_repo_match(x):
             return False
 
+    errorlist = []
     # convert old list format to new dict based format
     if isinstance(srclist, list):
         srcdict = {}
+        fnfallbackused = None
         for srcent in srclist:
             if 'filename' not in srcent:
                 # file collides for multiple !filename cases for compatibility
                 # yet we need them all processed, so not same dictionary key
                 srcent['filename'] = "cloud_config_sources.list"
                 key = util.rand_dict_key(srcdict, "cloud_config_sources.list")
+                if fnfallbackused is not None:
+                    errorlist.append(["multiple apt_source entries without",
+                                      "filename will conflict: %s vs %s" %
+                                      (srcent, fnfallbackused)])
+                fnfallbackused = srcent
             else:
                 # all with filename use that as key (matching new format)
                 key = srcent['filename']
@@ -231,7 +238,6 @@ def add_sources(srclist, template_params=None, aa_repo_match=None):
     else:
         srcdict = srclist
 
-    errorlist = []
     for filename in srcdict:
         ent = srcdict[filename]
         if 'filename' not in ent:
-- 
cgit v1.2.3


From 4ed5251d17ee7a44ce12d38d9b3d4fa554279419 Mon Sep 17 00:00:00 2001
From: Christian Ehrhardt <christian.ehrhardt@canonical.com>
Date: Mon, 23 May 2016 15:06:48 +0200
Subject: fix issue with dictionary style apt_sources handling filenames

---
 cloudinit/config/cc_apt_configure.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'cloudinit/config')

diff --git a/cloudinit/config/cc_apt_configure.py b/cloudinit/config/cc_apt_configure.py
index 327b543e..a25d6af1 100644
--- a/cloudinit/config/cc_apt_configure.py
+++ b/cloudinit/config/cc_apt_configure.py
@@ -241,7 +241,7 @@ def add_sources(srclist, template_params=None, aa_repo_match=None):
     for filename in srcdict:
         ent = srcdict[filename]
         if 'filename' not in ent:
-            ent[filename] = filename
+            ent['filename'] = filename
 
         # keys can be added without specifying a source
         try:
-- 
cgit v1.2.3


From 2945e028477ddb031d9a51ada16d5b992380242a Mon Sep 17 00:00:00 2001
From: Christian Ehrhardt <christian.ehrhardt@canonical.com>
Date: Mon, 23 May 2016 16:12:18 +0200
Subject: make sure we only handle list or dict apt_sources and bail out for
 others

---
 cloudinit/config/cc_apt_configure.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

(limited to 'cloudinit/config')

diff --git a/cloudinit/config/cc_apt_configure.py b/cloudinit/config/cc_apt_configure.py
index a25d6af1..dd199471 100644
--- a/cloudinit/config/cc_apt_configure.py
+++ b/cloudinit/config/cc_apt_configure.py
@@ -217,8 +217,8 @@ def add_sources(srclist, template_params=None, aa_repo_match=None):
 
     errorlist = []
     # convert old list format to new dict based format
+    srcdict = {}
     if isinstance(srclist, list):
-        srcdict = {}
         fnfallbackused = None
         for srcent in srclist:
             if 'filename' not in srcent:
@@ -235,8 +235,10 @@ def add_sources(srclist, template_params=None, aa_repo_match=None):
                 # all with filename use that as key (matching new format)
                 key = srcent['filename']
             srcdict[key] = srcent
-    else:
+    elif isinstance(srclist, dict):
         srcdict = srclist
+    else:
+        errorlist.append(["srclist", "unknown apt_sources format"])
 
     for filename in srcdict:
         ent = srcdict[filename]
-- 
cgit v1.2.3


From 2a20382249c35f368f931dc2bcd8225ea1fc4e84 Mon Sep 17 00:00:00 2001
From: Christian Ehrhardt <christian.ehrhardt@canonical.com>
Date: Tue, 24 May 2016 18:55:35 +0200
Subject: integrate further smaller review feedback

---
 cloudinit/config/cc_apt_configure.py | 52 +++++++++++++++++++-----------------
 1 file changed, 27 insertions(+), 25 deletions(-)

(limited to 'cloudinit/config')

diff --git a/cloudinit/config/cc_apt_configure.py b/cloudinit/config/cc_apt_configure.py
index 8f22c446..02b16336 100644
--- a/cloudinit/config/cc_apt_configure.py
+++ b/cloudinit/config/cc_apt_configure.py
@@ -21,7 +21,6 @@
 import glob
 import os
 import re
-import tempfile
 
 from cloudinit import templater
 from cloudinit import util
@@ -179,44 +178,29 @@ def add_key_raw(key):
     """
     try:
         util.subp(('apt-key', 'add', '-'), key)
-    except:
+    except util.ProcessExecutionError:
         raise Exception('failed add key')
 
 
 def add_key(ent):
     """
-    add key to the system as defiend in ent (if any)
-    suppords raw keys or keyid's
-    The latter will as a first step fetched to get the raw key
+    add key to the system as defined in ent (if any)
+    supports raw keys or keyid's
+    The latter will as a first step fetch the raw key from a keyserver
     """
     if 'keyid' in ent and 'key' not in ent:
         keyserver = "keyserver.ubuntu.com"
         if 'keyserver' in ent:
             keyserver = ent['keyserver']
-        try:
-            ent['key'] = getkeybyid(ent['keyid'], keyserver)
-        except:
-            raise Exception('failed to get key from %s' % keyserver)
+        ent['key'] = getkeybyid(ent['keyid'], keyserver)
 
     if 'key' in ent:
         add_key_raw(ent['key'])
 
-
-def add_sources(srclist, template_params=None, aa_repo_match=None):
-    """
-    add entries in /etc/apt/sources.list.d for each abbreviated
-    sources.list entry in 'srclist'.  When rendering template, also
-    include the values in dictionary searchList
+def convert_to_new_format(srclist, errorlist):
+    """ convert_to_new_format
+        convert the old list based format to the new dict based one
     """
-    if template_params is None:
-        template_params = {}
-
-    if aa_repo_match is None:
-        def aa_repo_match(x):
-            return False
-
-    errorlist = []
-    # convert old list format to new dict based format
     srcdict = {}
     if isinstance(srclist, list):
         fnfallbackused = None
@@ -240,6 +224,24 @@ def add_sources(srclist, template_params=None, aa_repo_match=None):
     else:
         errorlist.append(["srclist", "unknown apt_sources format"])
 
+    return srcdict
+
+def add_sources(srclist, template_params=None, aa_repo_match=None):
+    """
+    add entries in /etc/apt/sources.list.d for each abbreviated
+    sources.list entry in 'srclist'.  When rendering template, also
+    include the values in dictionary searchList
+    """
+    if template_params is None:
+        template_params = {}
+
+    if aa_repo_match is None:
+        def aa_repo_match(x):
+            return False
+
+    errorlist = []
+    srcdict = convert_to_new_format(srclist, errorlist)
+
     for filename in srcdict:
         ent = srcdict[filename]
         if 'filename' not in ent:
@@ -257,7 +259,7 @@ def add_sources(srclist, template_params=None, aa_repo_match=None):
         source = ent['source']
         source = templater.render_string(source, template_params)
 
-        if not ent['filename'].startswith("/"):
+        if not ent['filename'].startswith(os.path.sep):
             ent['filename'] = os.path.join("/etc/apt/sources.list.d/",
                                            ent['filename'])
 
-- 
cgit v1.2.3


From c2cfbd6831912e6dfd6b16ed21afb929592ce51a Mon Sep 17 00:00:00 2001
From: Christian Ehrhardt <christian.ehrhardt@canonical.com>
Date: Tue, 24 May 2016 19:06:55 +0200
Subject: pacify pep8 regarding the new changes

---
 cloudinit/config/cc_apt_configure.py | 2 ++
 1 file changed, 2 insertions(+)

(limited to 'cloudinit/config')

diff --git a/cloudinit/config/cc_apt_configure.py b/cloudinit/config/cc_apt_configure.py
index 02b16336..ffbf7513 100644
--- a/cloudinit/config/cc_apt_configure.py
+++ b/cloudinit/config/cc_apt_configure.py
@@ -197,6 +197,7 @@ def add_key(ent):
     if 'key' in ent:
         add_key_raw(ent['key'])
 
+
 def convert_to_new_format(srclist, errorlist):
     """ convert_to_new_format
         convert the old list based format to the new dict based one
@@ -226,6 +227,7 @@ def convert_to_new_format(srclist, errorlist):
 
     return srcdict
 
+
 def add_sources(srclist, template_params=None, aa_repo_match=None):
     """
     add entries in /etc/apt/sources.list.d for each abbreviated
-- 
cgit v1.2.3


From aa236033b159b691f5ec31885750a8167c63b2a1 Mon Sep 17 00:00:00 2001
From: Christian Ehrhardt <christian.ehrhardt@canonical.com>
Date: Mon, 30 May 2016 12:27:40 +0200
Subject: drop errorlist from convert_to_new_format

---
 cloudinit/config/cc_apt_configure.py                    | 12 +++---------
 tests/unittests/test_handler/test_handler_apt_source.py |  6 ++----
 2 files changed, 5 insertions(+), 13 deletions(-)

(limited to 'cloudinit/config')

diff --git a/cloudinit/config/cc_apt_configure.py b/cloudinit/config/cc_apt_configure.py
index ffbf7513..2dd48844 100644
--- a/cloudinit/config/cc_apt_configure.py
+++ b/cloudinit/config/cc_apt_configure.py
@@ -198,24 +198,18 @@ def add_key(ent):
         add_key_raw(ent['key'])
 
 
-def convert_to_new_format(srclist, errorlist):
+def convert_to_new_format(srclist):
     """ convert_to_new_format
         convert the old list based format to the new dict based one
     """
     srcdict = {}
     if isinstance(srclist, list):
-        fnfallbackused = None
         for srcent in srclist:
             if 'filename' not in srcent:
                 # file collides for multiple !filename cases for compatibility
                 # yet we need them all processed, so not same dictionary key
                 srcent['filename'] = "cloud_config_sources.list"
                 key = util.rand_dict_key(srcdict, "cloud_config_sources.list")
-                if fnfallbackused is not None:
-                    errorlist.append(["multiple apt_source entries without",
-                                      "filename will conflict: %s vs %s" %
-                                      (srcent, fnfallbackused)])
-                fnfallbackused = srcent
             else:
                 # all with filename use that as key (matching new format)
                 key = srcent['filename']
@@ -223,7 +217,7 @@ def convert_to_new_format(srclist, errorlist):
     elif isinstance(srclist, dict):
         srcdict = srclist
     else:
-        errorlist.append(["srclist", "unknown apt_sources format"])
+        raise ValueError("unknown apt_sources format")
 
     return srcdict
 
@@ -242,7 +236,7 @@ def add_sources(srclist, template_params=None, aa_repo_match=None):
             return False
 
     errorlist = []
-    srcdict = convert_to_new_format(srclist, errorlist)
+    srcdict = convert_to_new_format(srclist)
 
     for filename in srcdict:
         ent = srcdict[filename]
diff --git a/tests/unittests/test_handler/test_handler_apt_source.py b/tests/unittests/test_handler/test_handler_apt_source.py
index 753e86a8..4536c5b2 100644
--- a/tests/unittests/test_handler/test_handler_apt_source.py
+++ b/tests/unittests/test_handler/test_handler_apt_source.py
@@ -529,7 +529,6 @@ class TestAptSourceConfig(TestCase):
                 'filename': self.aptlistfile2}
         cfg3 = {'source': 'deb $MIRROR $RELEASE universe',
                 'filename': self.aptlistfile3}
-        errorlist = []
         checkcfg = {self.aptlistfile: {'filename': self.aptlistfile,
                                        'source': 'deb $MIRROR $RELEASE '
                                                  'multiverse'},
@@ -539,11 +538,10 @@ class TestAptSourceConfig(TestCase):
                                         'source': 'deb $MIRROR $RELEASE '
                                                   'universe'}}
 
-        newcfg = cc_apt_configure.convert_to_new_format([cfg1, cfg2, cfg3],
-                                                        errorlist)
+        newcfg = cc_apt_configure.convert_to_new_format([cfg1, cfg2, cfg3])
         self.assertEqual(newcfg, checkcfg)
 
-        newcfg2 = cc_apt_configure.convert_to_new_format(newcfg, errorlist)
+        newcfg2 = cc_apt_configure.convert_to_new_format(newcfg)
         self.assertEqual(newcfg2, checkcfg)
 
 
-- 
cgit v1.2.3


From 3c85315373306729443ef79fd8e54af46a7bc849 Mon Sep 17 00:00:00 2001
From: Christian Ehrhardt <christian.ehrhardt@canonical.com>
Date: Mon, 30 May 2016 13:07:22 +0200
Subject: fix EXPORT_GPG_KEYID for existing keys

This was broken for keys already existing in the local keyring.
There instead of the keycontent it reported the header like:
 pub   1024R/03683F77 2009-10-27
 uid                  Launchpad PPA for Scott Moser
---
 cloudinit/config/cc_apt_configure.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'cloudinit/config')

diff --git a/cloudinit/config/cc_apt_configure.py b/cloudinit/config/cc_apt_configure.py
index 2dd48844..d603f417 100644
--- a/cloudinit/config/cc_apt_configure.py
+++ b/cloudinit/config/cc_apt_configure.py
@@ -40,7 +40,7 @@ EXPORT_GPG_KEYID = """
     k=${1} ks=${2};
     exec 2>/dev/null
     [ -n "$k" ] || exit 1;
-    armour=$(gpg --list-keys --armour "${k}")
+    armour=$(gpg --export --armour "${k}")
     if [ -z "${armour}" ]; then
        gpg --keyserver ${ks} --recv "${k}" >/dev/null &&
           armour=$(gpg --export --armour "${k}") &&
-- 
cgit v1.2.3


From 80931f7008971c9a7705c054fabc29fec7a133e2 Mon Sep 17 00:00:00 2001
From: Scott Moser <smoser@ubuntu.com>
Date: Fri, 3 Jun 2016 15:27:32 -0400
Subject: fix tox -e flake8

---
 cloudinit/config/cc_apt_configure.py               |  4 +-
 .../test_handler_apt_configure_sources_list.py     | 17 +++---
 .../test_handler/test_handler_apt_source.py        | 66 +++++++++++-----------
 3 files changed, 43 insertions(+), 44 deletions(-)

(limited to 'cloudinit/config')

diff --git a/cloudinit/config/cc_apt_configure.py b/cloudinit/config/cc_apt_configure.py
index d603f417..7a9777c0 100644
--- a/cloudinit/config/cc_apt_configure.py
+++ b/cloudinit/config/cc_apt_configure.py
@@ -199,8 +199,8 @@ def add_key(ent):
 
 
 def convert_to_new_format(srclist):
-    """ convert_to_new_format
-        convert the old list based format to the new dict based one
+    """convert_to_new_format
+       convert the old list based format to the new dict based one
     """
     srcdict = {}
     if isinstance(srclist, list):
diff --git a/tests/unittests/test_handler/test_handler_apt_configure_sources_list.py b/tests/unittests/test_handler/test_handler_apt_configure_sources_list.py
index 5cf386f8..5d0417a2 100644
--- a/tests/unittests/test_handler/test_handler_apt_configure_sources_list.py
+++ b/tests/unittests/test_handler/test_handler_apt_configure_sources_list.py
@@ -3,7 +3,6 @@ Test templating of sources list
 """
 import logging
 import os
-import re
 import shutil
 import tempfile
 
@@ -62,14 +61,14 @@ deb-src http://archive.ubuntu.com/ubuntu/ fakerelease main restricted
 
 
 def load_tfile_or_url(*args, **kwargs):
-    """ load_tfile_or_url
+    """load_tfile_or_url
     load file and return content after decoding
     """
     return util.decode_binary(util.read_file_or_url(*args, **kwargs).contents)
 
 
 class TestAptSourceConfigSourceList(t_help.FilesystemMockingTestCase):
-    """ TestAptSourceConfigSourceList
+    """TestAptSourceConfigSourceList
     Main Class to test sources list rendering
     """
     def setUp(self):
@@ -89,7 +88,7 @@ class TestAptSourceConfigSourceList(t_help.FilesystemMockingTestCase):
         return cloud.Cloud(myds, paths, {}, mydist, None)
 
     def apt_source_list(self, distro, mirror, mirrorcheck=None):
-        """ apt_source_list
+        """apt_source_list
         Test rendering of a source.list from template for a given distro
         """
         if mirrorcheck is None:
@@ -115,19 +114,19 @@ class TestAptSourceConfigSourceList(t_help.FilesystemMockingTestCase):
             {'codename': '', 'primary': mirrorcheck, 'mirror': mirrorcheck})
 
     def test_apt_source_list_debian(self):
-        """ test_apt_source_list_debian
+        """test_apt_source_list_debian
         Test rendering of a source.list from template for debian
         """
         self.apt_source_list('debian', 'http://httpredir.debian.org/debian')
 
     def test_apt_source_list_ubuntu(self):
-        """ test_apt_source_list_ubuntu
+        """test_apt_source_list_ubuntu
         Test rendering of a source.list from template for ubuntu
         """
         self.apt_source_list('ubuntu', 'http://archive.ubuntu.com/ubuntu/')
 
     def test_apt_srcl_debian_mirrorfail(self):
-        """ test_apt_source_list_debian_mirrorfail
+        """test_apt_source_list_debian_mirrorfail
         Test rendering of a source.list from template for debian
         """
         self.apt_source_list('debian', ['http://does.not.exist',
@@ -135,7 +134,7 @@ class TestAptSourceConfigSourceList(t_help.FilesystemMockingTestCase):
                              'http://httpredir.debian.org/debian')
 
     def test_apt_srcl_ubuntu_mirrorfail(self):
-        """ test_apt_source_list_ubuntu_mirrorfail
+        """test_apt_source_list_ubuntu_mirrorfail
         Test rendering of a source.list from template for ubuntu
         """
         self.apt_source_list('ubuntu', ['http://does.not.exist',
@@ -143,7 +142,7 @@ class TestAptSourceConfigSourceList(t_help.FilesystemMockingTestCase):
                              'http://archive.ubuntu.com/ubuntu/')
 
     def test_apt_srcl_custom(self):
-        """ test_apt_srcl_custom
+        """test_apt_srcl_custom
         Test rendering from a custom source.list template
         """
         cfg = util.load_yaml(YAML_TEXT_CUSTOM_SL)
diff --git a/tests/unittests/test_handler/test_handler_apt_source.py b/tests/unittests/test_handler/test_handler_apt_source.py
index fe2ffae5..4dbe69f0 100644
--- a/tests/unittests/test_handler/test_handler_apt_source.py
+++ b/tests/unittests/test_handler/test_handler_apt_source.py
@@ -33,14 +33,14 @@ S0ORP6HXET3+jC8BMG4tBWCTK/XEZw==
 
 
 def load_tfile_or_url(*args, **kwargs):
-    """ load_tfile_or_url
+    """load_tfile_or_url
     load file and return content after decoding
     """
     return util.decode_binary(util.read_file_or_url(*args, **kwargs).contents)
 
 
 class TestAptSourceConfig(TestCase):
-    """ TestAptSourceConfig
+    """TestAptSourceConfig
     Main Class to test apt_source configs
     """
     def setUp(self):
@@ -57,7 +57,7 @@ class TestAptSourceConfig(TestCase):
 
     @staticmethod
     def _get_default_params():
-        """ get_default_params
+        """get_default_params
         Get the most basic default mrror and release info to be used in tests
         """
         params = {}
@@ -66,7 +66,7 @@ class TestAptSourceConfig(TestCase):
         return params
 
     def myjoin(self, *args, **kwargs):
-        """ myjoin - redir into writable tmpdir"""
+        """myjoin - redir into writable tmpdir"""
         if (args[0] == "/etc/apt/sources.list.d/" and
                 args[1] == "cloud_config_sources.list" and
                 len(args) == 2):
@@ -75,7 +75,7 @@ class TestAptSourceConfig(TestCase):
             return self.join(*args, **kwargs)
 
     def apt_src_basic(self, filename, cfg):
-        """ apt_src_basic
+        """apt_src_basic
         Test Fix deb source string, has to overwrite mirror conf in params
         """
         params = self._get_default_params()
@@ -92,7 +92,7 @@ class TestAptSourceConfig(TestCase):
                                   contents, flags=re.IGNORECASE))
 
     def test_apt_src_basic(self):
-        """ test_apt_src_basic
+        """test_apt_src_basic
         Test Fix deb source string, has to overwrite mirror conf in params.
         Test with a filename provided in config.
         """
@@ -103,7 +103,7 @@ class TestAptSourceConfig(TestCase):
         self.apt_src_basic(self.aptlistfile, [cfg])
 
     def test_apt_src_basic_dict(self):
-        """ test_apt_src_basic_dict
+        """test_apt_src_basic_dict
         Test Fix deb source string, has to overwrite mirror conf in params.
         Test with a filename provided in config.
         Provided in a dictionary with filename being the key (new format)
@@ -115,7 +115,7 @@ class TestAptSourceConfig(TestCase):
         self.apt_src_basic(self.aptlistfile, cfg)
 
     def apt_src_basic_tri(self, cfg):
-        """ apt_src_basic_tri
+        """apt_src_basic_tri
         Test Fix three deb source string, has to overwrite mirror conf in
         params. Test with filenames provided in config.
         generic part to check three files with different content
@@ -137,7 +137,7 @@ class TestAptSourceConfig(TestCase):
                                   contents, flags=re.IGNORECASE))
 
     def test_apt_src_basic_tri(self):
-        """ test_apt_src_basic_tri
+        """test_apt_src_basic_tri
         Test Fix three deb source string, has to overwrite mirror conf in
         params. Test with filenames provided in config.
         """
@@ -156,7 +156,7 @@ class TestAptSourceConfig(TestCase):
         self.apt_src_basic_tri([cfg1, cfg2, cfg3])
 
     def test_apt_src_basic_dict_tri(self):
-        """ test_apt_src_basic_dict_tri
+        """test_apt_src_basic_dict_tri
         Test Fix three deb source string, has to overwrite mirror conf in
         params. Test with filenames provided in config.
         Provided in a dictionary with filename being the key (new format)
@@ -176,7 +176,7 @@ class TestAptSourceConfig(TestCase):
         self.apt_src_basic_tri(cfg)
 
     def test_apt_src_basic_nofn(self):
-        """ test_apt_src_basic_nofn
+        """test_apt_src_basic_nofn
         Test Fix deb source string, has to overwrite mirror conf in params.
         Test without a filename provided in config and test for known fallback.
         """
@@ -187,7 +187,7 @@ class TestAptSourceConfig(TestCase):
             self.apt_src_basic(self.fallbackfn, [cfg])
 
     def apt_src_replacement(self, filename, cfg):
-        """ apt_src_replace
+        """apt_src_replace
         Test Autoreplacement of MIRROR and RELEASE in source specs
         """
         params = self._get_default_params()
@@ -202,7 +202,7 @@ class TestAptSourceConfig(TestCase):
                                   contents, flags=re.IGNORECASE))
 
     def test_apt_src_replace(self):
-        """ test_apt_src_replace
+        """test_apt_src_replace
         Test Autoreplacement of MIRROR and RELEASE in source specs with
         Filename being set
         """
@@ -211,7 +211,7 @@ class TestAptSourceConfig(TestCase):
         self.apt_src_replacement(self.aptlistfile, [cfg])
 
     def apt_src_replace_tri(self, cfg):
-        """ apt_src_replace_tri
+        """apt_src_replace_tri
         Test three autoreplacements of MIRROR and RELEASE in source specs with
         generic part
         """
@@ -231,7 +231,7 @@ class TestAptSourceConfig(TestCase):
                                   contents, flags=re.IGNORECASE))
 
     def test_apt_src_replace_tri(self):
-        """ test_apt_src_replace_tri
+        """test_apt_src_replace_tri
         Test three autoreplacements of MIRROR and RELEASE in source specs with
         Filename being set
         """
@@ -244,7 +244,7 @@ class TestAptSourceConfig(TestCase):
         self.apt_src_replace_tri([cfg1, cfg2, cfg3])
 
     def test_apt_src_replace_dict_tri(self):
-        """ test_apt_src_replace_dict_tri
+        """test_apt_src_replace_dict_tri
         Test three autoreplacements of MIRROR and RELEASE in source specs with
         Filename being set
         Provided in a dictionary with filename being the key (new format)
@@ -252,13 +252,13 @@ class TestAptSourceConfig(TestCase):
         filenames to be overwritten inside the directory entry.
         """
         cfg = {self.aptlistfile: {'source': 'deb $MIRROR $RELEASE multiverse'},
-               'notused':        {'source': 'deb $MIRROR $RELEASE main',
-                                  'filename': self.aptlistfile2},
+               'notused': {'source': 'deb $MIRROR $RELEASE main',
+                           'filename': self.aptlistfile2},
                self.aptlistfile3: {'source': 'deb $MIRROR $RELEASE universe'}}
         self.apt_src_replace_tri(cfg)
 
     def test_apt_src_replace_nofn(self):
-        """ test_apt_src_replace_nofn
+        """test_apt_src_replace_nofn
         Test Autoreplacement of MIRROR and RELEASE in source specs with
         No filename being set
         """
@@ -267,7 +267,7 @@ class TestAptSourceConfig(TestCase):
             self.apt_src_replacement(self.fallbackfn, [cfg])
 
     def apt_src_keyid(self, filename, cfg, keynum):
-        """ apt_src_keyid
+        """apt_src_keyid
         Test specification of a source + keyid
         """
         params = self._get_default_params()
@@ -293,7 +293,7 @@ class TestAptSourceConfig(TestCase):
                                   contents, flags=re.IGNORECASE))
 
     def test_apt_src_keyid(self):
-        """ test_apt_src_keyid
+        """test_apt_src_keyid
         Test specification of a source + keyid with filename being set
         """
         cfg = {'source': ('deb '
@@ -305,7 +305,7 @@ class TestAptSourceConfig(TestCase):
         self.apt_src_keyid(self.aptlistfile, [cfg], 1)
 
     def test_apt_src_keyid_tri(self):
-        """ test_apt_src_keyid_tri
+        """test_apt_src_keyid_tri
         Test specification of a source + keyid with filename being set
         Setting three of such, check for content and keys
         """
@@ -345,7 +345,7 @@ class TestAptSourceConfig(TestCase):
                                   contents, flags=re.IGNORECASE))
 
     def test_apt_src_keyid_nofn(self):
-        """ test_apt_src_keyid_nofn
+        """test_apt_src_keyid_nofn
         Test specification of a source + keyid without filename being set
         """
         cfg = {'source': ('deb '
@@ -357,7 +357,7 @@ class TestAptSourceConfig(TestCase):
             self.apt_src_keyid(self.fallbackfn, [cfg], 1)
 
     def apt_src_key(self, filename, cfg):
-        """ apt_src_key
+        """apt_src_key
         Test specification of a source + key
         """
         params = self._get_default_params()
@@ -378,7 +378,7 @@ class TestAptSourceConfig(TestCase):
                                   contents, flags=re.IGNORECASE))
 
     def test_apt_src_key(self):
-        """ test_apt_src_key
+        """test_apt_src_key
         Test specification of a source + key with filename being set
         """
         cfg = {'source': ('deb '
@@ -390,7 +390,7 @@ class TestAptSourceConfig(TestCase):
         self.apt_src_key(self.aptlistfile, cfg)
 
     def test_apt_src_key_nofn(self):
-        """ test_apt_src_key_nofn
+        """test_apt_src_key_nofn
         Test specification of a source + key without filename being set
         """
         cfg = {'source': ('deb '
@@ -402,7 +402,7 @@ class TestAptSourceConfig(TestCase):
             self.apt_src_key(self.fallbackfn, cfg)
 
     def test_apt_src_keyonly(self):
-        """ test_apt_src_keyonly
+        """test_apt_src_keyonly
         Test specification key without source
         """
         params = self._get_default_params()
@@ -419,7 +419,7 @@ class TestAptSourceConfig(TestCase):
         self.assertFalse(os.path.isfile(self.aptlistfile))
 
     def test_apt_src_keyidonly(self):
-        """ test_apt_src_keyidonly
+        """test_apt_src_keyidonly
         Test specification of a keyid without source
         """
         params = self._get_default_params()
@@ -436,7 +436,7 @@ class TestAptSourceConfig(TestCase):
         self.assertFalse(os.path.isfile(self.aptlistfile))
 
     def test_apt_src_keyid_real(self):
-        """ test_apt_src_keyid_real
+        """test_apt_src_keyid_real
         Test specification of a keyid without source incl
         up to addition of the key (nothing but add_key_raw mocked)
         """
@@ -454,7 +454,7 @@ class TestAptSourceConfig(TestCase):
         self.assertFalse(os.path.isfile(self.aptlistfile))
 
     def test_apt_src_longkeyid_real(self):
-        """ test_apt_src_longkeyid_real
+        """test_apt_src_longkeyid_real
         Test specification of a long key fingerprint without source incl
         up to addition of the key (nothing but add_key_raw mocked)
         """
@@ -472,7 +472,7 @@ class TestAptSourceConfig(TestCase):
         self.assertFalse(os.path.isfile(self.aptlistfile))
 
     def test_apt_src_ppa(self):
-        """ test_apt_src_ppa
+        """test_apt_src_ppa
         Test specification of a ppa
         """
         params = self._get_default_params()
@@ -491,7 +491,7 @@ class TestAptSourceConfig(TestCase):
         self.assertFalse(os.path.isfile(self.aptlistfile))
 
     def test_apt_src_ppa_tri(self):
-        """ test_apt_src_ppa_tri
+        """test_apt_src_ppa_tri
         Test specification of a ppa
         """
         params = self._get_default_params()
@@ -519,7 +519,7 @@ class TestAptSourceConfig(TestCase):
         self.assertFalse(os.path.isfile(self.aptlistfile3))
 
     def test_convert_to_new_format(self):
-        """ test_convert_to_new_format
+        """test_convert_to_new_format
         Test the conversion of old to new format
         And the noop conversion of new to new format as well
         """
-- 
cgit v1.2.3


From dca09871e61186deb216ce5cb1fc5db3b69c9fc2 Mon Sep 17 00:00:00 2001
From: Christian Ehrhardt <christian.ehrhardt@canonical.com>
Date: Mon, 6 Jun 2016 09:18:17 +0200
Subject: pythonify getkeybyid and move it to uitl.py

---
 cloudinit/config/cc_apt_configure.py | 27 +------------------------
 cloudinit/util.py                    | 38 ++++++++++++++++++++++++++++++++++++
 2 files changed, 39 insertions(+), 26 deletions(-)

(limited to 'cloudinit/config')

diff --git a/cloudinit/config/cc_apt_configure.py b/cloudinit/config/cc_apt_configure.py
index 7a9777c0..2f270662 100644
--- a/cloudinit/config/cc_apt_configure.py
+++ b/cloudinit/config/cc_apt_configure.py
@@ -34,21 +34,6 @@ APT_PROXY_FN = "/etc/apt/apt.conf.d/95cloud-init-proxy"
 # this will match 'XXX:YYY' (ie, 'cloud-archive:foo' or 'ppa:bar')
 ADD_APT_REPO_MATCH = r"^[\w-]+:\w"
 
-# A temporary shell program to get a given gpg key
-# from a given keyserver
-EXPORT_GPG_KEYID = """
-    k=${1} ks=${2};
-    exec 2>/dev/null
-    [ -n "$k" ] || exit 1;
-    armour=$(gpg --export --armour "${k}")
-    if [ -z "${armour}" ]; then
-       gpg --keyserver ${ks} --recv "${k}" >/dev/null &&
-          armour=$(gpg --export --armour "${k}") &&
-          gpg --batch --yes --delete-keys "${k}"
-    fi
-    [ -n "${armour}" ] && echo "${armour}"
-"""
-
 
 def handle(name, cfg, cloud, log, _args):
     if util.is_false(cfg.get('apt_configure_enabled', True)):
@@ -108,16 +93,6 @@ def handle(name, cfg, cloud, log, _args):
             util.logexc(log, "Failed to run debconf-set-selections")
 
 
-# get gpg keyid from keyserver
-def getkeybyid(keyid, keyserver):
-    with util.ExtendedTemporaryFile(suffix='.sh', mode="w+", ) as fh:
-        fh.write(EXPORT_GPG_KEYID)
-        fh.flush()
-        cmd = ['/bin/sh', fh.name, keyid, keyserver]
-        (stdout, _stderr) = util.subp(cmd)
-        return stdout.strip()
-
-
 def mirror2lists_fileprefix(mirror):
     string = mirror
     # take off http:// or ftp://
@@ -192,7 +167,7 @@ def add_key(ent):
         keyserver = "keyserver.ubuntu.com"
         if 'keyserver' in ent:
             keyserver = ent['keyserver']
-        ent['key'] = getkeybyid(ent['keyid'], keyserver)
+        ent['key'] = util.getkeybyid(ent['keyid'], keyserver)
 
     if 'key' in ent:
         add_key_raw(ent['key'])
diff --git a/cloudinit/util.py b/cloudinit/util.py
index d6b80dbe..d3b14f72 100644
--- a/cloudinit/util.py
+++ b/cloudinit/util.py
@@ -2234,3 +2234,41 @@ def message_from_string(string):
     if sys.version_info[:2] < (2, 7):
         return email.message_from_file(six.StringIO(string))
     return email.message_from_string(string)
+
+
+def gpg_export_armour(key):
+    """Export gpg key, armoured key gets returned"""
+    (armour, _) = subp(["gpg", "--export", "--armour", key], capture=True)
+    return armour
+
+
+def gpg_recv_key(key, keyserver):
+    """Receive gpg key from the specified keyserver"""
+    try:
+        subp(["gpg", "--keyserver", keyserver, "--recv", key],
+             capture=True)
+    except ProcessExecutionError as error:
+        raise ValueError('Failed to import key %s from server %s - error %s' %
+                         (key, keyserver, error))
+
+
+def gpg_delete_key(key):
+    """Delete the specified key from the local gpg ring"""
+    subp(["gpg", "--batch", "--yes", "--delete-keys", key], capture=False)
+
+
+def getkeybyid(keyid, keyserver):
+    """get gpg keyid from keyserver"""
+    armour = gpg_export_armour(keyid)
+    if not armour:
+        try:
+            gpg_recv_key(keyid, keyserver=keyserver)
+        except ValueError:
+            LOG.exception('Failed to obtain gpg key %s', keyid)
+            raise
+
+        armour = gpg_export_armour(keyid)
+        # delete just imported key to leave environment as it was before
+        gpg_delete_key(keyid)
+
+    return armour.rstrip('\n')
-- 
cgit v1.2.3


From 89688593e9358433cd9383d1bdfae12dbcd58f72 Mon Sep 17 00:00:00 2001
From: Christian Ehrhardt <christian.ehrhardt@canonical.com>
Date: Mon, 6 Jun 2016 13:36:23 +0200
Subject: add_key_raw - make exceptions more specific

---
 cloudinit/config/cc_apt_configure.py               |  4 +--
 cloudinit/util.py                                  |  1 +
 .../test_handler/test_handler_apt_source.py        | 38 +++++++++++++++++++---
 3 files changed, 36 insertions(+), 7 deletions(-)

(limited to 'cloudinit/config')

diff --git a/cloudinit/config/cc_apt_configure.py b/cloudinit/config/cc_apt_configure.py
index 2f270662..a9ac6ea8 100644
--- a/cloudinit/config/cc_apt_configure.py
+++ b/cloudinit/config/cc_apt_configure.py
@@ -154,7 +154,7 @@ def add_key_raw(key):
     try:
         util.subp(('apt-key', 'add', '-'), key)
     except util.ProcessExecutionError:
-        raise Exception('failed add key')
+        raise ValueError('failed to add apt GPG Key to apt keyring')
 
 
 def add_key(ent):
@@ -221,7 +221,7 @@ def add_sources(srclist, template_params=None, aa_repo_match=None):
         # keys can be added without specifying a source
         try:
             add_key(ent)
-        except Exception as detail:
+        except ValueError as detail:
             errorlist.append([ent, detail])
 
         if 'source' not in ent:
diff --git a/cloudinit/util.py b/cloudinit/util.py
index d3b14f72..a1622946 100644
--- a/cloudinit/util.py
+++ b/cloudinit/util.py
@@ -2244,6 +2244,7 @@ def gpg_export_armour(key):
 
 def gpg_recv_key(key, keyserver):
     """Receive gpg key from the specified keyserver"""
+    print("ORIGINAL gpg_recv_key")
     try:
         subp(["gpg", "--keyserver", keyserver, "--recv", key],
              capture=True)
diff --git a/tests/unittests/test_handler/test_handler_apt_source.py b/tests/unittests/test_handler/test_handler_apt_source.py
index a4878124..65e375a0 100644
--- a/tests/unittests/test_handler/test_handler_apt_source.py
+++ b/tests/unittests/test_handler/test_handler_apt_source.py
@@ -4,6 +4,7 @@ Testing various config variations of the apt_source config
 import os
 import re
 import shutil
+import socket
 import tempfile
 
 try:
@@ -56,6 +57,7 @@ class TestAptSourceConfig(TestCase):
         # mock fallback filename into writable tmp dir
         self.fallbackfn = os.path.join(self.tmp, "etc/apt/sources.list.d/",
                                        "cloud_config_sources.list")
+        self.orig_gpg_recv_key = util.gpg_recv_key
 
         patcher = mock.patch("cloudinit.config.cc_apt_configure.get_release")
         get_rel = patcher.start()
@@ -445,14 +447,40 @@ class TestAptSourceConfig(TestCase):
     def apt_src_keyid_real(self, cfg, expectedkey):
         """apt_src_keyid_real
         Test specification of a keyid without source including
-        up to addition of the key (nothing but add_key_raw mocked to keep the
+        up to addition of the key (add_key_raw mocked to keep the
         environment as is)
         """
         params = self._get_default_params()
-        with mock.patch.object(cc_apt_configure, 'add_key_raw') as mockobj:
-            cc_apt_configure.add_sources([cfg], params)
 
-        mockobj.assert_called_with(expectedkey)
+        def fake_gpg_recv_key(self, key, keyserver):
+            """try original gpg_recv_key, but allow fall back"""
+            try:
+                print("Try orig orig_gpg_recv_key")
+                self.orig_gpg_recv_key(key, keyserver)
+            except ValueError:
+                print("Fail, test net")
+                # if this is a networking issue mock it's effect
+                testsock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+                try:
+                    testsock.connect((keyserver, 80))
+                    testsock.close()
+                except socket.error:
+                    # as fallback add the known key as a working recv would
+                    print("Fallback import expectedkey")
+                    util.subp(("gpg", "--import", "-"), EXPECTEDKEY)
+
+        print("FOO")
+        with mock.patch.object(cc_apt_configure, 'add_key_raw') as mockkey:
+            with mock.patch.object(util, 'gpg_recv_key',
+                                   side_effect=fake_gpg_recv_key) as mockrecv:
+                cc_apt_configure.add_sources([cfg], params)
+
+        # since we might mock the recv path ensure it is called right
+        mockrecv.assert_called_with(cfg['keyid'],
+                                    keyserver=cfg.get('keyserver',
+                                                      'keyserver.ubuntu.com'))
+        # no matter if really imported or faked, ensure we add the right key
+        mockkey.assert_called_with(expectedkey)
 
         # filename should be ignored on key only
         self.assertFalse(os.path.isfile(self.aptlistfile))
@@ -477,7 +505,7 @@ class TestAptSourceConfig(TestCase):
         """test_apt_src_longkeyid_ks_real - Test long keyid from other ks"""
         keyid = "B59D 5F15 97A5 04B7 E230  6DCA 0620 BBCF 0368 3F77"
         cfg = {'keyid': keyid,
-               'keyserver': 'keys.gnupg.net',
+               'keyserver': 'knorz.gnupg.net',
                'filename': self.aptlistfile}
 
         self.apt_src_keyid_real(cfg, EXPECTEDKEY)
-- 
cgit v1.2.3


From c1ad09609805c6c4f5262eb21533799af814379a Mon Sep 17 00:00:00 2001
From: Christian Ehrhardt <christian.ehrhardt@canonical.com>
Date: Mon, 6 Jun 2016 14:23:55 +0200
Subject: rename add_key / add_source to add_apt_key / add_apt_source

The functions clearly are apt specific so the name should reflect that.
---
 cloudinit/config/cc_apt_configure.py               | 14 ++++++------
 .../test_handler/test_handler_apt_source.py        | 25 +++++++++++-----------
 2 files changed, 20 insertions(+), 19 deletions(-)

(limited to 'cloudinit/config')

diff --git a/cloudinit/config/cc_apt_configure.py b/cloudinit/config/cc_apt_configure.py
index a9ac6ea8..91a117f5 100644
--- a/cloudinit/config/cc_apt_configure.py
+++ b/cloudinit/config/cc_apt_configure.py
@@ -79,8 +79,8 @@ def handle(name, cfg, cloud, log, _args):
             def matcher(x):
                 return False
 
-        errors = add_sources(cfg['apt_sources'], params,
-                             aa_repo_match=matcher)
+        errors = add_apt_sources(cfg['apt_sources'], params,
+                                 aa_repo_match=matcher)
         for e in errors:
             log.warn("Add source error: %s", ':'.join(e))
 
@@ -146,7 +146,7 @@ def generate_sources_list(cfg, codename, mirrors, cloud, log):
     templater.render_to_file(template_fn, '/etc/apt/sources.list', params)
 
 
-def add_key_raw(key):
+def add_apt_key_raw(key):
     """
     actual adding of a key as defined in key argument
     to the system
@@ -157,7 +157,7 @@ def add_key_raw(key):
         raise ValueError('failed to add apt GPG Key to apt keyring')
 
 
-def add_key(ent):
+def add_apt_key(ent):
     """
     add key to the system as defined in ent (if any)
     supports raw keys or keyid's
@@ -170,7 +170,7 @@ def add_key(ent):
         ent['key'] = util.getkeybyid(ent['keyid'], keyserver)
 
     if 'key' in ent:
-        add_key_raw(ent['key'])
+        add_apt_key_raw(ent['key'])
 
 
 def convert_to_new_format(srclist):
@@ -197,7 +197,7 @@ def convert_to_new_format(srclist):
     return srcdict
 
 
-def add_sources(srclist, template_params=None, aa_repo_match=None):
+def add_apt_sources(srclist, template_params=None, aa_repo_match=None):
     """
     add entries in /etc/apt/sources.list.d for each abbreviated
     sources.list entry in 'srclist'.  When rendering template, also
@@ -220,7 +220,7 @@ def add_sources(srclist, template_params=None, aa_repo_match=None):
 
         # keys can be added without specifying a source
         try:
-            add_key(ent)
+            add_apt_key(ent)
         except ValueError as detail:
             errorlist.append([ent, detail])
 
diff --git a/tests/unittests/test_handler/test_handler_apt_source.py b/tests/unittests/test_handler/test_handler_apt_source.py
index c7eeb64c..4a720213 100644
--- a/tests/unittests/test_handler/test_handler_apt_source.py
+++ b/tests/unittests/test_handler/test_handler_apt_source.py
@@ -89,7 +89,7 @@ class TestAptSourceConfig(TestCase):
         """
         params = self._get_default_params()
 
-        cc_apt_configure.add_sources(cfg, params)
+        cc_apt_configure.add_apt_sources(cfg, params)
 
         self.assertTrue(os.path.isfile(filename))
 
@@ -200,7 +200,7 @@ class TestAptSourceConfig(TestCase):
         Test Autoreplacement of MIRROR and RELEASE in source specs
         """
         params = self._get_default_params()
-        cc_apt_configure.add_sources(cfg, params)
+        cc_apt_configure.add_apt_sources(cfg, params)
 
         self.assertTrue(os.path.isfile(filename))
 
@@ -283,7 +283,7 @@ class TestAptSourceConfig(TestCase):
 
         with mock.patch.object(util, 'subp',
                                return_value=('fakekey 1234', '')) as mockobj:
-            cc_apt_configure.add_sources(cfg, params)
+            cc_apt_configure.add_apt_sources(cfg, params)
 
         # check if it added the right ammount of keys
         calls = []
@@ -372,7 +372,7 @@ class TestAptSourceConfig(TestCase):
         params = self._get_default_params()
 
         with mock.patch.object(util, 'subp') as mockobj:
-            cc_apt_configure.add_sources([cfg], params)
+            cc_apt_configure.add_apt_sources([cfg], params)
 
         mockobj.assert_called_with(('apt-key', 'add', '-'), 'fakekey 4321')
 
@@ -419,7 +419,7 @@ class TestAptSourceConfig(TestCase):
                'filename': self.aptlistfile}
 
         with mock.patch.object(util, 'subp') as mockobj:
-            cc_apt_configure.add_sources([cfg], params)
+            cc_apt_configure.add_apt_sources([cfg], params)
 
         mockobj.assert_called_once_with(('apt-key', 'add', '-'),
                                         'fakekey 4242')
@@ -437,7 +437,7 @@ class TestAptSourceConfig(TestCase):
 
         with mock.patch.object(util, 'subp',
                                return_value=('fakekey 1212', '')) as mockobj:
-            cc_apt_configure.add_sources([cfg], params)
+            cc_apt_configure.add_apt_sources([cfg], params)
 
         mockobj.assert_called_with(('apt-key', 'add', '-'), 'fakekey 1212')
 
@@ -447,7 +447,7 @@ class TestAptSourceConfig(TestCase):
     def apt_src_keyid_real(self, cfg, expectedkey):
         """apt_src_keyid_real
         Test specification of a keyid without source including
-        up to addition of the key (add_key_raw mocked to keep the
+        up to addition of the key (add_apt_key_raw mocked to keep the
         environment as is)
         """
         params = self._get_default_params()
@@ -466,10 +466,10 @@ class TestAptSourceConfig(TestCase):
                     # as fallback add the known key as a working recv would
                     util.subp(("gpg", "--import", "-"), EXPECTEDKEY)
 
-        with mock.patch.object(cc_apt_configure, 'add_key_raw') as mockkey:
+        with mock.patch.object(cc_apt_configure, 'add_apt_key_raw') as mockkey:
             with mock.patch.object(util, 'gpg_recv_key',
                                    side_effect=fake_gpg_recv_key) as mockrecv:
-                cc_apt_configure.add_sources([cfg], params)
+                cc_apt_configure.add_apt_sources([cfg], params)
 
         # since we might mock the recv path ensure it is called right
         mockrecv.assert_called_with(cfg['keyid'],
@@ -518,7 +518,8 @@ class TestAptSourceConfig(TestCase):
         matcher = re.compile(r'^[\w-]+:\w').search
 
         with mock.patch.object(util, 'subp') as mockobj:
-            cc_apt_configure.add_sources([cfg], params, aa_repo_match=matcher)
+            cc_apt_configure.add_apt_sources([cfg], params,
+                                             aa_repo_match=matcher)
         mockobj.assert_called_once_with(['add-apt-repository',
                                          'ppa:smoser/cloud-init-test'])
 
@@ -541,8 +542,8 @@ class TestAptSourceConfig(TestCase):
         matcher = re.compile(r'^[\w-]+:\w').search
 
         with mock.patch.object(util, 'subp') as mockobj:
-            cc_apt_configure.add_sources([cfg1, cfg2, cfg3], params,
-                                         aa_repo_match=matcher)
+            cc_apt_configure.add_apt_sources([cfg1, cfg2, cfg3], params,
+                                             aa_repo_match=matcher)
         calls = [call(['add-apt-repository', 'ppa:smoser/cloud-init-test']),
                  call(['add-apt-repository', 'ppa:smoser/cloud-init-test2']),
                  call(['add-apt-repository', 'ppa:smoser/cloud-init-test3'])]
-- 
cgit v1.2.3


From 71ea8d200da164df6f5e28d28aea424b57fd8f1f Mon Sep 17 00:00:00 2001
From: Christian Ehrhardt <christian.ehrhardt@canonical.com>
Date: Mon, 6 Jun 2016 15:18:59 +0200
Subject: rename fucntion to mirrorurl_to_apt_fileprefix to reflect what it
 actually does

---
 cloudinit/config/cc_apt_configure.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

(limited to 'cloudinit/config')

diff --git a/cloudinit/config/cc_apt_configure.py b/cloudinit/config/cc_apt_configure.py
index 91a117f5..be6324a2 100644
--- a/cloudinit/config/cc_apt_configure.py
+++ b/cloudinit/config/cc_apt_configure.py
@@ -93,7 +93,7 @@ def handle(name, cfg, cloud, log, _args):
             util.logexc(log, "Failed to run debconf-set-selections")
 
 
-def mirror2lists_fileprefix(mirror):
+def mirrorurl_to_apt_fileprefix(mirror):
     string = mirror
     # take off http:// or ftp://
     if string.endswith("/"):
@@ -110,8 +110,8 @@ def rename_apt_lists(old_mirrors, new_mirrors, lists_d="/var/lib/apt/lists"):
         nmirror = new_mirrors.get(name)
         if not nmirror:
             continue
-        oprefix = os.path.join(lists_d, mirror2lists_fileprefix(omirror))
-        nprefix = os.path.join(lists_d, mirror2lists_fileprefix(nmirror))
+        oprefix = os.path.join(lists_d, mirrorurl_to_apt_fileprefix(omirror))
+        nprefix = os.path.join(lists_d, mirrorurl_to_apt_fileprefix(nmirror))
         if oprefix == nprefix:
             continue
         olen = len(oprefix)
-- 
cgit v1.2.3


From b89ac6b7caeeade5ad21137773ac4496cdaea2c5 Mon Sep 17 00:00:00 2001
From: Christian Ehrhardt <christian.ehrhardt@canonical.com>
Date: Thu, 9 Jun 2016 09:18:35 +0200
Subject: move gpg functions into gpg.py

This helps for cleaner code structuring.
ALong that makeing sure all these functions have a gpg_prefix.
---
 cloudinit/config/cc_apt_configure.py               |  3 +-
 cloudinit/gpg.py                                   | 64 ++++++++++++++++++++++
 cloudinit/util.py                                  | 38 -------------
 .../test_handler/test_handler_apt_source.py        |  4 +-
 4 files changed, 68 insertions(+), 41 deletions(-)
 create mode 100644 cloudinit/gpg.py

(limited to 'cloudinit/config')

diff --git a/cloudinit/config/cc_apt_configure.py b/cloudinit/config/cc_apt_configure.py
index be6324a2..ba080930 100644
--- a/cloudinit/config/cc_apt_configure.py
+++ b/cloudinit/config/cc_apt_configure.py
@@ -24,6 +24,7 @@ import re
 
 from cloudinit import templater
 from cloudinit import util
+from cloudinit import gpg
 
 distros = ['ubuntu', 'debian']
 
@@ -167,7 +168,7 @@ def add_apt_key(ent):
         keyserver = "keyserver.ubuntu.com"
         if 'keyserver' in ent:
             keyserver = ent['keyserver']
-        ent['key'] = util.getkeybyid(ent['keyid'], keyserver)
+        ent['key'] = gpg.gpg_getkeybyid(ent['keyid'], keyserver)
 
     if 'key' in ent:
         add_apt_key_raw(ent['key'])
diff --git a/cloudinit/gpg.py b/cloudinit/gpg.py
new file mode 100644
index 00000000..620dfb19
--- /dev/null
+++ b/cloudinit/gpg.py
@@ -0,0 +1,64 @@
+"""gpg.py - Collection of gpg key related functions"""
+# vi: ts=4 expandtab
+#
+#    Copyright (C) 2016 Canonical Ltd.
+#
+#    Author: Scott Moser <scott.moser@canonical.com>
+#    Author: Juerg Haefliger <juerg.haefliger@hp.com>
+#    Author: Joshua Harlow <harlowja@yahoo-inc.com>
+#    Author: Christian Ehrhardt <christian.ehrhardt@canonical.com>
+#
+#    This program is free software: you can redistribute it and/or modify
+#    it under the terms of the GNU General Public License version 3, as
+#    published by the Free Software Foundation.
+#
+#    This program is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU General Public License for more details.
+#
+#    You should have received a copy of the GNU General Public License
+#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+from cloudinit import util
+from cloudinit import log as logging
+
+LOG = logging.getLogger(__name__)
+
+
+def gpg_export_armour(key):
+    """Export gpg key, armoured key gets returned"""
+    (armour, _) = util.subp(["gpg", "--export", "--armour", key], capture=True)
+    return armour
+
+
+def gpg_recv_key(key, keyserver):
+    """Receive gpg key from the specified keyserver"""
+    try:
+        util.subp(["gpg", "--keyserver", keyserver, "--recv", key],
+                  capture=True)
+    except util.ProcessExecutionError as error:
+        raise ValueError('Failed to import key %s from server %s - error %s' %
+                         (key, keyserver, error))
+
+
+def gpg_delete_key(key):
+    """Delete the specified key from the local gpg ring"""
+    util.subp(["gpg", "--batch", "--yes", "--delete-keys", key], capture=True)
+
+
+def gpg_getkeybyid(keyid, keyserver):
+    """get gpg keyid from keyserver"""
+    armour = gpg_export_armour(keyid)
+    if not armour:
+        try:
+            gpg_recv_key(keyid, keyserver=keyserver)
+        except ValueError:
+            LOG.exception('Failed to obtain gpg key %s', keyid)
+            raise
+
+        armour = gpg_export_armour(keyid)
+        # delete just imported key to leave environment as it was before
+        gpg_delete_key(keyid)
+
+    return armour.rstrip('\n')
diff --git a/cloudinit/util.py b/cloudinit/util.py
index 6d16532d..d6b80dbe 100644
--- a/cloudinit/util.py
+++ b/cloudinit/util.py
@@ -2234,41 +2234,3 @@ def message_from_string(string):
     if sys.version_info[:2] < (2, 7):
         return email.message_from_file(six.StringIO(string))
     return email.message_from_string(string)
-
-
-def gpg_export_armour(key):
-    """Export gpg key, armoured key gets returned"""
-    (armour, _) = subp(["gpg", "--export", "--armour", key], capture=True)
-    return armour
-
-
-def gpg_recv_key(key, keyserver):
-    """Receive gpg key from the specified keyserver"""
-    try:
-        subp(["gpg", "--keyserver", keyserver, "--recv", key],
-             capture=True)
-    except ProcessExecutionError as error:
-        raise ValueError('Failed to import key %s from server %s - error %s' %
-                         (key, keyserver, error))
-
-
-def gpg_delete_key(key):
-    """Delete the specified key from the local gpg ring"""
-    subp(["gpg", "--batch", "--yes", "--delete-keys", key], capture=True)
-
-
-def getkeybyid(keyid, keyserver):
-    """get gpg keyid from keyserver"""
-    armour = gpg_export_armour(keyid)
-    if not armour:
-        try:
-            gpg_recv_key(keyid, keyserver=keyserver)
-        except ValueError:
-            LOG.exception('Failed to obtain gpg key %s', keyid)
-            raise
-
-        armour = gpg_export_armour(keyid)
-        # delete just imported key to leave environment as it was before
-        gpg_delete_key(keyid)
-
-    return armour.rstrip('\n')
diff --git a/tests/unittests/test_handler/test_handler_apt_source.py b/tests/unittests/test_handler/test_handler_apt_source.py
index 3f87fad3..b09dd8ab 100644
--- a/tests/unittests/test_handler/test_handler_apt_source.py
+++ b/tests/unittests/test_handler/test_handler_apt_source.py
@@ -14,6 +14,7 @@ from mock import call
 
 from cloudinit.config import cc_apt_configure
 from cloudinit import util
+from cloudinit import gpg
 
 from ..helpers import TestCase
 
@@ -58,7 +59,6 @@ class TestAptSourceConfig(TestCase):
         # mock fallback filename into writable tmp dir
         self.fallbackfn = os.path.join(self.tmp, "etc/apt/sources.list.d/",
                                        "cloud_config_sources.list")
-        self.orig_gpg_recv_key = util.gpg_recv_key
 
         patcher = mock.patch("cloudinit.config.cc_apt_configure.get_release")
         get_rel = patcher.start()
@@ -407,7 +407,7 @@ class TestAptSourceConfig(TestCase):
         params = self._get_default_params()
 
         with mock.patch.object(cc_apt_configure, 'add_apt_key_raw') as mockkey:
-            with mock.patch.object(util, 'getkeybyid',
+            with mock.patch.object(gpg, 'gpg_getkeybyid',
                                    return_value=expectedkey) as mockgetkey:
                 cc_apt_configure.add_apt_sources([cfg], params)
 
-- 
cgit v1.2.3


From 7e527b1b2f3fda558fb0f3a6958c42dde4716079 Mon Sep 17 00:00:00 2001
From: Scott Moser <smoser@ubuntu.com>
Date: Fri, 10 Jun 2016 13:22:59 -0400
Subject: minor changes prior to merge

a.) remove 'gpg_' from function names in new gpg module.
b.) use --recv-keys rather than --recv
    --recv-keys is  more obvious and works back to precise at least.
c.) do not trim trailing '\n' from a armour'd key.
---
 cloudinit/config/cc_apt_configure.py               |  4 ++--
 cloudinit/gpg.py                                   | 24 ++++++++++------------
 .../test_handler/test_handler_apt_source.py        |  4 ++--
 3 files changed, 15 insertions(+), 17 deletions(-)

(limited to 'cloudinit/config')

diff --git a/cloudinit/config/cc_apt_configure.py b/cloudinit/config/cc_apt_configure.py
index ba080930..96c4a43d 100644
--- a/cloudinit/config/cc_apt_configure.py
+++ b/cloudinit/config/cc_apt_configure.py
@@ -22,9 +22,9 @@ import glob
 import os
 import re
 
+from cloudinit import gpg
 from cloudinit import templater
 from cloudinit import util
-from cloudinit import gpg
 
 distros = ['ubuntu', 'debian']
 
@@ -168,7 +168,7 @@ def add_apt_key(ent):
         keyserver = "keyserver.ubuntu.com"
         if 'keyserver' in ent:
             keyserver = ent['keyserver']
-        ent['key'] = gpg.gpg_getkeybyid(ent['keyid'], keyserver)
+        ent['key'] = gpg.get_key_by_id(ent['keyid'], keyserver)
 
     if 'key' in ent:
         add_apt_key_raw(ent['key'])
diff --git a/cloudinit/gpg.py b/cloudinit/gpg.py
index baa8b534..6a76d785 100644
--- a/cloudinit/gpg.py
+++ b/cloudinit/gpg.py
@@ -4,8 +4,6 @@
 #    Copyright (C) 2016 Canonical Ltd.
 #
 #    Author: Scott Moser <scott.moser@canonical.com>
-#    Author: Juerg Haefliger <juerg.haefliger@hp.com>
-#    Author: Joshua Harlow <harlowja@yahoo-inc.com>
 #    Author: Christian Ehrhardt <christian.ehrhardt@canonical.com>
 #
 #    This program is free software: you can redistribute it and/or modify
@@ -20,13 +18,13 @@
 #    You should have received a copy of the GNU General Public License
 #    along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-from cloudinit import util
 from cloudinit import log as logging
+from cloudinit import util
 
 LOG = logging.getLogger(__name__)
 
 
-def gpg_export_armour(key):
+def export_armour(key):
     """Export gpg key, armoured key gets returned"""
     try:
         (armour, _) = util.subp(["gpg", "--export", "--armour", key],
@@ -38,11 +36,11 @@ def gpg_export_armour(key):
     return armour
 
 
-def gpg_recv_key(key, keyserver):
+def receive_key(key, keyserver):
     """Receive gpg key from the specified keyserver"""
     LOG.debug('Receive gpg key "%s"', key)
     try:
-        util.subp(["gpg", "--keyserver", keyserver, "--recv", key],
+        util.subp(["gpg", "--keyserver", keyserver, "--recv-keys", key],
                   capture=True)
     except util.ProcessExecutionError as error:
         raise ValueError(('Failed to import key "%s" '
@@ -50,7 +48,7 @@ def gpg_recv_key(key, keyserver):
                          (key, keyserver, error))
 
 
-def gpg_delete_key(key):
+def delete_key(key):
     """Delete the specified key from the local gpg ring"""
     try:
         util.subp(["gpg", "--batch", "--yes", "--delete-keys", key],
@@ -59,18 +57,18 @@ def gpg_delete_key(key):
         LOG.warn('Failed delete key "%s": %s', key, error)
 
 
-def gpg_getkeybyid(keyid, keyserver):
+def get_key_by_id(keyid, keyserver="keyserver.ubuntu.com"):
     """get gpg keyid from keyserver"""
-    armour = gpg_export_armour(keyid)
+    armour = export_armour(keyid)
     if not armour:
         try:
-            gpg_recv_key(keyid, keyserver=keyserver)
-            armour = gpg_export_armour(keyid)
+            receive_key(keyid, keyserver=keyserver)
+            armour = export_armour(keyid)
         except ValueError:
             LOG.exception('Failed to obtain gpg key %s', keyid)
             raise
         finally:
             # delete just imported key to leave environment as it was before
-            gpg_delete_key(keyid)
+            delete_key(keyid)
 
-    return armour.rstrip('\n')
+    return armour
diff --git a/tests/unittests/test_handler/test_handler_apt_source.py b/tests/unittests/test_handler/test_handler_apt_source.py
index 5b52f709..99a4d860 100644
--- a/tests/unittests/test_handler/test_handler_apt_source.py
+++ b/tests/unittests/test_handler/test_handler_apt_source.py
@@ -13,8 +13,8 @@ except ImportError:
 from mock import call
 
 from cloudinit.config import cc_apt_configure
-from cloudinit import util
 from cloudinit import gpg
+from cloudinit import util
 
 from ..helpers import TestCase
 
@@ -405,7 +405,7 @@ class TestAptSourceConfig(TestCase):
         params = self._get_default_params()
 
         with mock.patch.object(cc_apt_configure, 'add_apt_key_raw') as mockkey:
-            with mock.patch.object(gpg, 'gpg_getkeybyid',
+            with mock.patch.object(gpg, 'get_key_by_id',
                                    return_value=expectedkey) as mockgetkey:
                 cc_apt_configure.add_apt_sources([cfg], params)
 
-- 
cgit v1.2.3


From d4b587ebf500ddc2259fffc94a3c69c199c9a427 Mon Sep 17 00:00:00 2001
From: Scott Moser <smoser@ubuntu.com>
Date: Wed, 15 Jun 2016 22:50:12 -0400
Subject: fix some errors reported by pylint

pylint --errors-only found several errors.  Some of the changes
here represent real errors, others just code that pylint did
not like.
---
 cloudinit/config/cc_apt_configure.py      |  3 ++-
 cloudinit/config/cc_growpart.py           | 10 +++++-----
 cloudinit/distros/__init__.py             |  2 +-
 cloudinit/distros/arch.py                 |  4 ++--
 cloudinit/sources/DataSourceOVF.py        | 10 +++++-----
 cloudinit/sources/DataSourceOpenNebula.py |  2 +-
 cloudinit/sources/helpers/openstack.py    |  2 +-
 7 files changed, 17 insertions(+), 16 deletions(-)

(limited to 'cloudinit/config')

diff --git a/cloudinit/config/cc_apt_configure.py b/cloudinit/config/cc_apt_configure.py
index 96c4a43d..05ad4b03 100644
--- a/cloudinit/config/cc_apt_configure.py
+++ b/cloudinit/config/cc_apt_configure.py
@@ -208,8 +208,9 @@ def add_apt_sources(srclist, template_params=None, aa_repo_match=None):
         template_params = {}
 
     if aa_repo_match is None:
-        def aa_repo_match(x):
+        def _aa_repo_match(x):
             return False
+        aa_repo_match = _aa_repo_match
 
     errorlist = []
     srcdict = convert_to_new_format(srclist)
diff --git a/cloudinit/config/cc_growpart.py b/cloudinit/config/cc_growpart.py
index 859d69f1..40560f11 100644
--- a/cloudinit/config/cc_growpart.py
+++ b/cloudinit/config/cc_growpart.py
@@ -36,13 +36,13 @@ DEFAULT_CONFIG = {
 }
 
 
-def enum(**enums):
-    return type('Enum', (), enums)
+class RESIZE(object):
+    SKIPPED = "SKIPPED"
+    CHANGED = "CHANGED"
+    NOCHANGE = "NOCHANGE"
+    FAILED = "FAILED"
 
 
-RESIZE = enum(SKIPPED="SKIPPED", CHANGED="CHANGED", NOCHANGE="NOCHANGE",
-              FAILED="FAILED")
-
 LOG = logging.getLogger(__name__)
 
 
diff --git a/cloudinit/distros/__init__.py b/cloudinit/distros/__init__.py
index 5c29c804..14b500f8 100644
--- a/cloudinit/distros/__init__.py
+++ b/cloudinit/distros/__init__.py
@@ -454,7 +454,7 @@ class Distro(object):
             keys = kwargs['ssh_authorized_keys']
             if isinstance(keys, six.string_types):
                 keys = [keys]
-            if isinstance(keys, dict):
+            elif isinstance(keys, dict):
                 keys = list(keys.values())
             if keys is not None:
                 if not isinstance(keys, (tuple, list, set)):
diff --git a/cloudinit/distros/arch.py b/cloudinit/distros/arch.py
index 93a2e008..66209f22 100644
--- a/cloudinit/distros/arch.py
+++ b/cloudinit/distros/arch.py
@@ -196,6 +196,6 @@ def convert_resolv_conf(settings):
     """Returns a settings string formatted for resolv.conf."""
     result = ''
     if isinstance(settings, list):
-        for ns in list:
+        for ns in settings:
             result = result + 'nameserver %s\n' % ns
-        return result
+    return result
diff --git a/cloudinit/sources/DataSourceOVF.py b/cloudinit/sources/DataSourceOVF.py
index f2bb9366..43347cfb 100644
--- a/cloudinit/sources/DataSourceOVF.py
+++ b/cloudinit/sources/DataSourceOVF.py
@@ -37,16 +37,16 @@ from cloudinit.sources.helpers.vmware.imc.config_file \
     import ConfigFile
 from cloudinit.sources.helpers.vmware.imc.config_nic \
     import NicConfigurator
+from cloudinit.sources.helpers.vmware.imc.guestcust_error \
+    import GuestCustErrorEnum
 from cloudinit.sources.helpers.vmware.imc.guestcust_event \
     import GuestCustEventEnum
 from cloudinit.sources.helpers.vmware.imc.guestcust_state \
     import GuestCustStateEnum
-from cloudinit.sourceshelpers.vmware.imc.guestcust_error \
-    import GuestCustErrorEnum
-from cloudinit.sourceshelpers.vmware.imc.guestcust_util import (
-    set_customization_status,
+from cloudinit.sources.helpers.vmware.imc.guestcust_util import (
+    enable_nics,
     get_nics_to_enable,
-    enable_nics
+    set_customization_status
 )
 
 LOG = logging.getLogger(__name__)
diff --git a/cloudinit/sources/DataSourceOpenNebula.py b/cloudinit/sources/DataSourceOpenNebula.py
index 8f85b115..7b3a76b9 100644
--- a/cloudinit/sources/DataSourceOpenNebula.py
+++ b/cloudinit/sources/DataSourceOpenNebula.py
@@ -104,7 +104,7 @@ class DataSourceOpenNebula(sources.DataSource):
 
     def get_hostname(self, fqdn=False, resolve_ip=None):
         if resolve_ip is None:
-            if self.dsmode == sources.DSMODE_NET:
+            if self.dsmode == sources.DSMODE_NETWORK:
                 resolve_ip = True
             else:
                 resolve_ip = False
diff --git a/cloudinit/sources/helpers/openstack.py b/cloudinit/sources/helpers/openstack.py
index 494335b3..d52cb56a 100644
--- a/cloudinit/sources/helpers/openstack.py
+++ b/cloudinit/sources/helpers/openstack.py
@@ -157,7 +157,7 @@ class BaseReader(object):
         pass
 
     @abc.abstractmethod
-    def _path_read(self, path):
+    def _path_read(self, path, decode=False):
         pass
 
     @abc.abstractmethod
-- 
cgit v1.2.3