Fix apt configure unittests to run in more environments

As well as some improvements that were found along testing them and due to
the fact that we review some of that code again in the scope of curtin
currently.

Tests:
 - add a test for an alternate keyserver
 - harden mirrorfail tests to detect and skip if no network is available
 - improve apt_source related tests to work on CentOS7

Changes:
 - gpg key handling is now in python instead of a shell blob and moved
   to its own module.
 - packages/bddeb has an option to sign as someone else than smoser
 - make exception handling of apt_source features more specific
   (do not catch broad 'Exception')
 - rename some functions to reflect better what they actually do
 - capture some helper subp calls output to avoid spilling into stdout when
   not intended

2 files changed, 88 insertions, 38 deletions

diff --git a/cloudinit/config/cc_apt_configure.py b/cloudinit/config/cc_apt_configure.py
index 7a9777c0..96c4a43d 100644
--- a/cloudinit/config/cc_apt_configure.py
+++ b/cloudinit/config/cc_apt_configure.py
@@ -22,6 +22,7 @@ import glob
 import os
 import re
 
+from cloudinit import gpg
 from cloudinit import templater
 from cloudinit import util
 
@@ -34,21 +35,6 @@ APT_PROXY_FN = "/etc/apt/apt.conf.d/95cloud-init-proxy"
 # this will match 'XXX:YYY' (ie, 'cloud-archive:foo' or 'ppa:bar')
 ADD_APT_REPO_MATCH = r"^[\w-]+:\w"
 
-# A temporary shell program to get a given gpg key
-# from a given keyserver
-EXPORT_GPG_KEYID = """
-    k=${1} ks=${2};
-    exec 2>/dev/null
-    [ -n "$k" ] || exit 1;
-    armour=$(gpg --export --armour "${k}")
-    if [ -z "${armour}" ]; then
-       gpg --keyserver ${ks} --recv "${k}" >/dev/null &&
-          armour=$(gpg --export --armour "${k}") &&
-          gpg --batch --yes --delete-keys "${k}"
-    fi
-    [ -n "${armour}" ] && echo "${armour}"
-"""
-
 
 def handle(name, cfg, cloud, log, _args):
     if util.is_false(cfg.get('apt_configure_enabled', True)):
@@ -94,8 +80,8 @@ def handle(name, cfg, cloud, log, _args):
             def matcher(x):
                 return False
 
-        errors = add_sources(cfg['apt_sources'], params,
-                             aa_repo_match=matcher)
+        errors = add_apt_sources(cfg['apt_sources'], params,
+                                 aa_repo_match=matcher)
         for e in errors:
             log.warn("Add source error: %s", ':'.join(e))
 
@@ -108,17 +94,7 @@ def handle(name, cfg, cloud, log, _args):
             util.logexc(log, "Failed to run debconf-set-selections")
 
 
-# get gpg keyid from keyserver
-def getkeybyid(keyid, keyserver):
-    with util.ExtendedTemporaryFile(suffix='.sh', mode="w+", ) as fh:
-        fh.write(EXPORT_GPG_KEYID)
-        fh.flush()
-        cmd = ['/bin/sh', fh.name, keyid, keyserver]
-        (stdout, _stderr) = util.subp(cmd)
-        return stdout.strip()
-
-
-def mirror2lists_fileprefix(mirror):
+def mirrorurl_to_apt_fileprefix(mirror):
     string = mirror
     # take off http:// or ftp://
     if string.endswith("/"):
@@ -135,8 +111,8 @@ def rename_apt_lists(old_mirrors, new_mirrors, lists_d="/var/lib/apt/lists"):
         nmirror = new_mirrors.get(name)
         if not nmirror:
             continue
-        oprefix = os.path.join(lists_d, mirror2lists_fileprefix(omirror))
-        nprefix = os.path.join(lists_d, mirror2lists_fileprefix(nmirror))
+        oprefix = os.path.join(lists_d, mirrorurl_to_apt_fileprefix(omirror))
+        nprefix = os.path.join(lists_d, mirrorurl_to_apt_fileprefix(nmirror))
         if oprefix == nprefix:
             continue
         olen = len(oprefix)
@@ -171,7 +147,7 @@ def generate_sources_list(cfg, codename, mirrors, cloud, log):
     templater.render_to_file(template_fn, '/etc/apt/sources.list', params)
 
 
-def add_key_raw(key):
+def add_apt_key_raw(key):
     """
     actual adding of a key as defined in key argument
     to the system
@@ -179,10 +155,10 @@ def add_key_raw(key):
     try:
         util.subp(('apt-key', 'add', '-'), key)
     except util.ProcessExecutionError:
-        raise Exception('failed add key')
+        raise ValueError('failed to add apt GPG Key to apt keyring')
 
 
-def add_key(ent):
+def add_apt_key(ent):
     """
     add key to the system as defined in ent (if any)
     supports raw keys or keyid's
@@ -192,10 +168,10 @@ def add_key(ent):
         keyserver = "keyserver.ubuntu.com"
         if 'keyserver' in ent:
             keyserver = ent['keyserver']
-        ent['key'] = getkeybyid(ent['keyid'], keyserver)
+        ent['key'] = gpg.get_key_by_id(ent['keyid'], keyserver)
 
     if 'key' in ent:
-        add_key_raw(ent['key'])
+        add_apt_key_raw(ent['key'])
 
 
 def convert_to_new_format(srclist):
@@ -222,7 +198,7 @@ def convert_to_new_format(srclist):
     return srcdict
 
 
-def add_sources(srclist, template_params=None, aa_repo_match=None):
+def add_apt_sources(srclist, template_params=None, aa_repo_match=None):
     """
     add entries in /etc/apt/sources.list.d for each abbreviated
     sources.list entry in 'srclist'.  When rendering template, also
@@ -245,8 +221,8 @@ def add_sources(srclist, template_params=None, aa_repo_match=None):
 
         # keys can be added without specifying a source
         try:
-            add_key(ent)
-        except Exception as detail:
+            add_apt_key(ent)
+        except ValueError as detail:
             errorlist.append([ent, detail])
 
         if 'source' not in ent:
diff --git a/cloudinit/gpg.py b/cloudinit/gpg.py
new file mode 100644
index 00000000..6a76d785
--- /dev/null
+++ b/cloudinit/gpg.py
@@ -0,0 +1,74 @@
+"""gpg.py - Collection of gpg key related functions"""
+# vi: ts=4 expandtab
+#
+#    Copyright (C) 2016 Canonical Ltd.
+#
+#    Author: Scott Moser <scott.moser@canonical.com>
+#    Author: Christian Ehrhardt <christian.ehrhardt@canonical.com>
+#
+#    This program is free software: you can redistribute it and/or modify
+#    it under the terms of the GNU General Public License version 3, as
+#    published by the Free Software Foundation.
+#
+#    This program is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU General Public License for more details.
+#
+#    You should have received a copy of the GNU General Public License
+#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+from cloudinit import log as logging
+from cloudinit import util
+
+LOG = logging.getLogger(__name__)
+
+
+def export_armour(key):
+    """Export gpg key, armoured key gets returned"""
+    try:
+        (armour, _) = util.subp(["gpg", "--export", "--armour", key],
+                                capture=True)
+    except util.ProcessExecutionError as error:
+        # debug, since it happens for any key not on the system initially
+        LOG.debug('Failed to export armoured key "%s": %s', key, error)
+        armour = None
+    return armour
+
+
+def receive_key(key, keyserver):
+    """Receive gpg key from the specified keyserver"""
+    LOG.debug('Receive gpg key "%s"', key)
+    try:
+        util.subp(["gpg", "--keyserver", keyserver, "--recv-keys", key],
+                  capture=True)
+    except util.ProcessExecutionError as error:
+        raise ValueError(('Failed to import key "%s" '
+                          'from server "%s" - error %s') %
+                         (key, keyserver, error))
+
+
+def delete_key(key):
+    """Delete the specified key from the local gpg ring"""
+    try:
+        util.subp(["gpg", "--batch", "--yes", "--delete-keys", key],
+                  capture=True)
+    except util.ProcessExecutionError as error:
+        LOG.warn('Failed delete key "%s": %s', key, error)
+
+
+def get_key_by_id(keyid, keyserver="keyserver.ubuntu.com"):
+    """get gpg keyid from keyserver"""
+    armour = export_armour(keyid)
+    if not armour:
+        try:
+            receive_key(keyid, keyserver=keyserver)
+            armour = export_armour(keyid)
+        except ValueError:
+            LOG.exception('Failed to obtain gpg key %s', keyid)
+            raise
+        finally:
+            # delete just imported key to leave environment as it was before
+            delete_key(keyid)
+
+    return armour