Fix apt configure unittests to run in more environments

As well as some improvements that were found along testing them and due to
the fact that we review some of that code again in the scope of curtin
currently.

Tests:
 - add a test for an alternate keyserver
 - harden mirrorfail tests to detect and skip if no network is available
 - improve apt_source related tests to work on CentOS7

Changes:
 - gpg key handling is now in python instead of a shell blob and moved
   to its own module.
 - packages/bddeb has an option to sign as someone else than smoser
 - make exception handling of apt_source features more specific
   (do not catch broad 'Exception')
 - rename some functions to reflect better what they actually do
 - capture some helper subp calls output to avoid spilling into stdout when
   not intended

1 files changed, 14 insertions, 38 deletions

diff --git a/cloudinit/config/cc_apt_configure.py b/cloudinit/config/cc_apt_configure.py
index 7a9777c0..96c4a43d 100644
--- a/cloudinit/config/cc_apt_configure.py
+++ b/cloudinit/config/cc_apt_configure.py
@@ -22,6 +22,7 @@ import glob
 import os
 import re
 
+from cloudinit import gpg
 from cloudinit import templater
 from cloudinit import util
 
@@ -34,21 +35,6 @@ APT_PROXY_FN = "/etc/apt/apt.conf.d/95cloud-init-proxy"
 # this will match 'XXX:YYY' (ie, 'cloud-archive:foo' or 'ppa:bar')
 ADD_APT_REPO_MATCH = r"^[\w-]+:\w"
 
-# A temporary shell program to get a given gpg key
-# from a given keyserver
-EXPORT_GPG_KEYID = """
-    k=${1} ks=${2};
-    exec 2>/dev/null
-    [ -n "$k" ] || exit 1;
-    armour=$(gpg --export --armour "${k}")
-    if [ -z "${armour}" ]; then
-       gpg --keyserver ${ks} --recv "${k}" >/dev/null &&
-          armour=$(gpg --export --armour "${k}") &&
-          gpg --batch --yes --delete-keys "${k}"
-    fi
-    [ -n "${armour}" ] && echo "${armour}"
-"""
-
 
 def handle(name, cfg, cloud, log, _args):
     if util.is_false(cfg.get('apt_configure_enabled', True)):
@@ -94,8 +80,8 @@ def handle(name, cfg, cloud, log, _args):
             def matcher(x):
                 return False
 
-        errors = add_sources(cfg['apt_sources'], params,
-                             aa_repo_match=matcher)
+        errors = add_apt_sources(cfg['apt_sources'], params,
+                                 aa_repo_match=matcher)
         for e in errors:
             log.warn("Add source error: %s", ':'.join(e))
 
@@ -108,17 +94,7 @@ def handle(name, cfg, cloud, log, _args):
             util.logexc(log, "Failed to run debconf-set-selections")
 
 
-# get gpg keyid from keyserver
-def getkeybyid(keyid, keyserver):
-    with util.ExtendedTemporaryFile(suffix='.sh', mode="w+", ) as fh:
-        fh.write(EXPORT_GPG_KEYID)
-        fh.flush()
-        cmd = ['/bin/sh', fh.name, keyid, keyserver]
-        (stdout, _stderr) = util.subp(cmd)
-        return stdout.strip()
-
-
-def mirror2lists_fileprefix(mirror):
+def mirrorurl_to_apt_fileprefix(mirror):
     string = mirror
     # take off http:// or ftp://
     if string.endswith("/"):
@@ -135,8 +111,8 @@ def rename_apt_lists(old_mirrors, new_mirrors, lists_d="/var/lib/apt/lists"):
         nmirror = new_mirrors.get(name)
         if not nmirror:
             continue
-        oprefix = os.path.join(lists_d, mirror2lists_fileprefix(omirror))
-        nprefix = os.path.join(lists_d, mirror2lists_fileprefix(nmirror))
+        oprefix = os.path.join(lists_d, mirrorurl_to_apt_fileprefix(omirror))
+        nprefix = os.path.join(lists_d, mirrorurl_to_apt_fileprefix(nmirror))
         if oprefix == nprefix:
             continue
         olen = len(oprefix)
@@ -171,7 +147,7 @@ def generate_sources_list(cfg, codename, mirrors, cloud, log):
     templater.render_to_file(template_fn, '/etc/apt/sources.list', params)
 
 
-def add_key_raw(key):
+def add_apt_key_raw(key):
     """
     actual adding of a key as defined in key argument
     to the system
@@ -179,10 +155,10 @@ def add_key_raw(key):
     try:
         util.subp(('apt-key', 'add', '-'), key)
     except util.ProcessExecutionError:
-        raise Exception('failed add key')
+        raise ValueError('failed to add apt GPG Key to apt keyring')
 
 
-def add_key(ent):
+def add_apt_key(ent):
     """
     add key to the system as defined in ent (if any)
     supports raw keys or keyid's
@@ -192,10 +168,10 @@ def add_key(ent):
         keyserver = "keyserver.ubuntu.com"
         if 'keyserver' in ent:
             keyserver = ent['keyserver']
-        ent['key'] = getkeybyid(ent['keyid'], keyserver)
+        ent['key'] = gpg.get_key_by_id(ent['keyid'], keyserver)
 
     if 'key' in ent:
-        add_key_raw(ent['key'])
+        add_apt_key_raw(ent['key'])
 
 
 def convert_to_new_format(srclist):
@@ -222,7 +198,7 @@ def convert_to_new_format(srclist):
     return srcdict
 
 
-def add_sources(srclist, template_params=None, aa_repo_match=None):
+def add_apt_sources(srclist, template_params=None, aa_repo_match=None):
     """
     add entries in /etc/apt/sources.list.d for each abbreviated
     sources.list entry in 'srclist'.  When rendering template, also
@@ -245,8 +221,8 @@ def add_sources(srclist, template_params=None, aa_repo_match=None):
 
         # keys can be added without specifying a source
         try:
-            add_key(ent)
-        except Exception as detail:
+            add_apt_key(ent)
+        except ValueError as detail:
             errorlist.append([ent, detail])
 
         if 'source' not in ent: