Apt sources configuration improvements

- keyid-only (no source statement)
- key only (no source statement)
- custom source.list template
- support long gpg key fingerprints with spaces
- fix issue with key's that were already in the local gpg keyring
- allowing a new format to specify apt_sources in a dictionary instead of a
  list to allow merging of configurations

LP: #1574113

1 files changed, 117 insertions, 10 deletions

diff --git a/doc/examples/cloud-config.txt b/doc/examples/cloud-config.txt
index 1236796c..62b297bc 100644
--- a/doc/examples/cloud-config.txt
+++ b/doc/examples/cloud-config.txt
@@ -72,14 +72,87 @@ apt_pipelining: False
 # then apt_mirror above will have no effect
 apt_preserve_sources_list: true
 
+# Provide a custom template for rendering sources.list
+# Default: a default template for Ubuntu/Debain will be used as packaged in
+#  Ubuntu: /etc/cloud/templates/sources.list.ubuntu.tmpl
+#  Debian: /etc/cloud/templates/sources.list.debian.tmpl
+#  Others: n/a
+# This will follow the normal mirror/codename replacement rules before
+# being written to disk.
+apt_custom_sources_list: |
+    ## template:jinja
+    ## Note, this file is written by cloud-init on first boot of an instance
+    ## modifications made here will not survive a re-bundle.
+    ## if you wish to make changes you can:
+    ## a.) add 'apt_preserve_sources_list: true' to /etc/cloud/cloud.cfg
+    ##     or do the same in user-data
+    ## b.) add sources in /etc/apt/sources.list.d
+    ## c.) make changes to template file /etc/cloud/templates/sources.list.tmpl
+    deb {{mirror}} {{codename}} main restricted
+    deb-src {{mirror}} {{codename}} main restricted
+
+    # could drop some of the usually used entries
+
+    # could refer to other mirrors
+    deb http://ddebs.ubuntu.com {{codename}} main restricted universe multiverse
+    deb http://ddebs.ubuntu.com {{codename}}-updates main restricted universe multiverse
+    deb http://ddebs.ubuntu.com {{codename}}-proposed main restricted universe multiverse
+
+    # or even more uncommon examples like local or NFS mounted repos,
+    # eventually whatever is compatible with sources.list syntax
+    deb file:/home/apt/debian unstable main contrib non-free
+
 # 'source' entries in apt-sources that match this python regex
 # expression will be passed to add-apt-repository
 add_apt_repo_match: '^[\w-]+:\w'
 
+# 'apt_sources' is a dictionary
+# The key is the filename and will be prepended by /etc/apt/sources.list.d/ if
+# it doesn't start with a '/'.
+# There are certain cases - where no content is written into a source.list file
+# where the filename will be ignored - yet it can still be used as index for
+# merging.
+# The value it maps to is a dictionary with the following optional entries:
+#   source: a sources.list entry (some variable replacements apply)
+#   keyid: providing a key to import via shortid or fingerprint
+#   key: providing a raw PGP key
+#   keyserver: keyserver to fetch keys from, default is keyserver.ubuntu.com
+#   filename: for compatibility with the older format (now the key to this
+#               dictionary is the filename). If specified this overwrites the
+#               filename given as key.
+
+# the new "filename: {specification-dictionary}, filename2: ..." format allows
+# better merging between multiple input files than a list like:
+# cloud-config1
+# sources:
+    s1: {'key': 'key1', 'source': 'source1'}
+# cloud-config2
+# sources:
+    s2: {'key': 'key2'}
+    s1: {filename: 'foo'}
+# this would be merged to
+#sources:
+#    s1:
+#        filename: foo
+#        key: key1
+#        source: source1
+#    s2:
+#        key: key2
+# Be aware that this style of merging is not the default (for backward
+# compatibility reasons). You should specify the following merge_how to get
+# this more complete and modern merging behaviour:
+#   merge_how: "list()+dict()+str()"
+# This would then also be equivalent to the config merging used in curtin
+# (https://launchpad.net/curtin).
+
+# for more details see below in the various examples
+
 apt_sources:
- - source: "deb http://ppa.launchpad.net/byobu/ppa/ubuntu karmic main"
+ byobu-ppa.list:
+   source: "deb http://ppa.launchpad.net/byobu/ppa/ubuntu karmic main"
    keyid: F430BBA5 # GPG key ID published on a key server
-   filename: byobu-ppa.list
+ # adding a source.list line, importing a gpg key for a given key id and
+ # storing it in the file /etc/apt/sources.list.d/byobu-ppa.list
 
  # PPA shortcut:
  #  * Setup correct apt sources.list line
@@ -87,7 +160,9 @@ apt_sources:
  #
  #  See https://help.launchpad.net/Packaging/PPA for more information
  #  this requires 'add-apt-repository'
- - source: "ppa:smoser/ppa"    # Quote the string
+ #  due to that the filename key is ignored in this case
+ ignored1:
+   source: "ppa:smoser/ppa"    # Quote the string
 
  # Custom apt repository:
  #  * all that is required is 'source'
@@ -95,29 +170,60 @@ apt_sources:
  #  * [optional] Import the apt signing key from the keyserver 
  #  * Defaults:
  #    + keyserver: keyserver.ubuntu.com
- #    + filename: cloud_config_sources.list
  #
  #    See sources.list man page for more information about the format
- - source: deb http://archive.ubuntu.com/ubuntu karmic-backports main universe multiverse restricted
+ my-repo.list:
+   source: deb http://archive.ubuntu.com/ubuntu karmic-backports main universe multiverse restricted
 
  # sources can use $MIRROR and $RELEASE and they will be replaced
  # with the local mirror for this cloud, and the running release
  # the entry below would be possibly turned into:
- # - source: deb http://us-east-1.ec2.archive.ubuntu.com/ubuntu natty multiverse
- - source: deb $MIRROR $RELEASE multiverse
+ # source: deb http://us-east-1.ec2.archive.ubuntu.com/ubuntu natty multiverse
+ my-repo.list:
+   source: deb $MIRROR $RELEASE multiverse
 
  # this would have the same end effect as 'ppa:byobu/ppa'
- - source: "deb http://ppa.launchpad.net/byobu/ppa/ubuntu karmic main"
+ my-repo.list:
+   source: "deb http://ppa.launchpad.net/byobu/ppa/ubuntu karmic main"
    keyid: F430BBA5 # GPG key ID published on a key server
    filename: byobu-ppa.list
 
+ # this would only import the key without adding a ppa or other source spec
+ # since this doesn't generate a source.list file the filename key is ignored
+ ignored2:
+   keyid: F430BBA5 # GPG key ID published on a key server
+
+ # In general keyid's can also be specified via their long fingerprints
+ # since this doesn't generate a source.list file the filename key is ignored
+ ignored3:
+   keyid: B59D 5F15 97A5 04B7 E230  6DCA 0620 BBCF 0368 3F77
+
  # Custom apt repository:
  #  * The apt signing key can also be specified
  #    by providing a pgp public key block
- #  * Providing the PBG key here is the most robust method for
+ #  * Providing the PGP key here is the most robust method for
  #    specifying a key, as it removes dependency on a remote key server
+ my-repo.list:
+   source: deb http://ppa.launchpad.net/alestic/ppa/ubuntu karmic main 
+   key: | # The value needs to start with -----BEGIN PGP PUBLIC KEY BLOCK-----
+      -----BEGIN PGP PUBLIC KEY BLOCK-----
+      Version: SKS 1.0.10
+
+      mI0ESpA3UQEEALdZKVIMq0j6qWAXAyxSlF63SvPVIgxHPb9Nk0DZUixn+akqytxG4zKCONz6
+      qLjoBBfHnynyVLfT4ihg9an1PqxRnTO+JKQxl8NgKGz6Pon569GtAOdWNKw15XKinJTDLjnj
+      9y96ljJqRcpV9t/WsIcdJPcKFR5voHTEoABE2aEXABEBAAG0GUxhdW5jaHBhZCBQUEEgZm9y
+      IEFsZXN0aWOItgQTAQIAIAUCSpA3UQIbAwYLCQgHAwIEFQIIAwQWAgMBAh4BAheAAAoJEA7H
+      5Qi+CcVxWZ8D/1MyYvfj3FJPZUm2Yo1zZsQ657vHI9+pPouqflWOayRR9jbiyUFIn0VdQBrP
+      t0FwvnOFArUovUWoKAEdqR8hPy3M3APUZjl5K4cMZR/xaMQeQRZ5CHpS4DBKURKAHC0ltS5o
+      uBJKQOZm5iltJp15cgyIkBkGe8Mx18VFyVglAZey
+      =Y2oI
+      -----END PGP PUBLIC KEY BLOCK-----
 
- - source: deb http://ppa.launchpad.net/alestic/ppa/ubuntu karmic main 
+ # Custom gpg key:
+ #  * As with keyid, a key may also be specified without a related source.
+ #  * all other facts mentioned above still apply
+ # since this doesn't generate a source.list file the filename key is ignored
+ ignored4:
    key: | # The value needs to start with -----BEGIN PGP PUBLIC KEY BLOCK-----
       -----BEGIN PGP PUBLIC KEY BLOCK-----
       Version: SKS 1.0.10
@@ -132,6 +238,7 @@ apt_sources:
       =Y2oI
       -----END PGP PUBLIC KEY BLOCK-----
 
+
 ## apt config via system_info:
 # under the 'system_info', you can further customize cloud-init's interaction
 # with apt.