Apt: add new apt configuration format

This adds an improved apt configuration format that is fully backwards
compatible with previous behavior. This is mostly copied from curtin's
implementation.

It does:
 * clean up and centralizes many of the top level 'apt_*' values that
   previously existed into a single top level 'apt'key.
 * support a 'source' in apt/sources/entry that has only a key
 * documents new features and adds tests.

See the added doc/examples/cloud-config-apt.txt for more information.

1 files changed, 1 insertions, 250 deletions

diff --git a/doc/examples/cloud-config.txt b/doc/examples/cloud-config.txt
index 3cc9c055..190029e4 100644
--- a/doc/examples/cloud-config.txt
+++ b/doc/examples/cloud-config.txt
@@ -18,256 +18,7 @@ package_upgrade: true
 # Aliases: apt_reboot_if_required
 package_reboot_if_required: true
 
-# Add apt repositories
-#
-# Default: auto select based on cloud metadata
-#  in ec2, the default is <region>.archive.ubuntu.com
-# apt_mirror:
-#   use the provided mirror
-# apt_mirror_search:
-#   search the list for the first mirror.
-#   this is currently very limited, only verifying that
-#   the mirror is dns resolvable or an IP address
-#
-# if neither apt_mirror nor apt_mirror search is set (the default)
-# then use the mirror provided by the DataSource found.
-# In EC2, that means using <region>.ec2.archive.ubuntu.com
-# 
-# if no mirror is provided by the DataSource, and 'apt_mirror_search_dns' is
-# true, then search for dns names '<distro>-mirror' in each of
-# - fqdn of this host per cloud metadata
-# - localdomain
-# - no domain (which would search domains listed in /etc/resolv.conf)
-# If there is a dns entry for <distro>-mirror, then it is assumed that there
-# is a distro mirror at http://<distro>-mirror.<domain>/<distro>
-#
-# That gives the cloud provider the opportunity to set mirrors of a distro
-# up and expose them only by creating dns entries.
-#
-# if none of that is found, then the default distro mirror is used
-apt_mirror: http://us.archive.ubuntu.com/ubuntu/
-apt_mirror_search: 
- - http://local-mirror.mydomain
- - http://archive.ubuntu.com
-
-apt_mirror_search_dns: False
-
-# apt_proxy (configure Acquire::HTTP::Proxy)
-# 'apt_http_proxy' is an alias for 'apt_proxy'.
-# Also, available are 'apt_ftp_proxy' and 'apt_https_proxy'.
-# These affect Acquire::FTP::Proxy and Acquire::HTTPS::Proxy respectively
-apt_proxy: http://my.apt.proxy:3128
-
-# apt_pipelining (configure Acquire::http::Pipeline-Depth)
-# Default: disables HTTP pipelining. Certain web servers, such
-# as S3 do not pipeline properly (LP: #948461).
-# Valid options:
-#   False/default: Disables pipelining for APT
-#   None/Unchanged: Use OS default
-#   Number: Set pipelining to some number (not recommended)
-apt_pipelining: False
-
-# Preserve existing /etc/apt/sources.list
-# Default: overwrite sources_list with mirror.  If this is true
-# then apt_mirror above will have no effect
-apt_preserve_sources_list: true
-
-# Provide a custom template for rendering sources.list
-# Default: a default template for Ubuntu/Debain will be used as packaged in
-#  Ubuntu: /etc/cloud/templates/sources.list.ubuntu.tmpl
-#  Debian: /etc/cloud/templates/sources.list.debian.tmpl
-#  Others: n/a
-# This will follow the normal mirror/codename replacement rules before
-# being written to disk.
-apt_custom_sources_list: |
-    ## template:jinja
-    ## Note, this file is written by cloud-init on first boot of an instance
-    ## modifications made here will not survive a re-bundle.
-    ## if you wish to make changes you can:
-    ## a.) add 'apt_preserve_sources_list: true' to /etc/cloud/cloud.cfg
-    ##     or do the same in user-data
-    ## b.) add sources in /etc/apt/sources.list.d
-    ## c.) make changes to template file /etc/cloud/templates/sources.list.tmpl
-    deb {{mirror}} {{codename}} main restricted
-    deb-src {{mirror}} {{codename}} main restricted
-
-    # could drop some of the usually used entries
-
-    # could refer to other mirrors
-    deb http://ddebs.ubuntu.com {{codename}} main restricted universe multiverse
-    deb http://ddebs.ubuntu.com {{codename}}-updates main restricted universe multiverse
-    deb http://ddebs.ubuntu.com {{codename}}-proposed main restricted universe multiverse
-
-    # or even more uncommon examples like local or NFS mounted repos,
-    # eventually whatever is compatible with sources.list syntax
-    deb file:/home/apt/debian unstable main contrib non-free
-
-# 'source' entries in apt-sources that match this python regex
-# expression will be passed to add-apt-repository
-add_apt_repo_match: '^[\w-]+:\w'
-
-# 'apt_sources' is a dictionary
-# The key is the filename and will be prepended by /etc/apt/sources.list.d/ if
-# it doesn't start with a '/'.
-# There are certain cases - where no content is written into a source.list file
-# where the filename will be ignored - yet it can still be used as index for
-# merging.
-# The value it maps to is a dictionary with the following optional entries:
-#   source: a sources.list entry (some variable replacements apply)
-#   keyid: providing a key to import via shortid or fingerprint
-#   key: providing a raw PGP key
-#   keyserver: keyserver to fetch keys from, default is keyserver.ubuntu.com
-#   filename: for compatibility with the older format (now the key to this
-#               dictionary is the filename). If specified this overwrites the
-#               filename given as key.
-
-# the new "filename: {specification-dictionary}, filename2: ..." format allows
-# better merging between multiple input files than a list like:
-# cloud-config1
-# sources:
-#    s1: {'key': 'key1', 'source': 'source1'}
-# cloud-config2
-# sources:
-#    s2: {'key': 'key2'}
-#    s1: {filename: 'foo'}
-# this would be merged to
-#sources:
-#    s1:
-#        filename: foo
-#        key: key1
-#        source: source1
-#    s2:
-#        key: key2
-# Be aware that this style of merging is not the default (for backward
-# compatibility reasons). You should specify the following merge_how to get
-# this more complete and modern merging behaviour:
-#   merge_how: "list()+dict()+str()"
-# This would then also be equivalent to the config merging used in curtin
-# (https://launchpad.net/curtin).
-
-# for more details see below in the various examples
-
-apt_sources:
- byobu-ppa.list:
-   source: "deb http://ppa.launchpad.net/byobu/ppa/ubuntu karmic main"
-   keyid: F430BBA5 # GPG key ID published on a key server
- # adding a source.list line, importing a gpg key for a given key id and
- # storing it in the file /etc/apt/sources.list.d/byobu-ppa.list
-
- # PPA shortcut:
- #  * Setup correct apt sources.list line
- #  * Import the signing key from LP
- #
- #  See https://help.launchpad.net/Packaging/PPA for more information
- #  this requires 'add-apt-repository'
- #  due to that the filename key is ignored in this case
- ignored1:
-   source: "ppa:smoser/ppa"    # Quote the string
-
- # Custom apt repository:
- #  * all that is required is 'source'
- #  * Creates a file in /etc/apt/sources.list.d/ for the sources list entry
- #  * [optional] Import the apt signing key from the keyserver 
- #  * Defaults:
- #    + keyserver: keyserver.ubuntu.com
- #
- #    See sources.list man page for more information about the format
- my-repo.list:
-   source: deb http://archive.ubuntu.com/ubuntu karmic-backports main universe multiverse restricted
-
- # sources can use $MIRROR and $RELEASE and they will be replaced
- # with the local mirror for this cloud, and the running release
- # the entry below would be possibly turned into:
- # source: deb http://us-east-1.ec2.archive.ubuntu.com/ubuntu natty multiverse
- my-repo.list:
-   source: deb $MIRROR $RELEASE multiverse
-
- # this would have the same end effect as 'ppa:byobu/ppa'
- my-repo.list:
-   source: "deb http://ppa.launchpad.net/byobu/ppa/ubuntu karmic main"
-   keyid: F430BBA5 # GPG key ID published on a key server
-   filename: byobu-ppa.list
-
- # this would only import the key without adding a ppa or other source spec
- # since this doesn't generate a source.list file the filename key is ignored
- ignored2:
-   keyid: F430BBA5 # GPG key ID published on a key server
-
- # In general keyid's can also be specified via their long fingerprints
- # since this doesn't generate a source.list file the filename key is ignored
- ignored3:
-   keyid: B59D 5F15 97A5 04B7 E230  6DCA 0620 BBCF 0368 3F77
-
- # Custom apt repository:
- #  * The apt signing key can also be specified
- #    by providing a pgp public key block
- #  * Providing the PGP key here is the most robust method for
- #    specifying a key, as it removes dependency on a remote key server
- my-repo.list:
-   source: deb http://ppa.launchpad.net/alestic/ppa/ubuntu karmic main 
-   key: | # The value needs to start with -----BEGIN PGP PUBLIC KEY BLOCK-----
-      -----BEGIN PGP PUBLIC KEY BLOCK-----
-      Version: SKS 1.0.10
-
-      mI0ESpA3UQEEALdZKVIMq0j6qWAXAyxSlF63SvPVIgxHPb9Nk0DZUixn+akqytxG4zKCONz6
-      qLjoBBfHnynyVLfT4ihg9an1PqxRnTO+JKQxl8NgKGz6Pon569GtAOdWNKw15XKinJTDLjnj
-      9y96ljJqRcpV9t/WsIcdJPcKFR5voHTEoABE2aEXABEBAAG0GUxhdW5jaHBhZCBQUEEgZm9y
-      IEFsZXN0aWOItgQTAQIAIAUCSpA3UQIbAwYLCQgHAwIEFQIIAwQWAgMBAh4BAheAAAoJEA7H
-      5Qi+CcVxWZ8D/1MyYvfj3FJPZUm2Yo1zZsQ657vHI9+pPouqflWOayRR9jbiyUFIn0VdQBrP
-      t0FwvnOFArUovUWoKAEdqR8hPy3M3APUZjl5K4cMZR/xaMQeQRZ5CHpS4DBKURKAHC0ltS5o
-      uBJKQOZm5iltJp15cgyIkBkGe8Mx18VFyVglAZey
-      =Y2oI
-      -----END PGP PUBLIC KEY BLOCK-----
-
- # Custom gpg key:
- #  * As with keyid, a key may also be specified without a related source.
- #  * all other facts mentioned above still apply
- # since this doesn't generate a source.list file the filename key is ignored
- ignored4:
-   key: | # The value needs to start with -----BEGIN PGP PUBLIC KEY BLOCK-----
-      -----BEGIN PGP PUBLIC KEY BLOCK-----
-      Version: SKS 1.0.10
-
-      mI0ESpA3UQEEALdZKVIMq0j6qWAXAyxSlF63SvPVIgxHPb9Nk0DZUixn+akqytxG4zKCONz6
-      qLjoBBfHnynyVLfT4ihg9an1PqxRnTO+JKQxl8NgKGz6Pon569GtAOdWNKw15XKinJTDLjnj
-      9y96ljJqRcpV9t/WsIcdJPcKFR5voHTEoABE2aEXABEBAAG0GUxhdW5jaHBhZCBQUEEgZm9y
-      IEFsZXN0aWOItgQTAQIAIAUCSpA3UQIbAwYLCQgHAwIEFQIIAwQWAgMBAh4BAheAAAoJEA7H
-      5Qi+CcVxWZ8D/1MyYvfj3FJPZUm2Yo1zZsQ657vHI9+pPouqflWOayRR9jbiyUFIn0VdQBrP
-      t0FwvnOFArUovUWoKAEdqR8hPy3M3APUZjl5K4cMZR/xaMQeQRZ5CHpS4DBKURKAHC0ltS5o
-      uBJKQOZm5iltJp15cgyIkBkGe8Mx18VFyVglAZey
-      =Y2oI
-      -----END PGP PUBLIC KEY BLOCK-----
-
-
-## apt config via system_info:
-# under the 'system_info', you can further customize cloud-init's interaction
-# with apt. 
-#   system_info:
-#    apt_get_command: [command, argument, argument]
-#    apt_get_upgrade_subcommand: dist-upgrade
-#
-# apt_get_command:
-#  To specify a different 'apt-get' command, set 'apt_get_command'.
-#  This must be a list, and the subcommand (update, upgrade) is appended to it.
-#  default is:
-#    ['apt-get', '--option=Dpkg::Options::=--force-confold',
-#     '--option=Dpkg::options::=--force-unsafe-io', '--assume-yes', '--quiet']
-#
-# apt_get_upgrade_subcommand:
-#  Specify a different subcommand for 'upgrade. The default is 'dist-upgrade'.
-#  This is the subcommand that is invoked if package_upgrade is set to true above.
-#
-# apt_get_wrapper:
-#   command: eatmydata
-#   enabled: [True, False, "auto"]
-#  
-
-# Install additional packages on first boot
-#
-# Default: none
-#
-# if packages are specified, this apt_update will be set to true
-#
+# For 'apt' specific config, see cloud-config-apt.txt
 packages:
  - pwgen
  - pastebinit