Schema a d (#1211)

Migrate from legacy schema or define new schema in
cloud-init-schema.json, adding extensive schema tests for:
- cc_apt_configure
- cc_bootcmd
- cc_byobu
- cc_ca_certs
- cc_chef
- cc_debug
- cc_disable_ec2_metadata
- cc_disk_setup

Deprecate config hyphenated schema keys in favor of underscores:
  - ca_certs and ca_certs.remove_defaults instead of
    ca-certs and ca-certs.remove-defaults

- Continue to honor deprecated config keys but emit DEPRECATION
  warnings in logs for continued use of the deprecated keys:
  - apt_sources key
  - any apt v1 or v2 keys
  - use or ca-certs or ca_certs.remove-defaults

- Extend apt_configure schema
   - Define more strict schema below object opaque keys using
     patternProperties
  -  create common $def apt_configure.mirror for reuse in 'primary'
     and 'security' schema definitions within cc_apt_configure

 Co-Authored-by: James Falcon <james.falcon@canonical.com>

1 files changed, 13 insertions, 301 deletions

diff --git a/cloudinit/config/cc_apt_configure.py b/cloudinit/config/cc_apt_configure.py
index 37077a9f..7fe0e343 100644
--- a/cloudinit/config/cc_apt_configure.py
+++ b/cloudinit/config/cc_apt_configure.py
@@ -17,7 +17,7 @@ from textwrap import dedent
 from cloudinit import gpg
 from cloudinit import log as logging
 from cloudinit import subp, templater, util
-from cloudinit.config.schema import get_meta_doc, validate_cloudconfig_schema
+from cloudinit.config.schema import get_meta_doc
 from cloudinit.settings import PER_INSTANCE
 
 LOG = logging.getLogger(__name__)
@@ -31,33 +31,6 @@ CLOUD_INIT_GPG_DIR = "/etc/apt/cloud-init.gpg.d/"
 
 frequency = PER_INSTANCE
 distros = ["ubuntu", "debian"]
-mirror_property = {
-    "type": "array",
-    "items": {
-        "type": "object",
-        "additionalProperties": False,
-        "required": ["arches"],
-        "properties": {
-            "arches": {
-                "type": "array",
-                "items": {"type": "string"},
-                "minItems": 1,
-            },
-            "uri": {"type": "string", "format": "uri"},
-            "search": {
-                "type": "array",
-                "items": {"type": "string", "format": "uri"},
-                "minItems": 1,
-            },
-            "search_dns": {
-                "type": "boolean",
-            },
-            "keyid": {"type": "string"},
-            "key": {"type": "string"},
-            "keyserver": {"type": "string"},
-        },
-    },
-}
 
 meta = {
     "id": "cc_apt_configure",
@@ -147,275 +120,7 @@ meta = {
     "frequency": frequency,
 }
 
-schema = {
-    "type": "object",
-    "properties": {
-        "apt": {
-            "type": "object",
-            "additionalProperties": False,
-            "properties": {
-                "preserve_sources_list": {
-                    "type": "boolean",
-                    "default": False,
-                    "description": dedent(
-                        """\
-                        By default, cloud-init will generate a new sources
-                        list in ``/etc/apt/sources.list.d`` based on any
-                        changes specified in cloud config. To disable this
-                        behavior and preserve the sources list from the
-                        pristine image, set ``preserve_sources_list``
-                        to ``true``.
-
-                        The ``preserve_sources_list`` option overrides
-                        all other config keys that would alter
-                        ``sources.list`` or ``sources.list.d``,
-                        **except** for additional sources to be added
-                        to ``sources.list.d``."""
-                    ),
-                },
-                "disable_suites": {
-                    "type": "array",
-                    "items": {"type": "string"},
-                    "uniqueItems": True,
-                    "description": dedent(
-                        """\
-                        Entries in the sources list can be disabled using
-                        ``disable_suites``, which takes a list of suites
-                        to be disabled. If the string ``$RELEASE`` is
-                        present in a suite in the ``disable_suites`` list,
-                        it will be replaced with the release name. If a
-                        suite specified in ``disable_suites`` is not
-                        present in ``sources.list`` it will be ignored.
-                        For convenience, several aliases are provided for
-                        ``disable_suites``:
-
-                            - ``updates`` => ``$RELEASE-updates``
-                            - ``backports`` => ``$RELEASE-backports``
-                            - ``security`` => ``$RELEASE-security``
-                            - ``proposed`` => ``$RELEASE-proposed``
-                            - ``release`` => ``$RELEASE``.
-
-                        When a suite is disabled using ``disable_suites``,
-                        its entry in ``sources.list`` is not deleted; it
-                        is just commented out."""
-                    ),
-                },
-                "primary": {
-                    **mirror_property,
-                    "description": dedent(
-                        """\
-                        The primary and security archive mirrors can
-                        be specified using the ``primary`` and
-                        ``security`` keys, respectively. Both the
-                        ``primary`` and ``security`` keys take a list
-                        of configs, allowing mirrors to be specified
-                        on a per-architecture basis. Each config is a
-                        dictionary which must have an entry for
-                        ``arches``, specifying which architectures
-                        that config entry is for. The keyword
-                        ``default`` applies to any architecture not
-                        explicitly listed. The mirror url can be specified
-                        with the ``uri`` key, or a list of mirrors to
-                        check can be provided in order, with the first
-                        mirror that can be resolved being selected. This
-                        allows the same configuration to be used in
-                        different environment, with different hosts used
-                        for a local apt mirror. If no mirror is provided
-                        by ``uri`` or ``search``, ``search_dns`` may be
-                        used to search for dns names in the format
-                        ``<distro>-mirror`` in each of the following:
-
-                            - fqdn of this host per cloud metadata,
-                            - localdomain,
-                            - domains listed in ``/etc/resolv.conf``.
-
-                        If there is a dns entry for ``<distro>-mirror``,
-                        then it is assumed that there is a distro mirror
-                        at ``http://<distro>-mirror.<domain>/<distro>``.
-                        If the ``primary`` key is defined, but not the
-                        ``security`` key, then then configuration for
-                        ``primary`` is also used for ``security``.
-                        If ``search_dns`` is used for the ``security``
-                        key, the search pattern will be
-                        ``<distro>-security-mirror``.
-
-                        Each mirror may also specify a key to import via
-                        any of the following optional keys:
-
-                            - ``keyid``: a key to import via shortid or \
-                                  fingerprint.
-                            - ``key``: a raw PGP key.
-                            - ``keyserver``: alternate keyserver to pull \
-                                    ``keyid`` key from.
-
-                        If no mirrors are specified, or all lookups fail,
-                        then default mirrors defined in the datasource
-                        are used. If none are present in the datasource
-                        either the following defaults are used:
-
-                            - ``primary`` => \
-                            ``http://archive.ubuntu.com/ubuntu``.
-                            - ``security`` => \
-                            ``http://security.ubuntu.com/ubuntu``
-                        """
-                    ),
-                },
-                "security": {
-                    **mirror_property,
-                    "description": dedent(
-                        """\
-                        Please refer to the primary config documentation"""
-                    ),
-                },
-                "add_apt_repo_match": {
-                    "type": "string",
-                    "default": ADD_APT_REPO_MATCH,
-                    "description": dedent(
-                        """\
-                        All source entries in ``apt-sources`` that match
-                        regex in ``add_apt_repo_match`` will be added to
-                        the system using ``add-apt-repository``. If
-                        ``add_apt_repo_match`` is not specified, it
-                        defaults to ``{}``""".format(
-                            ADD_APT_REPO_MATCH
-                        )
-                    ),
-                },
-                "debconf_selections": {
-                    "type": "object",
-                    "items": {"type": "string"},
-                    "description": dedent(
-                        """\
-                        Debconf additional configurations can be specified as a
-                        dictionary under the ``debconf_selections`` config
-                        key, with each key in the dict representing a
-                        different set of configurations. The value of each key
-                        must be a string containing all the debconf
-                        configurations that must be applied. We will bundle
-                        all of the values and pass them to
-                        ``debconf-set-selections``. Therefore, each value line
-                        must be a valid entry for ``debconf-set-selections``,
-                        meaning that they must possess for distinct fields:
-
-                        ``pkgname question type answer``
-
-                        Where:
-
-                            - ``pkgname`` is the name of the package.
-                            - ``question`` the name of the questions.
-                            - ``type`` is the type of question.
-                            - ``answer`` is the value used to ansert the \
-                            question.
-
-                        For example: \
-                        ``ippackage ippackage/ip string 127.0.01``
-                    """
-                    ),
-                },
-                "sources_list": {
-                    "type": "string",
-                    "description": dedent(
-                        """\
-                       Specifies a custom template for rendering
-                       ``sources.list`` . If no ``sources_list`` template
-                       is given, cloud-init will use sane default. Within
-                       this template, the following strings will be
-                       replaced with the appropriate values:
-
-                            - ``$MIRROR``
-                            - ``$RELEASE``
-                            - ``$PRIMARY``
-                            - ``$SECURITY``
-                            - ``$KEY_FILE``"""
-                    ),
-                },
-                "conf": {
-                    "type": "string",
-                    "description": dedent(
-                        """\
-                        Specify configuration for apt, such as proxy
-                        configuration. This configuration is specified as a
-                        string. For multiline apt configuration, make sure
-                        to follow yaml syntax."""
-                    ),
-                },
-                "https_proxy": {
-                    "type": "string",
-                    "description": dedent(
-                        """\
-                        More convenient way to specify https apt proxy.
-                        https proxy url is specified in the format
-                        ``https://[[user][:pass]@]host[:port]/``."""
-                    ),
-                },
-                "http_proxy": {
-                    "type": "string",
-                    "description": dedent(
-                        """\
-                        More convenient way to specify http apt proxy.
-                        http proxy url is specified in the format
-                        ``http://[[user][:pass]@]host[:port]/``."""
-                    ),
-                },
-                "proxy": {
-                    "type": "string",
-                    "description": "Alias for defining a http apt proxy.",
-                },
-                "ftp_proxy": {
-                    "type": "string",
-                    "description": dedent(
-                        """\
-                        More convenient way to specify ftp apt proxy.
-                        ftp proxy url is specified in the format
-                        ``ftp://[[user][:pass]@]host[:port]/``."""
-                    ),
-                },
-                "sources": {
-                    "type": "object",
-                    "items": {"type": "string"},
-                    "description": dedent(
-                        """\
-                        Source list entries can be specified as a
-                        dictionary under the ``sources`` config key, with
-                        each key in the dict representing a different source
-                        file. The key of each source entry will be used
-                        as an id that can be referenced in other config
-                        entries, as well as the filename for the source's
-                        configuration under ``/etc/apt/sources.list.d``.
-                        If the name does not end with ``.list``, it will
-                        be appended. If there is no configuration for a
-                        key in ``sources``, no file will be written, but
-                        the key may still be referred to as an id in other
-                        ``sources`` entries.
-
-                        Each entry under ``sources`` is a dictionary which
-                        may contain any of the following optional keys:
-
-                            - ``source``: a sources.list entry \
-                                  (some variable replacements apply).
-                            - ``keyid``: a key to import via shortid or \
-                                  fingerprint.
-                            - ``key``: a raw PGP key.
-                            - ``keyserver``: alternate keyserver to pull \
-                                    ``keyid`` key from.
-                            - ``filename``: specify the name of the .list file
-
-                        The ``source`` key supports variable
-                        replacements for the following strings:
-
-                            - ``$MIRROR``
-                            - ``$PRIMARY``
-                            - ``$SECURITY``
-                            - ``$RELEASE``
-                            - ``$KEY_FILE``"""
-                    ),
-                },
-            },
-        }
-    },
-}
-
-__doc__ = get_meta_doc(meta, schema)
+__doc__ = get_meta_doc(meta)
 
 
 # place where apt stores cached repository data
@@ -474,7 +179,6 @@ def handle(name, ocfg, cloud, log, _):
             )
         )
 
-    validate_cloudconfig_schema(cfg, schema)
     apply_debconf_selections(cfg, target)
     apply_apt(cfg, cloud, target)
 
@@ -889,6 +593,10 @@ def add_apt_sources(
 def convert_v1_to_v2_apt_format(srclist):
     """convert v1 apt format to v2 (dict in apt_sources)"""
     srcdict = {}
+    LOG.warning(
+        "DEPRECATION: 'apt_sources' deprecated config key found."
+        " Use 'apt' instead"
+    )
     if isinstance(srclist, list):
         LOG.debug("apt config: convert V1 to V2 format (source list to dict)")
         for srcent in srclist:
@@ -963,15 +671,19 @@ def convert_v2_to_v3_apt_format(oldcfg):
     # no old config, so no new one to be created
     if not needtoconvert:
         return oldcfg
-    LOG.debug(
-        "apt config: convert V2 to V3 format for keys '%s'",
+    LOG.warning(
+        "DEPRECATION apt: converted deprecated config V2 to V3 format for"
+        " keys '%s'. Use updated config keys.",
         ", ".join(needtoconvert),
     )
 
     # if old AND new config are provided, prefer the new one (LP #1616831)
     newaptcfg = oldcfg.get("apt", None)
     if newaptcfg is not None:
-        LOG.debug("apt config: V1/2 and V3 format specified, preferring V3")
+        LOG.warning(
+            "DEPRECATION: apt config: deprecated V1/2 and V3 format specified,"
+            " preferring V3"
+        )
         for oldkey in needtoconvert:
             newkey = mapoldkeys[oldkey]
             verify = oldcfg[oldkey]  # drop, but keep a ref for verification