diff options
| author | Chad Smith <chad.smith@canonical.com> | 2022-01-31 20:45:29 -0700 |
|---|---|---|
| committer | GitHub <noreply@github.com> | 2022-01-31 20:45:29 -0700 |
| commit | af7eb1deab12c7208853c5d18b55228e0ba29c4d (patch) | |
| tree | db4f4b836a972f72aa4fdddf3840c136bc1abb57 | |
| parent | 46a0126e874927353e83b385b58ab054e58667cc (diff) | |
| download | vyos-cloud-init-af7eb1deab12c7208853c5d18b55228e0ba29c4d.tar.gz vyos-cloud-init-af7eb1deab12c7208853c5d18b55228e0ba29c4d.zip | |
Schema a d (#1211)
Migrate from legacy schema or define new schema in
cloud-init-schema.json, adding extensive schema tests for:
- cc_apt_configure
- cc_bootcmd
- cc_byobu
- cc_ca_certs
- cc_chef
- cc_debug
- cc_disable_ec2_metadata
- cc_disk_setup
Deprecate config hyphenated schema keys in favor of underscores:
- ca_certs and ca_certs.remove_defaults instead of
ca-certs and ca-certs.remove-defaults
- Continue to honor deprecated config keys but emit DEPRECATION
warnings in logs for continued use of the deprecated keys:
- apt_sources key
- any apt v1 or v2 keys
- use or ca-certs or ca_certs.remove-defaults
- Extend apt_configure schema
- Define more strict schema below object opaque keys using
patternProperties
- create common $def apt_configure.mirror for reuse in 'primary'
and 'security' schema definitions within cc_apt_configure
Co-Authored-by: James Falcon <james.falcon@canonical.com>
21 files changed, 1443 insertions, 901 deletions
diff --git a/cloudinit/config/cc_apt_configure.py b/cloudinit/config/cc_apt_configure.py index 37077a9f..7fe0e343 100644 --- a/cloudinit/config/cc_apt_configure.py +++ b/cloudinit/config/cc_apt_configure.py @@ -17,7 +17,7 @@ from textwrap import dedent from cloudinit import gpg from cloudinit import log as logging from cloudinit import subp, templater, util -from cloudinit.config.schema import get_meta_doc, validate_cloudconfig_schema +from cloudinit.config.schema import get_meta_doc from cloudinit.settings import PER_INSTANCE LOG = logging.getLogger(__name__) @@ -31,33 +31,6 @@ CLOUD_INIT_GPG_DIR = "/etc/apt/cloud-init.gpg.d/" frequency = PER_INSTANCE distros = ["ubuntu", "debian"] -mirror_property = { - "type": "array", - "items": { - "type": "object", - "additionalProperties": False, - "required": ["arches"], - "properties": { - "arches": { - "type": "array", - "items": {"type": "string"}, - "minItems": 1, - }, - "uri": {"type": "string", "format": "uri"}, - "search": { - "type": "array", - "items": {"type": "string", "format": "uri"}, - "minItems": 1, - }, - "search_dns": { - "type": "boolean", - }, - "keyid": {"type": "string"}, - "key": {"type": "string"}, - "keyserver": {"type": "string"}, - }, - }, -} meta = { "id": "cc_apt_configure", @@ -147,275 +120,7 @@ meta = { "frequency": frequency, } -schema = { - "type": "object", - "properties": { - "apt": { - "type": "object", - "additionalProperties": False, - "properties": { - "preserve_sources_list": { - "type": "boolean", - "default": False, - "description": dedent( - """\ - By default, cloud-init will generate a new sources - list in ``/etc/apt/sources.list.d`` based on any - changes specified in cloud config. To disable this - behavior and preserve the sources list from the - pristine image, set ``preserve_sources_list`` - to ``true``. - - The ``preserve_sources_list`` option overrides - all other config keys that would alter - ``sources.list`` or ``sources.list.d``, - **except** for additional sources to be added - to ``sources.list.d``.""" - ), - }, - "disable_suites": { - "type": "array", - "items": {"type": "string"}, - "uniqueItems": True, - "description": dedent( - """\ - Entries in the sources list can be disabled using - ``disable_suites``, which takes a list of suites - to be disabled. If the string ``$RELEASE`` is - present in a suite in the ``disable_suites`` list, - it will be replaced with the release name. If a - suite specified in ``disable_suites`` is not - present in ``sources.list`` it will be ignored. - For convenience, several aliases are provided for - ``disable_suites``: - - - ``updates`` => ``$RELEASE-updates`` - - ``backports`` => ``$RELEASE-backports`` - - ``security`` => ``$RELEASE-security`` - - ``proposed`` => ``$RELEASE-proposed`` - - ``release`` => ``$RELEASE``. - - When a suite is disabled using ``disable_suites``, - its entry in ``sources.list`` is not deleted; it - is just commented out.""" - ), - }, - "primary": { - **mirror_property, - "description": dedent( - """\ - The primary and security archive mirrors can - be specified using the ``primary`` and - ``security`` keys, respectively. Both the - ``primary`` and ``security`` keys take a list - of configs, allowing mirrors to be specified - on a per-architecture basis. Each config is a - dictionary which must have an entry for - ``arches``, specifying which architectures - that config entry is for. The keyword - ``default`` applies to any architecture not - explicitly listed. The mirror url can be specified - with the ``uri`` key, or a list of mirrors to - check can be provided in order, with the first - mirror that can be resolved being selected. This - allows the same configuration to be used in - different environment, with different hosts used - for a local apt mirror. If no mirror is provided - by ``uri`` or ``search``, ``search_dns`` may be - used to search for dns names in the format - ``<distro>-mirror`` in each of the following: - - - fqdn of this host per cloud metadata, - - localdomain, - - domains listed in ``/etc/resolv.conf``. - - If there is a dns entry for ``<distro>-mirror``, - then it is assumed that there is a distro mirror - at ``http://<distro>-mirror.<domain>/<distro>``. - If the ``primary`` key is defined, but not the - ``security`` key, then then configuration for - ``primary`` is also used for ``security``. - If ``search_dns`` is used for the ``security`` - key, the search pattern will be - ``<distro>-security-mirror``. - - Each mirror may also specify a key to import via - any of the following optional keys: - - - ``keyid``: a key to import via shortid or \ - fingerprint. - - ``key``: a raw PGP key. - - ``keyserver``: alternate keyserver to pull \ - ``keyid`` key from. - - If no mirrors are specified, or all lookups fail, - then default mirrors defined in the datasource - are used. If none are present in the datasource - either the following defaults are used: - - - ``primary`` => \ - ``http://archive.ubuntu.com/ubuntu``. - - ``security`` => \ - ``http://security.ubuntu.com/ubuntu`` - """ - ), - }, - "security": { - **mirror_property, - "description": dedent( - """\ - Please refer to the primary config documentation""" - ), - }, - "add_apt_repo_match": { - "type": "string", - "default": ADD_APT_REPO_MATCH, - "description": dedent( - """\ - All source entries in ``apt-sources`` that match - regex in ``add_apt_repo_match`` will be added to - the system using ``add-apt-repository``. If - ``add_apt_repo_match`` is not specified, it - defaults to ``{}``""".format( - ADD_APT_REPO_MATCH - ) - ), - }, - "debconf_selections": { - "type": "object", - "items": {"type": "string"}, - "description": dedent( - """\ - Debconf additional configurations can be specified as a - dictionary under the ``debconf_selections`` config - key, with each key in the dict representing a - different set of configurations. The value of each key - must be a string containing all the debconf - configurations that must be applied. We will bundle - all of the values and pass them to - ``debconf-set-selections``. Therefore, each value line - must be a valid entry for ``debconf-set-selections``, - meaning that they must possess for distinct fields: - - ``pkgname question type answer`` - - Where: - - - ``pkgname`` is the name of the package. - - ``question`` the name of the questions. - - ``type`` is the type of question. - - ``answer`` is the value used to ansert the \ - question. - - For example: \ - ``ippackage ippackage/ip string 127.0.01`` - """ - ), - }, - "sources_list": { - "type": "string", - "description": dedent( - """\ - Specifies a custom template for rendering - ``sources.list`` . If no ``sources_list`` template - is given, cloud-init will use sane default. Within - this template, the following strings will be - replaced with the appropriate values: - - - ``$MIRROR`` - - ``$RELEASE`` - - ``$PRIMARY`` - - ``$SECURITY`` - - ``$KEY_FILE``""" - ), - }, - "conf": { - "type": "string", - "description": dedent( - """\ - Specify configuration for apt, such as proxy - configuration. This configuration is specified as a - string. For multiline apt configuration, make sure - to follow yaml syntax.""" - ), - }, - "https_proxy": { - "type": "string", - "description": dedent( - """\ - More convenient way to specify https apt proxy. - https proxy url is specified in the format - ``https://[[user][:pass]@]host[:port]/``.""" - ), - }, - "http_proxy": { - "type": "string", - "description": dedent( - """\ - More convenient way to specify http apt proxy. - http proxy url is specified in the format - ``http://[[user][:pass]@]host[:port]/``.""" - ), - }, - "proxy": { - "type": "string", - "description": "Alias for defining a http apt proxy.", - }, - "ftp_proxy": { - "type": "string", - "description": dedent( - """\ - More convenient way to specify ftp apt proxy. - ftp proxy url is specified in the format - ``ftp://[[user][:pass]@]host[:port]/``.""" - ), - }, - "sources": { - "type": "object", - "items": {"type": "string"}, - "description": dedent( - """\ - Source list entries can be specified as a - dictionary under the ``sources`` config key, with - each key in the dict representing a different source - file. The key of each source entry will be used - as an id that can be referenced in other config - entries, as well as the filename for the source's - configuration under ``/etc/apt/sources.list.d``. - If the name does not end with ``.list``, it will - be appended. If there is no configuration for a - key in ``sources``, no file will be written, but - the key may still be referred to as an id in other - ``sources`` entries. - - Each entry under ``sources`` is a dictionary which - may contain any of the following optional keys: - - - ``source``: a sources.list entry \ - (some variable replacements apply). - - ``keyid``: a key to import via shortid or \ - fingerprint. - - ``key``: a raw PGP key. - - ``keyserver``: alternate keyserver to pull \ - ``keyid`` key from. - - ``filename``: specify the name of the .list file - - The ``source`` key supports variable - replacements for the following strings: - - - ``$MIRROR`` - - ``$PRIMARY`` - - ``$SECURITY`` - - ``$RELEASE`` - - ``$KEY_FILE``""" - ), - }, - }, - } - }, -} - -__doc__ = get_meta_doc(meta, schema) +__doc__ = get_meta_doc(meta) # place where apt stores cached repository data @@ -474,7 +179,6 @@ def handle(name, ocfg, cloud, log, _): ) ) - validate_cloudconfig_schema(cfg, schema) apply_debconf_selections(cfg, target) apply_apt(cfg, cloud, target) @@ -889,6 +593,10 @@ def add_apt_sources( def convert_v1_to_v2_apt_format(srclist): """convert v1 apt format to v2 (dict in apt_sources)""" srcdict = {} + LOG.warning( + "DEPRECATION: 'apt_sources' deprecated config key found." + " Use 'apt' instead" + ) if isinstance(srclist, list): LOG.debug("apt config: convert V1 to V2 format (source list to dict)") for srcent in srclist: @@ -963,15 +671,19 @@ def convert_v2_to_v3_apt_format(oldcfg): # no old config, so no new one to be created if not needtoconvert: return oldcfg - LOG.debug( - "apt config: convert V2 to V3 format for keys '%s'", + LOG.warning( + "DEPRECATION apt: converted deprecated config V2 to V3 format for" + " keys '%s'. Use updated config keys.", ", ".join(needtoconvert), ) # if old AND new config are provided, prefer the new one (LP #1616831) newaptcfg = oldcfg.get("apt", None) if newaptcfg is not None: - LOG.debug("apt config: V1/2 and V3 format specified, preferring V3") + LOG.warning( + "DEPRECATION: apt config: deprecated V1/2 and V3 format specified," + " preferring V3" + ) for oldkey in needtoconvert: newkey = mapoldkeys[oldkey] verify = oldcfg[oldkey] # drop, but keep a ref for verification diff --git a/cloudinit/config/cc_bootcmd.py b/cloudinit/config/cc_bootcmd.py index bff11a24..3a239376 100644 --- a/cloudinit/config/cc_bootcmd.py +++ b/cloudinit/config/cc_bootcmd.py @@ -13,17 +13,11 @@ import os from textwrap import dedent from cloudinit import subp, temp_utils, util -from cloudinit.config.schema import get_meta_doc, validate_cloudconfig_schema +from cloudinit.config.schema import get_meta_doc from cloudinit.settings import PER_ALWAYS frequency = PER_ALWAYS -# The schema definition for each cloud-config module is a strict contract for -# describing supported configuration parameters for each cloud-config section. -# It allows cloud-config to validate and alert users to invalid or ignored -# configuration options before actually attempting to deploy with said -# configuration. - distros = ["all"] meta = { @@ -62,25 +56,7 @@ meta = { "frequency": PER_ALWAYS, } -schema = { - "type": "object", - "properties": { - "bootcmd": { - "type": "array", - "items": { - "oneOf": [ - {"type": "array", "items": {"type": "string"}}, - {"type": "string"}, - ] - }, - "additionalItems": False, # Reject items of non-string non-list - "additionalProperties": False, - "minItems": 1, - } - }, -} - -__doc__ = get_meta_doc(meta, schema) # Supplement python help() +__doc__ = get_meta_doc(meta) def handle(name, cfg, cloud, log, _args): @@ -91,7 +67,6 @@ def handle(name, cfg, cloud, log, _args): ) return - validate_cloudconfig_schema(cfg, schema) with temp_utils.ExtendedTemporaryFile(suffix=".sh") as tmpf: try: content = util.shellify(cfg["bootcmd"]) diff --git a/cloudinit/config/cc_byobu.py b/cloudinit/config/cc_byobu.py index 53b6d0c8..b96736a4 100755 --- a/cloudinit/config/cc_byobu.py +++ b/cloudinit/config/cc_byobu.py @@ -6,11 +6,14 @@ # # This file is part of cloud-init. See LICENSE file for license information. -""" -Byobu ------ -**Summary:** enable/disable byobu system wide and for default user +"""Byobu: Enable/disable byobu system wide and for default user.""" + +from cloudinit import subp, util +from cloudinit.config.schema import get_meta_doc +from cloudinit.distros import ug_util +from cloudinit.settings import PER_INSTANCE +MODULE_DESCRIPTION = """\ This module controls whether byobu is enabled or disabled system wide and for the default system user. If byobu is to be enabled, this module will ensure it is installed. Likewise, if it is to be disabled, it will be removed if @@ -26,23 +29,24 @@ Valid configuration options for this module are: - ``disable``: disable byobu for all users - ``user``: alias for ``enable-user`` - ``system``: alias for ``enable-system`` - -**Internal name:** ``cc_byobu`` - -**Module frequency:** per instance - -**Supported distros:** ubuntu, debian - -**Config keys**:: - - byobu_by_default: <user/system> """ - -from cloudinit import subp, util -from cloudinit.distros import ug_util - distros = ["ubuntu", "debian"] +meta = { + "id": "cc_byobu", + "name": "Byobu", + "title": "Enable/disable byobu system wide and for default user", + "description": MODULE_DESCRIPTION, + "distros": distros, + "frequency": PER_INSTANCE, + "examples": [ + "byobu_by_default: enable-user", + "byobu_by_default: disable-system", + ], +} + +__doc__ = get_meta_doc(meta) + def handle(name, cfg, cloud, log, args): if len(args) != 0: diff --git a/cloudinit/config/cc_ca_certs.py b/cloudinit/config/cc_ca_certs.py index 9de065ab..c46d0fbe 100644 --- a/cloudinit/config/cc_ca_certs.py +++ b/cloudinit/config/cc_ca_certs.py @@ -2,46 +2,14 @@ # # This file is part of cloud-init. See LICENSE file for license information. -""" -CA Certs --------- -**Summary:** add ca certificates - -This module adds CA certificates to ``/etc/ca-certificates.conf`` and updates -the ssl cert cache using ``update-ca-certificates``. The default certificates -can be removed from the system with the configuration option -``remove-defaults``. - -.. note:: - certificates must be specified using valid yaml. in order to specify a - multiline certificate, the yaml multiline list syntax must be used - -.. note:: - For Alpine Linux the "remove-defaults" functionality works if the - ca-certificates package is installed but not if the - ca-certificates-bundle package is installed. - -**Internal name:** ``cc_ca_certs`` - -**Module frequency:** per instance - -**Supported distros:** alpine, debian, ubuntu, rhel - -**Config keys**:: - - ca-certs: - remove-defaults: <true/false> - trusted: - - <single line cert> - - | - -----BEGIN CERTIFICATE----- - YOUR-ORGS-TRUSTED-CA-CERT-HERE - -----END CERTIFICATE----- -""" +"""CA Certs: Add ca certificates.""" import os +from textwrap import dedent from cloudinit import subp, util +from cloudinit.config.schema import get_meta_doc +from cloudinit.settings import PER_INSTANCE DEFAULT_CONFIG = { "ca_cert_path": "/usr/share/ca-certificates/", @@ -60,9 +28,48 @@ DISTRO_OVERRIDES = { } } +MODULE_DESCRIPTION = """\ +This module adds CA certificates to ``/etc/ca-certificates.conf`` and updates +the ssl cert cache using ``update-ca-certificates``. The default certificates +can be removed from the system with the configuration option +``remove_defaults``. +.. note:: + certificates must be specified using valid yaml. in order to specify a + multiline certificate, the yaml multiline list syntax must be used + +.. note:: + For Alpine Linux the "remove_defaults" functionality works if the + ca-certificates package is installed but not if the + ca-certificates-bundle package is installed. +""" distros = ["alpine", "debian", "ubuntu", "rhel"] +meta = { + "id": "cc_ca_certs", + "name": "CA Certificates", + "title": "Add ca certificates", + "description": MODULE_DESCRIPTION, + "distros": distros, + "frequency": PER_INSTANCE, + "examples": [ + dedent( + """\ + ca_certs: + remove_defaults: true + trusted: + - single_line_cert + - | + -----BEGIN CERTIFICATE----- + YOUR-ORGS-TRUSTED-CA-CERT-HERE + -----END CERTIFICATE----- + """ + ) + ], +} + +__doc__ = get_meta_doc(meta) + def _distro_ca_certs_configs(distro_name): """Return a distro-specific ca_certs config dictionary @@ -162,20 +169,37 @@ def handle(name, cfg, cloud, log, _args): @param log: Pre-initialized Python logger object to use for logging. @param args: Any module arguments from cloud.cfg """ - # If there isn't a ca-certs section in the configuration don't do anything - if "ca-certs" not in cfg: + if "ca-certs" in cfg: + log.warning( + "DEPRECATION: key 'ca-certs' is now deprecated. Use 'ca_certs'" + " instead." + ) + elif "ca_certs" not in cfg: log.debug( - "Skipping module named %s, no 'ca-certs' key in configuration", + "Skipping module named %s, no 'ca_certs' key in configuration", name, ) return - ca_cert_cfg = cfg["ca-certs"] + if "ca-certs" in cfg and "ca_certs" in cfg: + log.warning( + "Found both ca-certs (deprecated) and ca_certs config keys." + " Ignoring ca-certs." + ) + ca_cert_cfg = cfg.get("ca_certs", cfg.get("ca-certs")) distro_cfg = _distro_ca_certs_configs(cloud.distro.name) - # If there is a remove-defaults option set to true, remove the system + # If there is a remove_defaults option set to true, remove the system # default trusted CA certs first. - if ca_cert_cfg.get("remove-defaults", False): + if "remove-defaults" in ca_cert_cfg: + log.warning( + "DEPRECATION: key 'ca-certs.remove-defaults' is now deprecated." + " Use 'ca_certs.remove_defaults' instead." + ) + if ca_cert_cfg.get("remove-defaults", False): + log.debug("Removing default certificates") + remove_default_ca_certs(cloud.distro.name, distro_cfg) + elif ca_cert_cfg.get("remove_defaults", False): log.debug("Removing default certificates") remove_default_ca_certs(cloud.distro.name, distro_cfg) diff --git a/cloudinit/config/cc_chef.py b/cloudinit/config/cc_chef.py index 67889683..aaf7eaf1 100644 --- a/cloudinit/config/cc_chef.py +++ b/cloudinit/config/cc_chef.py @@ -14,7 +14,7 @@ import os from textwrap import dedent from cloudinit import subp, temp_utils, templater, url_helper, util -from cloudinit.config.schema import get_meta_doc, validate_cloudconfig_schema +from cloudinit.config.schema import get_meta_doc from cloudinit.settings import PER_ALWAYS RUBY_VERSION_DEFAULT = "1.8" @@ -137,303 +137,7 @@ meta = { "frequency": frequency, } -schema = { - "type": "object", - "properties": { - "chef": { - "type": "object", - "additionalProperties": False, - "properties": { - "directories": { - "type": "array", - "items": {"type": "string"}, - "uniqueItems": True, - "description": dedent( - """\ - Create the necessary directories for chef to run. By - default, it creates the following directories: - - {chef_dirs}""" - ).format( - chef_dirs="\n".join( - [" - ``{}``".format(d) for d in CHEF_DIRS] - ) - ), - }, - "validation_cert": { - "type": "string", - "description": dedent( - """\ - Optional string to be written to file validation_key. - Special value ``system`` means set use existing file. - """ - ), - }, - "validation_key": { - "type": "string", - "default": CHEF_VALIDATION_PEM_PATH, - "description": dedent( - """\ - Optional path for validation_cert. default to - ``{}``.""".format( - CHEF_VALIDATION_PEM_PATH - ) - ), - }, - "firstboot_path": { - "type": "string", - "default": CHEF_FB_PATH, - "description": dedent( - """\ - Path to write run_list and initial_attributes keys that - should also be present in this configuration, defaults - to ``{}``.""".format( - CHEF_FB_PATH - ) - ), - }, - "exec": { - "type": "boolean", - "default": False, - "description": dedent( - """\ - define if we should run or not run chef (defaults to - false, unless a gem installed is requested where this - will then default to true).""" - ), - }, - "client_key": { - "type": "string", - "default": CHEF_RB_TPL_DEFAULTS["client_key"], - "description": dedent( - """\ - Optional path for client_cert. default to - ``{}``.""".format( - CHEF_RB_TPL_DEFAULTS["client_key"] - ) - ), - }, - "encrypted_data_bag_secret": { - "type": "string", - "default": None, - "description": dedent( - """\ - Specifies the location of the secret key used by chef - to encrypt data items. By default, this path is set - to None, meaning that chef will have to look at the - path ``{}`` for it. - """.format( - CHEF_ENCRYPTED_DATA_BAG_PATH - ) - ), - }, - "environment": { - "type": "string", - "default": CHEF_ENVIRONMENT, - "description": dedent( - """\ - Specifies which environment chef will use. By default, - it will use the ``{}`` configuration. - """.format( - CHEF_ENVIRONMENT - ) - ), - }, - "file_backup_path": { - "type": "string", - "default": CHEF_RB_TPL_DEFAULTS["file_backup_path"], - "description": dedent( - """\ - Specifies the location in which backup files are - stored. By default, it uses the - ``{}`` location.""".format( - CHEF_RB_TPL_DEFAULTS["file_backup_path"] - ) - ), - }, - "file_cache_path": { - "type": "string", - "default": CHEF_RB_TPL_DEFAULTS["file_cache_path"], - "description": dedent( - """\ - Specifies the location in which chef cache files will - be saved. By default, it uses the ``{}`` - location.""".format( - CHEF_RB_TPL_DEFAULTS["file_cache_path"] - ) - ), - }, - "json_attribs": { - "type": "string", - "default": CHEF_FB_PATH, - "description": dedent( - """\ - Specifies the location in which some chef json data is - stored. By default, it uses the - ``{}`` location.""".format( - CHEF_FB_PATH - ) - ), - }, - "log_level": { - "type": "string", - "default": CHEF_RB_TPL_DEFAULTS["log_level"], - "description": dedent( - """\ - Defines the level of logging to be stored in the log - file. By default this value is set to ``{}``. - """.format( - CHEF_RB_TPL_DEFAULTS["log_level"] - ) - ), - }, - "log_location": { - "type": "string", - "default": CHEF_RB_TPL_DEFAULTS["log_location"], - "description": dedent( - """\ - Specifies the location of the chef lof file. By - default, the location is specified at - ``{}``.""".format( - CHEF_RB_TPL_DEFAULTS["log_location"] - ) - ), - }, - "node_name": { - "type": "string", - "description": dedent( - """\ - The name of the node to run. By default, we will - use th instance id as the node name.""" - ), - }, - "omnibus_url": { - "type": "string", - "default": OMNIBUS_URL, - "description": dedent( - """\ - Omnibus URL if chef should be installed through - Omnibus. By default, it uses the - ``{}``.""".format( - OMNIBUS_URL - ) - ), - }, - "omnibus_url_retries": { - "type": "integer", - "default": OMNIBUS_URL_RETRIES, - "description": dedent( - """\ - The number of retries that will be attempted to reach - the Omnibus URL""" - ), - }, - "omnibus_version": { - "type": "string", - "description": dedent( - """\ - Optional version string to require for omnibus - install.""" - ), - }, - "pid_file": { - "type": "string", - "default": CHEF_RB_TPL_DEFAULTS["pid_file"], - "description": dedent( - """\ - The location in which a process identification - number (pid) is saved. By default, it saves - in the ``{}`` location.""".format( - CHEF_RB_TPL_DEFAULTS["pid_file"] - ) - ), - }, - "server_url": { - "type": "string", - "description": "The URL for the chef server", - }, - "show_time": { - "type": "boolean", - "default": True, - "description": "Show time in chef logs", - }, - "ssl_verify_mode": { - "type": "string", - "default": CHEF_RB_TPL_DEFAULTS["ssl_verify_mode"], - "description": dedent( - """\ - Set the verify mode for HTTPS requests. We can have - two possible values for this parameter: - - - ``:verify_none``: No validation of SSL \ - certificates. - - ``:verify_peer``: Validate all SSL certificates. - - By default, the parameter is set as ``{}``. - """.format( - CHEF_RB_TPL_DEFAULTS["ssl_verify_mode"] - ) - ), - }, - "validation_name": { - "type": "string", - "description": dedent( - """\ - The name of the chef-validator key that Chef Infra - Client uses to access the Chef Infra Server during - the initial Chef Infra Client run.""" - ), - }, - "force_install": { - "type": "boolean", - "default": False, - "description": dedent( - """\ - If set to ``True``, forces chef installation, even - if it is already installed.""" - ), - }, - "initial_attributes": { - "type": "object", - "items": {"type": "string"}, - "description": dedent( - """\ - Specify a list of initial attributes used by the - cookbooks.""" - ), - }, - "install_type": { - "type": "string", - "default": "packages", - "description": dedent( - """\ - The type of installation for chef. It can be one of - the following values: - - - ``packages`` - - ``gems`` - - ``omnibus``""" - ), - }, - "run_list": { - "type": "array", - "items": {"type": "string"}, - "description": "A run list for a first boot json.", - }, - "chef_license": { - "type": "string", - "description": dedent( - """\ - string that indicates if user accepts or not license - related to some of chef products""" - ), - }, - }, - } - }, -} - -__doc__ = get_meta_doc(meta, schema) +__doc__ = get_meta_doc(meta) def post_run_chef(chef_cfg, log): @@ -489,7 +193,6 @@ def handle(name, cfg, cloud, log, _args): ) return - validate_cloudconfig_schema(cfg, schema) chef_cfg = cfg["chef"] # Ensure the chef directories we use exist diff --git a/cloudinit/config/cc_debug.py b/cloudinit/config/cc_debug.py index d09fc129..1a3c9346 100644 --- a/cloudinit/config/cc_debug.py +++ b/cloudinit/config/cc_debug.py @@ -2,37 +2,47 @@ # # This file is part of cloud-init. See LICENSE file for license information. -""" -Debug ------ -**Summary:** helper to debug cloud-init *internal* datastructures. +"""Debug: Helper to debug cloud-init *internal* datastructures.""" + +import copy +from io import StringIO +from textwrap import dedent + +from cloudinit import safeyaml, type_utils, util +from cloudinit.config.schema import get_meta_doc +from cloudinit.distros import ALL_DISTROS +from cloudinit.settings import PER_INSTANCE + +SKIP_KEYS = frozenset(["log_cfgs"]) +MODULE_DESCRIPTION = """\ This module will enable for outputting various internal information that cloud-init sources provide to either a file or to the output console/log location that this cloud-init has been configured with when running. .. note:: Log configurations are not output. - -**Internal name:** ``cc_debug`` - -**Module frequency:** per instance - -**Supported distros:** all - -**Config keys**:: - - debug: - verbose: true/false (defaulting to true) - output: (location to write output, defaulting to console + log) """ -import copy -from io import StringIO - -from cloudinit import safeyaml, type_utils, util - -SKIP_KEYS = frozenset(["log_cfgs"]) +meta = { + "id": "cc_debug", + "name": "Debug", + "title": "Helper to debug cloud-init *internal* datastructures", + "description": MODULE_DESCRIPTION, + "distros": [ALL_DISTROS], + "frequency": PER_INSTANCE, + "examples": [ + dedent( + """\ + debug: + verbose: true + output: /tmp/my_debug.log + """ + ) + ], +} + +__doc__ = get_meta_doc(meta) def _make_header(text): @@ -53,7 +63,6 @@ def _dumps(obj): def handle(name, cfg, cloud, log, args): """Handler method activated by cloud-init.""" - verbose = util.get_cfg_by_path(cfg, ("debug", "verbose"), default=True) if args: # if args are provided (from cmdline) then explicitly set verbose diff --git a/cloudinit/config/cc_disable_ec2_metadata.py b/cloudinit/config/cc_disable_ec2_metadata.py index 5e528e81..6a5e7eda 100644 --- a/cloudinit/config/cc_disable_ec2_metadata.py +++ b/cloudinit/config/cc_disable_ec2_metadata.py @@ -6,34 +6,35 @@ # # This file is part of cloud-init. See LICENSE file for license information. -""" -Disable EC2 Metadata --------------------- -**Summary:** disable aws ec2 metadata +"""Disable EC2 Metadata: Disable AWS EC2 metadata.""" -This module can disable the ec2 datasource by rejecting the route to -``169.254.169.254``, the usual route to the datasource. This module is disabled -by default. - -**Internal name:** ``cc_disable_ec2_metadata`` - -**Module frequency:** always - -**Supported distros:** all - -**Config keys**:: - - disable_ec2_metadata: <true/false> -""" +from textwrap import dedent from cloudinit import subp, util +from cloudinit.config.schema import get_meta_doc +from cloudinit.distros import ALL_DISTROS from cloudinit.settings import PER_ALWAYS -frequency = PER_ALWAYS - REJECT_CMD_IF = ["route", "add", "-host", "169.254.169.254", "reject"] REJECT_CMD_IP = ["ip", "route", "add", "prohibit", "169.254.169.254"] +meta = { + "id": "cc_disable_ec2_metadata", + "name": "Disable EC2 Metadata", + "title": "Disable AWS EC2 Metadata", + "description": dedent( + """\ + This module can disable the ec2 datasource by rejecting the route to + ``169.254.169.254``, the usual route to the datasource. This module + is disabled by default.""" + ), + "distros": [ALL_DISTROS], + "frequency": PER_ALWAYS, + "examples": ["disable_ec2_metadata: true"], +} + +__doc__ = get_meta_doc(meta) + def handle(name, cfg, _cloud, log, _args): disabled = util.get_cfg_option_bool(cfg, "disable_ec2_metadata", False) diff --git a/cloudinit/config/cc_disk_setup.py b/cloudinit/config/cc_disk_setup.py index 4d527c7a..c59d00cd 100644 --- a/cloudinit/config/cc_disk_setup.py +++ b/cloudinit/config/cc_disk_setup.py @@ -5,110 +5,18 @@ # # This file is part of cloud-init. See LICENSE file for license information. -""" -Disk Setup ----------- -**Summary:** configure partitions and filesystems - -This module is able to configure simple partition tables and filesystems. - -.. note:: - for more detail about configuration options for disk setup, see the disk - setup example - -For convenience, aliases can be specified for disks using the -``device_aliases`` config key, which takes a dictionary of alias: path -mappings. There are automatic aliases for ``swap`` and ``ephemeral<X>``, where -``swap`` will always refer to the active swap partition and ``ephemeral<X>`` -will refer to the block device of the ephemeral image. - -Disk partitioning is done using the ``disk_setup`` directive. This config -directive accepts a dictionary where each key is either a path to a block -device or an alias specified in ``device_aliases``, and each value is the -configuration options for the device. The ``table_type`` option specifies the -partition table type, either ``mbr`` or ``gpt``. The ``layout`` option -specifies how partitions on the device are to be arranged. If ``layout`` is set -to ``true``, a single partition using all the space on the device will be -created. If set to ``false``, no partitions will be created. Partitions can be -specified by providing a list to ``layout``, where each entry in the list is -either a size or a list containing a size and the numerical value for a -partition type. The size for partitions is specified in **percentage** of disk -space, not in bytes (e.g. a size of 33 would take up 1/3 of the disk space). -The ``overwrite`` option controls whether this module tries to be safe about -writing partition tables or not. If ``overwrite: false`` is set, the device -will be checked for a partition table and for a file system and if either is -found, the operation will be skipped. If ``overwrite: true`` is set, no checks -will be performed. - -.. note:: - Using ``overwrite: true`` is dangerous and can lead to data loss, so double - check that the correct device has been specified if using this option. - -File system configuration is done using the ``fs_setup`` directive. This config -directive accepts a list of filesystem configs. The device to create the -filesystem on may be specified either as a path or as an alias in the format -``<alias name>.<y>`` where ``<y>`` denotes the partition number on the device. -The partition can also be specified by setting ``partition`` to the desired -partition number. The ``partition`` option may also be set to ``auto``, in -which this module will search for the existence of a filesystem matching the -``label``, ``type`` and ``device`` of the ``fs_setup`` entry and will skip -creating the filesystem if one is found. The ``partition`` option may also be -set to ``any``, in which case any file system that matches ``type`` and -``device`` will cause this module to skip filesystem creation for the -``fs_setup`` entry, regardless of ``label`` matching or not. To write a -filesystem directly to a device, use ``partition: none``. ``partition: none`` -will **always** write the filesystem, even when the ``label`` and -``filesystem`` are matched, and ``overwrite`` is ``false``. - -A label can be specified for the filesystem using -``label``, and the filesystem type can be specified using ``filesystem``. - -.. note:: - If specifying device using the ``<device name>.<partition number>`` format, - the value of ``partition`` will be overwritten. - -.. note:: - Using ``overwrite: true`` for filesystems is dangerous and can lead to data - loss, so double check the entry in ``fs_setup``. - -.. note:: - ``replace_fs`` is ignored unless ``partition`` is ``auto`` or ``any``. - -**Internal name:** ``cc_disk_setup`` - -**Module frequency:** per instance - -**Supported distros:** all - -**Config keys**:: - - device_aliases: - <alias name>: <device path> - disk_setup: - <alias name/path>: - table_type: <'mbr'/'gpt'> - layout: - - [33,82] - - 66 - overwrite: <true/false> - fs_setup: - - label: <label> - filesystem: <filesystem type> - device: <device> - partition: <"auto"/"any"/"none"/<partition number>> - overwrite: <true/false> - replace_fs: <filesystem type> -""" +"""Disk Setup: Configure partitions and filesystems.""" import logging import os import shlex +from textwrap import dedent from cloudinit import subp, util +from cloudinit.config.schema import get_meta_doc +from cloudinit.distros import ALL_DISTROS from cloudinit.settings import PER_INSTANCE -frequency = PER_INSTANCE - # Define the commands to use SFDISK_CMD = subp.which("sfdisk") SGDISK_CMD = subp.which("sgdisk") @@ -119,9 +27,64 @@ PARTPROBE_CMD = subp.which("partprobe") WIPEFS_CMD = subp.which("wipefs") LANG_C_ENV = {"LANG": "C"} - LOG = logging.getLogger(__name__) +MODULE_DESCRIPTION = """\ +This module is able to configure simple partition tables and filesystems. + +.. note:: + for more detail about configuration options for disk setup, see the disk + setup example + +For convenience, aliases can be specified for disks using the +``device_aliases`` config key, which takes a dictionary of alias: path +mappings. There are automatic aliases for ``swap`` and ``ephemeral<X>``, where +``swap`` will always refer to the active swap partition and ``ephemeral<X>`` +will refer to the block device of the ephemeral image. + +Disk partitioning is done using the ``disk_setup`` directive. This config +directive accepts a dictionary where each key is either a path to a block +device or an alias specified in ``device_aliases``, and each value is the +configuration options for the device. File system configuration is done using +the ``fs_setup`` directive. This config directive accepts a list of +filesystem configs. +""" + +meta = { + "id": "cc_disk_setup", + "name": "Disk Setup", + "title": "Configure partitions and filesystems", + "description": MODULE_DESCRIPTION, + "distros": [ALL_DISTROS], + "frequency": PER_INSTANCE, + "examples": [ + dedent( + """\ + device_aliases: + my_alias: /dev/sdb + disk_setup: + my_alias: + table_type: gpt + layout: [50, 50] + overwrite: true + fs_setup: + - label: fs1 + filesystem: ext4 + device: my_alias.1 + cmd: mkfs -t %(filesystem)s -L %(label)s %(device)s + - label: fs2 + device: my_alias.2 + filesystem: ext4 + mounts: + - ["my_alias.1", "/mnt1"] + - ["my_alias.2", "/mnt2"] + """ + ) + ], +} + +__doc__ = get_meta_doc(meta) + def handle(_name, cfg, cloud, log, _args): """ @@ -1008,7 +971,7 @@ def mkfs(fs_cfg): if not device: LOG.debug( - "No device aviable that matches request. " + "No device available that matches request. " "Skipping fs creation for %s", fs_cfg, ) diff --git a/cloudinit/config/cloud-init-schema.json b/cloudinit/config/cloud-init-schema.json index afaed285..2d43d06a 100644 --- a/cloudinit/config/cloud-init-schema.json +++ b/cloudinit/config/cloud-init-schema.json @@ -1,6 +1,57 @@ { "$schema": "http://json-schema.org/draft-04/schema#", "$defs": { + "apt_configure.mirror": { + "type": "array", + "items": { + "type": "object", + "additionalProperties": false, + "required": ["arches"], + "properties": { + "arches": { + "type": "array", + "items": {"type": "string"}, + "minItems": 1 + }, + "uri": {"type": "string", "format": "uri"}, + "search": { + "type": "array", + "items": {"type": "string", "format": "uri"}, + "minItems": 1 + }, + "search_dns": { + "type": "boolean" + }, + "keyid": {"type": "string"}, + "key": {"type": "string"}, + "keyserver": {"type": "string"} + } + }, + "minItems": 1 + }, + "ca_certs.properties": { + "type": "object", + "properties": { + "remove-defaults": { + "description": "Deprecated key name. Use remove_defaults instead.", + "type": "boolean", + "default": false + }, + "remove_defaults": { + "description": "Remove default CA certificates if true. Default: false", + "type": "boolean", + "default": false + }, + "trusted": { + "description": "List of trusted CA certificates to add.", + "type": "array", + "items": {"type": "string"}, + "minItems": 1 + } + }, + "additionalProperties": false, + "minProperties": 1 + }, "cc_apk_configure": { "type": "object", "properties": { @@ -28,7 +79,7 @@ "testing_enabled": { "type": "boolean", "default": false, - "description": "Whether to add the Testing repo to the repositories file. By default the Testing repo is not included. It is only recommended to use the Testing repo on a machine running the ``Edge`` version of Alpine as packages installed from Testing may have dependancies that conflict with those in non-Edge Main or Community repos." + "description": "Whether to add the Testing repo to the repositories file. By default the Testing repo is not included. It is only recommended to use the Testing repo on a machine running the ``Edge`` version of Alpine as packages installed from Testing may have dependencies that conflict with those in non-Edge Main or Community repos." }, "version": { "type": "string", @@ -49,6 +100,104 @@ } } }, + "cc_apt_configure": { + "properties": { + "apt": { + "type": "object", + "additionalProperties": false, + "minProperties": 1, + "properties": { + "preserve_sources_list": { + "type": "boolean", + "default": false, + "description": "By default, cloud-init will generate a new sources list in ``/etc/apt/sources.list.d`` based on any changes specified in cloud config. To disable this behavior and preserve the sources list from the pristine image, set ``preserve_sources_list`` to ``true``.\n\nThe ``preserve_sources_list`` option overrides all other config keys that would alter ``sources.list`` or ``sources.list.d``, **except** for additional sources to be added to ``sources.list.d``." + }, + "disable_suites": { + "type": "array", + "items": {"type": "string"}, + "minItems": 1, + "uniqueItems": true, + "description": "Entries in the sources list can be disabled using ``disable_suites``, which takes a list of suites to be disabled. If the string ``$RELEASE`` is present in a suite in the ``disable_suites`` list, it will be replaced with the release name. If a suite specified in ``disable_suites`` is not present in ``sources.list`` it will be ignored. For convenience, several aliases are provided for`` disable_suites``:\n\n - ``updates`` => ``$RELEASE-updates``\n - ``backports`` => ``$RELEASE-backports``\n - ``security`` => ``$RELEASE-security``\n - ``proposed`` => ``$RELEASE-proposed``\n - ``release`` => ``$RELEASE``.\n\nWhen a suite is disabled using ``disable_suites``, its entry in ``sources.list`` is not deleted; it is just commented out." + }, + "primary": { + "$ref": "#/$defs/apt_configure.mirror", + "description": "The primary and security archive mirrors can be specified using the ``primary`` and ``security`` keys, respectively. Both the ``primary`` and ``security`` keys take a list of configs, allowing mirrors to be specified on a per-architecture basis. Each config is a dictionary which must have an entry for ``arches``, specifying which architectures that config entry is for. The keyword ``default`` applies to any architecture not explicitly listed. The mirror url can be specified with the ``uri`` key, or a list of mirrors to check can be provided in order, with the first mirror that can be resolved being selected. This allows the same configuration to be used in different environment, with different hosts used for a local APT mirror. If no mirror is provided by ``uri`` or ``search``, ``search_dns`` may be used to search for dns names in the format ``<distro>-mirror`` in each of the following:\n\n - fqdn of this host per cloud metadata,\n - localdomain,\n - domains listed in ``/etc/resolv.conf``.\n\nIf there is a dns entry for ``<distro>-mirror``, then it is assumed that there is a distro mirror at ``http://<distro>-mirror.<domain>/<distro>``. If the ``primary`` key is defined, but not the ``security`` key, then then configuration for ``primary`` is also used for ``security``. If ``search_dns`` is used for the ``security`` key, the search pattern will be ``<distro>-security-mirror``.\n\nEach mirror may also specify a key to import via any of the following optional keys:\n\n - ``keyid``: a key to import via shortid or fingerprint.\n - ``key``: a raw PGP key.\n - ``keyserver``: alternate keyserver to pull ``keyid`` key from.\n\nIf no mirrors are specified, or all lookups fail, then default mirrors defined in the datasource are used. If none are present in the datasource either the following defaults are used:\n\n - ``primary`` => ``http://archive.ubuntu.com/ubuntu``.\n - ``security`` => ``http://security.ubuntu.com/ubuntu``" + }, + "security": { + "$ref": "#/$defs/apt_configure.mirror", + "description": "Please refer to the primary config documentation" + }, + "add_apt_repo_match": { + "type": "string", + "default": "^[\\w-]+:\\w", + "description": "All source entries in ``apt-sources`` that match regex in ``add_apt_repo_match`` will be added to the system using ``add-apt-repository``. If ``add_apt_repo_match`` is not specified, it defaults to ``^[\\w-]+:\\w``" + }, + "debconf_selections": { + "type": "object", + "minProperties": 1, + "patternProperties": { + "^.+$": { + "type": "string" + } + }, + "description": "Debconf additional configurations can be specified as a dictionary under the ``debconf_selections`` config key, with each key in the dict representing a different set of configurations. The value of each key must be a string containing all the debconf configurations that must be applied. We will bundle all of the values and pass them to ``debconf-set-selections``. Therefore, each value line must be a valid entry for ``debconf-set-selections``, meaning that they must possess for distinct fields:\n\n``pkgname question type answer``\n\nWhere:\n\n - ``pkgname`` is the name of the package.\n - ``question`` the name of the questions.\n - ``type`` is the type of question.\n - ``answer`` is the value used to answer the question.\n\nFor example: ``ippackage ippackage/ip string 127.0.01``" + }, + "sources_list": { + "type": "string", + "description": "Specifies a custom template for rendering ``sources.list`` . If no ``sources_list`` template is given, cloud-init will use sane default. Within this template, the following strings will be replaced with the appropriate values:\n\n - ``$MIRROR``\n - ``$RELEASE``\n - ``$PRIMARY``\n - ``$SECURITY``\n - ``$KEY_FILE``" + }, + "conf": { + "type": "string", + "description": "Specify configuration for apt, such as proxy configuration. This configuration is specified as a string. For multiline APT configuration, make sure to follow yaml syntax." + }, + "https_proxy": { + "type": "string", + "description": "More convenient way to specify https APT proxy. https proxy url is specified in the format ``https://[[user][:pass]@]host[:port]/``." + }, + "http_proxy": { + "type": "string", + "description": "More convenient way to specify http APT proxy. http proxy url is specified in the format ``http://[[user][:pass]@]host[:port]/``." + }, + "proxy": { + "type": "string", + "description": "Alias for defining a http APT proxy." + }, + "ftp_proxy": { + "type": "string", + "description": "More convenient way to specify ftp APT proxy. ftp proxy url is specified in the format ``ftp://[[user][:pass]@]host[:port]/``." + }, + "sources": { + "type": "object", + "patternProperties": { + "^.+$": { + "type": "object", + "properties": { + "source": { + "type": "string" + }, + "keyid": { + "type": "string" + }, + "key": { + "type": "string" + }, + "keyserver": { + "type": "string" + }, + "filename": { + "type": "string" + } + }, + "additionalProperties": false, + "minProperties": 1 + } + }, + "description": "Source list entries can be specified as a dictionary under the ``sources`` config key, with each key in the dict representing a different source file. The key of each source entry will be used as an id that can be referenced in other config entries, as well as the filename for the source's configuration under ``/etc/apt/sources.list.d``. If the name does not end with ``.list``, it will be appended. If there is no configuration for a key in ``sources``, no file will be written, but the key may still be referred to as an id in other ``sources`` entries.\n\nEach entry under ``sources`` is a dictionary which may contain any of the following optional keys:\n - ``source``: a sources.list entry (some variable replacements apply).\n - ``keyid``: a key to import via shortid or fingerprint.\n - ``key``: a raw PGP key.\n - ``keyserver``: alternate keyserver to pull ``keyid`` key from.\n - ``filename``: specify the name of the list file\n\nThe ``source`` key supports variable replacements for the following strings:\n\n - ``$MIRROR``\n - ``$PRIMARY``\n - ``$SECURITY``\n - ``$RELEASE``\n - ``$KEY_FILE``" + } + } + } + } + }, "cc_apt_pipelining": { "type": "object", "properties": { @@ -60,10 +209,352 @@ ] } } + }, + "cc_bootcmd": { + "type": "object", + "properties": { + "bootcmd": { + "type": "array", + "items": { + "oneOf": [ + {"type": "array", "items": {"type": "string"}}, + {"type": "string"} + ] + }, + "additionalItems": false, + "minItems": 1 + } + } + }, + "cc_byobu": { + "type": "object", + "properties": { + "byobu_by_default": { + "type": "string", + "enum": [ + "enable-system", + "enable-user", + "disable-system", + "disable-user", + "enable", + "disable", + "user", + "system" + ] + } + } + }, + "cc_ca_certs": { + "type": "object", + "properties": { + "ca_certs": { + "$ref": "#/$defs/ca_certs.properties" + }, + "ca-certs": { + "$ref": "#/$defs/ca_certs.properties" + } + } + }, + "cc_chef": { + "type": "object", + "properties": { + "chef": { + "type": "object", + "additionalProperties": false, + "minProperties": 1, + "properties": { + "directories": { + "type": "array", + "items": {"type": "string"}, + "minItems": 1, + "uniqueItems": true, + "description": "Create the necessary directories for chef to run. By default, it creates the following directories:\n\n - ``/etc/chef``\n - ``/var/log/chef``\n - ``/var/lib/chef``\n - ``/var/cache/chef``\n - ``/var/backups/chef``\n - ``/var/run/chef``" + }, + "validation_cert": { + "type": "string", + "description": "Optional string to be written to file validation_key. Special value ``system`` means set use existing file." + }, + "validation_key": { + "type": "string", + "default": "/etc/chef/validation.pem", + "description": "Optional path for validation_cert. default to ``/etc/chef/validation.pem``" + }, + "firstboot_path": { + "type": "string", + "default": "/etc/chef/firstboot.json", + "description": "Path to write run_list and initial_attributes keys that should also be present in this configuration, defaults to ``/etc/chef/firstboot.json``" + }, + "exec": { + "type": "boolean", + "default": false, + "description": "Set true if we should run or not run chef (defaults to false, unless a gem installed is requested where this will then default to true)." + }, + "client_key": { + "type": "string", + "default": "/etc/chef/client.pem", + "description": "Optional path for client_cert. Default to ``/etc/chef/client.pem``." + }, + "encrypted_data_bag_secret": { + "type": "string", + "default": null, + "description": "Specifies the location of the secret key used by chef to encrypt data items. By default, this path is set to null, meaning that chef will have to look at the path ``/etc/chef/encrypted_data_bag_secret`` for it." + }, + "environment": { + "type": "string", + "default": "_default", + "description": "Specifies which environment chef will use. By default, it will use the ``_default`` configuration." + }, + "file_backup_path": { + "type": "string", + "default": "/var/backups/chef", + "description": "Specifies the location in which backup files are stored. By default, it uses the ``/var/backups/chef`` location." + }, + "file_cache_path": { + "type": "string", + "default": "/var/cache/chef", + "description": "Specifies the location in which chef cache files will be saved. By default, it uses the ``/var/cache/chef`` location." + }, + "json_attribs": { + "type": "string", + "default": "/etc/chef/firstboot.json", + "description": "Specifies the location in which some chef json data is stored. By default, it uses the ``/etc/chef/firstboot.json`` location." + }, + "log_level": { + "type": "string", + "default": ":info", + "description": "Defines the level of logging to be stored in the log file. By default this value is set to ``:info``." + }, + "log_location": { + "type": "string", + "default": "/var/log/chef/client.log", + "description": "Specifies the location of the chef lof file. By default, the location is specified at ``/var/log/chef/client.log``." + }, + "node_name": { + "type": "string", + "description": "The name of the node to run. By default, we will use th instance id as the node name." + }, + "omnibus_url": { + "type": "string", + "default": "https://www.chef.io/chef/install.sh", + "description": "Omnibus URL if chef should be installed through Omnibus. By default, it uses the ``https://www.chef.io/chef/install.sh``." + }, + "omnibus_url_retries": { + "type": "integer", + "default": 5, + "description": "The number of retries that will be attempted to reach the Omnibus URL. Default is 5." + }, + "omnibus_version": { + "type": "string", + "description": "Optional version string to require for omnibus install." + }, + "pid_file": { + "type": "string", + "default": "/var/run/chef/client.pid", + "description": "The location in which a process identification number (pid) is saved. By default, it saves in the ``/var/run/chef/client.pid`` location." + }, + "server_url": { + "type": "string", + "description": "The URL for the chef server" + }, + "show_time": { + "type": "boolean", + "default": true, + "description": "Show time in chef logs" + }, + "ssl_verify_mode": { + "type": "string", + "default": ":verify_none", + "description": "Set the verify mode for HTTPS requests. We can have two possible values for this parameter:\n\n - ``:verify_none``: No validation of SSL certificates.\n - ``:verify_peer``: Validate all SSL certificates.\n\nBy default, the parameter is set as ``:verify_none``." + }, + "validation_name": { + "type": "string", + "description": "The name of the chef-validator key that Chef Infra Client uses to access the Chef Infra Server during the initial Chef Infra Client run." + }, + "force_install": { + "type": "boolean", + "default": false, + "description": "If set to ``true``, forces chef installation, even if it is already installed." + }, + "initial_attributes": { + "type": "object", + "items": {"type": "string"}, + "description": "Specify a list of initial attributes used by the cookbooks." + }, + "install_type": { + "type": "string", + "default": "packages", + "enum": [ + "packages", + "gems", + "omnibus" + ], + "description": "The type of installation for chef. It can be one of the following values:\n\n - ``packages``\n - ``gems``\n - ``omnibus``" + }, + "run_list": { + "type": "array", + "items": {"type": "string"}, + "description": "A run list for a first boot json." + }, + "chef_license": { + "type": "string", + "description": "string that indicates if user accepts or not license related to some of chef products" + } + } + } + } + }, + "cc_debug": { + "type": "object", + "properties": { + "debug": { + "additionalProperties": false, + "minProperties": 1, + "type": "object", + "properties": { + "verbose": { + "description": "Should always be true for this module", + "type": "boolean" + }, + "output": { + "description": "Location to write output. Defaults to console + log", + "type": "string" + } + } + } + } + }, + "cc_disable_ec2_metadata": { + "type": "object", + "properties": { + "disable_ec2_metadata": { + "default": false, + "description": "Set true to disable IPv4 routes to EC2 metadata. Default: false.", + "type": "boolean" + } + } + }, + "cc_disk_setup": { + "type": "object", + "properties": { + "device_aliases": { + "type": "object", + "patternProperties": { + "^.+$": { + "label": "<alias_name>", + "type": "string", + "description": "Path to disk to be aliased by this name." + } + } + }, + "disk_setup": { + "type": "object", + "patternProperties": { + "^.+$": { + "label": "<alias name/path>", + "type": "object", + "additionalProperties": false, + "properties": { + "table_type": { + "type": "string", + "default": "mbr", + "enum": ["mbr", "gpt"], + "description": "Specifies the partition table type, either ``mbr`` or ``gpt``. Default: ``mbr``." + }, + "layout": { + "type": ["string", "boolean", "array"], + "default": false, + "oneOf": [ + {"type": "string", "enum": ["remove"]}, + {"type": "boolean"}, + { + "type": "array", + "items": { + "oneOf": [ + {"type": "integer"}, + { + "type": "array", + "items": {"type": "integer"}, + "minItems": 2, + "maxItems": 2 + } + ] + } + } + ], + "description": "If set to ``true``, a single partition using all the space on the device will be created. If set to ``false``, no partitions will be created. If set to ``remove``, any existing partition table will be purged. Partitions can be specified by providing a list to ``layout``, where each entry in the list is either a size or a list containing a size and the numerical value for a partition type. The size for partitions is specified in **percentage** of disk space, not in bytes (e.g. a size of 33 would take up 1/3 of the disk space). Default: ``false``." + }, + "overwrite": { + "type": "boolean", + "default": false, + "description": "Controls whether this module tries to be safe about writing partition tables or not. If ``overwrite: false`` is set, the device will be checked for a partition table and for a file system and if either is found, the operation will be skipped. If ``overwrite: true`` is set, no checks will be performed. Using ``overwrite: true`` is **dangerous** and can lead to data loss, so double check that the correct device has been specified if using this option. Default: ``false``" + } + } + } + } + }, + "fs_setup": { + "type": "array", + "items": { + "type": "object", + "additionalProperties": false, + "properties": { + "label": { + "type": "string", + "description": "Label for the filesystem." + }, + "filesystem": { + "type": "string", + "description": "Filesystem type to create. E.g., ``ext4`` or ``btrfs``" + }, + "device": { + "type": "string", + "description": "Specified either as a path or as an alias in the format ``<alias name>.<y>`` where ``<y>`` denotes the partition number on the device. If specifying device using the ``<device name>.<partition number>`` format, the value of ``partition`` will be overwritten." + }, + "partition": { + "type": ["string", "integer"], + "oneOf": [ + { + "type": "string", + "enum": ["auto", "any", "none"] + }, + {"type": "integer"} + ], + "description": "The partition can be specified by setting ``partition`` to the desired partition number. The ``partition`` option may also be set to ``auto``, in which this module will search for the existence of a filesystem matching the ``label``, ``type`` and ``device`` of the ``fs_setup`` entry and will skip creating the filesystem if one is found. The ``partition`` option may also be set to ``any``, in which case any file system that matches ``type`` and ``device`` will cause this module to skip filesystem creation for the ``fs_setup`` entry, regardless of ``label`` matching or not. To write a filesystem directly to a device, use ``partition: none``. ``partition: none`` will **always** write the filesystem, even when the ``label`` and ``filesystem`` are matched, and ``overwrite`` is ``false``." + }, + "overwrite": { + "type": "boolean", + "description": "If ``true``, overwrite any existing filesystem. Using ``overwrite: true`` for filesystems is **dangerous** and can lead to data loss, so double check the entry in ``fs_setup``. Default: ``false``" + }, + "replace_fs": { + "type": "string", + "description": "Ignored unless ``partition`` is ``auto`` or ``any``. Default ``false``." + }, + "extra_opts": { + "type": ["array", "string"], + "items": {"type": "string"}, + "description": "Optional options to pass to the filesystem creation command. Ignored if you using ``cmd`` directly." + }, + "cmd": { + "type": ["array", "string"], + "items": {"type": "string"}, + "description": "Optional command to run to create the filesystem. Can include string substitutions of the other ``fs_setup`` config keys. This is only necessary if you need to override the default command." + } + } + } + } + } } }, "allOf": [ { "$ref": "#/$defs/cc_apk_configure" }, - { "$ref": "#/$defs/cc_apt_pipelining" } + { "$ref": "#/$defs/cc_apt_configure" }, + { "$ref": "#/$defs/cc_apt_pipelining" }, + { "$ref": "#/$defs/cc_bootcmd" }, + { "$ref": "#/$defs/cc_byobu" }, + { "$ref": "#/$defs/cc_ca_certs" }, + { "$ref": "#/$defs/cc_chef" }, + { "$ref": "#/$defs/cc_debug" }, + { "$ref": "#/$defs/cc_disable_ec2_metadata" }, + { "$ref": "#/$defs/cc_disk_setup" } ] } diff --git a/doc/examples/cloud-config-ca-certs.txt b/doc/examples/cloud-config-ca-certs.txt index 5e9115a0..9f7beb05 100644 --- a/doc/examples/cloud-config-ca-certs.txt +++ b/doc/examples/cloud-config-ca-certs.txt @@ -7,13 +7,13 @@ # Make sure that this file is valid yaml before starting instances. # It should be passed as user-data when starting the instance. -ca-certs: - # If present and set to True, the 'remove-defaults' parameter will remove +ca_certs: + # If present and set to True, the 'remove_defaults' parameter will remove # all the default trusted CA certificates that are normally shipped with # Ubuntu. # This is mainly for paranoid admins - most users will not need this # functionality. - remove-defaults: true + remove_defaults: true # If present, the 'trusted' parameter should contain a certificate (or list # of certificates) to add to the system as trusted CA certificates. diff --git a/doc/examples/cloud-config-disk-setup.txt b/doc/examples/cloud-config-disk-setup.txt index 5c6de77e..a36e6cfb 100644 --- a/doc/examples/cloud-config-disk-setup.txt +++ b/doc/examples/cloud-config-disk-setup.txt @@ -80,7 +80,7 @@ fs_setup: disk_setup: ephmeral0: table_type: 'mbr' - layout: 'auto' + layout: true /dev/xvdh: table_type: 'mbr' layout: diff --git a/tests/integration_tests/modules/test_ca_certs.py b/tests/integration_tests/modules/test_ca_certs.py index d514fc62..7247fd7d 100644 --- a/tests/integration_tests/modules/test_ca_certs.py +++ b/tests/integration_tests/modules/test_ca_certs.py @@ -12,8 +12,8 @@ import pytest USER_DATA = """\ #cloud-config -ca-certs: - remove-defaults: true +ca_certs: + remove_defaults: true trusted: - | -----BEGIN CERTIFICATE----- diff --git a/tests/unittests/config/test_cc_apt_configure.py b/tests/unittests/config/test_cc_apt_configure.py new file mode 100644 index 00000000..bd1bb963 --- /dev/null +++ b/tests/unittests/config/test_cc_apt_configure.py @@ -0,0 +1,202 @@ +# This file is part of cloud-init. See LICENSE file for license information. + +""" Tests for cc_apt_configure module """ + +import re + +import pytest + +from cloudinit.config.schema import ( + SchemaValidationError, + get_schema, + validate_cloudconfig_schema, +) +from tests.unittests.helpers import skipUnlessJsonSchema + + +class TestAPTConfigureSchema: + @pytest.mark.parametrize( + "config, error_msg", + ( + # Supplement valid schemas from examples tested in test_schema + ({"apt": {"preserve_sources_list": True}}, None), + # Invalid schemas + ( + {"apt": "nonobject"}, + "apt: 'nonobject' is not of type 'object", + ), + ( + {"apt": {"boguskey": True}}, + re.escape( + "apt: Additional properties are not allowed" + " ('boguskey' was unexpected)" + ), + ), + ({"apt": {}}, "apt: {} does not have enough properties"), + ( + {"apt": {"preserve_sources_list": 1}}, + "apt.preserve_sources_list: 1 is not of type 'boolean'", + ), + ( + {"apt": {"disable_suites": 1}}, + "apt.disable_suites: 1 is not of type 'array'", + ), + ( + {"apt": {"disable_suites": []}}, + re.escape("apt.disable_suites: [] is too short"), + ), + ( + {"apt": {"disable_suites": [1]}}, + "apt.disable_suites.0: 1 is not of type 'string'", + ), + ( + {"apt": {"disable_suites": ["a", "a"]}}, + re.escape( + "apt.disable_suites: ['a', 'a'] has non-unique elements" + ), + ), + # All apt: primary tests are applicable for "security" key too. + # Those apt:security tests are exercised in the unittest below + ( + {"apt": {"primary": "nonlist"}}, + "apt.primary: 'nonlist' is not of type 'array'", + ), + ( + {"apt": {"primary": []}}, + re.escape("apt.primary: [] is too short"), + ), + ( + {"apt": {"primary": ["nonobj"]}}, + "apt.primary.0: 'nonobj' is not of type 'object'", + ), + ( + {"apt": {"primary": [{}]}}, + "apt.primary.0: 'arches' is a required property", + ), + ( + {"apt": {"primary": [{"boguskey": True}]}}, + re.escape( + "apt.primary.0: Additional properties are not allowed" + " ('boguskey' was unexpected)" + ), + ), + ( + {"apt": {"primary": [{"arches": True}]}}, + "apt.primary.0.arches: True is not of type 'array'", + ), + ( + {"apt": {"primary": [{"uri": True}]}}, + "apt.primary.0.uri: True is not of type 'string'", + ), + ( + { + "apt": { + "primary": [ + {"arches": ["amd64"], "search": "non-array"} + ] + } + }, + "apt.primary.0.search: 'non-array' is not of type 'array'", + ), + ( + {"apt": {"primary": [{"arches": ["amd64"], "search": []}]}}, + re.escape("apt.primary.0.search: [] is too short"), + ), + ( + { + "apt": { + "primary": [{"arches": ["amd64"], "search_dns": "a"}] + } + }, + "apt.primary.0.search_dns: 'a' is not of type 'boolean'", + ), + ( + {"apt": {"primary": [{"arches": ["amd64"], "keyid": 1}]}}, + "apt.primary.0.keyid: 1 is not of type 'string'", + ), + ( + {"apt": {"primary": [{"arches": ["amd64"], "key": 1}]}}, + "apt.primary.0.key: 1 is not of type 'string'", + ), + ( + {"apt": {"primary": [{"arches": ["amd64"], "keyserver": 1}]}}, + "apt.primary.0.keyserver: 1 is not of type 'string'", + ), + ( + {"apt": {"add_apt_repo_match": True}}, + "apt.add_apt_repo_match: True is not of type 'string'", + ), + ( + {"apt": {"debconf_selections": True}}, + "apt.debconf_selections: True is not of type 'object'", + ), + ( + {"apt": {"debconf_selections": {}}}, + "apt.debconf_selections: {} does not have enough properties", + ), + ( + {"apt": {"sources_list": True}}, + "apt.sources_list: True is not of type 'string'", + ), + ( + {"apt": {"conf": True}}, + "apt.conf: True is not of type 'string'", + ), + ( + {"apt": {"http_proxy": True}}, + "apt.http_proxy: True is not of type 'string'", + ), + ( + {"apt": {"https_proxy": True}}, + "apt.https_proxy: True is not of type 'string'", + ), + ( + {"apt": {"proxy": True}}, + "apt.proxy: True is not of type 'string'", + ), + ( + {"apt": {"ftp_proxy": True}}, + "apt.ftp_proxy: True is not of type 'string'", + ), + ( + {"apt": {"sources": True}}, + "apt.sources: True is not of type 'object'", + ), + ( + {"apt": {"sources": {"opaquekey": True}}}, + "apt.sources.opaquekey: True is not of type 'object'", + ), + ( + {"apt": {"sources": {"opaquekey": {}}}}, + "apt.sources.opaquekey: {} does not have enough properties", + ), + ( + {"apt": {"sources": {"opaquekey": {"boguskey": True}}}}, + re.escape( + "apt.sources.opaquekey: Additional properties are not" + " allowed ('boguskey' was unexpected)" + ), + ), + ), + ) + @skipUnlessJsonSchema() + def test_schema_validation(self, config, error_msg): + schema = get_schema() + if error_msg is None: + validate_cloudconfig_schema(config, schema, strict=True) + else: + with pytest.raises(SchemaValidationError, match=error_msg): + validate_cloudconfig_schema(config, schema, strict=True) + # Note apt['primary'] and apt['security'] have same defition + # Avoid test setup duplicates by running same test using 'security' + if isinstance(config.get("apt"), dict) and config["apt"].get( + "primary" + ): + # To exercise security schema, rename test key from primary + config["apt"]["security"] = config["apt"].pop("primary") + error_msg = error_msg.replace("primary", "security") + with pytest.raises(SchemaValidationError, match=error_msg): + validate_cloudconfig_schema(config, schema, strict=True) + + +# vi: ts=4 expandtab diff --git a/tests/unittests/config/test_cc_bootcmd.py b/tests/unittests/config/test_cc_bootcmd.py index 17033596..34b16b85 100644 --- a/tests/unittests/config/test_cc_bootcmd.py +++ b/tests/unittests/config/test_cc_bootcmd.py @@ -1,15 +1,18 @@ # This file is part of cloud-init. See LICENSE file for license information. import logging +import re import tempfile +import pytest + from cloudinit import subp, util -from cloudinit.config.cc_bootcmd import handle, schema -from tests.unittests.helpers import ( - CiTestCase, - SchemaTestCaseMixin, - mock, - skipUnlessJsonSchema, +from cloudinit.config.cc_bootcmd import handle +from cloudinit.config.schema import ( + SchemaValidationError, + get_schema, + validate_cloudconfig_schema, ) +from tests.unittests.helpers import CiTestCase, mock, skipUnlessJsonSchema from tests.unittests.util import get_cloud LOG = logging.getLogger(__name__) @@ -65,44 +68,13 @@ class TestBootcmd(CiTestCase): str(context_manager.exception), ) - @skipUnlessJsonSchema() - def test_handler_schema_validation_warns_non_array_type(self): - """Schema validation warns of non-array type for bootcmd key. - - Schema validation is not strict, so bootcmd attempts to shellify the - invalid content. - """ - invalid_config = {"bootcmd": 1} - cc = get_cloud() - with self.assertRaises(TypeError): - handle("cc_bootcmd", invalid_config, cc, LOG, []) - self.assertIn( - "Invalid cloud-config provided:\nbootcmd: 1 is not of type" - " 'array'", - self.logs.getvalue(), - ) - self.assertIn("Failed to shellify", self.logs.getvalue()) - - @skipUnlessJsonSchema() - def test_handler_schema_validation_warns_non_array_item_type(self): - """Schema validation warns of non-array or string bootcmd items. - - Schema validation is not strict, so bootcmd attempts to shellify the - invalid content. - """ invalid_config = { "bootcmd": ["ls /", 20, ["wget", "http://stuff/blah"], {"a": "n"}] } cc = get_cloud() with self.assertRaises(TypeError) as context_manager: handle("cc_bootcmd", invalid_config, cc, LOG, []) - expected_warnings = [ - "bootcmd.1: 20 is not valid under any of the given schemas", - "bootcmd.3: {'a': 'n'} is not valid under any of the given schema", - ] logs = self.logs.getvalue() - for warning in expected_warnings: - self.assertIn(warning, logs) self.assertIn("Failed to shellify", logs) self.assertEqual( "Unable to shellify type 'int'. Expected list, string, tuple. " @@ -146,22 +118,48 @@ class TestBootcmd(CiTestCase): @skipUnlessJsonSchema() -class TestSchema(CiTestCase, SchemaTestCaseMixin): +class TestBootCMDSchema: """Directly test schema rather than through handle.""" - schema = schema - - def test_duplicates_are_fine_array_array(self): - """Duplicated commands array/array entries are allowed.""" - self.assertSchemaValid( - ["byebye", "byebye"], "command entries can be duplicate" - ) - - def test_duplicates_are_fine_array_string(self): - """Duplicated commands array/string entries are allowed.""" - self.assertSchemaValid( - ["echo bye", "echo bye"], "command entries can be duplicate." - ) + @pytest.mark.parametrize( + "config, error_msg", + ( + # Valid schemas tested by meta.examples in test_schema + # Invalid schemas + ( + {"bootcmd": 1}, + "Cloud config schema errors: bootcmd: 1 is not of type" + " 'array'", + ), + ({"bootcmd": []}, re.escape("bootcmd: [] is too short")), + ( + {"bootcmd": []}, + re.escape( + "Cloud config schema errors: bootcmd: [] is too short" + ), + ), + ( + { + "bootcmd": [ + "ls /", + 20, + ["wget", "http://stuff/blah"], + {"a": "n"}, + ] + }, + "Cloud config schema errors: bootcmd.1: 20 is not valid under" + " any of the given schemas, bootcmd.3: {'a': 'n'} is not" + " valid under any of the given schemas", + ), + ), + ) + @skipUnlessJsonSchema() + def test_schema_validation(self, config, error_msg): + """Assert expected schema validation and error messages.""" + # New-style schema $defs exist in config/cloud-init-schema*.json + schema = get_schema() + with pytest.raises(SchemaValidationError, match=error_msg): + validate_cloudconfig_schema(config, schema, strict=True) # vi: ts=4 expandtab diff --git a/tests/unittests/config/test_cc_byobu.py b/tests/unittests/config/test_cc_byobu.py new file mode 100644 index 00000000..fbdf3403 --- /dev/null +++ b/tests/unittests/config/test_cc_byobu.py @@ -0,0 +1,51 @@ +# This file is part of cloud-init. See LICENSE file for license information. + +import re + +import pytest + +from cloudinit.config.schema import ( + SchemaValidationError, + get_schema, + validate_cloudconfig_schema, +) +from tests.unittests.helpers import skipUnlessJsonSchema + + +class TestByobuSchema: + """Directly test schema rather than through handle.""" + + @pytest.mark.parametrize( + "config, error_msg", + ( + # Supplement valid schemas tested by meta.examples in test_schema + ({"byobu_by_default": "enable"}, None), + # Invalid schemas + ( + {"byobu_by_default": 1}, + "byobu_by_default: 1 is not of type 'string'", + ), + ( + {"byobu_by_default": "bogusenum"}, + re.escape( + "byobu_by_default: 'bogusenum' is not one of" + " ['enable-system', 'enable-user', 'disable-system'," + " 'disable-user', 'enable', 'disable'," + " 'user', 'system']" + ), + ), + ), + ) + @skipUnlessJsonSchema() + def test_schema_validation(self, config, error_msg): + """Assert expected schema validation and error messages.""" + # New-style schema $defs exist in config/cloud-init-schema*.json + schema = get_schema() + if error_msg is None: + validate_cloudconfig_schema(config, schema, strict=True) + else: + with pytest.raises(SchemaValidationError, match=error_msg): + validate_cloudconfig_schema(config, schema, strict=True) + + +# vi: ts=4 expandtab diff --git a/tests/unittests/config/test_cc_ca_certs.py b/tests/unittests/config/test_cc_ca_certs.py index c49922e6..39614635 100644 --- a/tests/unittests/config/test_cc_ca_certs.py +++ b/tests/unittests/config/test_cc_ca_certs.py @@ -1,14 +1,22 @@ # This file is part of cloud-init. See LICENSE file for license information. import logging +import re import shutil import tempfile import unittest from contextlib import ExitStack from unittest import mock +import pytest + from cloudinit import distros, helpers, subp, util from cloudinit.config import cc_ca_certs -from tests.unittests.helpers import TestCase +from cloudinit.config.schema import ( + SchemaValidationError, + get_schema, + validate_cloudconfig_schema, +) +from tests.unittests.helpers import TestCase, skipUnlessJsonSchema from tests.unittests.util import get_cloud @@ -128,7 +136,7 @@ class TestConfig(TestCase): def test_remove_default_ca_certs(self): """Test remove_defaults works as expected.""" - config = {"ca-certs": {"remove-defaults": True}} + config = {"ca_certs": {"remove_defaults": True}} for distro_name in cc_ca_certs.distros: self._mock_init() @@ -141,7 +149,7 @@ class TestConfig(TestCase): def test_no_remove_defaults_if_false(self): """Test remove_defaults is not called when config value is False.""" - config = {"ca-certs": {"remove-defaults": False}} + config = {"ca_certs": {"remove_defaults": False}} for distro_name in cc_ca_certs.distros: self._mock_init() @@ -154,7 +162,7 @@ class TestConfig(TestCase): def test_correct_order_for_remove_then_add(self): """Test remove_defaults is not called when config value is False.""" - config = {"ca-certs": {"remove-defaults": True, "trusted": ["CERT1"]}} + config = {"ca_certs": {"remove_defaults": True, "trusted": ["CERT1"]}} for distro_name in cc_ca_certs.distros: self._mock_init() @@ -406,4 +414,94 @@ class TestRemoveDefaultCaCerts(TestCase): ) +class TestCACertsSchema: + """Directly test schema rather than through handle.""" + + @pytest.mark.parametrize( + "config, error_msg", + ( + # Valid, yet deprecated schemas + ({"ca-certs": {"remove-defaults": True}}, None), + # Invalid schemas + ( + {"ca_certs": 1}, + "ca_certs: 1 is not of type 'object'", + ), + ( + {"ca_certs": {}}, + re.escape("ca_certs: {} does not have enough properties"), + ), + ( + {"ca_certs": {"boguskey": 1}}, + re.escape( + "ca_certs: Additional properties are not allowed" + " ('boguskey' was unexpected)" + ), + ), + ( + {"ca_certs": {"remove_defaults": 1}}, + "ca_certs.remove_defaults: 1 is not of type 'boolean'", + ), + ( + {"ca_certs": {"trusted": [1]}}, + "ca_certs.trusted.0: 1 is not of type 'string'", + ), + ( + {"ca_certs": {"trusted": []}}, + re.escape("ca_certs.trusted: [] is too short"), + ), + ), + ) + @skipUnlessJsonSchema() + def test_schema_validation(self, config, error_msg): + """Assert expected schema validation and error messages.""" + # New-style schema $defs exist in config/cloud-init-schema*.json + schema = get_schema() + if error_msg is None: + validate_cloudconfig_schema(config, schema, strict=True) + else: + with pytest.raises(SchemaValidationError, match=error_msg): + validate_cloudconfig_schema(config, schema, strict=True) + + @mock.patch.object(cc_ca_certs, "update_ca_certs") + def test_deprecate_key_warnings(self, update_ca_certs, caplog): + """Assert warnings are logged for deprecated keys.""" + log = logging.getLogger("CALogTest") + cloud = get_cloud("ubuntu") + cc_ca_certs.handle( + "IGNORE", {"ca-certs": {"remove-defaults": False}}, cloud, log, [] + ) + expected_warnings = [ + "DEPRECATION: key 'ca-certs' is now deprecated. Use 'ca_certs'" + " instead.", + "DEPRECATION: key 'ca-certs.remove-defaults' is now deprecated." + " Use 'ca_certs.remove_defaults' instead.", + ] + for warning in expected_warnings: + assert warning in caplog.text + assert 1 == update_ca_certs.call_count + + @mock.patch.object(cc_ca_certs, "update_ca_certs") + def test_duplicate_keys(self, update_ca_certs, caplog): + """Assert warnings are logged for deprecated keys.""" + log = logging.getLogger("CALogTest") + cloud = get_cloud("ubuntu") + cc_ca_certs.handle( + "IGNORE", + { + "ca-certs": {"remove-defaults": True}, + "ca_certs": {"remove_defaults": False}, + }, + cloud, + log, + [], + ) + expected_warning = ( + "Found both ca-certs (deprecated) and ca_certs config keys." + " Ignoring ca-certs." + ) + assert expected_warning in caplog.text + assert 1 == update_ca_certs.call_count + + # vi: ts=4 expandtab diff --git a/tests/unittests/config/test_cc_chef.py b/tests/unittests/config/test_cc_chef.py index 835974e5..f86be293 100644 --- a/tests/unittests/config/test_cc_chef.py +++ b/tests/unittests/config/test_cc_chef.py @@ -3,17 +3,25 @@ import json import logging import os +import re import httpretty +import pytest from cloudinit import util from cloudinit.config import cc_chef +from cloudinit.config.schema import ( + SchemaValidationError, + get_schema, + validate_cloudconfig_schema, +) from tests.unittests.helpers import ( FilesystemMockingTestCase, HttprettyTestCase, cloud_init_project_dir, mock, skipIf, + skipUnlessJsonSchema, ) from tests.unittests.util import get_cloud @@ -289,4 +297,168 @@ class TestChef(FilesystemMockingTestCase): self.assertEqual(expected_cert, util.load_file(v_path)) +@skipUnlessJsonSchema() +class TestBootCMDSchema: + """Directly test schema rather than through handle.""" + + @pytest.mark.parametrize( + "config, error_msg", + ( + # Valid schemas tested by meta.examples in test_schema + # Invalid schemas + ( + {"chef": 1}, + "chef: 1 is not of type 'object'", + ), + ( + {"chef": {}}, + re.escape(" chef: {} does not have enough properties"), + ), + ( + {"chef": {"boguskey": True}}, + re.escape( + "chef: Additional properties are not allowed" + " ('boguskey' was unexpected)" + ), + ), + ( + {"chef": {"directories": 1}}, + "chef.directories: 1 is not of type 'array'", + ), + ( + {"chef": {"directories": []}}, + re.escape("chef.directories: [] is too short"), + ), + ( + {"chef": {"directories": [1]}}, + "chef.directories.0: 1 is not of type 'string'", + ), + ( + {"chef": {"directories": ["a", "a"]}}, + re.escape( + "chef.directories: ['a', 'a'] has non-unique elements" + ), + ), + ( + {"chef": {"validation_cert": 1}}, + "chef.validation_cert: 1 is not of type 'string'", + ), + ( + {"chef": {"validation_key": 1}}, + "chef.validation_key: 1 is not of type 'string'", + ), + ( + {"chef": {"firstboot_path": 1}}, + "chef.firstboot_path: 1 is not of type 'string'", + ), + ( + {"chef": {"client_key": 1}}, + "chef.client_key: 1 is not of type 'string'", + ), + ( + {"chef": {"encrypted_data_bag_secret": 1}}, + "chef.encrypted_data_bag_secret: 1 is not of type 'string'", + ), + ( + {"chef": {"environment": 1}}, + "chef.environment: 1 is not of type 'string'", + ), + ( + {"chef": {"file_backup_path": 1}}, + "chef.file_backup_path: 1 is not of type 'string'", + ), + ( + {"chef": {"file_cache_path": 1}}, + "chef.file_cache_path: 1 is not of type 'string'", + ), + ( + {"chef": {"json_attribs": 1}}, + "chef.json_attribs: 1 is not of type 'string'", + ), + ( + {"chef": {"log_level": 1}}, + "chef.log_level: 1 is not of type 'string'", + ), + ( + {"chef": {"log_location": 1}}, + "chef.log_location: 1 is not of type 'string'", + ), + ( + {"chef": {"node_name": 1}}, + "chef.node_name: 1 is not of type 'string'", + ), + ( + {"chef": {"omnibus_url": 1}}, + "chef.omnibus_url: 1 is not of type 'string'", + ), + ( + {"chef": {"omnibus_url_retries": "one"}}, + "chef.omnibus_url_retries: 'one' is not of type 'integer'", + ), + ( + {"chef": {"omnibus_version": 1}}, + "chef.omnibus_version: 1 is not of type 'string'", + ), + ( + {"chef": {"omnibus_version": 1}}, + "chef.omnibus_version: 1 is not of type 'string'", + ), + ( + {"chef": {"pid_file": 1}}, + "chef.pid_file: 1 is not of type 'string'", + ), + ( + {"chef": {"server_url": 1}}, + "chef.server_url: 1 is not of type 'string'", + ), + ( + {"chef": {"show_time": 1}}, + "chef.show_time: 1 is not of type 'boolean'", + ), + ( + {"chef": {"ssl_verify_mode": 1}}, + "chef.ssl_verify_mode: 1 is not of type 'string'", + ), + ( + {"chef": {"validation_name": 1}}, + "chef.validation_name: 1 is not of type 'string'", + ), + ( + {"chef": {"force_install": 1}}, + "chef.force_install: 1 is not of type 'boolean'", + ), + ( + {"chef": {"initial_attributes": 1}}, + "chef.initial_attributes: 1 is not of type 'object'", + ), + ( + {"chef": {"install_type": 1}}, + "chef.install_type: 1 is not of type 'string'", + ), + ( + {"chef": {"install_type": "bogusenum"}}, + re.escape( + "chef.install_type: 'bogusenum' is not one of" + " ['packages', 'gems', 'omnibus']" + ), + ), + ( + {"chef": {"run_list": 1}}, + "chef.run_list: 1 is not of type 'array'", + ), + ( + {"chef": {"chef_license": 1}}, + "chef.chef_license: 1 is not of type 'string'", + ), + ), + ) + @skipUnlessJsonSchema() + def test_schema_validation(self, config, error_msg): + """Assert expected schema validation and error messages.""" + # New-style schema $defs exist in config/cloud-init-schema*.json + schema = get_schema() + with pytest.raises(SchemaValidationError, match=error_msg): + validate_cloudconfig_schema(config, schema, strict=True) + + # vi: ts=4 expandtab diff --git a/tests/unittests/config/test_cc_debug.py b/tests/unittests/config/test_cc_debug.py index 79a88561..fc8d43dc 100644 --- a/tests/unittests/config/test_cc_debug.py +++ b/tests/unittests/config/test_cc_debug.py @@ -2,12 +2,24 @@ # # This file is part of cloud-init. See LICENSE file for license information. import logging +import re import shutil import tempfile +import pytest + from cloudinit import util from cloudinit.config import cc_debug -from tests.unittests.helpers import FilesystemMockingTestCase, mock +from cloudinit.config.schema import ( + SchemaValidationError, + get_schema, + validate_cloudconfig_schema, +) +from tests.unittests.helpers import ( + FilesystemMockingTestCase, + mock, + skipUnlessJsonSchema, +) from tests.unittests.util import get_cloud LOG = logging.getLogger(__name__) @@ -57,4 +69,44 @@ class TestDebug(FilesystemMockingTestCase): ) +@skipUnlessJsonSchema() +class TestDebugSchema: + """Directly test schema rather than through handle.""" + + @pytest.mark.parametrize( + "config, error_msg", + ( + # Valid schemas tested by meta.examples in test_schema + # Invalid schemas + ({"debug": 1}, "debug: 1 is not of type 'object'"), + ( + {"debug": {}}, + re.escape("debug: {} does not have enough properties"), + ), + ( + {"debug": {"boguskey": True}}, + re.escape( + "Additional properties are not allowed ('boguskey' was" + " unexpected)" + ), + ), + ( + {"debug": {"verbose": 1}}, + "debug.verbose: 1 is not of type 'boolean'", + ), + ( + {"debug": {"output": 1}}, + "debug.output: 1 is not of type 'string'", + ), + ), + ) + @skipUnlessJsonSchema() + def test_schema_validation(self, config, error_msg): + """Assert expected schema validation and error messages.""" + # New-style schema $defs exist in config/cloud-init-schema*.json + schema = get_schema() + with pytest.raises(SchemaValidationError, match=error_msg): + validate_cloudconfig_schema(config, schema, strict=True) + + # vi: ts=4 expandtab diff --git a/tests/unittests/config/test_cc_disable_ec2_metadata.py b/tests/unittests/config/test_cc_disable_ec2_metadata.py index 3c3313a7..5755e29e 100644 --- a/tests/unittests/config/test_cc_disable_ec2_metadata.py +++ b/tests/unittests/config/test_cc_disable_ec2_metadata.py @@ -4,8 +4,15 @@ import logging +import pytest + import cloudinit.config.cc_disable_ec2_metadata as ec2_meta -from tests.unittests.helpers import CiTestCase, mock +from cloudinit.config.schema import ( + SchemaValidationError, + get_schema, + validate_cloudconfig_schema, +) +from tests.unittests.helpers import CiTestCase, mock, skipUnlessJsonSchema LOG = logging.getLogger(__name__) @@ -47,4 +54,28 @@ class TestEC2MetadataRoute(CiTestCase): m_subp.assert_not_called() +@skipUnlessJsonSchema() +class TestDisableEc2MetadataSchema: + """Directly test schema rather than through handle.""" + + @pytest.mark.parametrize( + "config, error_msg", + ( + # Valid schemas tested by meta.examples in test_schema + # Invalid schemas + ( + {"disable_ec2_metadata": 1}, + "disable_ec2_metadata: 1 is not of type 'boolean'", + ), + ), + ) + @skipUnlessJsonSchema() + def test_schema_validation(self, config, error_msg): + """Assert expected schema validation and error messages.""" + # New-style schema $defs exist in config/cloud-init-schema*.json + schema = get_schema() + with pytest.raises(SchemaValidationError, match=error_msg): + validate_cloudconfig_schema(config, schema, strict=True) + + # vi: ts=4 expandtab diff --git a/tests/unittests/config/test_cc_disk_setup.py b/tests/unittests/config/test_cc_disk_setup.py index 8a8d7195..f2796e83 100644 --- a/tests/unittests/config/test_cc_disk_setup.py +++ b/tests/unittests/config/test_cc_disk_setup.py @@ -1,9 +1,23 @@ # This file is part of cloud-init. See LICENSE file for license information. import random +import re + +import pytest from cloudinit.config import cc_disk_setup -from tests.unittests.helpers import CiTestCase, ExitStack, TestCase, mock +from cloudinit.config.schema import ( + SchemaValidationError, + get_schema, + validate_cloudconfig_schema, +) +from tests.unittests.helpers import ( + CiTestCase, + ExitStack, + TestCase, + mock, + skipUnlessJsonSchema, +) class TestIsDiskUsed(TestCase): @@ -283,5 +297,37 @@ class TestMkfsCommandHandling(CiTestCase): ) -# +@skipUnlessJsonSchema() +class TestDebugSchema: + """Directly test schema rather than through handle.""" + + @pytest.mark.parametrize( + "config, error_msg", + ( + # Valid schemas tested by meta.examples in test_schema + # Invalid schemas + ({"disk_setup": 1}, "disk_setup: 1 is not of type 'object'"), + ({"fs_setup": 1}, "fs_setup: 1 is not of type 'array'"), + ( + {"device_aliases": 1}, + "device_aliases: 1 is not of type 'object'", + ), + ( + {"debug": {"boguskey": True}}, + re.escape( + "Additional properties are not allowed ('boguskey' was" + " unexpected)" + ), + ), + ), + ) + @skipUnlessJsonSchema() + def test_schema_validation(self, config, error_msg): + """Assert expected schema validation and error messages.""" + # New-style schema $defs exist in config/cloud-init-schema*.json + schema = get_schema() + with pytest.raises(SchemaValidationError, match=error_msg): + validate_cloudconfig_schema(config, schema, strict=True) + + # vi: ts=4 expandtab diff --git a/tests/unittests/config/test_schema.py b/tests/unittests/config/test_schema.py index 1647f6e5..2f43d9e7 100644 --- a/tests/unittests/config/test_schema.py +++ b/tests/unittests/config/test_schema.py @@ -91,6 +91,13 @@ class TestGetSchema: "cc_apt_configure", "cc_apt_pipelining", "cc_bootcmd", + "cc_byobu", + "cc_ca_certs", + "cc_chef", + "cc_debug", + "cc_disable_ec2_metadata", + "cc_disk_setup", + "cc_install_hotplug", "cc_keyboard", "cc_locale", "cc_ntp", @@ -101,8 +108,6 @@ class TestGetSchema: "cc_ubuntu_drivers", "cc_write_files", "cc_zypper_add_repo", - "cc_chef", - "cc_install_hotplug", ] ) == sorted( [meta["id"] for meta in get_metas().values() if meta is not None] @@ -112,7 +117,15 @@ class TestGetSchema: # New style schema should be defined in static schema file in $defs expected_subschema_defs = [ {"$ref": "#/$defs/cc_apk_configure"}, + {"$ref": "#/$defs/cc_apt_configure"}, {"$ref": "#/$defs/cc_apt_pipelining"}, + {"$ref": "#/$defs/cc_bootcmd"}, + {"$ref": "#/$defs/cc_byobu"}, + {"$ref": "#/$defs/cc_ca_certs"}, + {"$ref": "#/$defs/cc_chef"}, + {"$ref": "#/$defs/cc_debug"}, + {"$ref": "#/$defs/cc_disable_ec2_metadata"}, + {"$ref": "#/$defs/cc_disk_setup"}, ] found_subschema_defs = [] legacy_schema_keys = [] @@ -125,9 +138,6 @@ class TestGetSchema: assert expected_subschema_defs == found_subschema_defs # This list will dwindle as we move legacy schema to new $defs assert [ - "apt", - "bootcmd", - "chef", "drivers", "keyboard", "locale", @@ -153,7 +163,7 @@ class TestLoadDoc: "module_name", ( "cc_apt_pipelining", # new style composite schema file - "cc_bootcmd", # legacy sub-schema defined in module + "cc_zypper_add_repo", # legacy sub-schema defined in module ), ) def test_report_docs_for_legacy_and_consolidated_schema(self, module_name): |