diff options
Diffstat (limited to 'cloudinit/config')
63 files changed, 3722 insertions, 2713 deletions
diff --git a/cloudinit/config/__init__.py b/cloudinit/config/__init__.py index 0ef9a748..ed124180 100644 --- a/cloudinit/config/__init__.py +++ b/cloudinit/config/__init__.py @@ -6,9 +6,8 @@ # # This file is part of cloud-init. See LICENSE file for license information. -from cloudinit.settings import (PER_INSTANCE, FREQUENCIES) - from cloudinit import log as logging +from cloudinit.settings import FREQUENCIES, PER_INSTANCE LOG = logging.getLogger(__name__) @@ -22,26 +21,27 @@ MOD_PREFIX = "cc_" def form_module_name(name): canon_name = name.replace("-", "_") if canon_name.lower().endswith(".py"): - canon_name = canon_name[0:(len(canon_name) - 3)] + canon_name = canon_name[0 : (len(canon_name) - 3)] canon_name = canon_name.strip() if not canon_name: return None if not canon_name.startswith(MOD_PREFIX): - canon_name = '%s%s' % (MOD_PREFIX, canon_name) + canon_name = "%s%s" % (MOD_PREFIX, canon_name) return canon_name def fixup_module(mod, def_freq=PER_INSTANCE): - if not hasattr(mod, 'frequency'): - setattr(mod, 'frequency', def_freq) + if not hasattr(mod, "frequency"): + setattr(mod, "frequency", def_freq) else: freq = mod.frequency if freq and freq not in FREQUENCIES: LOG.warning("Module %s has an unknown frequency %s", mod, freq) - if not hasattr(mod, 'distros'): - setattr(mod, 'distros', []) - if not hasattr(mod, 'osfamilies'): - setattr(mod, 'osfamilies', []) + if not hasattr(mod, "distros"): + setattr(mod, "distros", []) + if not hasattr(mod, "osfamilies"): + setattr(mod, "osfamilies", []) return mod + # vi: ts=4 expandtab diff --git a/cloudinit/config/cc_apk_configure.py b/cloudinit/config/cc_apk_configure.py index d227a58d..a615c814 100644 --- a/cloudinit/config/cc_apk_configure.py +++ b/cloudinit/config/cc_apk_configure.py @@ -9,9 +9,7 @@ from textwrap import dedent from cloudinit import log as logging -from cloudinit import temp_utils -from cloudinit import templater -from cloudinit import util +from cloudinit import temp_utils, templater, util from cloudinit.config.schema import get_meta_doc, validate_cloudconfig_schema from cloudinit.settings import PER_INSTANCE @@ -54,34 +52,41 @@ REPOSITORIES_TEMPLATE = """\ frequency = PER_INSTANCE -distros = ['alpine'] +distros = ["alpine"] meta = { - 'id': 'cc_apk_configure', - 'name': 'APK Configure', - 'title': 'Configure apk repositories file', - 'description': dedent("""\ + "id": "cc_apk_configure", + "name": "APK Configure", + "title": "Configure apk repositories file", + "description": dedent( + """\ This module handles configuration of the /etc/apk/repositories file. .. note:: To ensure that apk configuration is valid yaml, any strings containing special characters, especially ``:`` should be quoted. - """), - 'distros': distros, - 'examples': [ - dedent("""\ + """ + ), + "distros": distros, + "examples": [ + dedent( + """\ # Keep the existing /etc/apk/repositories file unaltered. apk_repos: preserve_repositories: true - """), - dedent("""\ + """ + ), + dedent( + """\ # Create repositories file for Alpine v3.12 main and community # using default mirror site. apk_repos: alpine_repo: community_enabled: true version: 'v3.12' - """), - dedent("""\ + """ + ), + dedent( + """\ # Create repositories file for Alpine Edge main, community, and # testing using a specified mirror site and also a local repo. apk_repos: @@ -91,21 +96,23 @@ meta = { testing_enabled: true version: 'edge' local_repo_base_url: 'https://my-local-server/local-alpine' - """), + """ + ), ], - 'frequency': frequency, + "frequency": frequency, } schema = { - 'type': 'object', - 'properties': { - 'apk_repos': { - 'type': 'object', - 'properties': { - 'preserve_repositories': { - 'type': 'boolean', - 'default': False, - 'description': dedent("""\ + "type": "object", + "properties": { + "apk_repos": { + "type": "object", + "properties": { + "preserve_repositories": { + "type": "boolean", + "default": False, + "description": dedent( + """\ By default, cloud-init will generate a new repositories file ``/etc/apk/repositories`` based on any valid configuration settings specified within a apk_repos @@ -116,33 +123,41 @@ schema = { The ``preserve_repositories`` option overrides all other config keys that would alter ``/etc/apk/repositories``. - """) + """ + ), }, - 'alpine_repo': { - 'type': ['object', 'null'], - 'properties': { - 'base_url': { - 'type': 'string', - 'default': DEFAULT_MIRROR, - 'description': dedent("""\ + "alpine_repo": { + "type": ["object", "null"], + "properties": { + "base_url": { + "type": "string", + "default": DEFAULT_MIRROR, + "description": dedent( + """\ The base URL of an Alpine repository, or mirror, to download official packages from. If not specified then it defaults to ``{}`` - """.format(DEFAULT_MIRROR)) + """.format( + DEFAULT_MIRROR + ) + ), }, - 'community_enabled': { - 'type': 'boolean', - 'default': False, - 'description': dedent("""\ + "community_enabled": { + "type": "boolean", + "default": False, + "description": dedent( + """\ Whether to add the Community repo to the repositories file. By default the Community repo is not included. - """) + """ + ), }, - 'testing_enabled': { - 'type': 'boolean', - 'default': False, - 'description': dedent("""\ + "testing_enabled": { + "type": "boolean", + "default": False, + "description": dedent( + """\ Whether to add the Testing repo to the repositories file. By default the Testing repo is not included. It is only recommended @@ -151,32 +166,37 @@ schema = { installed from Testing may have dependancies that conflict with those in non-Edge Main or Community repos." - """) + """ + ), }, - 'version': { - 'type': 'string', - 'description': dedent("""\ + "version": { + "type": "string", + "description": dedent( + """\ The Alpine version to use (e.g. ``v3.12`` or ``edge``) - """) + """ + ), }, }, - 'required': ['version'], - 'minProperties': 1, - 'additionalProperties': False, + "required": ["version"], + "minProperties": 1, + "additionalProperties": False, }, - 'local_repo_base_url': { - 'type': 'string', - 'description': dedent("""\ + "local_repo_base_url": { + "type": "string", + "description": dedent( + """\ The base URL of an Alpine repository containing unofficial packages - """) - } + """ + ), + }, }, - 'minProperties': 1, # Either preserve_repositories or alpine_repo - 'additionalProperties': False, + "minProperties": 1, # Either preserve_repositories or alpine_repo + "additionalProperties": False, } - } + }, } __doc__ = get_meta_doc(meta, schema) @@ -195,38 +215,44 @@ def handle(name, cfg, cloud, log, _args): # If there is no "apk_repos" section in the configuration # then do nothing. - apk_section = cfg.get('apk_repos') + apk_section = cfg.get("apk_repos") if not apk_section: - LOG.debug(("Skipping module named %s," - " no 'apk_repos' section found"), name) + LOG.debug( + "Skipping module named %s, no 'apk_repos' section found", name + ) return validate_cloudconfig_schema(cfg, schema) # If "preserve_repositories" is explicitly set to True in # the configuration do nothing. - if util.get_cfg_option_bool(apk_section, 'preserve_repositories', False): - LOG.debug(("Skipping module named %s," - " 'preserve_repositories' is set"), name) + if util.get_cfg_option_bool(apk_section, "preserve_repositories", False): + LOG.debug( + "Skipping module named %s, 'preserve_repositories' is set", name + ) return # If there is no "alpine_repo" subsection of "apk_repos" present in the # configuration then do nothing, as at least "version" is required to # create valid repositories entries. - alpine_repo = apk_section.get('alpine_repo') + alpine_repo = apk_section.get("alpine_repo") if not alpine_repo: - LOG.debug(("Skipping module named %s," - " no 'alpine_repo' configuration found"), name) + LOG.debug( + "Skipping module named %s, no 'alpine_repo' configuration found", + name, + ) return # If there is no "version" value present in configuration then do nothing. - alpine_version = alpine_repo.get('version') + alpine_version = alpine_repo.get("version") if not alpine_version: - LOG.debug(("Skipping module named %s," - " 'version' not specified in alpine_repo"), name) + LOG.debug( + "Skipping module named %s, 'version' not specified in alpine_repo", + name, + ) return - local_repo = apk_section.get('local_repo_base_url', '') + local_repo = apk_section.get("local_repo_base_url", "") _write_repositories_file(alpine_repo, alpine_version, local_repo) @@ -240,22 +266,23 @@ def _write_repositories_file(alpine_repo, alpine_version, local_repo): @param local_repo: A string containing the base URL of a local repo. """ - repo_file = '/etc/apk/repositories' + repo_file = "/etc/apk/repositories" - alpine_baseurl = alpine_repo.get('base_url', DEFAULT_MIRROR) + alpine_baseurl = alpine_repo.get("base_url", DEFAULT_MIRROR) - params = {'alpine_baseurl': alpine_baseurl, - 'alpine_version': alpine_version, - 'community_enabled': alpine_repo.get('community_enabled'), - 'testing_enabled': alpine_repo.get('testing_enabled'), - 'local_repo': local_repo} + params = { + "alpine_baseurl": alpine_baseurl, + "alpine_version": alpine_version, + "community_enabled": alpine_repo.get("community_enabled"), + "testing_enabled": alpine_repo.get("testing_enabled"), + "local_repo": local_repo, + } - tfile = temp_utils.mkstemp(prefix='template_name-', suffix=".tmpl") + tfile = temp_utils.mkstemp(prefix="template_name-", suffix=".tmpl") template_fn = tfile[1] # Filepath is second item in tuple util.write_file(template_fn, content=REPOSITORIES_TEMPLATE) - LOG.debug('Generating Alpine repository configuration file: %s', - repo_file) + LOG.debug("Generating Alpine repository configuration file: %s", repo_file) templater.render_to_file(template_fn, repo_file, params) # Clean up temporary template util.del_file(template_fn) diff --git a/cloudinit/config/cc_apt_configure.py b/cloudinit/config/cc_apt_configure.py index 2e844c2c..b0728517 100644 --- a/cloudinit/config/cc_apt_configure.py +++ b/cloudinit/config/cc_apt_configure.py @@ -10,16 +10,14 @@ import glob import os -import re import pathlib +import re from textwrap import dedent -from cloudinit.config.schema import get_meta_doc, validate_cloudconfig_schema from cloudinit import gpg from cloudinit import log as logging -from cloudinit import subp -from cloudinit import templater -from cloudinit import util +from cloudinit import subp, templater, util +from cloudinit.config.schema import get_meta_doc, validate_cloudconfig_schema from cloudinit.settings import PER_INSTANCE LOG = logging.getLogger(__name__) @@ -27,59 +25,46 @@ LOG = logging.getLogger(__name__) # this will match 'XXX:YYY' (ie, 'cloud-archive:foo' or 'ppa:bar') ADD_APT_REPO_MATCH = r"^[\w-]+:\w" -APT_LOCAL_KEYS = '/etc/apt/trusted.gpg' -APT_TRUSTED_GPG_DIR = '/etc/apt/trusted.gpg.d/' -CLOUD_INIT_GPG_DIR = '/etc/apt/cloud-init.gpg.d/' +APT_LOCAL_KEYS = "/etc/apt/trusted.gpg" +APT_TRUSTED_GPG_DIR = "/etc/apt/trusted.gpg.d/" +CLOUD_INIT_GPG_DIR = "/etc/apt/cloud-init.gpg.d/" frequency = PER_INSTANCE distros = ["ubuntu", "debian"] mirror_property = { - 'type': 'array', - 'items': { - 'type': 'object', - 'additionalProperties': False, - 'required': ['arches'], - 'properties': { - 'arches': { - 'type': 'array', - 'items': { - 'type': 'string' - }, - 'minItems': 1 - }, - 'uri': { - 'type': 'string', - 'format': 'uri' - }, - 'search': { - 'type': 'array', - 'items': { - 'type': 'string', - 'format': 'uri' - }, - 'minItems': 1 - }, - 'search_dns': { - 'type': 'boolean', + "type": "array", + "items": { + "type": "object", + "additionalProperties": False, + "required": ["arches"], + "properties": { + "arches": { + "type": "array", + "items": {"type": "string"}, + "minItems": 1, }, - 'keyid': { - 'type': 'string' + "uri": {"type": "string", "format": "uri"}, + "search": { + "type": "array", + "items": {"type": "string", "format": "uri"}, + "minItems": 1, }, - 'key': { - 'type': 'string' + "search_dns": { + "type": "boolean", }, - 'keyserver': { - 'type': 'string' - } - } - } + "keyid": {"type": "string"}, + "key": {"type": "string"}, + "keyserver": {"type": "string"}, + }, + }, } meta = { - 'id': 'cc_apt_configure', - 'name': 'Apt Configure', - 'title': 'Configure apt for the user', - 'description': dedent("""\ + "id": "cc_apt_configure", + "name": "Apt Configure", + "title": "Configure apt for the user", + "description": dedent( + """\ This module handles both configuration of apt options and adding source lists. There are configuration options such as ``apt_get_wrapper`` and ``apt_get_command`` that control how @@ -94,9 +79,12 @@ meta = { .. note:: For more information about apt configuration, see the - ``Additional apt configuration`` example."""), - 'distros': distros, - 'examples': [dedent("""\ + ``Additional apt configuration`` example.""" + ), + "distros": distros, + "examples": [ + dedent( + """\ apt: preserve_sources_list: false disable_suites: @@ -153,21 +141,24 @@ meta = { key: | ------BEGIN PGP PUBLIC KEY BLOCK------- <key data> - ------END PGP PUBLIC KEY BLOCK-------""")], - 'frequency': frequency, + ------END PGP PUBLIC KEY BLOCK-------""" + ) + ], + "frequency": frequency, } schema = { - 'type': 'object', - 'properties': { - 'apt': { - 'type': 'object', - 'additionalProperties': False, - 'properties': { - 'preserve_sources_list': { - 'type': 'boolean', - 'default': False, - 'description': dedent("""\ + "type": "object", + "properties": { + "apt": { + "type": "object", + "additionalProperties": False, + "properties": { + "preserve_sources_list": { + "type": "boolean", + "default": False, + "description": dedent( + """\ By default, cloud-init will generate a new sources list in ``/etc/apt/sources.list.d`` based on any changes specified in cloud config. To disable this @@ -179,15 +170,15 @@ schema = { all other config keys that would alter ``sources.list`` or ``sources.list.d``, **except** for additional sources to be added - to ``sources.list.d``.""") + to ``sources.list.d``.""" + ), }, - 'disable_suites': { - 'type': 'array', - 'items': { - 'type': 'string' - }, - 'uniqueItems': True, - 'description': dedent("""\ + "disable_suites": { + "type": "array", + "items": {"type": "string"}, + "uniqueItems": True, + "description": dedent( + """\ Entries in the sources list can be disabled using ``disable_suites``, which takes a list of suites to be disabled. If the string ``$RELEASE`` is @@ -206,11 +197,13 @@ schema = { When a suite is disabled using ``disable_suites``, its entry in ``sources.list`` is not deleted; it - is just commented out.""") + is just commented out.""" + ), }, - 'primary': { + "primary": { **mirror_property, - 'description': dedent("""\ + "description": dedent( + """\ The primary and security archive mirrors can be specified using the ``primary`` and ``security`` keys, respectively. Both the @@ -264,27 +257,35 @@ schema = { ``http://archive.ubuntu.com/ubuntu``. - ``security`` => \ ``http://security.ubuntu.com/ubuntu`` - """) + """ + ), }, - 'security': { + "security": { **mirror_property, - 'description': dedent("""\ - Please refer to the primary config documentation""") + "description": dedent( + """\ + Please refer to the primary config documentation""" + ), }, - 'add_apt_repo_match': { - 'type': 'string', - 'default': ADD_APT_REPO_MATCH, - 'description': dedent("""\ + "add_apt_repo_match": { + "type": "string", + "default": ADD_APT_REPO_MATCH, + "description": dedent( + """\ All source entries in ``apt-sources`` that match regex in ``add_apt_repo_match`` will be added to the system using ``add-apt-repository``. If ``add_apt_repo_match`` is not specified, it - defaults to ``{}``""".format(ADD_APT_REPO_MATCH)) + defaults to ``{}``""".format( + ADD_APT_REPO_MATCH + ) + ), }, - 'debconf_selections': { - 'type': 'object', - 'items': {'type': 'string'}, - 'description': dedent("""\ + "debconf_selections": { + "type": "object", + "items": {"type": "string"}, + "description": dedent( + """\ Debconf additional configurations can be specified as a dictionary under the ``debconf_selections`` config key, with each key in the dict representing a @@ -308,11 +309,13 @@ schema = { For example: \ ``ippackage ippackage/ip string 127.0.01`` - """) + """ + ), }, - 'sources_list': { - 'type': 'string', - 'description': dedent("""\ + "sources_list": { + "type": "string", + "description": dedent( + """\ Specifies a custom template for rendering ``sources.list`` . If no ``sources_list`` template is given, cloud-init will use sane default. Within @@ -323,45 +326,55 @@ schema = { - ``$RELEASE`` - ``$PRIMARY`` - ``$SECURITY`` - - ``$KEY_FILE``""") + - ``$KEY_FILE``""" + ), }, - 'conf': { - 'type': 'string', - 'description': dedent("""\ + "conf": { + "type": "string", + "description": dedent( + """\ Specify configuration for apt, such as proxy configuration. This configuration is specified as a string. For multiline apt configuration, make sure - to follow yaml syntax.""") + to follow yaml syntax.""" + ), }, - 'https_proxy': { - 'type': 'string', - 'description': dedent("""\ + "https_proxy": { + "type": "string", + "description": dedent( + """\ More convenient way to specify https apt proxy. https proxy url is specified in the format - ``https://[[user][:pass]@]host[:port]/``.""") + ``https://[[user][:pass]@]host[:port]/``.""" + ), }, - 'http_proxy': { - 'type': 'string', - 'description': dedent("""\ + "http_proxy": { + "type": "string", + "description": dedent( + """\ More convenient way to specify http apt proxy. http proxy url is specified in the format - ``http://[[user][:pass]@]host[:port]/``.""") + ``http://[[user][:pass]@]host[:port]/``.""" + ), }, - 'proxy': { - 'type': 'string', - 'description': 'Alias for defining a http apt proxy.' + "proxy": { + "type": "string", + "description": "Alias for defining a http apt proxy.", }, - 'ftp_proxy': { - 'type': 'string', - 'description': dedent("""\ + "ftp_proxy": { + "type": "string", + "description": dedent( + """\ More convenient way to specify ftp apt proxy. ftp proxy url is specified in the format - ``ftp://[[user][:pass]@]host[:port]/``.""") + ``ftp://[[user][:pass]@]host[:port]/``.""" + ), }, - 'sources': { - 'type': 'object', - 'items': {'type': 'string'}, - 'description': dedent("""\ + "sources": { + "type": "object", + "items": {"type": "string"}, + "description": dedent( + """\ Source list entries can be specified as a dictionary under the ``sources`` config key, with each key in the dict representing a different source @@ -394,11 +407,12 @@ schema = { - ``$PRIMARY`` - ``$SECURITY`` - ``$RELEASE`` - - ``$KEY_FILE``""") - } - } + - ``$KEY_FILE``""" + ), + }, + }, } - } + }, } __doc__ = get_meta_doc(meta, schema) @@ -415,18 +429,22 @@ APT_PROXY_FN = "/etc/apt/apt.conf.d/90cloud-init-aptproxy" DEFAULT_KEYSERVER = "keyserver.ubuntu.com" # Default archive mirrors -PRIMARY_ARCH_MIRRORS = {"PRIMARY": "http://archive.ubuntu.com/ubuntu/", - "SECURITY": "http://security.ubuntu.com/ubuntu/"} -PORTS_MIRRORS = {"PRIMARY": "http://ports.ubuntu.com/ubuntu-ports", - "SECURITY": "http://ports.ubuntu.com/ubuntu-ports"} -PRIMARY_ARCHES = ['amd64', 'i386'] -PORTS_ARCHES = ['s390x', 'arm64', 'armhf', 'powerpc', 'ppc64el', 'riscv64'] +PRIMARY_ARCH_MIRRORS = { + "PRIMARY": "http://archive.ubuntu.com/ubuntu/", + "SECURITY": "http://security.ubuntu.com/ubuntu/", +} +PORTS_MIRRORS = { + "PRIMARY": "http://ports.ubuntu.com/ubuntu-ports", + "SECURITY": "http://ports.ubuntu.com/ubuntu-ports", +} +PRIMARY_ARCHES = ["amd64", "i386"] +PORTS_ARCHES = ["s390x", "arm64", "armhf", "powerpc", "ppc64el", "riscv64"] def get_default_mirrors(arch=None, target=None): """returns the default mirrors for the target. These depend on the - architecture, for more see: - https://wiki.ubuntu.com/UbuntuDevelopment/PackageArchive#Ports""" + architecture, for more see: + https://wiki.ubuntu.com/UbuntuDevelopment/PackageArchive#Ports""" if arch is None: arch = util.get_dpkg_architecture(target) if arch in PRIMARY_ARCHES: @@ -438,8 +456,8 @@ def get_default_mirrors(arch=None, target=None): def handle(name, ocfg, cloud, log, _): """process the config for apt_config. This can be called from - curthooks if a global apt config was provided or via the "apt" - standalone command.""" + curthooks if a global apt config was provided or via the "apt" + standalone command.""" # keeping code close to curtin codebase via entry handler target = None if log is not None: @@ -447,12 +465,14 @@ def handle(name, ocfg, cloud, log, _): LOG = log # feed back converted config, but only work on the subset under 'apt' ocfg = convert_to_v3_apt_format(ocfg) - cfg = ocfg.get('apt', {}) + cfg = ocfg.get("apt", {}) if not isinstance(cfg, dict): raise ValueError( "Expected dictionary for 'apt' config, found {config_type}".format( - config_type=type(cfg))) + config_type=type(cfg) + ) + ) validate_cloudconfig_schema(cfg, schema) apply_debconf_selections(cfg, target) @@ -463,7 +483,7 @@ def _should_configure_on_empty_apt(): # if no config was provided, should apt configuration be done? if util.system_is_snappy(): return False, "system is snappy." - if not (subp.which('apt-get') or subp.which('apt')): + if not (subp.which("apt-get") or subp.which("apt")): return False, "no apt commands." return True, "Apt is available." @@ -478,12 +498,12 @@ def apply_apt(cfg, cloud, target): LOG.debug("handling apt config: %s", cfg) - release = util.lsb_release(target=target)['codename'] + release = util.lsb_release(target=target)["codename"] arch = util.get_dpkg_architecture(target) mirrors = find_apt_mirror_info(cfg, cloud, arch=arch) LOG.debug("Apt Mirror info: %s", mirrors) - if util.is_false(cfg.get('preserve_sources_list', False)): + if util.is_false(cfg.get("preserve_sources_list", False)): add_mirror_keys(cfg, target) generate_sources_list(cfg, release, mirrors, cloud) rename_apt_lists(mirrors, target, arch) @@ -494,25 +514,34 @@ def apply_apt(cfg, cloud, target): LOG.exception("Failed to apply proxy or apt config info:") # Process 'apt_source -> sources {dict}' - if 'sources' in cfg: + if "sources" in cfg: params = mirrors - params['RELEASE'] = release - params['MIRROR'] = mirrors["MIRROR"] + params["RELEASE"] = release + params["MIRROR"] = mirrors["MIRROR"] matcher = None - matchcfg = cfg.get('add_apt_repo_match', ADD_APT_REPO_MATCH) + matchcfg = cfg.get("add_apt_repo_match", ADD_APT_REPO_MATCH) if matchcfg: matcher = re.compile(matchcfg).search - add_apt_sources(cfg['sources'], cloud, target=target, - template_params=params, aa_repo_match=matcher) + add_apt_sources( + cfg["sources"], + cloud, + target=target, + template_params=params, + aa_repo_match=matcher, + ) def debconf_set_selections(selections, target=None): - if not selections.endswith(b'\n'): - selections += b'\n' - subp.subp(['debconf-set-selections'], data=selections, target=target, - capture=True) + if not selections.endswith(b"\n"): + selections += b"\n" + subp.subp( + ["debconf-set-selections"], + data=selections, + target=target, + capture=True, + ) def dpkg_reconfigure(packages, target=None): @@ -532,12 +561,20 @@ def dpkg_reconfigure(packages, target=None): unhandled.append(pkg) if len(unhandled): - LOG.warning("The following packages were installed and preseeded, " - "but cannot be unconfigured: %s", unhandled) + LOG.warning( + "The following packages were installed and preseeded, " + "but cannot be unconfigured: %s", + unhandled, + ) if len(to_config): - subp.subp(['dpkg-reconfigure', '--frontend=noninteractive'] + - list(to_config), data=None, target=target, capture=True) + subp.subp( + ["dpkg-reconfigure", "--frontend=noninteractive"] + + list(to_config), + data=None, + target=target, + capture=True, + ) def apply_debconf_selections(cfg, target=None): @@ -546,13 +583,12 @@ def apply_debconf_selections(cfg, target=None): # set1: | # cloud-init cloud-init/datasources multiselect MAAS # set2: pkg pkg/value string bar - selsets = cfg.get('debconf_selections') + selsets = cfg.get("debconf_selections") if not selsets: LOG.debug("debconf_selections was not set in config") return - selections = '\n'.join( - [selsets[key] for key in sorted(selsets.keys())]) + selections = "\n".join([selsets[key] for key in sorted(selsets.keys())]) debconf_set_selections(selections.encode(), target=target) # get a complete list of packages listed in input @@ -579,7 +615,8 @@ def apply_debconf_selections(cfg, target=None): def clean_cloud_init(target): """clean out any local cloud-init config""" flist = glob.glob( - subp.target_path(target, "/etc/cloud/cloud.cfg.d/*dpkg*")) + subp.target_path(target, "/etc/cloud/cloud.cfg.d/*dpkg*") + ) LOG.debug("cleaning cloud-init config from: %s", flist) for dpkg_cfg in flist: @@ -588,18 +625,18 @@ def clean_cloud_init(target): def mirrorurl_to_apt_fileprefix(mirror): """mirrorurl_to_apt_fileprefix - Convert a mirror url to the file prefix used by apt on disk to - store cache information for that mirror. - To do so do: - - take off ???:// - - drop tailing / - - convert in string / to _""" + Convert a mirror url to the file prefix used by apt on disk to + store cache information for that mirror. + To do so do: + - take off ???:// + - drop tailing / + - convert in string / to _""" string = mirror if string.endswith("/"): string = string[0:-1] pos = string.find("://") if pos >= 0: - string = string[pos + 3:] + string = string[pos + 3 :] string = string.replace("/", "_") return string @@ -631,8 +668,8 @@ def rename_apt_lists(new_mirrors, target, arch): def mirror_to_placeholder(tmpl, mirror, placeholder): """mirror_to_placeholder - replace the specified mirror in a template with a placeholder string - Checks for existance of the expected mirror and warns if not found""" + replace the specified mirror in a template with a placeholder string + Checks for existance of the expected mirror and warns if not found""" if mirror not in tmpl: LOG.warning("Expected mirror '%s' not found in: %s", mirror, tmpl) return tmpl.replace(mirror, placeholder) @@ -640,13 +677,15 @@ def mirror_to_placeholder(tmpl, mirror, placeholder): def map_known_suites(suite): """there are a few default names which will be auto-extended. - This comes at the inability to use those names literally as suites, - but on the other hand increases readability of the cfg quite a lot""" - mapping = {'updates': '$RELEASE-updates', - 'backports': '$RELEASE-backports', - 'security': '$RELEASE-security', - 'proposed': '$RELEASE-proposed', - 'release': '$RELEASE'} + This comes at the inability to use those names literally as suites, + but on the other hand increases readability of the cfg quite a lot""" + mapping = { + "updates": "$RELEASE-updates", + "backports": "$RELEASE-backports", + "security": "$RELEASE-security", + "proposed": "$RELEASE-proposed", + "release": "$RELEASE", + } try: retsuite = mapping[suite] except KeyError: @@ -656,14 +695,14 @@ def map_known_suites(suite): def disable_suites(disabled, src, release): """reads the config for suites to be disabled and removes those - from the template""" + from the template""" if not disabled: return src retsrc = src for suite in disabled: suite = map_known_suites(suite) - releasesuite = templater.render_string(suite, {'RELEASE': release}) + releasesuite = templater.render_string(suite, {"RELEASE": release}) LOG.debug("Disabling suite %s as %s", suite, releasesuite) newsrc = "" @@ -685,7 +724,7 @@ def disable_suites(disabled, src, release): break if cols[pcol] == releasesuite: - line = '# suite disabled by cloud-init: %s' % line + line = "# suite disabled by cloud-init: %s" % line newsrc += line retsrc = newsrc @@ -694,36 +733,38 @@ def disable_suites(disabled, src, release): def add_mirror_keys(cfg, target): """Adds any keys included in the primary/security mirror clauses""" - for key in ('primary', 'security'): + for key in ("primary", "security"): for mirror in cfg.get(key, []): add_apt_key(mirror, target, file_name=key) def generate_sources_list(cfg, release, mirrors, cloud): """generate_sources_list - create a source.list file based on a custom or default template - by replacing mirrors and release in the template""" + create a source.list file based on a custom or default template + by replacing mirrors and release in the template""" aptsrc = "/etc/apt/sources.list" - params = {'RELEASE': release, 'codename': release} + params = {"RELEASE": release, "codename": release} for k in mirrors: params[k] = mirrors[k] params[k.lower()] = mirrors[k] - tmpl = cfg.get('sources_list', None) + tmpl = cfg.get("sources_list", None) if tmpl is None: LOG.info("No custom template provided, fall back to builtin") - template_fn = cloud.get_template_filename('sources.list.%s' % - (cloud.distro.name)) + template_fn = cloud.get_template_filename( + "sources.list.%s" % (cloud.distro.name) + ) if not template_fn: - template_fn = cloud.get_template_filename('sources.list') + template_fn = cloud.get_template_filename("sources.list") if not template_fn: - LOG.warning("No template found, " - "not rendering /etc/apt/sources.list") + LOG.warning( + "No template found, not rendering /etc/apt/sources.list" + ) return tmpl = util.load_file(template_fn) rendered = templater.render_string(tmpl, params) - disabled = disable_suites(cfg.get('disable_suites'), rendered, release) + disabled = disable_suites(cfg.get("disable_suites"), rendered, release) util.write_file(aptsrc, disabled, mode=0o644) @@ -735,7 +776,7 @@ def add_apt_key_raw(key, file_name, hardened=False, target=None): LOG.debug("Adding key:\n'%s'", key) try: name = pathlib.Path(file_name).stem - return apt_key('add', output_file=name, data=key, hardened=hardened) + return apt_key("add", output_file=name, data=key, hardened=hardened) except subp.ProcessExecutionError: LOG.exception("failed to add apt GPG Key to apt keyring") raise @@ -747,26 +788,26 @@ def add_apt_key(ent, target=None, hardened=False, file_name=None): Supports raw keys or keyid's The latter will as a first step fetched to get the raw key """ - if 'keyid' in ent and 'key' not in ent: + if "keyid" in ent and "key" not in ent: keyserver = DEFAULT_KEYSERVER - if 'keyserver' in ent: - keyserver = ent['keyserver'] + if "keyserver" in ent: + keyserver = ent["keyserver"] - ent['key'] = gpg.getkeybyid(ent['keyid'], keyserver) + ent["key"] = gpg.getkeybyid(ent["keyid"], keyserver) - if 'key' in ent: + if "key" in ent: return add_apt_key_raw( - ent['key'], - file_name or ent['filename'], - hardened=hardened) + ent["key"], file_name or ent["filename"], hardened=hardened + ) def update_packages(cloud): cloud.distro.update_package_sources() -def add_apt_sources(srcdict, cloud, target=None, template_params=None, - aa_repo_match=None): +def add_apt_sources( + srcdict, cloud, target=None, template_params=None, aa_repo_match=None +): """ install keys and repo source .list files defined in 'sources' @@ -795,33 +836,34 @@ def add_apt_sources(srcdict, cloud, target=None, template_params=None, template_params = {} if aa_repo_match is None: - raise ValueError('did not get a valid repo matcher') + raise ValueError("did not get a valid repo matcher") if not isinstance(srcdict, dict): - raise TypeError('unknown apt format: %s' % (srcdict)) + raise TypeError("unknown apt format: %s" % (srcdict)) for filename in srcdict: ent = srcdict[filename] LOG.debug("adding source/key '%s'", ent) - if 'filename' not in ent: - ent['filename'] = filename + if "filename" not in ent: + ent["filename"] = filename - if 'source' in ent and '$KEY_FILE' in ent['source']: + if "source" in ent and "$KEY_FILE" in ent["source"]: key_file = add_apt_key(ent, target, hardened=True) - template_params['KEY_FILE'] = key_file + template_params["KEY_FILE"] = key_file else: key_file = add_apt_key(ent, target) - if 'source' not in ent: + if "source" not in ent: continue - source = ent['source'] + source = ent["source"] source = templater.render_string(source, template_params) - if not ent['filename'].startswith("/"): - ent['filename'] = os.path.join("/etc/apt/sources.list.d/", - ent['filename']) - if not ent['filename'].endswith(".list"): - ent['filename'] += ".list" + if not ent["filename"].startswith("/"): + ent["filename"] = os.path.join( + "/etc/apt/sources.list.d/", ent["filename"] + ) + if not ent["filename"].endswith(".list"): + ent["filename"] += ".list" if aa_repo_match(source): try: @@ -831,7 +873,7 @@ def add_apt_sources(srcdict, cloud, target=None, template_params=None, raise continue - sourcefn = subp.target_path(target, ent['filename']) + sourcefn = subp.target_path(target, ent["filename"]) try: contents = "%s\n" % (source) util.write_file(sourcefn, contents, omode="a") @@ -850,14 +892,14 @@ def convert_v1_to_v2_apt_format(srclist): if isinstance(srclist, list): LOG.debug("apt config: convert V1 to V2 format (source list to dict)") for srcent in srclist: - if 'filename' not in srcent: + if "filename" not in srcent: # file collides for multiple !filename cases for compatibility # yet we need them all processed, so not same dictionary key - srcent['filename'] = "cloud_config_sources.list" + srcent["filename"] = "cloud_config_sources.list" key = util.rand_dict_key(srcdict, "cloud_config_sources.list") else: # all with filename use that as key (matching new format) - key = srcent['filename'] + key = srcent["filename"] srcdict[key] = srcent elif isinstance(srclist, dict): srcdict = srclist @@ -869,7 +911,7 @@ def convert_v1_to_v2_apt_format(srclist): def convert_key(oldcfg, aptcfg, oldkey, newkey): """convert an old key to the new one if the old one exists - returns true if a key was found and converted""" + returns true if a key was found and converted""" if oldcfg.get(oldkey, None) is not None: aptcfg[newkey] = oldcfg.get(oldkey) del oldcfg[oldkey] @@ -879,33 +921,37 @@ def convert_key(oldcfg, aptcfg, oldkey, newkey): def convert_mirror(oldcfg, aptcfg): """convert old apt_mirror keys into the new more advanced mirror spec""" - keymap = [('apt_mirror', 'uri'), - ('apt_mirror_search', 'search'), - ('apt_mirror_search_dns', 'search_dns')] + keymap = [ + ("apt_mirror", "uri"), + ("apt_mirror_search", "search"), + ("apt_mirror_search_dns", "search_dns"), + ] converted = False - newmcfg = {'arches': ['default']} + newmcfg = {"arches": ["default"]} for oldkey, newkey in keymap: if convert_key(oldcfg, newmcfg, oldkey, newkey): converted = True # only insert new style config if anything was converted if converted: - aptcfg['primary'] = [newmcfg] + aptcfg["primary"] = [newmcfg] def convert_v2_to_v3_apt_format(oldcfg): """convert old to new keys and adapt restructured mirror spec""" - mapoldkeys = {'apt_sources': 'sources', - 'apt_mirror': None, - 'apt_mirror_search': None, - 'apt_mirror_search_dns': None, - 'apt_proxy': 'proxy', - 'apt_http_proxy': 'http_proxy', - 'apt_ftp_proxy': 'https_proxy', - 'apt_https_proxy': 'ftp_proxy', - 'apt_preserve_sources_list': 'preserve_sources_list', - 'apt_custom_sources_list': 'sources_list', - 'add_apt_repo_match': 'add_apt_repo_match'} + mapoldkeys = { + "apt_sources": "sources", + "apt_mirror": None, + "apt_mirror_search": None, + "apt_mirror_search_dns": None, + "apt_proxy": "proxy", + "apt_http_proxy": "http_proxy", + "apt_ftp_proxy": "https_proxy", + "apt_https_proxy": "ftp_proxy", + "apt_preserve_sources_list": "preserve_sources_list", + "apt_custom_sources_list": "sources_list", + "add_apt_repo_match": "add_apt_repo_match", + } needtoconvert = [] for oldkey in mapoldkeys: if oldkey in oldcfg: @@ -917,11 +963,13 @@ def convert_v2_to_v3_apt_format(oldcfg): # no old config, so no new one to be created if not needtoconvert: return oldcfg - LOG.debug("apt config: convert V2 to V3 format for keys '%s'", - ", ".join(needtoconvert)) + LOG.debug( + "apt config: convert V2 to V3 format for keys '%s'", + ", ".join(needtoconvert), + ) # if old AND new config are provided, prefer the new one (LP #1616831) - newaptcfg = oldcfg.get('apt', None) + newaptcfg = oldcfg.get("apt", None) if newaptcfg is not None: LOG.debug("apt config: V1/2 and V3 format specified, preferring V3") for oldkey in needtoconvert: @@ -932,10 +980,11 @@ def convert_v2_to_v3_apt_format(oldcfg): # no simple mapping or no collision on this particular key continue if verify != newaptcfg[newkey]: - raise ValueError("Old and New apt format defined with unequal " - "values %s vs %s @ %s" % (verify, - newaptcfg[newkey], - oldkey)) + raise ValueError( + "Old and New apt format defined with unequal " + "values %s vs %s @ %s" + % (verify, newaptcfg[newkey], oldkey) + ) # return conf after clearing conflicting V1/2 keys return oldcfg @@ -955,17 +1004,17 @@ def convert_v2_to_v3_apt_format(oldcfg): raise ValueError("old apt key '%s' left after conversion" % oldkey) # insert new format into config and return full cfg with only v3 content - oldcfg['apt'] = aptcfg + oldcfg["apt"] = aptcfg return oldcfg def convert_to_v3_apt_format(cfg): """convert the old list based format to the new dict based one. After that - convert the old dict keys/format to v3 a.k.a 'new apt config'""" + convert the old dict keys/format to v3 a.k.a 'new apt config'""" # V1 -> V2, the apt_sources entry from list to dict - apt_sources = cfg.get('apt_sources', None) + apt_sources = cfg.get("apt_sources", None) if apt_sources is not None: - cfg['apt_sources'] = convert_v1_to_v2_apt_format(apt_sources) + cfg["apt_sources"] = convert_v1_to_v2_apt_format(apt_sources) # V2 -> V3, move all former globals under the "apt" key # Restructure into new key names and mirror hierarchy @@ -997,7 +1046,12 @@ def search_for_mirror_dns(configured, mirrortype, cfg, cloud): if mydom: doms.append(".%s" % mydom) - doms.extend((".localdomain", "",)) + doms.extend( + ( + ".localdomain", + "", + ) + ) mirror_list = [] distro = cloud.distro.name @@ -1012,12 +1066,11 @@ def search_for_mirror_dns(configured, mirrortype, cfg, cloud): def update_mirror_info(pmirror, smirror, arch, cloud): """sets security mirror to primary if not defined. - returns defaults if no mirrors are defined""" + returns defaults if no mirrors are defined""" if pmirror is not None: if smirror is None: smirror = pmirror - return {'PRIMARY': pmirror, - 'SECURITY': smirror} + return {"PRIMARY": pmirror, "SECURITY": smirror} # None specified at all, get default mirrors from cloud mirror_info = cloud.datasource.get_package_mirror_info() @@ -1026,8 +1079,8 @@ def update_mirror_info(pmirror, smirror, arch, cloud): # arbitrary key/value pairs including 'primary' and 'security' keys. # caller expects dict with PRIMARY and SECURITY. m = mirror_info.copy() - m['PRIMARY'] = m['primary'] - m['SECURITY'] = m['security'] + m["PRIMARY"] = m["primary"] + m["SECURITY"] = m["security"] return m @@ -1037,7 +1090,7 @@ def update_mirror_info(pmirror, smirror, arch, cloud): def get_arch_mirrorconfig(cfg, mirrortype, arch): """out of a list of potential mirror configurations select - and return the one matching the architecture (or default)""" + and return the one matching the architecture (or default)""" # select the mirror specification (if-any) mirror_cfg_list = cfg.get(mirrortype, None) if mirror_cfg_list is None: @@ -1056,8 +1109,8 @@ def get_arch_mirrorconfig(cfg, mirrortype, arch): def get_mirror(cfg, mirrortype, arch, cloud): """pass the three potential stages of mirror specification - returns None is neither of them found anything otherwise the first - hit is returned""" + returns None is neither of them found anything otherwise the first + hit is returned""" mcfg = get_arch_mirrorconfig(cfg, mirrortype, arch) if mcfg is None: return None @@ -1073,18 +1126,19 @@ def get_mirror(cfg, mirrortype, arch, cloud): # fallback to search_dns if specified if mirror is None: # list of mirrors to try to resolve - mirror = search_for_mirror_dns(mcfg.get("search_dns", None), - mirrortype, cfg, cloud) + mirror = search_for_mirror_dns( + mcfg.get("search_dns", None), mirrortype, cfg, cloud + ) return mirror def find_apt_mirror_info(cfg, cloud, arch=None): """find_apt_mirror_info - find an apt_mirror given the cfg provided. - It can check for separate config of primary and security mirrors - If only primary is given security is assumed to be equal to primary - If the generic apt_mirror is given that is defining for both + find an apt_mirror given the cfg provided. + It can check for separate config of primary and security mirrors + If only primary is given security is assumed to be equal to primary + If the generic apt_mirror is given that is defining for both """ if arch is None: @@ -1105,32 +1159,35 @@ def find_apt_mirror_info(cfg, cloud, arch=None): def apply_apt_config(cfg, proxy_fname, config_fname): """apply_apt_config - Applies any apt*proxy config from if specified + Applies any apt*proxy config from if specified """ # Set up any apt proxy - cfgs = (('proxy', 'Acquire::http::Proxy "%s";'), - ('http_proxy', 'Acquire::http::Proxy "%s";'), - ('ftp_proxy', 'Acquire::ftp::Proxy "%s";'), - ('https_proxy', 'Acquire::https::Proxy "%s";')) + cfgs = ( + ("proxy", 'Acquire::http::Proxy "%s";'), + ("http_proxy", 'Acquire::http::Proxy "%s";'), + ("ftp_proxy", 'Acquire::ftp::Proxy "%s";'), + ("https_proxy", 'Acquire::https::Proxy "%s";'), + ) proxies = [fmt % cfg.get(name) for (name, fmt) in cfgs if cfg.get(name)] if len(proxies): LOG.debug("write apt proxy info to %s", proxy_fname) - util.write_file(proxy_fname, '\n'.join(proxies) + '\n') + util.write_file(proxy_fname, "\n".join(proxies) + "\n") elif os.path.isfile(proxy_fname): util.del_file(proxy_fname) LOG.debug("no apt proxy configured, removed %s", proxy_fname) - if cfg.get('conf', None): + if cfg.get("conf", None): LOG.debug("write apt config info to %s", config_fname) - util.write_file(config_fname, cfg.get('conf')) + util.write_file(config_fname, cfg.get("conf")) elif os.path.isfile(config_fname): util.del_file(config_fname) LOG.debug("no apt config configured, removed %s", config_fname) -def apt_key(command, output_file=None, data=None, hardened=False, - human_output=True): +def apt_key( + command, output_file=None, data=None, hardened=False, human_output=True +): """apt-key replacement commands implemented: 'add', 'list', 'finger' @@ -1153,32 +1210,36 @@ def apt_key(command, output_file=None, data=None, hardened=False, key_files = [APT_LOCAL_KEYS] if os.path.isfile(APT_LOCAL_KEYS) else [] for file in os.listdir(APT_TRUSTED_GPG_DIR): - if file.endswith('.gpg') or file.endswith('.asc'): + if file.endswith(".gpg") or file.endswith(".asc"): key_files.append(APT_TRUSTED_GPG_DIR + file) - return key_files if key_files else '' + return key_files if key_files else "" def apt_key_add(): """apt-key add <file> returns filepath to new keyring, or '/dev/null' when an error occurs """ - file_name = '/dev/null' + file_name = "/dev/null" if not output_file: util.logexc( - LOG, 'Unknown filename, failed to add key: "{}"'.format(data)) + LOG, 'Unknown filename, failed to add key: "{}"'.format(data) + ) else: try: - key_dir = \ + key_dir = ( CLOUD_INIT_GPG_DIR if hardened else APT_TRUSTED_GPG_DIR + ) stdout = gpg.dearmor(data) - file_name = '{}{}.gpg'.format(key_dir, output_file) + file_name = "{}{}.gpg".format(key_dir, output_file) util.write_file(file_name, stdout) except subp.ProcessExecutionError: - util.logexc(LOG, 'Gpg error, failed to add key: {}'.format( - data)) + util.logexc( + LOG, "Gpg error, failed to add key: {}".format(data) + ) except UnicodeDecodeError: - util.logexc(LOG, 'Decode error, failed to add key: {}'.format( - data)) + util.logexc( + LOG, "Decode error, failed to add key: {}".format(data) + ) return file_name def apt_key_list(): @@ -1193,19 +1254,20 @@ def apt_key(command, output_file=None, data=None, hardened=False, key_list.append(gpg.list(key_file, human_output=human_output)) except subp.ProcessExecutionError as error: LOG.warning('Failed to list key "%s": %s', key_file, error) - return '\n'.join(key_list) + return "\n".join(key_list) - if command == 'add': + if command == "add": return apt_key_add() - elif command == 'finger' or command == 'list': + elif command == "finger" or command == "list": return apt_key_list() else: raise ValueError( - 'apt_key() commands add, list, and finger are currently supported') + "apt_key() commands add, list, and finger are currently supported" + ) CONFIG_CLEANERS = { - 'cloud-init': clean_cloud_init, + "cloud-init": clean_cloud_init, } # vi: ts=4 expandtab diff --git a/cloudinit/config/cc_apt_pipelining.py b/cloudinit/config/cc_apt_pipelining.py index aa186ce2..569849d1 100644 --- a/cloudinit/config/cc_apt_pipelining.py +++ b/cloudinit/config/cc_apt_pipelining.py @@ -29,17 +29,19 @@ not recommended. apt_pipelining: <false/none/unchanged/os/number> """ -from cloudinit.settings import PER_INSTANCE from cloudinit import util +from cloudinit.settings import PER_INSTANCE frequency = PER_INSTANCE -distros = ['ubuntu', 'debian'] +distros = ["ubuntu", "debian"] DEFAULT_FILE = "/etc/apt/apt.conf.d/90cloud-init-pipelining" -APT_PIPE_TPL = ("//Written by cloud-init per 'apt_pipelining'\n" - 'Acquire::http::Pipeline-Depth "%s";\n') +APT_PIPE_TPL = ( + "//Written by cloud-init per 'apt_pipelining'\n" + 'Acquire::http::Pipeline-Depth "%s";\n' +) # Acquire::http::Pipeline-Depth can be a value # from 0 to 5 indicating how many outstanding requests APT should send. @@ -49,7 +51,7 @@ APT_PIPE_TPL = ("//Written by cloud-init per 'apt_pipelining'\n" def handle(_name, cfg, _cloud, log, _args): - apt_pipe_value = util.get_cfg_option_str(cfg, "apt_pipelining", 'os') + apt_pipe_value = util.get_cfg_option_str(cfg, "apt_pipelining", "os") apt_pipe_value_s = str(apt_pipe_value).lower().strip() if apt_pipe_value_s == "false": @@ -69,4 +71,5 @@ def write_apt_snippet(setting, log, f_name): util.write_file(f_name, file_contents) log.debug("Wrote %s with apt pipeline depth setting %s", f_name, setting) + # vi: ts=4 expandtab diff --git a/cloudinit/config/cc_bootcmd.py b/cloudinit/config/cc_bootcmd.py index 06f7a26e..bff11a24 100644 --- a/cloudinit/config/cc_bootcmd.py +++ b/cloudinit/config/cc_bootcmd.py @@ -12,11 +12,9 @@ import os from textwrap import dedent +from cloudinit import subp, temp_utils, util from cloudinit.config.schema import get_meta_doc, validate_cloudconfig_schema from cloudinit.settings import PER_ALWAYS -from cloudinit import temp_utils -from cloudinit import subp -from cloudinit import util frequency = PER_ALWAYS @@ -26,13 +24,14 @@ frequency = PER_ALWAYS # configuration options before actually attempting to deploy with said # configuration. -distros = ['all'] +distros = ["all"] meta = { - 'id': 'cc_bootcmd', - 'name': 'Bootcmd', - 'title': 'Run arbitrary commands early in the boot process', - 'description': dedent("""\ + "id": "cc_bootcmd", + "name": "Bootcmd", + "title": "Run arbitrary commands early in the boot process", + "description": dedent( + """\ This module runs arbitrary commands very early in the boot process, only slightly after a boothook would run. This is very similar to a boothook, but more user friendly. The environment variable @@ -48,31 +47,37 @@ meta = { when writing files, do not use /tmp dir as it races with systemd-tmpfiles-clean LP: #1707222. Use /run/somedir instead. - """), - 'distros': distros, - 'examples': [dedent("""\ + """ + ), + "distros": distros, + "examples": [ + dedent( + """\ bootcmd: - echo 192.168.1.130 us.archive.ubuntu.com > /etc/hosts - [ cloud-init-per, once, mymkfs, mkfs, /dev/vdb ] - """)], - 'frequency': PER_ALWAYS, + """ + ) + ], + "frequency": PER_ALWAYS, } schema = { - 'type': 'object', - 'properties': { - 'bootcmd': { - 'type': 'array', - 'items': { - 'oneOf': [ - {'type': 'array', 'items': {'type': 'string'}}, - {'type': 'string'}] + "type": "object", + "properties": { + "bootcmd": { + "type": "array", + "items": { + "oneOf": [ + {"type": "array", "items": {"type": "string"}}, + {"type": "string"}, + ] }, - 'additionalItems': False, # Reject items of non-string non-list - 'additionalProperties': False, - 'minItems': 1, + "additionalItems": False, # Reject items of non-string non-list + "additionalProperties": False, + "minItems": 1, } - } + }, } __doc__ = get_meta_doc(meta, schema) # Supplement python help() @@ -81,8 +86,9 @@ __doc__ = get_meta_doc(meta, schema) # Supplement python help() def handle(name, cfg, cloud, log, _args): if "bootcmd" not in cfg: - log.debug(("Skipping module named %s," - " no 'bootcmd' key in configuration"), name) + log.debug( + "Skipping module named %s, no 'bootcmd' key in configuration", name + ) return validate_cloudconfig_schema(cfg, schema) @@ -99,11 +105,12 @@ def handle(name, cfg, cloud, log, _args): env = os.environ.copy() iid = cloud.get_instance_id() if iid: - env['INSTANCE_ID'] = str(iid) - cmd = ['/bin/sh', tmpf.name] + env["INSTANCE_ID"] = str(iid) + cmd = ["/bin/sh", tmpf.name] subp.subp(cmd, env=env, capture=False) except Exception: util.logexc(log, "Failed to run bootcmd module %s", name) raise + # vi: ts=4 expandtab diff --git a/cloudinit/config/cc_byobu.py b/cloudinit/config/cc_byobu.py index 9fdaeba1..53b6d0c8 100755 --- a/cloudinit/config/cc_byobu.py +++ b/cloudinit/config/cc_byobu.py @@ -38,11 +38,10 @@ Valid configuration options for this module are: byobu_by_default: <user/system> """ +from cloudinit import subp, util from cloudinit.distros import ug_util -from cloudinit import subp -from cloudinit import util -distros = ['ubuntu', 'debian'] +distros = ["ubuntu", "debian"] def handle(name, cfg, cloud, log, args): @@ -58,8 +57,14 @@ def handle(name, cfg, cloud, log, args): if value == "user" or value == "system": value = "enable-%s" % value - valid = ("enable-user", "enable-system", "enable", - "disable-user", "disable-system", "disable") + valid = ( + "enable-user", + "enable-system", + "enable", + "disable-user", + "disable-system", + "disable", + ) if value not in valid: log.warning("Unknown value %s for byobu_by_default", value) @@ -81,13 +86,16 @@ def handle(name, cfg, cloud, log, args): (users, _groups) = ug_util.normalize_users_groups(cfg, cloud.distro) (user, _user_config) = ug_util.extract_default(users) if not user: - log.warning(("No default byobu user provided, " - "can not launch %s for the default user"), bl_inst) + log.warning( + "No default byobu user provided, " + "can not launch %s for the default user", + bl_inst, + ) else: - shcmd += " sudo -Hu \"%s\" byobu-launcher-%s" % (user, bl_inst) + shcmd += ' sudo -Hu "%s" byobu-launcher-%s' % (user, bl_inst) shcmd += " || X=$(($X+1)); " if mod_sys: - shcmd += "echo \"%s\" | debconf-set-selections" % dc_val + shcmd += 'echo "%s" | debconf-set-selections' % dc_val shcmd += " && dpkg-reconfigure byobu --frontend=noninteractive" shcmd += " || X=$(($X+1)); " @@ -96,4 +104,5 @@ def handle(name, cfg, cloud, log, args): log.debug("Setting byobu to %s", value) subp.subp(cmd, capture=False) + # vi: ts=4 expandtab diff --git a/cloudinit/config/cc_ca_certs.py b/cloudinit/config/cc_ca_certs.py index bd7bead9..9de065ab 100644 --- a/cloudinit/config/cc_ca_certs.py +++ b/cloudinit/config/cc_ca_certs.py @@ -41,28 +41,27 @@ can be removed from the system with the configuration option import os -from cloudinit import subp -from cloudinit import util +from cloudinit import subp, util DEFAULT_CONFIG = { - 'ca_cert_path': '/usr/share/ca-certificates/', - 'ca_cert_filename': 'cloud-init-ca-certs.crt', - 'ca_cert_config': '/etc/ca-certificates.conf', - 'ca_cert_system_path': '/etc/ssl/certs/', - 'ca_cert_update_cmd': ['update-ca-certificates'] + "ca_cert_path": "/usr/share/ca-certificates/", + "ca_cert_filename": "cloud-init-ca-certs.crt", + "ca_cert_config": "/etc/ca-certificates.conf", + "ca_cert_system_path": "/etc/ssl/certs/", + "ca_cert_update_cmd": ["update-ca-certificates"], } DISTRO_OVERRIDES = { - 'rhel': { - 'ca_cert_path': '/usr/share/pki/ca-trust-source/', - 'ca_cert_filename': 'anchors/cloud-init-ca-certs.crt', - 'ca_cert_config': None, - 'ca_cert_system_path': '/etc/pki/ca-trust/', - 'ca_cert_update_cmd': ['update-ca-trust'] + "rhel": { + "ca_cert_path": "/usr/share/pki/ca-trust-source/", + "ca_cert_filename": "anchors/cloud-init-ca-certs.crt", + "ca_cert_config": None, + "ca_cert_system_path": "/etc/pki/ca-trust/", + "ca_cert_update_cmd": ["update-ca-trust"], } } -distros = ['alpine', 'debian', 'ubuntu', 'rhel'] +distros = ["alpine", "debian", "ubuntu", "rhel"] def _distro_ca_certs_configs(distro_name): @@ -72,8 +71,9 @@ def _distro_ca_certs_configs(distro_name): @returns: Dict of distro configurations for ca-cert. """ cfg = DISTRO_OVERRIDES.get(distro_name, DEFAULT_CONFIG) - cfg['ca_cert_full_path'] = os.path.join(cfg['ca_cert_path'], - cfg['ca_cert_filename']) + cfg["ca_cert_full_path"] = os.path.join( + cfg["ca_cert_path"], cfg["ca_cert_filename"] + ) return cfg @@ -83,7 +83,7 @@ def update_ca_certs(distro_cfg): @param distro_cfg: A hash providing _distro_ca_certs_configs function. """ - subp.subp(distro_cfg['ca_cert_update_cmd'], capture=False) + subp.subp(distro_cfg["ca_cert_update_cmd"], capture=False) def add_ca_certs(distro_cfg, certs): @@ -98,9 +98,9 @@ def add_ca_certs(distro_cfg, certs): return # First ensure they are strings... cert_file_contents = "\n".join([str(c) for c in certs]) - util.write_file(distro_cfg['ca_cert_full_path'], - cert_file_contents, - mode=0o644) + util.write_file( + distro_cfg["ca_cert_full_path"], cert_file_contents, mode=0o644 + ) update_cert_config(distro_cfg) @@ -110,23 +110,27 @@ def update_cert_config(distro_cfg): @param distro_cfg: A hash providing _distro_ca_certs_configs function. """ - if distro_cfg['ca_cert_config'] is None: + if distro_cfg["ca_cert_config"] is None: return - if os.stat(distro_cfg['ca_cert_config']).st_size == 0: + if os.stat(distro_cfg["ca_cert_config"]).st_size == 0: # If the CA_CERT_CONFIG file is empty (i.e. all existing # CA certs have been deleted) then simply output a single # line with the cloud-init cert filename. - out = "%s\n" % distro_cfg['ca_cert_filename'] + out = "%s\n" % distro_cfg["ca_cert_filename"] else: # Append cert filename to CA_CERT_CONFIG file. # We have to strip the content because blank lines in the file # causes subsequent entries to be ignored. (LP: #1077020) - orig = util.load_file(distro_cfg['ca_cert_config']) - cr_cont = '\n'.join([line for line in orig.splitlines() - if line != distro_cfg['ca_cert_filename']]) - out = "%s\n%s\n" % (cr_cont.rstrip(), - distro_cfg['ca_cert_filename']) - util.write_file(distro_cfg['ca_cert_config'], out, omode="wb") + orig = util.load_file(distro_cfg["ca_cert_config"]) + cr_cont = "\n".join( + [ + line + for line in orig.splitlines() + if line != distro_cfg["ca_cert_filename"] + ] + ) + out = "%s\n%s\n" % (cr_cont.rstrip(), distro_cfg["ca_cert_filename"]) + util.write_file(distro_cfg["ca_cert_config"], out, omode="wb") def remove_default_ca_certs(distro_name, distro_cfg): @@ -137,14 +141,15 @@ def remove_default_ca_certs(distro_name, distro_cfg): @param distro_name: String providing the distro class name. @param distro_cfg: A hash providing _distro_ca_certs_configs function. """ - util.delete_dir_contents(distro_cfg['ca_cert_path']) - util.delete_dir_contents(distro_cfg['ca_cert_system_path']) - util.write_file(distro_cfg['ca_cert_config'], "", mode=0o644) + util.delete_dir_contents(distro_cfg["ca_cert_path"]) + util.delete_dir_contents(distro_cfg["ca_cert_system_path"]) + util.write_file(distro_cfg["ca_cert_config"], "", mode=0o644) - if distro_name in ['debian', 'ubuntu']: + if distro_name in ["debian", "ubuntu"]: debconf_sel = ( - "ca-certificates ca-certificates/trust_new_crts " + "select no") - subp.subp(('debconf-set-selections', '-'), debconf_sel) + "ca-certificates ca-certificates/trust_new_crts " + "select no" + ) + subp.subp(("debconf-set-selections", "-"), debconf_sel) def handle(name, cfg, cloud, log, _args): @@ -159,11 +164,13 @@ def handle(name, cfg, cloud, log, _args): """ # If there isn't a ca-certs section in the configuration don't do anything if "ca-certs" not in cfg: - log.debug(("Skipping module named %s," - " no 'ca-certs' key in configuration"), name) + log.debug( + "Skipping module named %s, no 'ca-certs' key in configuration", + name, + ) return - ca_cert_cfg = cfg['ca-certs'] + ca_cert_cfg = cfg["ca-certs"] distro_cfg = _distro_ca_certs_configs(cloud.distro.name) # If there is a remove-defaults option set to true, remove the system @@ -183,4 +190,5 @@ def handle(name, cfg, cloud, log, _args): log.debug("Updating certificates") update_ca_certs(distro_cfg) + # vi: ts=4 expandtab diff --git a/cloudinit/config/cc_chef.py b/cloudinit/config/cc_chef.py index ed734d1c..67889683 100644 --- a/cloudinit/config/cc_chef.py +++ b/cloudinit/config/cc_chef.py @@ -13,87 +13,91 @@ import json import os from textwrap import dedent -from cloudinit import subp +from cloudinit import subp, temp_utils, templater, url_helper, util from cloudinit.config.schema import get_meta_doc, validate_cloudconfig_schema -from cloudinit import templater -from cloudinit import temp_utils -from cloudinit import url_helper -from cloudinit import util from cloudinit.settings import PER_ALWAYS - RUBY_VERSION_DEFAULT = "1.8" -CHEF_DIRS = tuple([ - '/etc/chef', - '/var/log/chef', - '/var/lib/chef', - '/var/cache/chef', - '/var/backups/chef', - '/var/run/chef', -]) -REQUIRED_CHEF_DIRS = tuple([ - '/etc/chef', -]) +CHEF_DIRS = tuple( + [ + "/etc/chef", + "/var/log/chef", + "/var/lib/chef", + "/var/cache/chef", + "/var/backups/chef", + "/var/run/chef", + ] +) +REQUIRED_CHEF_DIRS = tuple( + [ + "/etc/chef", + ] +) # Used if fetching chef from a omnibus style package OMNIBUS_URL = "https://www.chef.io/chef/install.sh" OMNIBUS_URL_RETRIES = 5 -CHEF_VALIDATION_PEM_PATH = '/etc/chef/validation.pem' -CHEF_ENCRYPTED_DATA_BAG_PATH = '/etc/chef/encrypted_data_bag_secret' -CHEF_ENVIRONMENT = '_default' -CHEF_FB_PATH = '/etc/chef/firstboot.json' +CHEF_VALIDATION_PEM_PATH = "/etc/chef/validation.pem" +CHEF_ENCRYPTED_DATA_BAG_PATH = "/etc/chef/encrypted_data_bag_secret" +CHEF_ENVIRONMENT = "_default" +CHEF_FB_PATH = "/etc/chef/firstboot.json" CHEF_RB_TPL_DEFAULTS = { # These are ruby symbols... - 'ssl_verify_mode': ':verify_none', - 'log_level': ':info', + "ssl_verify_mode": ":verify_none", + "log_level": ":info", # These are not symbols... - 'log_location': '/var/log/chef/client.log', - 'validation_key': CHEF_VALIDATION_PEM_PATH, - 'validation_cert': None, - 'client_key': '/etc/chef/client.pem', - 'json_attribs': CHEF_FB_PATH, - 'file_cache_path': '/var/cache/chef', - 'file_backup_path': '/var/backups/chef', - 'pid_file': '/var/run/chef/client.pid', - 'show_time': True, - 'encrypted_data_bag_secret': None, + "log_location": "/var/log/chef/client.log", + "validation_key": CHEF_VALIDATION_PEM_PATH, + "validation_cert": None, + "client_key": "/etc/chef/client.pem", + "json_attribs": CHEF_FB_PATH, + "file_cache_path": "/var/cache/chef", + "file_backup_path": "/var/backups/chef", + "pid_file": "/var/run/chef/client.pid", + "show_time": True, + "encrypted_data_bag_secret": None, } -CHEF_RB_TPL_BOOL_KEYS = frozenset(['show_time']) -CHEF_RB_TPL_PATH_KEYS = frozenset([ - 'log_location', - 'validation_key', - 'client_key', - 'file_cache_path', - 'json_attribs', - 'pid_file', - 'encrypted_data_bag_secret', -]) +CHEF_RB_TPL_BOOL_KEYS = frozenset(["show_time"]) +CHEF_RB_TPL_PATH_KEYS = frozenset( + [ + "log_location", + "validation_key", + "client_key", + "file_cache_path", + "json_attribs", + "pid_file", + "encrypted_data_bag_secret", + ] +) CHEF_RB_TPL_KEYS = list(CHEF_RB_TPL_DEFAULTS.keys()) CHEF_RB_TPL_KEYS.extend(CHEF_RB_TPL_BOOL_KEYS) CHEF_RB_TPL_KEYS.extend(CHEF_RB_TPL_PATH_KEYS) -CHEF_RB_TPL_KEYS.extend([ - 'server_url', - 'node_name', - 'environment', - 'validation_name', - 'chef_license', -]) +CHEF_RB_TPL_KEYS.extend( + [ + "server_url", + "node_name", + "environment", + "validation_name", + "chef_license", + ] +) CHEF_RB_TPL_KEYS = frozenset(CHEF_RB_TPL_KEYS) -CHEF_RB_PATH = '/etc/chef/client.rb' -CHEF_EXEC_PATH = '/usr/bin/chef-client' -CHEF_EXEC_DEF_ARGS = tuple(['-d', '-i', '1800', '-s', '20']) +CHEF_RB_PATH = "/etc/chef/client.rb" +CHEF_EXEC_PATH = "/usr/bin/chef-client" +CHEF_EXEC_DEF_ARGS = tuple(["-d", "-i", "1800", "-s", "20"]) frequency = PER_ALWAYS distros = ["all"] meta = { - 'id': 'cc_chef', - 'name': 'Chef', - 'title': 'module that configures, starts and installs chef', - 'description': dedent("""\ + "id": "cc_chef", + "name": "Chef", + "title": "module that configures, starts and installs chef", + "description": dedent( + """\ This module enables chef to be installed (from packages, gems, or from omnibus). Before this occurs, chef configuration is written to disk (validation.pem, client.pem, firstboot.json, @@ -101,9 +105,12 @@ meta = { /var/log/chef and so-on). If configured, chef will be installed and started in either daemon or non-daemon mode. If run in non-daemon mode, post run actions are executed to do - finishing activities such as removing validation.pem."""), - 'distros': distros, - 'examples': [dedent(""" + finishing activities such as removing validation.pem.""" + ), + "distros": distros, + "examples": [ + dedent( + """ chef: directories: - /etc/chef @@ -124,180 +131,237 @@ meta = { omnibus_url_retries: 2 server_url: https://chef.yourorg.com:4000 ssl_verify_mode: :verify_peer - validation_name: yourorg-validator""")], - 'frequency': frequency, + validation_name: yourorg-validator""" + ) + ], + "frequency": frequency, } schema = { - 'type': 'object', - 'properties': { - 'chef': { - 'type': 'object', - 'additionalProperties': False, - 'properties': { - 'directories': { - 'type': 'array', - 'items': { - 'type': 'string' - }, - 'uniqueItems': True, - 'description': dedent("""\ + "type": "object", + "properties": { + "chef": { + "type": "object", + "additionalProperties": False, + "properties": { + "directories": { + "type": "array", + "items": {"type": "string"}, + "uniqueItems": True, + "description": dedent( + """\ Create the necessary directories for chef to run. By default, it creates the following directories: - {chef_dirs}""").format( + {chef_dirs}""" + ).format( chef_dirs="\n".join( [" - ``{}``".format(d) for d in CHEF_DIRS] ) - ) + ), }, - 'validation_cert': { - 'type': 'string', - 'description': dedent("""\ + "validation_cert": { + "type": "string", + "description": dedent( + """\ Optional string to be written to file validation_key. Special value ``system`` means set use existing file. - """) + """ + ), }, - 'validation_key': { - 'type': 'string', - 'default': CHEF_VALIDATION_PEM_PATH, - 'description': dedent("""\ + "validation_key": { + "type": "string", + "default": CHEF_VALIDATION_PEM_PATH, + "description": dedent( + """\ Optional path for validation_cert. default to - ``{}``.""".format(CHEF_VALIDATION_PEM_PATH)) + ``{}``.""".format( + CHEF_VALIDATION_PEM_PATH + ) + ), }, - 'firstboot_path': { - 'type': 'string', - 'default': CHEF_FB_PATH, - 'description': dedent("""\ + "firstboot_path": { + "type": "string", + "default": CHEF_FB_PATH, + "description": dedent( + """\ Path to write run_list and initial_attributes keys that should also be present in this configuration, defaults - to ``{}``.""".format(CHEF_FB_PATH)) + to ``{}``.""".format( + CHEF_FB_PATH + ) + ), }, - 'exec': { - 'type': 'boolean', - 'default': False, - 'description': dedent("""\ + "exec": { + "type": "boolean", + "default": False, + "description": dedent( + """\ define if we should run or not run chef (defaults to false, unless a gem installed is requested where this - will then default to true).""") + will then default to true).""" + ), }, - 'client_key': { - 'type': 'string', - 'default': CHEF_RB_TPL_DEFAULTS['client_key'], - 'description': dedent("""\ + "client_key": { + "type": "string", + "default": CHEF_RB_TPL_DEFAULTS["client_key"], + "description": dedent( + """\ Optional path for client_cert. default to - ``{}``.""".format(CHEF_RB_TPL_DEFAULTS['client_key'])) + ``{}``.""".format( + CHEF_RB_TPL_DEFAULTS["client_key"] + ) + ), }, - 'encrypted_data_bag_secret': { - 'type': 'string', - 'default': None, - 'description': dedent("""\ + "encrypted_data_bag_secret": { + "type": "string", + "default": None, + "description": dedent( + """\ Specifies the location of the secret key used by chef to encrypt data items. By default, this path is set to None, meaning that chef will have to look at the path ``{}`` for it. - """.format(CHEF_ENCRYPTED_DATA_BAG_PATH)) + """.format( + CHEF_ENCRYPTED_DATA_BAG_PATH + ) + ), }, - 'environment': { - 'type': 'string', - 'default': CHEF_ENVIRONMENT, - 'description': dedent("""\ + "environment": { + "type": "string", + "default": CHEF_ENVIRONMENT, + "description": dedent( + """\ Specifies which environment chef will use. By default, it will use the ``{}`` configuration. - """.format(CHEF_ENVIRONMENT)) + """.format( + CHEF_ENVIRONMENT + ) + ), }, - 'file_backup_path': { - 'type': 'string', - 'default': CHEF_RB_TPL_DEFAULTS['file_backup_path'], - 'description': dedent("""\ + "file_backup_path": { + "type": "string", + "default": CHEF_RB_TPL_DEFAULTS["file_backup_path"], + "description": dedent( + """\ Specifies the location in which backup files are stored. By default, it uses the ``{}`` location.""".format( - CHEF_RB_TPL_DEFAULTS['file_backup_path'])) + CHEF_RB_TPL_DEFAULTS["file_backup_path"] + ) + ), }, - 'file_cache_path': { - 'type': 'string', - 'default': CHEF_RB_TPL_DEFAULTS['file_cache_path'], - 'description': dedent("""\ + "file_cache_path": { + "type": "string", + "default": CHEF_RB_TPL_DEFAULTS["file_cache_path"], + "description": dedent( + """\ Specifies the location in which chef cache files will be saved. By default, it uses the ``{}`` location.""".format( - CHEF_RB_TPL_DEFAULTS['file_cache_path'])) + CHEF_RB_TPL_DEFAULTS["file_cache_path"] + ) + ), }, - 'json_attribs': { - 'type': 'string', - 'default': CHEF_FB_PATH, - 'description': dedent("""\ + "json_attribs": { + "type": "string", + "default": CHEF_FB_PATH, + "description": dedent( + """\ Specifies the location in which some chef json data is stored. By default, it uses the - ``{}`` location.""".format(CHEF_FB_PATH)) + ``{}`` location.""".format( + CHEF_FB_PATH + ) + ), }, - 'log_level': { - 'type': 'string', - 'default': CHEF_RB_TPL_DEFAULTS['log_level'], - 'description': dedent("""\ + "log_level": { + "type": "string", + "default": CHEF_RB_TPL_DEFAULTS["log_level"], + "description": dedent( + """\ Defines the level of logging to be stored in the log file. By default this value is set to ``{}``. - """.format(CHEF_RB_TPL_DEFAULTS['log_level'])) + """.format( + CHEF_RB_TPL_DEFAULTS["log_level"] + ) + ), }, - 'log_location': { - 'type': 'string', - 'default': CHEF_RB_TPL_DEFAULTS['log_location'], - 'description': dedent("""\ + "log_location": { + "type": "string", + "default": CHEF_RB_TPL_DEFAULTS["log_location"], + "description": dedent( + """\ Specifies the location of the chef lof file. By default, the location is specified at ``{}``.""".format( - CHEF_RB_TPL_DEFAULTS['log_location'])) + CHEF_RB_TPL_DEFAULTS["log_location"] + ) + ), }, - 'node_name': { - 'type': 'string', - 'description': dedent("""\ + "node_name": { + "type": "string", + "description": dedent( + """\ The name of the node to run. By default, we will - use th instance id as the node name.""") + use th instance id as the node name.""" + ), }, - 'omnibus_url': { - 'type': 'string', - 'default': OMNIBUS_URL, - 'description': dedent("""\ + "omnibus_url": { + "type": "string", + "default": OMNIBUS_URL, + "description": dedent( + """\ Omnibus URL if chef should be installed through Omnibus. By default, it uses the - ``{}``.""".format(OMNIBUS_URL)) + ``{}``.""".format( + OMNIBUS_URL + ) + ), }, - 'omnibus_url_retries': { - 'type': 'integer', - 'default': OMNIBUS_URL_RETRIES, - 'description': dedent("""\ + "omnibus_url_retries": { + "type": "integer", + "default": OMNIBUS_URL_RETRIES, + "description": dedent( + """\ The number of retries that will be attempted to reach - the Omnibus URL""") + the Omnibus URL""" + ), }, - 'omnibus_version': { - 'type': 'string', - 'description': dedent("""\ + "omnibus_version": { + "type": "string", + "description": dedent( + """\ Optional version string to require for omnibus - install.""") + install.""" + ), }, - 'pid_file': { - 'type': 'string', - 'default': CHEF_RB_TPL_DEFAULTS['pid_file'], - 'description': dedent("""\ + "pid_file": { + "type": "string", + "default": CHEF_RB_TPL_DEFAULTS["pid_file"], + "description": dedent( + """\ The location in which a process identification number (pid) is saved. By default, it saves in the ``{}`` location.""".format( - CHEF_RB_TPL_DEFAULTS['pid_file'])) + CHEF_RB_TPL_DEFAULTS["pid_file"] + ) + ), }, - 'server_url': { - 'type': 'string', - 'description': 'The URL for the chef server' + "server_url": { + "type": "string", + "description": "The URL for the chef server", }, - 'show_time': { - 'type': 'boolean', - 'default': True, - 'description': 'Show time in chef logs' + "show_time": { + "type": "boolean", + "default": True, + "description": "Show time in chef logs", }, - 'ssl_verify_mode': { - 'type': 'string', - 'default': CHEF_RB_TPL_DEFAULTS['ssl_verify_mode'], - 'description': dedent("""\ + "ssl_verify_mode": { + "type": "string", + "default": CHEF_RB_TPL_DEFAULTS["ssl_verify_mode"], + "description": dedent( + """\ Set the verify mode for HTTPS requests. We can have two possible values for this parameter: @@ -306,67 +370,76 @@ schema = { - ``:verify_peer``: Validate all SSL certificates. By default, the parameter is set as ``{}``. - """.format(CHEF_RB_TPL_DEFAULTS['ssl_verify_mode'])) + """.format( + CHEF_RB_TPL_DEFAULTS["ssl_verify_mode"] + ) + ), }, - 'validation_name': { - 'type': 'string', - 'description': dedent("""\ + "validation_name": { + "type": "string", + "description": dedent( + """\ The name of the chef-validator key that Chef Infra Client uses to access the Chef Infra Server during - the initial Chef Infra Client run.""") + the initial Chef Infra Client run.""" + ), }, - 'force_install': { - 'type': 'boolean', - 'default': False, - 'description': dedent("""\ + "force_install": { + "type": "boolean", + "default": False, + "description": dedent( + """\ If set to ``True``, forces chef installation, even - if it is already installed.""") + if it is already installed.""" + ), }, - 'initial_attributes': { - 'type': 'object', - 'items': { - 'type': 'string' - }, - 'description': dedent("""\ + "initial_attributes": { + "type": "object", + "items": {"type": "string"}, + "description": dedent( + """\ Specify a list of initial attributes used by the - cookbooks.""") + cookbooks.""" + ), }, - 'install_type': { - 'type': 'string', - 'default': 'packages', - 'description': dedent("""\ + "install_type": { + "type": "string", + "default": "packages", + "description": dedent( + """\ The type of installation for chef. It can be one of the following values: - ``packages`` - ``gems`` - - ``omnibus``""") + - ``omnibus``""" + ), }, - 'run_list': { - 'type': 'array', - 'items': { - 'type': 'string' - }, - 'description': 'A run list for a first boot json.' + "run_list": { + "type": "array", + "items": {"type": "string"}, + "description": "A run list for a first boot json.", }, "chef_license": { - 'type': 'string', - 'description': dedent("""\ + "type": "string", + "description": dedent( + """\ string that indicates if user accepts or not license - related to some of chef products""") - } - } + related to some of chef products""" + ), + }, + }, } - } + }, } __doc__ = get_meta_doc(meta, schema) def post_run_chef(chef_cfg, log): - delete_pem = util.get_cfg_option_bool(chef_cfg, - 'delete_validation_post_exec', - default=False) + delete_pem = util.get_cfg_option_bool( + chef_cfg, "delete_validation_post_exec", default=False + ) if delete_pem and os.path.isfile(CHEF_VALIDATION_PEM_PATH): os.unlink(CHEF_VALIDATION_PEM_PATH) @@ -389,16 +462,20 @@ def get_template_params(iid, chef_cfg, log): else: params[k] = util.get_cfg_option_str(chef_cfg, k) # These ones are overwritten to be exact values... - params.update({ - 'generated_by': util.make_header(), - 'node_name': util.get_cfg_option_str(chef_cfg, 'node_name', - default=iid), - 'environment': util.get_cfg_option_str(chef_cfg, 'environment', - default='_default'), - # These two are mandatory... - 'server_url': chef_cfg['server_url'], - 'validation_name': chef_cfg['validation_name'], - }) + params.update( + { + "generated_by": util.make_header(), + "node_name": util.get_cfg_option_str( + chef_cfg, "node_name", default=iid + ), + "environment": util.get_cfg_option_str( + chef_cfg, "environment", default="_default" + ), + # These two are mandatory... + "server_url": chef_cfg["server_url"], + "validation_name": chef_cfg["validation_name"], + } + ) return params @@ -406,35 +483,38 @@ def handle(name, cfg, cloud, log, _args): """Handler method activated by cloud-init.""" # If there isn't a chef key in the configuration don't do anything - if 'chef' not in cfg: - log.debug(("Skipping module named %s," - " no 'chef' key in configuration"), name) + if "chef" not in cfg: + log.debug( + "Skipping module named %s, no 'chef' key in configuration", name + ) return validate_cloudconfig_schema(cfg, schema) - chef_cfg = cfg['chef'] + chef_cfg = cfg["chef"] # Ensure the chef directories we use exist - chef_dirs = util.get_cfg_option_list(chef_cfg, 'directories') + chef_dirs = util.get_cfg_option_list(chef_cfg, "directories") if not chef_dirs: chef_dirs = list(CHEF_DIRS) for d in itertools.chain(chef_dirs, REQUIRED_CHEF_DIRS): util.ensure_dir(d) - vkey_path = chef_cfg.get('validation_key', CHEF_VALIDATION_PEM_PATH) - vcert = chef_cfg.get('validation_cert') + vkey_path = chef_cfg.get("validation_key", CHEF_VALIDATION_PEM_PATH) + vcert = chef_cfg.get("validation_cert") # special value 'system' means do not overwrite the file # but still render the template to contain 'validation_key' if vcert: if vcert != "system": util.write_file(vkey_path, vcert) elif not os.path.isfile(vkey_path): - log.warning("chef validation_cert provided as 'system', but " - "validation_key path '%s' does not exist.", - vkey_path) + log.warning( + "chef validation_cert provided as 'system', but " + "validation_key path '%s' does not exist.", + vkey_path, + ) # Create the chef config from template - template_fn = cloud.get_template_filename('chef_client.rb') + template_fn = cloud.get_template_filename("chef_client.rb") if template_fn: iid = str(cloud.datasource.get_instance_id()) params = get_template_params(iid, chef_cfg, log) @@ -448,32 +528,33 @@ def handle(name, cfg, cloud, log, _args): util.ensure_dirs(param_paths) templater.render_to_file(template_fn, CHEF_RB_PATH, params) else: - log.warning("No template found, not rendering to %s", - CHEF_RB_PATH) + log.warning("No template found, not rendering to %s", CHEF_RB_PATH) # Set the firstboot json - fb_filename = util.get_cfg_option_str(chef_cfg, 'firstboot_path', - default=CHEF_FB_PATH) + fb_filename = util.get_cfg_option_str( + chef_cfg, "firstboot_path", default=CHEF_FB_PATH + ) if not fb_filename: log.info("First boot path empty, not writing first boot json file") else: initial_json = {} - if 'run_list' in chef_cfg: - initial_json['run_list'] = chef_cfg['run_list'] - if 'initial_attributes' in chef_cfg: - initial_attributes = chef_cfg['initial_attributes'] + if "run_list" in chef_cfg: + initial_json["run_list"] = chef_cfg["run_list"] + if "initial_attributes" in chef_cfg: + initial_attributes = chef_cfg["initial_attributes"] for k in list(initial_attributes.keys()): initial_json[k] = initial_attributes[k] util.write_file(fb_filename, json.dumps(initial_json)) # Try to install chef, if its not already installed... - force_install = util.get_cfg_option_bool(chef_cfg, - 'force_install', default=False) + force_install = util.get_cfg_option_bool( + chef_cfg, "force_install", default=False + ) installed = subp.is_exe(CHEF_EXEC_PATH) if not installed or force_install: run = install_chef(cloud, chef_cfg, log) elif installed: - run = util.get_cfg_option_bool(chef_cfg, 'exec', default=False) + run = util.get_cfg_option_bool(chef_cfg, "exec", default=False) else: run = False if run: @@ -482,18 +563,21 @@ def handle(name, cfg, cloud, log, _args): def run_chef(chef_cfg, log): - log.debug('Running chef-client') + log.debug("Running chef-client") cmd = [CHEF_EXEC_PATH] - if 'exec_arguments' in chef_cfg: - cmd_args = chef_cfg['exec_arguments'] + if "exec_arguments" in chef_cfg: + cmd_args = chef_cfg["exec_arguments"] if isinstance(cmd_args, (list, tuple)): cmd.extend(cmd_args) elif isinstance(cmd_args, str): cmd.append(cmd_args) else: - log.warning("Unknown type %s provided for chef" - " 'exec_arguments' expected list, tuple," - " or string", type(cmd_args)) + log.warning( + "Unknown type %s provided for chef" + " 'exec_arguments' expected list, tuple," + " or string", + type(cmd_args), + ) cmd.extend(CHEF_EXEC_DEF_ARGS) else: cmd.extend(CHEF_EXEC_DEF_ARGS) @@ -507,16 +591,16 @@ def subp_blob_in_tempfile(blob, *args, **kwargs): The 'args' argument to subp will be updated with the full path to the filename as the first argument. """ - basename = kwargs.pop('basename', "subp_blob") + basename = kwargs.pop("basename", "subp_blob") - if len(args) == 0 and 'args' not in kwargs: + if len(args) == 0 and "args" not in kwargs: args = [tuple()] # Use tmpdir over tmpfile to avoid 'text file busy' on execute with temp_utils.tempdir(needs_exe=True) as tmpd: tmpf = os.path.join(tmpd, basename) - if 'args' in kwargs: - kwargs['args'] = [tmpf] + list(kwargs['args']) + if "args" in kwargs: + kwargs["args"] = [tmpf] + list(kwargs["args"]) else: args = list(args) args[0] = [tmpf] + args[0] @@ -543,36 +627,39 @@ def install_chef_from_omnibus(url=None, retries=None, omnibus_version=None): if omnibus_version is None: args = [] else: - args = ['-v', omnibus_version] + args = ["-v", omnibus_version] content = url_helper.readurl(url=url, retries=retries).contents return subp_blob_in_tempfile( - blob=content, args=args, - basename='chef-omnibus-install', capture=False) + blob=content, args=args, basename="chef-omnibus-install", capture=False + ) def install_chef(cloud, chef_cfg, log): # If chef is not installed, we install chef based on 'install_type' - install_type = util.get_cfg_option_str(chef_cfg, 'install_type', - 'packages') - run = util.get_cfg_option_bool(chef_cfg, 'exec', default=False) + install_type = util.get_cfg_option_str( + chef_cfg, "install_type", "packages" + ) + run = util.get_cfg_option_bool(chef_cfg, "exec", default=False) if install_type == "gems": # This will install and run the chef-client from gems - chef_version = util.get_cfg_option_str(chef_cfg, 'version', None) - ruby_version = util.get_cfg_option_str(chef_cfg, 'ruby_version', - RUBY_VERSION_DEFAULT) + chef_version = util.get_cfg_option_str(chef_cfg, "version", None) + ruby_version = util.get_cfg_option_str( + chef_cfg, "ruby_version", RUBY_VERSION_DEFAULT + ) install_chef_from_gems(ruby_version, chef_version, cloud.distro) # Retain backwards compat, by preferring True instead of False # when not provided/overriden... - run = util.get_cfg_option_bool(chef_cfg, 'exec', default=True) - elif install_type == 'packages': + run = util.get_cfg_option_bool(chef_cfg, "exec", default=True) + elif install_type == "packages": # This will install and run the chef-client from packages - cloud.distro.install_packages(('chef',)) - elif install_type == 'omnibus': + cloud.distro.install_packages(("chef",)) + elif install_type == "omnibus": omnibus_version = util.get_cfg_option_str(chef_cfg, "omnibus_version") install_chef_from_omnibus( url=util.get_cfg_option_str(chef_cfg, "omnibus_url"), retries=util.get_cfg_option_int(chef_cfg, "omnibus_url_retries"), - omnibus_version=omnibus_version) + omnibus_version=omnibus_version, + ) else: log.warning("Unknown chef install type '%s'", install_type) run = False @@ -581,25 +668,47 @@ def install_chef(cloud, chef_cfg, log): def get_ruby_packages(version): # return a list of packages needed to install ruby at version - pkgs = ['ruby%s' % version, 'ruby%s-dev' % version] + pkgs = ["ruby%s" % version, "ruby%s-dev" % version] if version == "1.8": - pkgs.extend(('libopenssl-ruby1.8', 'rubygems1.8')) + pkgs.extend(("libopenssl-ruby1.8", "rubygems1.8")) return pkgs def install_chef_from_gems(ruby_version, chef_version, distro): distro.install_packages(get_ruby_packages(ruby_version)) - if not os.path.exists('/usr/bin/gem'): - util.sym_link('/usr/bin/gem%s' % ruby_version, '/usr/bin/gem') - if not os.path.exists('/usr/bin/ruby'): - util.sym_link('/usr/bin/ruby%s' % ruby_version, '/usr/bin/ruby') + if not os.path.exists("/usr/bin/gem"): + util.sym_link("/usr/bin/gem%s" % ruby_version, "/usr/bin/gem") + if not os.path.exists("/usr/bin/ruby"): + util.sym_link("/usr/bin/ruby%s" % ruby_version, "/usr/bin/ruby") if chef_version: - subp.subp(['/usr/bin/gem', 'install', 'chef', - '-v %s' % chef_version, '--no-ri', - '--no-rdoc', '--bindir', '/usr/bin', '-q'], capture=False) + subp.subp( + [ + "/usr/bin/gem", + "install", + "chef", + "-v %s" % chef_version, + "--no-ri", + "--no-rdoc", + "--bindir", + "/usr/bin", + "-q", + ], + capture=False, + ) else: - subp.subp(['/usr/bin/gem', 'install', 'chef', - '--no-ri', '--no-rdoc', '--bindir', - '/usr/bin', '-q'], capture=False) + subp.subp( + [ + "/usr/bin/gem", + "install", + "chef", + "--no-ri", + "--no-rdoc", + "--bindir", + "/usr/bin", + "-q", + ], + capture=False, + ) + # vi: ts=4 expandtab diff --git a/cloudinit/config/cc_debug.py b/cloudinit/config/cc_debug.py index 4d5a6aa2..d09fc129 100644 --- a/cloudinit/config/cc_debug.py +++ b/cloudinit/config/cc_debug.py @@ -30,18 +30,16 @@ location that this cloud-init has been configured with when running. import copy from io import StringIO -from cloudinit import type_utils -from cloudinit import util -from cloudinit import safeyaml +from cloudinit import safeyaml, type_utils, util -SKIP_KEYS = frozenset(['log_cfgs']) +SKIP_KEYS = frozenset(["log_cfgs"]) def _make_header(text): header = StringIO() header.write("-" * 80) header.write("\n") - header.write(text.center(80, ' ')) + header.write(text.center(80, " ")) header.write("\n") header.write("-" * 80) header.write("\n") @@ -56,17 +54,16 @@ def _dumps(obj): def handle(name, cfg, cloud, log, args): """Handler method activated by cloud-init.""" - verbose = util.get_cfg_by_path(cfg, ('debug', 'verbose'), default=True) + verbose = util.get_cfg_by_path(cfg, ("debug", "verbose"), default=True) if args: # if args are provided (from cmdline) then explicitly set verbose out_file = args[0] verbose = True else: - out_file = util.get_cfg_by_path(cfg, ('debug', 'output')) + out_file = util.get_cfg_by_path(cfg, ("debug", "output")) if not verbose: - log.debug(("Skipping module named %s," - " verbose printing disabled"), name) + log.debug("Skipping module named %s, verbose printing disabled", name) return # Clean out some keys that we just don't care about showing... dump_cfg = copy.deepcopy(cfg) @@ -85,8 +82,9 @@ def handle(name, cfg, cloud, log, args): to_print.write(_dumps(cloud.datasource.metadata)) to_print.write("\n") to_print.write(_make_header("Misc")) - to_print.write("Datasource: %s\n" % - (type_utils.obj_name(cloud.datasource))) + to_print.write( + "Datasource: %s\n" % (type_utils.obj_name(cloud.datasource)) + ) to_print.write("Distro: %s\n" % (type_utils.obj_name(cloud.distro))) to_print.write("Hostname: %s\n" % (cloud.get_hostname(True))) to_print.write("Instance ID: %s\n" % (cloud.get_instance_id())) @@ -102,4 +100,5 @@ def handle(name, cfg, cloud, log, args): else: util.multi_log("".join(content_to_file), console=True, stderr=False) + # vi: ts=4 expandtab diff --git a/cloudinit/config/cc_disable_ec2_metadata.py b/cloudinit/config/cc_disable_ec2_metadata.py index 61c769b3..5e528e81 100644 --- a/cloudinit/config/cc_disable_ec2_metadata.py +++ b/cloudinit/config/cc_disable_ec2_metadata.py @@ -26,32 +26,35 @@ by default. disable_ec2_metadata: <true/false> """ -from cloudinit import subp -from cloudinit import util - +from cloudinit import subp, util from cloudinit.settings import PER_ALWAYS frequency = PER_ALWAYS -REJECT_CMD_IF = ['route', 'add', '-host', '169.254.169.254', 'reject'] -REJECT_CMD_IP = ['ip', 'route', 'add', 'prohibit', '169.254.169.254'] +REJECT_CMD_IF = ["route", "add", "-host", "169.254.169.254", "reject"] +REJECT_CMD_IP = ["ip", "route", "add", "prohibit", "169.254.169.254"] def handle(name, cfg, _cloud, log, _args): disabled = util.get_cfg_option_bool(cfg, "disable_ec2_metadata", False) if disabled: reject_cmd = None - if subp.which('ip'): + if subp.which("ip"): reject_cmd = REJECT_CMD_IP - elif subp.which('ifconfig'): + elif subp.which("ifconfig"): reject_cmd = REJECT_CMD_IF else: - log.error(('Neither "route" nor "ip" command found, unable to ' - 'manipulate routing table')) + log.error( + 'Neither "route" nor "ip" command found, unable to ' + "manipulate routing table" + ) return subp.subp(reject_cmd, capture=False) else: - log.debug(("Skipping module named %s," - " disabling the ec2 route not enabled"), name) + log.debug( + "Skipping module named %s, disabling the ec2 route not enabled", + name, + ) + # vi: ts=4 expandtab diff --git a/cloudinit/config/cc_disk_setup.py b/cloudinit/config/cc_disk_setup.py index 440f05f1..4d527c7a 100644 --- a/cloudinit/config/cc_disk_setup.py +++ b/cloudinit/config/cc_disk_setup.py @@ -100,13 +100,13 @@ A label can be specified for the filesystem using replace_fs: <filesystem type> """ -from cloudinit.settings import PER_INSTANCE -from cloudinit import util -from cloudinit import subp import logging import os import shlex +from cloudinit import subp, util +from cloudinit.settings import PER_INSTANCE + frequency = PER_INSTANCE # Define the commands to use @@ -118,7 +118,7 @@ BLKDEV_CMD = subp.which("blockdev") PARTPROBE_CMD = subp.which("partprobe") WIPEFS_CMD = subp.which("wipefs") -LANG_C_ENV = {'LANG': 'C'} +LANG_C_ENV = {"LANG": "C"} LOG = logging.getLogger(__name__) @@ -145,9 +145,12 @@ def handle(_name, cfg, cloud, log, _args): try: log.debug("Creating new partition table/disk") - util.log_time(logfunc=LOG.debug, - msg="Creating partition on %s" % disk, - func=mkpart, args=(disk, definition)) + util.log_time( + logfunc=LOG.debug, + msg="Creating partition on %s" % disk, + func=mkpart, + args=(disk, definition), + ) except Exception as e: util.logexc(LOG, "Failed partitioning operation\n%s" % e) @@ -162,10 +165,13 @@ def handle(_name, cfg, cloud, log, _args): try: log.debug("Creating new filesystem.") - device = definition.get('device') - util.log_time(logfunc=LOG.debug, - msg="Creating fs for %s" % device, - func=mkfs, args=(definition,)) + device = definition.get("device") + util.log_time( + logfunc=LOG.debug, + msg="Creating fs for %s" % device, + func=mkfs, + args=(definition,), + ) except Exception as e: util.logexc(LOG, "Failed during filesystem operation\n%s" % e) @@ -178,16 +184,22 @@ def update_disk_setup_devices(disk_setup, tformer): if transformed is None or transformed == origname: continue if transformed in disk_setup: - LOG.info("Replacing %s in disk_setup for translation of %s", - origname, transformed) + LOG.info( + "Replacing %s in disk_setup for translation of %s", + origname, + transformed, + ) del disk_setup[transformed] disk_setup[transformed] = disk_setup[origname] if isinstance(disk_setup[transformed], dict): - disk_setup[transformed]['_origname'] = origname + disk_setup[transformed]["_origname"] = origname del disk_setup[origname] - LOG.debug("updated disk_setup device entry '%s' to '%s'", - origname, transformed) + LOG.debug( + "updated disk_setup device entry '%s' to '%s'", + origname, + transformed, + ) def update_fs_setup_devices(disk_setup, tformer): @@ -198,7 +210,7 @@ def update_fs_setup_devices(disk_setup, tformer): LOG.warning("entry in disk_setup not a dict: %s", definition) continue - origname = definition.get('device') + origname = definition.get("device") if origname is None: continue @@ -208,19 +220,24 @@ def update_fs_setup_devices(disk_setup, tformer): tformed = tformer(dev) if tformed is not None: dev = tformed - LOG.debug("%s is mapped to disk=%s part=%s", - origname, tformed, part) - definition['_origname'] = origname - definition['device'] = tformed + LOG.debug( + "%s is mapped to disk=%s part=%s", origname, tformed, part + ) + definition["_origname"] = origname + definition["device"] = tformed if part: # In origname with <dev>.N, N overrides 'partition' key. - if 'partition' in definition: - LOG.warning("Partition '%s' from dotted device name '%s' " - "overrides 'partition' key in %s", part, origname, - definition) - definition['_partition'] = definition['partition'] - definition['partition'] = part + if "partition" in definition: + LOG.warning( + "Partition '%s' from dotted device name '%s' " + "overrides 'partition' key in %s", + part, + origname, + definition, + ) + definition["_partition"] = definition["partition"] + definition["partition"] = part def value_splitter(values, start=None): @@ -232,7 +249,7 @@ def value_splitter(values, start=None): if start: _values = _values[start:] - for key, value in [x.split('=') for x in _values]: + for key, value in [x.split("=") for x in _values]: yield key, value @@ -251,11 +268,16 @@ def enumerate_disk(device, nodeps=False): name: the device name, i.e. sda """ - lsblk_cmd = [LSBLK_CMD, '--pairs', '--output', 'NAME,TYPE,FSTYPE,LABEL', - device] + lsblk_cmd = [ + LSBLK_CMD, + "--pairs", + "--output", + "NAME,TYPE,FSTYPE,LABEL", + device, + ] if nodeps: - lsblk_cmd.append('--nodeps') + lsblk_cmd.append("--nodeps") info = None try: @@ -269,10 +291,10 @@ def enumerate_disk(device, nodeps=False): for part in parts: d = { - 'name': None, - 'type': None, - 'fstype': None, - 'label': None, + "name": None, + "type": None, + "fstype": None, + "label": None, } for key, value in value_splitter(part): @@ -303,9 +325,9 @@ def is_device_valid(name, partition=False): LOG.warning("Query against device %s failed", name) return False - if partition and d_type == 'part': + if partition and d_type == "part": return True - elif not partition and d_type == 'disk': + elif not partition and d_type == "disk": return True return False @@ -321,7 +343,7 @@ def check_fs(device): """ out, label, fs_type, uuid = None, None, None, None - blkid_cmd = [BLKID_CMD, '-c', '/dev/null', device] + blkid_cmd = [BLKID_CMD, "-c", "/dev/null", device] try: out, _err = subp.subp(blkid_cmd, rcs=[0, 2]) except Exception as e: @@ -332,11 +354,11 @@ def check_fs(device): if out: if len(out.splitlines()) == 1: for key, value in value_splitter(out, start=1): - if key.lower() == 'label': + if key.lower() == "label": label = value - elif key.lower() == 'type': + elif key.lower() == "type": fs_type = value - elif key.lower() == 'uuid': + elif key.lower() == "uuid": uuid = value return label, fs_type, uuid @@ -350,8 +372,14 @@ def is_filesystem(device): return fs_type -def find_device_node(device, fs_type=None, label=None, valid_targets=None, - label_match=True, replace_fs=None): +def find_device_node( + device, + fs_type=None, + label=None, + valid_targets=None, + label_match=True, + replace_fs=None, +): """ Find a device that is either matches the spec, or the first @@ -366,31 +394,32 @@ def find_device_node(device, fs_type=None, label=None, valid_targets=None, label = "" if not valid_targets: - valid_targets = ['disk', 'part'] + valid_targets = ["disk", "part"] raw_device_used = False for d in enumerate_disk(device): - if d['fstype'] == replace_fs and label_match is False: + if d["fstype"] == replace_fs and label_match is False: # We found a device where we want to replace the FS - return ('/dev/%s' % d['name'], False) + return ("/dev/%s" % d["name"], False) - if (d['fstype'] == fs_type and - ((label_match and d['label'] == label) or not label_match)): + if d["fstype"] == fs_type and ( + (label_match and d["label"] == label) or not label_match + ): # If we find a matching device, we return that - return ('/dev/%s' % d['name'], True) + return ("/dev/%s" % d["name"], True) - if d['type'] in valid_targets: + if d["type"] in valid_targets: - if d['type'] != 'disk' or d['fstype']: + if d["type"] != "disk" or d["fstype"]: raw_device_used = True - if d['type'] == 'disk': + if d["type"] == "disk": # Skip the raw disk, its the default pass - elif not d['fstype']: - return ('/dev/%s' % d['name'], False) + elif not d["fstype"]: + return ("/dev/%s" % d["name"], False) if not raw_device_used: return (device, False) @@ -433,7 +462,7 @@ def get_dyn_func(*args): if len(args) < 2: raise Exception("Unable to determine dynamic funcation name") - func_name = (args[0] % args[1]) + func_name = args[0] % args[1] func_args = args[2:] try: @@ -448,8 +477,8 @@ def get_dyn_func(*args): def get_hdd_size(device): try: - size_in_bytes, _ = subp.subp([BLKDEV_CMD, '--getsize64', device]) - sector_size, _ = subp.subp([BLKDEV_CMD, '--getss', device]) + size_in_bytes, _ = subp.subp([BLKDEV_CMD, "--getsize64", device]) + sector_size, _ = subp.subp([BLKDEV_CMD, "--getss", device]) except Exception as e: raise Exception("Failed to get %s size\n%s" % (device, e)) from e @@ -481,13 +510,13 @@ def check_partition_mbr_layout(device, layout): if device in _line[0]: # We don't understand extended partitions yet - if _line[-1].lower() in ['extended', 'empty']: + if _line[-1].lower() in ["extended", "empty"]: continue # Find the partition types type_label = None for x in sorted(range(1, len(_line)), reverse=True): - if _line[x].isdigit() and _line[x] != '/': + if _line[x].isdigit() and _line[x] != "/": type_label = _line[x] break @@ -496,7 +525,7 @@ def check_partition_mbr_layout(device, layout): def check_partition_gpt_layout(device, layout): - prt_cmd = [SGDISK_CMD, '-p', device] + prt_cmd = [SGDISK_CMD, "-p", device] try: out, _err = subp.subp(prt_cmd, update_env=LANG_C_ENV) except Exception as e: @@ -522,7 +551,7 @@ def check_partition_gpt_layout(device, layout): # Number Start (sector) End (sector) Size Code Name # 1 2048 206847 100.0 MiB 0700 Microsoft basic data for line in out_lines: - if line.strip().startswith('Number'): + if line.strip().startswith("Number"): break codes = [line.strip().split()[5] for line in out_lines] @@ -545,10 +574,16 @@ def check_partition_layout(table_type, device, layout): function called check_partition_%s_layout """ found_layout = get_dyn_func( - "check_partition_%s_layout", table_type, device, layout) - - LOG.debug("called check_partition_%s_layout(%s, %s), returned: %s", - table_type, device, layout, found_layout) + "check_partition_%s_layout", table_type, device, layout + ) + + LOG.debug( + "called check_partition_%s_layout(%s, %s), returned: %s", + table_type, + device, + layout, + found_layout, + ) if isinstance(layout, bool): # if we are using auto partitioning, or "True" be happy # if a single partition exists. @@ -559,10 +594,12 @@ def check_partition_layout(table_type, device, layout): elif len(found_layout) == len(layout): # This just makes sure that the number of requested # partitions and the type labels are right - layout_types = [str(x[1]) if isinstance(x, (tuple, list)) else None - for x in layout] - LOG.debug("Layout types=%s. Found types=%s", - layout_types, found_layout) + layout_types = [ + str(x[1]) if isinstance(x, (tuple, list)) else None for x in layout + ] + LOG.debug( + "Layout types=%s. Found types=%s", layout_types, found_layout + ) for itype, ftype in zip(layout_types, found_layout): if itype is not None and str(ftype) != str(itype): return False @@ -588,8 +625,9 @@ def get_partition_mbr_layout(size, layout): # Create a single partition return "0," - if ((len(layout) == 0 and isinstance(layout, list)) or - not isinstance(layout, list)): + if (len(layout) == 0 and isinstance(layout, list)) or not isinstance( + layout, list + ): raise Exception("Partition layout is invalid") last_part_num = len(layout) @@ -617,8 +655,10 @@ def get_partition_mbr_layout(size, layout): sfdisk_definition = "\n".join(part_definition) if len(part_definition) > 4: - raise Exception("Calculated partition definition is too big\n%s" % - sfdisk_definition) + raise Exception( + "Calculated partition definition is too big\n%s" + % sfdisk_definition + ) return sfdisk_definition @@ -632,14 +672,15 @@ def get_partition_gpt_layout(size, layout): if isinstance(partition, list): if len(partition) != 2: raise Exception( - "Partition was incorrectly defined: %s" % partition) + "Partition was incorrectly defined: %s" % partition + ) percent, partition_type = partition else: percent = partition partition_type = None part_size = int(float(size) * (float(percent) / 100)) - partition_specs.append((partition_type, [0, '+{}'.format(part_size)])) + partition_specs.append((partition_type, [0, "+{}".format(part_size)])) # The last partition should use up all remaining space partition_specs[-1][-1][-1] = 0 @@ -649,7 +690,7 @@ def get_partition_gpt_layout(size, layout): def purge_disk_ptable(device): # wipe the first and last megabyte of a disk (or file) # gpt stores partition table both at front and at end. - null = '\0' + null = "\0" start_len = 1024 * 1024 end_len = 1024 * 1024 with open(device, "rb+") as fp: @@ -668,14 +709,14 @@ def purge_disk(device): # wipe any file systems first for d in enumerate_disk(device): - if d['type'] not in ["disk", "crypt"]: - wipefs_cmd = [WIPEFS_CMD, "--all", "/dev/%s" % d['name']] + if d["type"] not in ["disk", "crypt"]: + wipefs_cmd = [WIPEFS_CMD, "--all", "/dev/%s" % d["name"]] try: - LOG.info("Purging filesystem on /dev/%s", d['name']) + LOG.info("Purging filesystem on /dev/%s", d["name"]) subp.subp(wipefs_cmd) except Exception as e: raise Exception( - "Failed FS purge of /dev/%s" % d['name'] + "Failed FS purge of /dev/%s" % d["name"] ) from e purge_disk_ptable(device) @@ -701,7 +742,7 @@ def read_parttbl(device): if PARTPROBE_CMD is not None: probe_cmd = [PARTPROBE_CMD, device] else: - probe_cmd = [BLKDEV_CMD, '--rereadpt', device] + probe_cmd = [BLKDEV_CMD, "--rereadpt", device] util.udevadm_settle() try: subp.subp(probe_cmd) @@ -730,17 +771,24 @@ def exec_mkpart_mbr(device, layout): def exec_mkpart_gpt(device, layout): try: - subp.subp([SGDISK_CMD, '-Z', device]) + subp.subp([SGDISK_CMD, "-Z", device]) for index, (partition_type, (start, end)) in enumerate(layout): index += 1 - subp.subp([SGDISK_CMD, - '-n', '{}:{}:{}'.format(index, start, end), device]) + subp.subp( + [ + SGDISK_CMD, + "-n", + "{}:{}:{}".format(index, start, end), + device, + ] + ) if partition_type is not None: # convert to a 4 char (or more) string right padded with 0 # 82 -> 8200. 'Linux' -> 'Linux' pinput = str(partition_type).ljust(4, "0") subp.subp( - [SGDISK_CMD, '-t', '{}:{}'.format(index, pinput), device]) + [SGDISK_CMD, "-t", "{}:{}".format(index, pinput), device] + ) except Exception: LOG.warning("Failed to partition device %s", device) raise @@ -766,8 +814,10 @@ def assert_and_settle_device(device): if not os.path.exists(device): util.udevadm_settle() if not os.path.exists(device): - raise RuntimeError("Device %s did not exist and was not created " - "with a udevadm settle." % device) + raise RuntimeError( + "Device %s did not exist and was not created " + "with a udevadm settle." % device + ) # Whether or not the device existed above, it is possible that udev # events that would populate udev database (for reading by lsdname) have @@ -794,9 +844,9 @@ def mkpart(device, definition): device = os.path.realpath(device) LOG.debug("Checking values for %s definition", device) - overwrite = definition.get('overwrite', False) - layout = definition.get('layout', False) - table_type = definition.get('table_type', 'mbr') + overwrite = definition.get("overwrite", False) + layout = definition.get("layout", False) + table_type = definition.get("table_type", "mbr") # Check if the default device is a partition or not LOG.debug("Checking against default devices") @@ -809,7 +859,8 @@ def mkpart(device, definition): LOG.debug("Checking if device %s is a valid device", device) if not is_device_valid(device): raise Exception( - 'Device {device} is not a disk device!'.format(device=device)) + "Device {device} is not a disk device!".format(device=device) + ) # Remove the partition table entries if isinstance(layout, str) and layout.lower() == "remove": @@ -845,21 +896,21 @@ def lookup_force_flag(fs): A force flag might be -F or -F, this look it up """ flags = { - 'ext': '-F', - 'btrfs': '-f', - 'xfs': '-f', - 'reiserfs': '-f', - 'swap': '-f', + "ext": "-F", + "btrfs": "-f", + "xfs": "-f", + "reiserfs": "-f", + "swap": "-f", } - if 'ext' in fs.lower(): - fs = 'ext' + if "ext" in fs.lower(): + fs = "ext" if fs.lower() in flags: return flags[fs] LOG.warning("Force flag for %s is unknown.", fs) - return '' + return "" def mkfs(fs_cfg): @@ -883,14 +934,14 @@ def mkfs(fs_cfg): When 'cmd' is provided then no other parameter is required. """ - label = fs_cfg.get('label') - device = fs_cfg.get('device') - partition = str(fs_cfg.get('partition', 'any')) - fs_type = fs_cfg.get('filesystem') - fs_cmd = fs_cfg.get('cmd', []) - fs_opts = fs_cfg.get('extra_opts', []) - fs_replace = fs_cfg.get('replace_fs', False) - overwrite = fs_cfg.get('overwrite', False) + label = fs_cfg.get("label") + device = fs_cfg.get("device") + partition = str(fs_cfg.get("partition", "any")) + fs_type = fs_cfg.get("filesystem") + fs_cmd = fs_cfg.get("cmd", []) + fs_opts = fs_cfg.get("extra_opts", []) + fs_replace = fs_cfg.get("replace_fs", False) + overwrite = fs_cfg.get("overwrite", False) # ensure that we get a real device rather than a symbolic link assert_and_settle_device(device) @@ -903,14 +954,19 @@ def mkfs(fs_cfg): # Handle manual definition of partition if partition.isdigit(): device = "%s%s" % (device, partition) - LOG.debug("Manual request of partition %s for %s", - partition, device) + LOG.debug( + "Manual request of partition %s for %s", partition, device + ) # Check to see if the fs already exists LOG.debug("Checking device %s", device) check_label, check_fstype, _ = check_fs(device) - LOG.debug("Device '%s' has check_label='%s' check_fstype=%s", - device, check_label, check_fstype) + LOG.debug( + "Device '%s' has check_label='%s' check_fstype=%s", + device, + check_label, + check_fstype, + ) if check_label == label and check_fstype == fs_type: LOG.debug("Existing file system found at %s", device) @@ -924,19 +980,23 @@ def mkfs(fs_cfg): else: LOG.debug("Device %s is cleared for formating", device) - elif partition and str(partition).lower() in ('auto', 'any'): + elif partition and str(partition).lower() in ("auto", "any"): # For auto devices, we match if the filesystem does exist odevice = device LOG.debug("Identifying device to create %s filesytem on", label) # any mean pick the first match on the device with matching fs_type label_match = True - if partition.lower() == 'any': + if partition.lower() == "any": label_match = False - device, reuse = find_device_node(device, fs_type=fs_type, label=label, - label_match=label_match, - replace_fs=fs_replace) + device, reuse = find_device_node( + device, + fs_type=fs_type, + label=label, + label_match=label_match, + replace_fs=fs_replace, + ) LOG.debug("Automatic device for %s identified as %s", odevice, device) if reuse: @@ -947,18 +1007,25 @@ def mkfs(fs_cfg): LOG.debug("Replacing file system on %s as instructed.", device) if not device: - LOG.debug("No device aviable that matches request. " - "Skipping fs creation for %s", fs_cfg) + LOG.debug( + "No device aviable that matches request. " + "Skipping fs creation for %s", + fs_cfg, + ) return - elif not partition or str(partition).lower() == 'none': + elif not partition or str(partition).lower() == "none": LOG.debug("Using the raw device to place filesystem %s on", label) else: LOG.debug("Error in device identification handling.") return - LOG.debug("File system type '%s' with label '%s' will be created on %s", - fs_type, label, device) + LOG.debug( + "File system type '%s' with label '%s' will be created on %s", + fs_type, + label, + device, + ) # Make sure the device is defined if not device: @@ -969,26 +1036,29 @@ def mkfs(fs_cfg): if not (fs_type or fs_cmd): raise Exception( "No way to create filesystem '{label}'. fs_type or fs_cmd " - "must be set.".format(label=label)) + "must be set.".format(label=label) + ) # Create the commands shell = False if fs_cmd: - fs_cmd = fs_cfg['cmd'] % { - 'label': label, - 'filesystem': fs_type, - 'device': device, + fs_cmd = fs_cfg["cmd"] % { + "label": label, + "filesystem": fs_type, + "device": device, } shell = True if overwrite: LOG.warning( "fs_setup:overwrite ignored because cmd was specified: %s", - fs_cmd) + fs_cmd, + ) if fs_opts: LOG.warning( "fs_setup:extra_opts ignored because cmd was specified: %s", - fs_cmd) + fs_cmd, + ) else: # Find the mkfs command mkfs_cmd = subp.which("mkfs.%s" % fs_type) @@ -996,8 +1066,11 @@ def mkfs(fs_cfg): mkfs_cmd = subp.which("mk%s" % fs_type) if not mkfs_cmd: - LOG.warning("Cannot create fstype '%s'. No mkfs.%s command", - fs_type, fs_type) + LOG.warning( + "Cannot create fstype '%s'. No mkfs.%s command", + fs_type, + fs_type, + ) return fs_cmd = [mkfs_cmd, device] @@ -1022,4 +1095,5 @@ def mkfs(fs_cfg): except Exception as e: raise Exception("Failed to exec of '%s':\n%s" % (fs_cmd, e)) from e + # vi: ts=4 expandtab diff --git a/cloudinit/config/cc_emit_upstart.py b/cloudinit/config/cc_emit_upstart.py index 40eee052..a928082b 100644 --- a/cloudinit/config/cc_emit_upstart.py +++ b/cloudinit/config/cc_emit_upstart.py @@ -24,12 +24,12 @@ user configuration should be required. import os from cloudinit import log as logging -from cloudinit.settings import PER_ALWAYS from cloudinit import subp +from cloudinit.settings import PER_ALWAYS frequency = PER_ALWAYS -distros = ['ubuntu', 'debian'] +distros = ["ubuntu", "debian"] LOG = logging.getLogger(__name__) @@ -39,15 +39,18 @@ def is_upstart_system(): return False myenv = os.environ.copy() - if 'UPSTART_SESSION' in myenv: - del myenv['UPSTART_SESSION'] - check_cmd = ['initctl', 'version'] + if "UPSTART_SESSION" in myenv: + del myenv["UPSTART_SESSION"] + check_cmd = ["initctl", "version"] try: (out, _err) = subp.subp(check_cmd, env=myenv) - return 'upstart' in out + return "upstart" in out except subp.ProcessExecutionError as e: - LOG.debug("'%s' returned '%s', not using upstart", - ' '.join(check_cmd), e.exit_code) + LOG.debug( + "'%s' returned '%s', not using upstart", + " ".join(check_cmd), + e.exit_code, + ) return False @@ -56,7 +59,7 @@ def handle(name, _cfg, cloud, log, args): if not event_names: # Default to the 'cloud-config' # event for backwards compat. - event_names = ['cloud-config'] + event_names = ["cloud-config"] if not is_upstart_system(): log.debug("not upstart system, '%s' disabled", name) @@ -64,11 +67,12 @@ def handle(name, _cfg, cloud, log, args): cfgpath = cloud.paths.get_ipath_cur("cloud_config") for n in event_names: - cmd = ['initctl', 'emit', str(n), 'CLOUD_CFG=%s' % cfgpath] + cmd = ["initctl", "emit", str(n), "CLOUD_CFG=%s" % cfgpath] try: subp.subp(cmd) except Exception as e: # TODO(harlowja), use log exception from utils?? log.warning("Emission of upstart event %s failed due to: %s", n, e) + # vi: ts=4 expandtab diff --git a/cloudinit/config/cc_fan.py b/cloudinit/config/cc_fan.py index 91f50e22..50a81744 100644 --- a/cloudinit/config/cc_fan.py +++ b/cloudinit/config/cc_fan.py @@ -38,60 +38,62 @@ If cloud-init sees a ``fan`` entry in cloud-config it will: """ from cloudinit import log as logging +from cloudinit import subp, util from cloudinit.settings import PER_INSTANCE -from cloudinit import subp -from cloudinit import util LOG = logging.getLogger(__name__) frequency = PER_INSTANCE BUILTIN_CFG = { - 'config': None, - 'config_path': '/etc/network/fan', + "config": None, + "config_path": "/etc/network/fan", } def stop_update_start(distro, service, config_file, content): try: - distro.manage_service('stop', service) + distro.manage_service("stop", service) stop_failed = False except subp.ProcessExecutionError as e: stop_failed = True LOG.warning("failed to stop %s: %s", service, e) - if not content.endswith('\n'): - content += '\n' + if not content.endswith("\n"): + content += "\n" util.write_file(config_file, content, omode="w") try: - distro.manage_service('start', service) + distro.manage_service("start", service) if stop_failed: LOG.warning("success: %s started", service) except subp.ProcessExecutionError as e: LOG.warning("failed to start %s: %s", service, e) - distro.manage_service('enable', service) + distro.manage_service("enable", service) def handle(name, cfg, cloud, log, args): - cfgin = cfg.get('fan') + cfgin = cfg.get("fan") if not cfgin: cfgin = {} mycfg = util.mergemanydict([cfgin, BUILTIN_CFG]) - if not mycfg.get('config'): + if not mycfg.get("config"): LOG.debug("%s: no 'fan' config entry. disabling", name) return - util.write_file(mycfg.get('config_path'), mycfg.get('config'), omode="w") + util.write_file(mycfg.get("config_path"), mycfg.get("config"), omode="w") distro = cloud.distro - if not subp.which('fanctl'): - distro.install_packages(['ubuntu-fan']) + if not subp.which("fanctl"): + distro.install_packages(["ubuntu-fan"]) stop_update_start( distro, - service='ubuntu-fan', config_file=mycfg.get('config_path'), - content=mycfg.get('config')) + service="ubuntu-fan", + config_file=mycfg.get("config_path"), + content=mycfg.get("config"), + ) + # vi: ts=4 expandtab diff --git a/cloudinit/config/cc_final_message.py b/cloudinit/config/cc_final_message.py index 4fa5297e..f443ccd8 100644 --- a/cloudinit/config/cc_final_message.py +++ b/cloudinit/config/cc_final_message.py @@ -31,10 +31,7 @@ specified as a jinja template with the following variables set: """ -from cloudinit import templater -from cloudinit import util -from cloudinit import version - +from cloudinit import templater, util, version from cloudinit.settings import PER_ALWAYS frequency = PER_ALWAYS @@ -49,7 +46,7 @@ FINAL_MESSAGE_DEF = ( def handle(_name, cfg, cloud, log, args): - msg_in = '' + msg_in = "" if len(args) != 0: msg_in = str(args[0]) else: @@ -64,14 +61,18 @@ def handle(_name, cfg, cloud, log, args): cver = version.version_string() try: subs = { - 'uptime': uptime, - 'timestamp': ts, - 'version': cver, - 'datasource': str(cloud.datasource), + "uptime": uptime, + "timestamp": ts, + "version": cver, + "datasource": str(cloud.datasource), } subs.update(dict([(k.upper(), v) for k, v in subs.items()])) - util.multi_log("%s\n" % (templater.render_string(msg_in, subs)), - console=False, stderr=True, log=log) + util.multi_log( + "%s\n" % (templater.render_string(msg_in, subs)), + console=False, + stderr=True, + log=log, + ) except Exception: util.logexc(log, "Failed to render final message template") @@ -85,4 +86,5 @@ def handle(_name, cfg, cloud, log, args): if cloud.datasource.is_disconnected: log.warning("Used fallback datasource") + # vi: ts=4 expandtab diff --git a/cloudinit/config/cc_foo.py b/cloudinit/config/cc_foo.py index 924b967c..3c307153 100644 --- a/cloudinit/config/cc_foo.py +++ b/cloudinit/config/cc_foo.py @@ -53,4 +53,5 @@ frequency = PER_INSTANCE def handle(name, _cfg, _cloud, log, _args): log.debug("Hi from module %s", name) + # vi: ts=4 expandtab diff --git a/cloudinit/config/cc_growpart.py b/cloudinit/config/cc_growpart.py index 1ddc9dc7..43334caa 100644 --- a/cloudinit/config/cc_growpart.py +++ b/cloudinit/config/cc_growpart.py @@ -70,17 +70,15 @@ import re import stat from cloudinit import log as logging +from cloudinit import subp, temp_utils, util from cloudinit.settings import PER_ALWAYS -from cloudinit import subp -from cloudinit import temp_utils -from cloudinit import util frequency = PER_ALWAYS DEFAULT_CONFIG = { - 'mode': 'auto', - 'devices': ['/'], - 'ignore_growroot_disabled': False, + "mode": "auto", + "devices": ["/"], + "ignore_growroot_disabled": False, } @@ -131,7 +129,7 @@ class ResizeFailedException(Exception): class ResizeGrowPart(object): def available(self): myenv = os.environ.copy() - myenv['LANG'] = 'C' + myenv["LANG"] = "C" try: (out, _err) = subp.subp(["growpart", "--help"], env=myenv) @@ -144,7 +142,7 @@ class ResizeGrowPart(object): def resize(self, diskdev, partnum, partdev): myenv = os.environ.copy() - myenv['LANG'] = 'C' + myenv["LANG"] = "C" before = get_size(partdev) # growpart uses tmp dir to store intermediate states @@ -153,14 +151,19 @@ class ResizeGrowPart(object): growpart_tmp = os.path.join(tmpd, "growpart") if not os.path.exists(growpart_tmp): os.mkdir(growpart_tmp, 0o700) - myenv['TMPDIR'] = growpart_tmp + myenv["TMPDIR"] = growpart_tmp try: - subp.subp(["growpart", '--dry-run', diskdev, partnum], - env=myenv) + subp.subp( + ["growpart", "--dry-run", diskdev, partnum], env=myenv + ) except subp.ProcessExecutionError as e: if e.exit_code != 1: - util.logexc(LOG, "Failed growpart --dry-run for (%s, %s)", - diskdev, partnum) + util.logexc( + LOG, + "Failed growpart --dry-run for (%s, %s)", + diskdev, + partnum, + ) raise ResizeFailedException(e) from e return (before, before) @@ -176,7 +179,7 @@ class ResizeGrowPart(object): class ResizeGpart(object): def available(self): myenv = os.environ.copy() - myenv['LANG'] = 'C' + myenv["LANG"] = "C" try: (_out, err) = subp.subp(["gpart", "help"], env=myenv, rcs=[0, 1]) @@ -234,11 +237,11 @@ def device_part_info(devpath): # the device, like /dev/vtbd0p2. if util.is_FreeBSD(): freebsd_part = "/dev/" + util.find_freebsd_part(devpath) - m = re.search('^(/dev/.+)p([0-9])$', freebsd_part) + m = re.search("^(/dev/.+)p([0-9])$", freebsd_part) return (m.group(1), m.group(2)) elif util.is_DragonFlyBSD(): dragonflybsd_part = "/dev/" + util.find_dragonflybsd_part(devpath) - m = re.search('^(/dev/.+)s([0-9])$', dragonflybsd_part) + m = re.search("^(/dev/.+)s([0-9])$", dragonflybsd_part) return (m.group(1), m.group(2)) if not os.path.exists(syspath): @@ -275,7 +278,7 @@ def devent2dev(devent): container = util.is_container() # Ensure the path is a block device. - if (dev == "/dev/root" and not container): + if dev == "/dev/root" and not container: dev = util.rootdev_from_cmdline(util.get_cmdline()) if dev is None: if os.path.exists(dev): @@ -293,65 +296,102 @@ def resize_devices(resizer, devices): try: blockdev = devent2dev(devent) except ValueError as e: - info.append((devent, RESIZE.SKIPPED, - "unable to convert to device: %s" % e,)) + info.append( + ( + devent, + RESIZE.SKIPPED, + "unable to convert to device: %s" % e, + ) + ) continue try: statret = os.stat(blockdev) except OSError as e: - info.append((devent, RESIZE.SKIPPED, - "stat of '%s' failed: %s" % (blockdev, e),)) + info.append( + ( + devent, + RESIZE.SKIPPED, + "stat of '%s' failed: %s" % (blockdev, e), + ) + ) continue - if (not stat.S_ISBLK(statret.st_mode) and - not stat.S_ISCHR(statret.st_mode)): - info.append((devent, RESIZE.SKIPPED, - "device '%s' not a block device" % blockdev,)) + if not stat.S_ISBLK(statret.st_mode) and not stat.S_ISCHR( + statret.st_mode + ): + info.append( + ( + devent, + RESIZE.SKIPPED, + "device '%s' not a block device" % blockdev, + ) + ) continue try: (disk, ptnum) = device_part_info(blockdev) except (TypeError, ValueError) as e: - info.append((devent, RESIZE.SKIPPED, - "device_part_info(%s) failed: %s" % (blockdev, e),)) + info.append( + ( + devent, + RESIZE.SKIPPED, + "device_part_info(%s) failed: %s" % (blockdev, e), + ) + ) continue try: (old, new) = resizer.resize(disk, ptnum, blockdev) if old == new: - info.append((devent, RESIZE.NOCHANGE, - "no change necessary (%s, %s)" % (disk, ptnum),)) + info.append( + ( + devent, + RESIZE.NOCHANGE, + "no change necessary (%s, %s)" % (disk, ptnum), + ) + ) else: - info.append((devent, RESIZE.CHANGED, - "changed (%s, %s) from %s to %s" % - (disk, ptnum, old, new),)) + info.append( + ( + devent, + RESIZE.CHANGED, + "changed (%s, %s) from %s to %s" + % (disk, ptnum, old, new), + ) + ) except ResizeFailedException as e: - info.append((devent, RESIZE.FAILED, - "failed to resize: disk=%s, ptnum=%s: %s" % - (disk, ptnum, e),)) + info.append( + ( + devent, + RESIZE.FAILED, + "failed to resize: disk=%s, ptnum=%s: %s" + % (disk, ptnum, e), + ) + ) return info def handle(_name, cfg, _cloud, log, _args): - if 'growpart' not in cfg: - log.debug("No 'growpart' entry in cfg. Using default: %s" % - DEFAULT_CONFIG) - cfg['growpart'] = DEFAULT_CONFIG + if "growpart" not in cfg: + log.debug( + "No 'growpart' entry in cfg. Using default: %s" % DEFAULT_CONFIG + ) + cfg["growpart"] = DEFAULT_CONFIG - mycfg = cfg.get('growpart') + mycfg = cfg.get("growpart") if not isinstance(mycfg, dict): log.warning("'growpart' in config was not a dict") return - mode = mycfg.get('mode', "auto") + mode = mycfg.get("mode", "auto") if util.is_false(mode): log.debug("growpart disabled: mode=%s" % mode) return - if util.is_false(mycfg.get('ignore_growroot_disabled', False)): + if util.is_false(mycfg.get("ignore_growroot_disabled", False)): if os.path.isfile("/etc/growroot-disabled"): log.debug("growpart disabled: /etc/growroot-disabled exists") log.debug("use ignore_growroot_disabled to ignore") @@ -370,8 +410,12 @@ def handle(_name, cfg, _cloud, log, _args): raise e return - resized = util.log_time(logfunc=log.debug, msg="resize_devices", - func=resize_devices, args=(resizer, devices)) + resized = util.log_time( + logfunc=log.debug, + msg="resize_devices", + func=resize_devices, + args=(resizer, devices), + ) for (entry, action, msg) in resized: if action == RESIZE.CHANGED: log.info("'%s' resized: %s" % (entry, msg)) @@ -379,6 +423,6 @@ def handle(_name, cfg, _cloud, log, _args): log.debug("'%s' %s: %s" % (entry, action, msg)) -RESIZERS = (('growpart', ResizeGrowPart), ('gpart', ResizeGpart)) +RESIZERS = (("growpart", ResizeGrowPart), ("gpart", ResizeGpart)) # vi: ts=4 expandtab diff --git a/cloudinit/config/cc_grub_dpkg.py b/cloudinit/config/cc_grub_dpkg.py index eb03c664..ad7243d9 100644 --- a/cloudinit/config/cc_grub_dpkg.py +++ b/cloudinit/config/cc_grub_dpkg.py @@ -43,11 +43,10 @@ seeded with empty values, and install_devices_empty is set to true. import os -from cloudinit import subp -from cloudinit import util +from cloudinit import subp, util from cloudinit.subp import ProcessExecutionError -distros = ['ubuntu', 'debian'] +distros = ["ubuntu", "debian"] def fetch_idevs(log): @@ -60,8 +59,9 @@ def fetch_idevs(log): try: # get the root disk where the /boot directory resides. - disk = subp.subp(['grub-probe', '-t', 'disk', '/boot'], - capture=True)[0].strip() + disk = subp.subp(["grub-probe", "-t", "disk", "/boot"], capture=True)[ + 0 + ].strip() except ProcessExecutionError as e: # grub-common may not be installed, especially on containers # FileNotFoundError is a nested exception of ProcessExecutionError @@ -81,26 +81,30 @@ def fetch_idevs(log): if not disk or not os.path.exists(disk): # If we failed to detect a disk, we can return early - return '' + return "" try: # check if disk exists and use udevadm to fetch symlinks - devices = subp.subp( - ['udevadm', 'info', '--root', '--query=symlink', disk], - capture=True - )[0].strip().split() + devices = ( + subp.subp( + ["udevadm", "info", "--root", "--query=symlink", disk], + capture=True, + )[0] + .strip() + .split() + ) except Exception: util.logexc( log, "udevadm DEVLINKS symlink query failed for disk='%s'", disk ) - log.debug('considering these device symlinks: %s', ','.join(devices)) + log.debug("considering these device symlinks: %s", ",".join(devices)) # filter symlinks for /dev/disk/by-id entries - devices = [dev for dev in devices if 'disk/by-id' in dev] - log.debug('filtered to these disk/by-id symlinks: %s', ','.join(devices)) + devices = [dev for dev in devices if "disk/by-id" in dev] + log.debug("filtered to these disk/by-id symlinks: %s", ",".join(devices)) # select first device if there is one, else fall back to plain name idevs = sorted(devices)[0] if devices else disk - log.debug('selected %s', idevs) + log.debug("selected %s", idevs) return idevs @@ -111,14 +115,15 @@ def handle(name, cfg, _cloud, log, _args): if not mycfg: mycfg = {} - enabled = mycfg.get('enabled', True) + enabled = mycfg.get("enabled", True) if util.is_false(enabled): log.debug("%s disabled by config grub_dpkg/enabled=%s", name, enabled) return idevs = util.get_cfg_option_str(mycfg, "grub-pc/install_devices", None) idevs_empty = util.get_cfg_option_str( - mycfg, "grub-pc/install_devices_empty", None) + mycfg, "grub-pc/install_devices_empty", None + ) if idevs is None: idevs = fetch_idevs(log) @@ -128,16 +133,21 @@ def handle(name, cfg, _cloud, log, _args): # now idevs and idevs_empty are set to determined values # or, those set by user - dconf_sel = (("grub-pc grub-pc/install_devices string %s\n" - "grub-pc grub-pc/install_devices_empty boolean %s\n") % - (idevs, idevs_empty)) + dconf_sel = ( + "grub-pc grub-pc/install_devices string %s\n" + "grub-pc grub-pc/install_devices_empty boolean %s\n" + % (idevs, idevs_empty) + ) - log.debug("Setting grub debconf-set-selections with '%s','%s'" % - (idevs, idevs_empty)) + log.debug( + "Setting grub debconf-set-selections with '%s','%s'" + % (idevs, idevs_empty) + ) try: - subp.subp(['debconf-set-selections'], dconf_sel) + subp.subp(["debconf-set-selections"], dconf_sel) except Exception: util.logexc(log, "Failed to run debconf-set-selections for grub-dpkg") + # vi: ts=4 expandtab diff --git a/cloudinit/config/cc_install_hotplug.py b/cloudinit/config/cc_install_hotplug.py index 9b4075cc..952d9f13 100644 --- a/cloudinit/config/cc_install_hotplug.py +++ b/cloudinit/config/cc_install_hotplug.py @@ -3,15 +3,12 @@ import os from textwrap import dedent -from cloudinit import util -from cloudinit import subp -from cloudinit import stages +from cloudinit import stages, subp, util from cloudinit.config.schema import get_meta_doc, validate_cloudconfig_schema from cloudinit.distros import ALL_DISTROS -from cloudinit.event import EventType, EventScope +from cloudinit.event import EventScope, EventType from cloudinit.settings import PER_INSTANCE - frequency = PER_INSTANCE distros = [ALL_DISTROS] @@ -19,7 +16,8 @@ meta = { "id": "cc_install_hotplug", "name": "Install Hotplug", "title": "Install hotplug if supported and enabled", - "description": dedent("""\ + "description": dedent( + """\ This module will install the udev rules to enable hotplug if supported by the datasource and enabled in the userdata. The udev rules will be installed as @@ -32,21 +30,26 @@ meta = { network configuration. Currently supported datasources: Openstack, EC2 - """), + """ + ), "distros": distros, "examples": [ - dedent("""\ + dedent( + """\ # Enable hotplug of network devices updates: network: when: ["hotplug"] - """), - dedent("""\ + """ + ), + dedent( + """\ # Enable network hotplug alongside boot event updates: network: when: ["boot", "hotplug"] - """), + """ + ), ], "frequency": frequency, } @@ -74,14 +77,14 @@ schema = { "boot-legacy", "boot", "hotplug", - ] - } + ], + }, } - } + }, } - } + }, } - } + }, } __doc__ = get_meta_doc(meta, schema) @@ -100,14 +103,15 @@ LABEL="cloudinit_end" def handle(_name, cfg, cloud, log, _args): validate_cloudconfig_schema(cfg, schema) network_hotplug_enabled = ( - 'updates' in cfg and - 'network' in cfg['updates'] and - 'when' in cfg['updates']['network'] and - 'hotplug' in cfg['updates']['network']['when'] + "updates" in cfg + and "network" in cfg["updates"] + and "when" in cfg["updates"]["network"] + and "hotplug" in cfg["updates"]["network"]["when"] ) hotplug_supported = EventType.HOTPLUG in ( - cloud.datasource.get_supported_events( - [EventType.HOTPLUG]).get(EventScope.NETWORK, set()) + cloud.datasource.get_supported_events([EventType.HOTPLUG]).get( + EventScope.NETWORK, set() + ) ) hotplug_enabled = stages.update_event_enabled( datasource=cloud.datasource, diff --git a/cloudinit/config/cc_keys_to_console.py b/cloudinit/config/cc_keys_to_console.py index d72b5244..ab35e136 100644 --- a/cloudinit/config/cc_keys_to_console.py +++ b/cloudinit/config/cc_keys_to_console.py @@ -38,49 +38,53 @@ host keys are not written to console. import os +from cloudinit import subp, util from cloudinit.settings import PER_INSTANCE -from cloudinit import subp -from cloudinit import util frequency = PER_INSTANCE # This is a tool that cloud init provides -HELPER_TOOL_TPL = '%s/cloud-init/write-ssh-key-fingerprints' +HELPER_TOOL_TPL = "%s/cloud-init/write-ssh-key-fingerprints" def _get_helper_tool_path(distro): try: base_lib = distro.usr_lib_exec except AttributeError: - base_lib = '/usr/lib' + base_lib = "/usr/lib" return HELPER_TOOL_TPL % base_lib def handle(name, cfg, cloud, log, _args): if util.is_false(cfg.get("ssh", {}).get("emit_keys_to_console", True)): - log.debug(("Skipping module named %s, " - "logging of SSH host keys disabled"), name) + log.debug( + "Skipping module named %s, logging of SSH host keys disabled", name + ) return helper_path = _get_helper_tool_path(cloud.distro) if not os.path.exists(helper_path): - log.warning(("Unable to activate module %s," - " helper tool not found at %s"), name, helper_path) + log.warning( + "Unable to activate module %s, helper tool not found at %s", + name, + helper_path, + ) return - fp_blacklist = util.get_cfg_option_list(cfg, - "ssh_fp_console_blacklist", []) - key_blacklist = util.get_cfg_option_list(cfg, - "ssh_key_console_blacklist", - ["ssh-dss"]) + fp_blacklist = util.get_cfg_option_list( + cfg, "ssh_fp_console_blacklist", [] + ) + key_blacklist = util.get_cfg_option_list( + cfg, "ssh_key_console_blacklist", ["ssh-dss"] + ) try: - cmd = [helper_path, ','.join(fp_blacklist), ','.join(key_blacklist)] + cmd = [helper_path, ",".join(fp_blacklist), ",".join(key_blacklist)] (stdout, _stderr) = subp.subp(cmd) - util.multi_log("%s\n" % (stdout.strip()), - stderr=False, console=True) + util.multi_log("%s\n" % (stdout.strip()), stderr=False, console=True) except Exception: log.warning("Writing keys to the system console failed!") raise + # vi: ts=4 expandtab diff --git a/cloudinit/config/cc_landscape.py b/cloudinit/config/cc_landscape.py index 299c4d01..03ebf411 100644 --- a/cloudinit/config/cc_landscape.py +++ b/cloudinit/config/cc_landscape.py @@ -60,10 +60,7 @@ from io import BytesIO from configobj import ConfigObj -from cloudinit import type_utils -from cloudinit import subp -from cloudinit import util - +from cloudinit import subp, type_utils, util from cloudinit.settings import PER_INSTANCE frequency = PER_INSTANCE @@ -71,15 +68,15 @@ frequency = PER_INSTANCE LSC_CLIENT_CFG_FILE = "/etc/landscape/client.conf" LS_DEFAULT_FILE = "/etc/default/landscape-client" -distros = ['ubuntu'] +distros = ["ubuntu"] # defaults taken from stock client.conf in landscape-client 11.07.1.1-0ubuntu2 LSC_BUILTIN_CFG = { - 'client': { - 'log_level': "info", - 'url': "https://landscape.canonical.com/message-system", - 'ping_url': "http://landscape.canonical.com/ping", - 'data_path': "/var/lib/landscape/client", + "client": { + "log_level": "info", + "url": "https://landscape.canonical.com/message-system", + "ping_url": "http://landscape.canonical.com/ping", + "data_path": "/var/lib/landscape/client", } } @@ -97,11 +94,13 @@ def handle(_name, cfg, cloud, log, _args): raise RuntimeError( "'landscape' key existed in config, but not a dictionary type," " is a {_type} instead".format( - _type=type_utils.obj_name(ls_cloudcfg))) + _type=type_utils.obj_name(ls_cloudcfg) + ) + ) if not ls_cloudcfg: return - cloud.distro.install_packages(('landscape-client',)) + cloud.distro.install_packages(("landscape-client",)) merge_data = [ LSC_BUILTIN_CFG, @@ -135,4 +134,5 @@ def merge_together(objs): cfg.merge(ConfigObj(obj)) return cfg + # vi: ts=4 expandtab diff --git a/cloudinit/config/cc_locale.py b/cloudinit/config/cc_locale.py index 7fed9abd..487f58f7 100644 --- a/cloudinit/config/cc_locale.py +++ b/cloudinit/config/cc_locale.py @@ -14,45 +14,48 @@ from cloudinit import util from cloudinit.config.schema import get_meta_doc, validate_cloudconfig_schema from cloudinit.settings import PER_INSTANCE - frequency = PER_INSTANCE -distros = ['all'] +distros = ["all"] meta = { - 'id': 'cc_locale', - 'name': 'Locale', - 'title': 'Set system locale', - 'description': dedent( + "id": "cc_locale", + "name": "Locale", + "title": "Set system locale", + "description": dedent( """\ Configure the system locale and apply it system wide. By default use the locale specified by the datasource.""" ), - 'distros': distros, - 'examples': [ - dedent("""\ + "distros": distros, + "examples": [ + dedent( + """\ # Set the locale to ar_AE locale: ar_AE - """), - dedent("""\ + """ + ), + dedent( + """\ # Set the locale to fr_CA in /etc/alternate_path/locale locale: fr_CA locale_configfile: /etc/alternate_path/locale - """), + """ + ), ], - 'frequency': frequency, + "frequency": frequency, } schema = { - 'type': 'object', - 'properties': { - 'locale': { - 'type': 'string', - 'description': ( + "type": "object", + "properties": { + "locale": { + "type": "string", + "description": ( "The locale to set as the system's locale (e.g. ar_PS)" ), }, - 'locale_configfile': { - 'type': 'string', - 'description': ( + "locale_configfile": { + "type": "string", + "description": ( "The file in which to write the locale configuration (defaults" " to the distro's default location)" ), @@ -70,8 +73,9 @@ def handle(name, cfg, cloud, log, args): locale = util.get_cfg_option_str(cfg, "locale", cloud.get_locale()) if util.is_false(locale): - log.debug("Skipping module named %s, disabled by config: %s", - name, locale) + log.debug( + "Skipping module named %s, disabled by config: %s", name, locale + ) return validate_cloudconfig_schema(cfg, schema) @@ -80,4 +84,5 @@ def handle(name, cfg, cloud, log, args): locale_cfgfile = util.get_cfg_option_str(cfg, "locale_configfile") cloud.distro.apply_locale(locale, locale_cfgfile) + # vi: ts=4 expandtab diff --git a/cloudinit/config/cc_lxd.py b/cloudinit/config/cc_lxd.py index 486037d9..13ddcbe9 100644 --- a/cloudinit/config/cc_lxd.py +++ b/cloudinit/config/cc_lxd.py @@ -47,12 +47,12 @@ lxd-bridge will be configured accordingly. domain: <domain> """ -from cloudinit import log as logging -from cloudinit import subp -from cloudinit import util import os -distros = ['ubuntu'] +from cloudinit import log as logging +from cloudinit import subp, util + +distros = ["ubuntu"] LOG = logging.getLogger(__name__) @@ -61,36 +61,42 @@ _DEFAULT_NETWORK_NAME = "lxdbr0" def handle(name, cfg, cloud, log, args): # Get config - lxd_cfg = cfg.get('lxd') + lxd_cfg = cfg.get("lxd") if not lxd_cfg: - log.debug("Skipping module named %s, not present or disabled by cfg", - name) + log.debug( + "Skipping module named %s, not present or disabled by cfg", name + ) return if not isinstance(lxd_cfg, dict): - log.warning("lxd config must be a dictionary. found a '%s'", - type(lxd_cfg)) + log.warning( + "lxd config must be a dictionary. found a '%s'", type(lxd_cfg) + ) return # Grab the configuration - init_cfg = lxd_cfg.get('init') + init_cfg = lxd_cfg.get("init") if not isinstance(init_cfg, dict): - log.warning("lxd/init config must be a dictionary. found a '%s'", - type(init_cfg)) + log.warning( + "lxd/init config must be a dictionary. found a '%s'", + type(init_cfg), + ) init_cfg = {} - bridge_cfg = lxd_cfg.get('bridge', {}) + bridge_cfg = lxd_cfg.get("bridge", {}) if not isinstance(bridge_cfg, dict): - log.warning("lxd/bridge config must be a dictionary. found a '%s'", - type(bridge_cfg)) + log.warning( + "lxd/bridge config must be a dictionary. found a '%s'", + type(bridge_cfg), + ) bridge_cfg = {} # Install the needed packages packages = [] if not subp.which("lxd"): - packages.append('lxd') + packages.append("lxd") - if init_cfg.get("storage_backend") == "zfs" and not subp.which('zfs'): - packages.append('zfsutils-linux') + if init_cfg.get("storage_backend") == "zfs" and not subp.which("zfs"): + packages.append("zfsutils-linux") if len(packages): try: @@ -102,23 +108,30 @@ def handle(name, cfg, cloud, log, args): # Set up lxd if init config is given if init_cfg: init_keys = ( - 'network_address', 'network_port', 'storage_backend', - 'storage_create_device', 'storage_create_loop', - 'storage_pool', 'trust_password') - subp.subp(['lxd', 'waitready', '--timeout=300']) - cmd = ['lxd', 'init', '--auto'] + "network_address", + "network_port", + "storage_backend", + "storage_create_device", + "storage_create_loop", + "storage_pool", + "trust_password", + ) + subp.subp(["lxd", "waitready", "--timeout=300"]) + cmd = ["lxd", "init", "--auto"] for k in init_keys: if init_cfg.get(k): - cmd.extend(["--%s=%s" % - (k.replace('_', '-'), str(init_cfg[k]))]) + cmd.extend( + ["--%s=%s" % (k.replace("_", "-"), str(init_cfg[k]))] + ) subp.subp(cmd) # Set up lxd-bridge if bridge config is given dconf_comm = "debconf-communicate" if bridge_cfg: net_name = bridge_cfg.get("name", _DEFAULT_NETWORK_NAME) - if os.path.exists("/etc/default/lxd-bridge") \ - and subp.which(dconf_comm): + if os.path.exists("/etc/default/lxd-bridge") and subp.which( + dconf_comm + ): # Bridge configured through packaging debconf = bridge_to_debconf(bridge_cfg) @@ -126,39 +139,47 @@ def handle(name, cfg, cloud, log, args): # Update debconf database try: log.debug("Setting lxd debconf via " + dconf_comm) - data = "\n".join(["set %s %s" % (k, v) - for k, v in debconf.items()]) + "\n" - subp.subp(['debconf-communicate'], data) + data = ( + "\n".join( + ["set %s %s" % (k, v) for k, v in debconf.items()] + ) + + "\n" + ) + subp.subp(["debconf-communicate"], data) except Exception: - util.logexc(log, "Failed to run '%s' for lxd with" % - dconf_comm) + util.logexc( + log, "Failed to run '%s' for lxd with" % dconf_comm + ) # Remove the existing configuration file (forces re-generation) util.del_file("/etc/default/lxd-bridge") # Run reconfigure log.debug("Running dpkg-reconfigure for lxd") - subp.subp(['dpkg-reconfigure', 'lxd', - '--frontend=noninteractive']) + subp.subp(["dpkg-reconfigure", "lxd", "--frontend=noninteractive"]) else: # Built-in LXD bridge support cmd_create, cmd_attach = bridge_to_cmd(bridge_cfg) maybe_cleanup_default( - net_name=net_name, did_init=bool(init_cfg), - create=bool(cmd_create), attach=bool(cmd_attach)) + net_name=net_name, + did_init=bool(init_cfg), + create=bool(cmd_create), + attach=bool(cmd_attach), + ) if cmd_create: - log.debug("Creating lxd bridge: %s" % - " ".join(cmd_create)) + log.debug("Creating lxd bridge: %s" % " ".join(cmd_create)) _lxc(cmd_create) if cmd_attach: - log.debug("Setting up default lxd bridge: %s" % - " ".join(cmd_attach)) + log.debug( + "Setting up default lxd bridge: %s" % " ".join(cmd_attach) + ) _lxc(cmd_attach) elif bridge_cfg: raise RuntimeError( - "Unable to configure lxd bridge without %s." + dconf_comm) + "Unable to configure lxd bridge without %s." + dconf_comm + ) def bridge_to_debconf(bridge_cfg): @@ -180,33 +201,32 @@ def bridge_to_debconf(bridge_cfg): if bridge_cfg.get("ipv4_address"): debconf["lxd/bridge-ipv4"] = "true" - debconf["lxd/bridge-ipv4-address"] = \ - bridge_cfg.get("ipv4_address") - debconf["lxd/bridge-ipv4-netmask"] = \ - bridge_cfg.get("ipv4_netmask") - debconf["lxd/bridge-ipv4-dhcp-first"] = \ - bridge_cfg.get("ipv4_dhcp_first") - debconf["lxd/bridge-ipv4-dhcp-last"] = \ - bridge_cfg.get("ipv4_dhcp_last") - debconf["lxd/bridge-ipv4-dhcp-leases"] = \ - bridge_cfg.get("ipv4_dhcp_leases") - debconf["lxd/bridge-ipv4-nat"] = \ - bridge_cfg.get("ipv4_nat", "true") + debconf["lxd/bridge-ipv4-address"] = bridge_cfg.get("ipv4_address") + debconf["lxd/bridge-ipv4-netmask"] = bridge_cfg.get("ipv4_netmask") + debconf["lxd/bridge-ipv4-dhcp-first"] = bridge_cfg.get( + "ipv4_dhcp_first" + ) + debconf["lxd/bridge-ipv4-dhcp-last"] = bridge_cfg.get( + "ipv4_dhcp_last" + ) + debconf["lxd/bridge-ipv4-dhcp-leases"] = bridge_cfg.get( + "ipv4_dhcp_leases" + ) + debconf["lxd/bridge-ipv4-nat"] = bridge_cfg.get("ipv4_nat", "true") if bridge_cfg.get("ipv6_address"): debconf["lxd/bridge-ipv6"] = "true" - debconf["lxd/bridge-ipv6-address"] = \ - bridge_cfg.get("ipv6_address") - debconf["lxd/bridge-ipv6-netmask"] = \ - bridge_cfg.get("ipv6_netmask") - debconf["lxd/bridge-ipv6-nat"] = \ - bridge_cfg.get("ipv6_nat", "false") + debconf["lxd/bridge-ipv6-address"] = bridge_cfg.get("ipv6_address") + debconf["lxd/bridge-ipv6-netmask"] = bridge_cfg.get("ipv6_netmask") + debconf["lxd/bridge-ipv6-nat"] = bridge_cfg.get( + "ipv6_nat", "false" + ) if bridge_cfg.get("domain"): debconf["lxd/bridge-domain"] = bridge_cfg.get("domain") else: - raise Exception("invalid bridge mode \"%s\"" % bridge_cfg.get("mode")) + raise Exception('invalid bridge mode "%s"' % bridge_cfg.get("mode")) return debconf @@ -217,37 +237,41 @@ def bridge_to_cmd(bridge_cfg): bridge_name = bridge_cfg.get("name", _DEFAULT_NETWORK_NAME) cmd_create = [] - cmd_attach = ["network", "attach-profile", bridge_name, - "default", "eth0"] + cmd_attach = ["network", "attach-profile", bridge_name, "default", "eth0"] if bridge_cfg.get("mode") == "existing": return None, cmd_attach if bridge_cfg.get("mode") != "new": - raise Exception("invalid bridge mode \"%s\"" % bridge_cfg.get("mode")) + raise Exception('invalid bridge mode "%s"' % bridge_cfg.get("mode")) cmd_create = ["network", "create", bridge_name] if bridge_cfg.get("ipv4_address") and bridge_cfg.get("ipv4_netmask"): - cmd_create.append("ipv4.address=%s/%s" % - (bridge_cfg.get("ipv4_address"), - bridge_cfg.get("ipv4_netmask"))) + cmd_create.append( + "ipv4.address=%s/%s" + % (bridge_cfg.get("ipv4_address"), bridge_cfg.get("ipv4_netmask")) + ) if bridge_cfg.get("ipv4_nat", "true") == "true": cmd_create.append("ipv4.nat=true") - if bridge_cfg.get("ipv4_dhcp_first") and \ - bridge_cfg.get("ipv4_dhcp_last"): - dhcp_range = "%s-%s" % (bridge_cfg.get("ipv4_dhcp_first"), - bridge_cfg.get("ipv4_dhcp_last")) + if bridge_cfg.get("ipv4_dhcp_first") and bridge_cfg.get( + "ipv4_dhcp_last" + ): + dhcp_range = "%s-%s" % ( + bridge_cfg.get("ipv4_dhcp_first"), + bridge_cfg.get("ipv4_dhcp_last"), + ) cmd_create.append("ipv4.dhcp.ranges=%s" % dhcp_range) else: cmd_create.append("ipv4.address=none") if bridge_cfg.get("ipv6_address") and bridge_cfg.get("ipv6_netmask"): - cmd_create.append("ipv6.address=%s/%s" % - (bridge_cfg.get("ipv6_address"), - bridge_cfg.get("ipv6_netmask"))) + cmd_create.append( + "ipv6.address=%s/%s" + % (bridge_cfg.get("ipv6_address"), bridge_cfg.get("ipv6_netmask")) + ) if bridge_cfg.get("ipv6_nat", "false") == "true": cmd_create.append("ipv6.nat=true") @@ -262,14 +286,17 @@ def bridge_to_cmd(bridge_cfg): def _lxc(cmd): - env = {'LC_ALL': 'C', - 'HOME': os.environ.get('HOME', '/root'), - 'USER': os.environ.get('USER', 'root')} - subp.subp(['lxc'] + list(cmd) + ["--force-local"], update_env=env) + env = { + "LC_ALL": "C", + "HOME": os.environ.get("HOME", "/root"), + "USER": os.environ.get("USER", "root"), + } + subp.subp(["lxc"] + list(cmd) + ["--force-local"], update_env=env) -def maybe_cleanup_default(net_name, did_init, create, attach, - profile="default", nic_name="eth0"): +def maybe_cleanup_default( + net_name, did_init, create, attach, profile="default", nic_name="eth0" +): """Newer versions of lxc (3.0.1+) create a lxdbr0 network when 'lxd init --auto' is run. Older versions did not. @@ -306,4 +333,5 @@ def maybe_cleanup_default(net_name, did_init, create, attach, raise e LOG.debug(msg, nic_name, profile, fail_assume_enoent) + # vi: ts=4 expandtab diff --git a/cloudinit/config/cc_mcollective.py b/cloudinit/config/cc_mcollective.py index 41ea4fc9..1b0158ec 100644 --- a/cloudinit/config/cc_mcollective.py +++ b/cloudinit/config/cc_mcollective.py @@ -56,18 +56,21 @@ import io from configobj import ConfigObj from cloudinit import log as logging -from cloudinit import subp -from cloudinit import util +from cloudinit import subp, util PUBCERT_FILE = "/etc/mcollective/ssl/server-public.pem" PRICERT_FILE = "/etc/mcollective/ssl/server-private.pem" -SERVER_CFG = '/etc/mcollective/server.cfg' +SERVER_CFG = "/etc/mcollective/server.cfg" LOG = logging.getLogger(__name__) -def configure(config, server_cfg=SERVER_CFG, - pubcert_file=PUBCERT_FILE, pricert_file=PRICERT_FILE): +def configure( + config, + server_cfg=SERVER_CFG, + pubcert_file=PUBCERT_FILE, + pricert_file=PRICERT_FILE, +): # Read server.cfg (if it exists) values from the # original file in order to be able to mix the rest up. try: @@ -77,20 +80,20 @@ def configure(config, server_cfg=SERVER_CFG, if e.errno != errno.ENOENT: raise else: - LOG.debug("Did not find file %s (starting with an empty" - " config)", server_cfg) + LOG.debug( + "Did not find file %s (starting with an empty config)", + server_cfg, + ) mcollective_config = ConfigObj() for (cfg_name, cfg) in config.items(): - if cfg_name == 'public-cert': + if cfg_name == "public-cert": util.write_file(pubcert_file, cfg, mode=0o644) - mcollective_config[ - 'plugin.ssl_server_public'] = pubcert_file - mcollective_config['securityprovider'] = 'ssl' - elif cfg_name == 'private-cert': + mcollective_config["plugin.ssl_server_public"] = pubcert_file + mcollective_config["securityprovider"] = "ssl" + elif cfg_name == "private-cert": util.write_file(pricert_file, cfg, mode=0o600) - mcollective_config[ - 'plugin.ssl_server_private'] = pricert_file - mcollective_config['securityprovider'] = 'ssl' + mcollective_config["plugin.ssl_server_private"] = pricert_file + mcollective_config["securityprovider"] = "ssl" else: if isinstance(cfg, str): # Just set it in the 'main' section @@ -126,21 +129,24 @@ def configure(config, server_cfg=SERVER_CFG, def handle(name, cfg, cloud, log, _args): # If there isn't a mcollective key in the configuration don't do anything - if 'mcollective' not in cfg: - log.debug(("Skipping module named %s, " - "no 'mcollective' key in configuration"), name) + if "mcollective" not in cfg: + log.debug( + "Skipping module named %s, no 'mcollective' key in configuration", + name, + ) return - mcollective_cfg = cfg['mcollective'] + mcollective_cfg = cfg["mcollective"] # Start by installing the mcollective package ... cloud.distro.install_packages(("mcollective",)) # ... and then update the mcollective configuration - if 'conf' in mcollective_cfg: - configure(config=mcollective_cfg['conf']) + if "conf" in mcollective_cfg: + configure(config=mcollective_cfg["conf"]) # restart mcollective to handle updated config - subp.subp(['service', 'mcollective', 'restart'], capture=False) + subp.subp(["service", "mcollective", "restart"], capture=False) + # vi: ts=4 expandtab diff --git a/cloudinit/config/cc_migrator.py b/cloudinit/config/cc_migrator.py index 79bcc27d..4fafb4af 100644 --- a/cloudinit/config/cc_migrator.py +++ b/cloudinit/config/cc_migrator.py @@ -29,16 +29,14 @@ false`` in config. import os import shutil -from cloudinit import helpers -from cloudinit import util - +from cloudinit import helpers, util from cloudinit.settings import PER_ALWAYS frequency = PER_ALWAYS def _migrate_canon_sems(cloud): - paths = (cloud.paths.get_ipath('sem'), cloud.paths.get_cpath('sem')) + paths = (cloud.paths.get_ipath("sem"), cloud.paths.get_cpath("sem")) am_adjusted = 0 for sem_path in paths: if not sem_path or not os.path.exists(sem_path): @@ -57,12 +55,12 @@ def _migrate_canon_sems(cloud): def _migrate_legacy_sems(cloud, log): legacy_adjust = { - 'apt-update-upgrade': [ - 'apt-configure', - 'package-update-upgrade-install', + "apt-update-upgrade": [ + "apt-configure", + "package-update-upgrade-install", ], } - paths = (cloud.paths.get_ipath('sem'), cloud.paths.get_cpath('sem')) + paths = (cloud.paths.get_ipath("sem"), cloud.paths.get_cpath("sem")) for sem_path in paths: if not sem_path or not os.path.exists(sem_path): continue @@ -78,8 +76,9 @@ def _migrate_legacy_sems(cloud, log): util.del_file(os.path.join(sem_path, p)) (_name, freq) = os.path.splitext(p) for m in migrate_to: - log.debug("Migrating %s => %s with the same frequency", - p, m) + log.debug( + "Migrating %s => %s with the same frequency", p, m + ) with sem_helper.lock(m, freq): pass @@ -90,8 +89,10 @@ def handle(name, cfg, cloud, log, _args): log.debug("Skipping module named %s, migration disabled", name) return sems_moved = _migrate_canon_sems(cloud) - log.debug("Migrated %s semaphore files to there canonicalized names", - sems_moved) + log.debug( + "Migrated %s semaphore files to there canonicalized names", sems_moved + ) _migrate_legacy_sems(cloud, log) + # vi: ts=4 expandtab diff --git a/cloudinit/config/cc_mounts.py b/cloudinit/config/cc_mounts.py index eeb008d2..ec2e46ff 100644 --- a/cloudinit/config/cc_mounts.py +++ b/cloudinit/config/cc_mounts.py @@ -62,15 +62,12 @@ swap file is created. maxsize: <size in bytes> """ -from string import whitespace - import logging import os import re +from string import whitespace -from cloudinit import type_utils -from cloudinit import subp -from cloudinit import util +from cloudinit import subp, type_utils, util # Shortname matches 'sda', 'sda1', 'xvda', 'hda', 'sdb', xvdb, vda, vdd1, sr0 DEVICE_NAME_FILTER = r"^([x]{0,1}[shv]d[a-z][0-9]*|sr[0-9]+)$" @@ -105,21 +102,25 @@ def is_network_device(name): def _get_nth_partition_for_device(device_path, partition_number): - potential_suffixes = [str(partition_number), 'p%s' % (partition_number,), - '-part%s' % (partition_number,)] + potential_suffixes = [ + str(partition_number), + "p%s" % (partition_number,), + "-part%s" % (partition_number,), + ] for suffix in potential_suffixes: - potential_partition_device = '%s%s' % (device_path, suffix) + potential_partition_device = "%s%s" % (device_path, suffix) if os.path.exists(potential_partition_device): return potential_partition_device return None def _is_block_device(device_path, partition_path=None): - device_name = os.path.realpath(device_path).split('/')[-1] - sys_path = os.path.join('/sys/block/', device_name) + device_name = os.path.realpath(device_path).split("/")[-1] + sys_path = os.path.join("/sys/block/", device_name) if partition_path is not None: sys_path = os.path.join( - sys_path, os.path.realpath(partition_path).split('/')[-1]) + sys_path, os.path.realpath(partition_path).split("/")[-1] + ) return os.path.exists(sys_path) @@ -159,8 +160,9 @@ def sanitize_devname(startname, transformer, log, aliases=None): if partition_number is None: partition_path = _get_nth_partition_for_device(device_path, 1) else: - partition_path = _get_nth_partition_for_device(device_path, - partition_number) + partition_path = _get_nth_partition_for_device( + device_path, partition_number + ) if partition_path is None: return None @@ -174,12 +176,12 @@ def sanitize_devname(startname, transformer, log, aliases=None): def suggested_swapsize(memsize=None, maxsize=None, fsys=None): # make a suggestion on the size of swap for this system. if memsize is None: - memsize = util.read_meminfo()['total'] + memsize = util.read_meminfo()["total"] GB = 2 ** 30 sugg_max = 8 * GB - info = {'avail': 'na', 'max_in': maxsize, 'mem': memsize} + info = {"avail": "na", "max_in": maxsize, "mem": memsize} if fsys is None and maxsize is None: # set max to 8GB default if no filesystem given @@ -187,18 +189,18 @@ def suggested_swapsize(memsize=None, maxsize=None, fsys=None): elif fsys: statvfs = os.statvfs(fsys) avail = statvfs.f_frsize * statvfs.f_bfree - info['avail'] = avail + info["avail"] = avail if maxsize is None: # set to 25% of filesystem space maxsize = min(int(avail / 4), sugg_max) - elif maxsize > ((avail * .9)): + elif maxsize > ((avail * 0.9)): # set to 90% of available disk space - maxsize = int(avail * .9) + maxsize = int(avail * 0.9) elif maxsize is None: maxsize = sugg_max - info['max'] = maxsize + info["max"] = maxsize formulas = [ # < 1G: swap = double memory @@ -226,7 +228,7 @@ def suggested_swapsize(memsize=None, maxsize=None, fsys=None): if size is not None: size = maxsize - info['size'] = size + info["size"] = size MB = 2 ** 20 pinfo = {} @@ -236,9 +238,14 @@ def suggested_swapsize(memsize=None, maxsize=None, fsys=None): else: pinfo[k] = v - LOG.debug("suggest %s swap for %s memory with '%s'" - " disk given max=%s [max=%s]'", pinfo['size'], pinfo['mem'], - pinfo['avail'], pinfo['max_in'], pinfo['max']) + LOG.debug( + "suggest %s swap for %s memory with '%s' disk given max=%s [max=%s]'", + pinfo["size"], + pinfo["mem"], + pinfo["avail"], + pinfo["max_in"], + pinfo["max"], + ) return size @@ -248,14 +255,23 @@ def create_swapfile(fname: str, size: str) -> None: errmsg = "Failed to create swapfile '%s' of size %sMB via %s: %s" def create_swap(fname, size, method): - LOG.debug("Creating swapfile in '%s' on fstype '%s' using '%s'", - fname, fstype, method) + LOG.debug( + "Creating swapfile in '%s' on fstype '%s' using '%s'", + fname, + fstype, + method, + ) if method == "fallocate": - cmd = ['fallocate', '-l', '%sM' % size, fname] + cmd = ["fallocate", "-l", "%sM" % size, fname] elif method == "dd": - cmd = ['dd', 'if=/dev/zero', 'of=%s' % fname, 'bs=1M', - 'count=%s' % size] + cmd = [ + "dd", + "if=/dev/zero", + "of=%s" % fname, + "bs=1M", + "count=%s" % size, + ] try: subp.subp(cmd, capture=True) @@ -269,8 +285,9 @@ def create_swapfile(fname: str, size: str) -> None: fstype = util.get_mount_info(swap_dir)[1] - if (fstype == "xfs" and - util.kernel_version() < (4, 18)) or fstype == "btrfs": + if ( + fstype == "xfs" and util.kernel_version() < (4, 18) + ) or fstype == "btrfs": create_swap(fname, size, "dd") else: try: @@ -282,7 +299,7 @@ def create_swapfile(fname: str, size: str) -> None: if os.path.exists(fname): util.chmod(fname, 0o600) try: - subp.subp(['mkswap', fname]) + subp.subp(["mkswap", fname]) except subp.ProcessExecutionError: util.del_file(fname) raise @@ -297,37 +314,42 @@ def setup_swapfile(fname, size=None, maxsize=None): swap_dir = os.path.dirname(fname) if str(size).lower() == "auto": try: - memsize = util.read_meminfo()['total'] + memsize = util.read_meminfo()["total"] except IOError: LOG.debug("Not creating swap: failed to read meminfo") return util.ensure_dir(swap_dir) - size = suggested_swapsize(fsys=swap_dir, maxsize=maxsize, - memsize=memsize) + size = suggested_swapsize( + fsys=swap_dir, maxsize=maxsize, memsize=memsize + ) mibsize = str(int(size / (2 ** 20))) if not size: LOG.debug("Not creating swap: suggested size was 0") return - util.log_time(LOG.debug, msg="Setting up swap file", func=create_swapfile, - args=[fname, mibsize]) + util.log_time( + LOG.debug, + msg="Setting up swap file", + func=create_swapfile, + args=[fname, mibsize], + ) return fname def handle_swapcfg(swapcfg): """handle the swap config, calling setup_swap if necessary. - return None or (filename, size) + return None or (filename, size) """ if not isinstance(swapcfg, dict): LOG.warning("input for swap config was not a dict.") return None - fname = swapcfg.get('filename', '/swap.img') - size = swapcfg.get('size', 0) - maxsize = swapcfg.get('maxsize', None) + fname = swapcfg.get("filename", "/swap.img") + size = swapcfg.get("size", 0) + maxsize = swapcfg.get("maxsize", None) if not (size and fname): LOG.debug("no need to setup swap") @@ -335,8 +357,10 @@ def handle_swapcfg(swapcfg): if os.path.exists(fname): if not os.path.exists("/proc/swaps"): - LOG.debug("swap file %s exists, but no /proc/swaps exists, " - "being safe", fname) + LOG.debug( + "swap file %s exists, but no /proc/swaps exists, being safe", + fname, + ) return fname try: for line in util.load_file("/proc/swaps").splitlines(): @@ -345,8 +369,9 @@ def handle_swapcfg(swapcfg): return fname LOG.debug("swap file %s exists, but not in /proc/swaps", fname) except Exception: - LOG.warning("swap file %s exists. Error reading /proc/swaps", - fname) + LOG.warning( + "swap file %s exists. Error reading /proc/swaps", fname + ) return fname try: @@ -373,8 +398,10 @@ def handle(_name, cfg, cloud, log, _args): defvals = cfg.get("mount_default_fields", defvals) # these are our default set of mounts - defmnts = [["ephemeral0", "/mnt", "auto", defvals[3], "0", "2"], - ["swap", "none", "swap", "sw", "0", "0"]] + defmnts = [ + ["ephemeral0", "/mnt", "auto", defvals[3], "0", "2"], + ["swap", "none", "swap", "sw", "0", "0"], + ] cfgmnt = [] if "mounts" in cfg: @@ -404,13 +431,17 @@ def handle(_name, cfg, cloud, log, _args): for i in range(len(cfgmnt)): # skip something that wasn't a list if not isinstance(cfgmnt[i], list): - log.warning("Mount option %s not a list, got a %s instead", - (i + 1), type_utils.obj_name(cfgmnt[i])) + log.warning( + "Mount option %s not a list, got a %s instead", + (i + 1), + type_utils.obj_name(cfgmnt[i]), + ) continue start = str(cfgmnt[i][0]) - sanitized = sanitize_devname(start, cloud.device_name_to_device, log, - aliases=device_aliases) + sanitized = sanitize_devname( + start, cloud.device_name_to_device, log, aliases=device_aliases + ) if sanitized != start: log.debug("changed %s => %s" % (start, sanitized)) @@ -418,8 +449,11 @@ def handle(_name, cfg, cloud, log, _args): log.debug("Ignoring nonexistent named mount %s", start) continue elif sanitized in fstab_devs: - log.info("Device %s already defined in fstab: %s", - sanitized, fstab_devs[sanitized]) + log.info( + "Device %s already defined in fstab: %s", + sanitized, + fstab_devs[sanitized], + ) continue cfgmnt[i][0] = sanitized @@ -452,8 +486,9 @@ def handle(_name, cfg, cloud, log, _args): # entry has the same device name for defmnt in defmnts: start = defmnt[0] - sanitized = sanitize_devname(start, cloud.device_name_to_device, log, - aliases=device_aliases) + sanitized = sanitize_devname( + start, cloud.device_name_to_device, log, aliases=device_aliases + ) if sanitized != start: log.debug("changed default device %s => %s" % (start, sanitized)) @@ -461,8 +496,11 @@ def handle(_name, cfg, cloud, log, _args): log.debug("Ignoring nonexistent default named mount %s", start) continue elif sanitized in fstab_devs: - log.debug("Device %s already defined in fstab: %s", - sanitized, fstab_devs[sanitized]) + log.debug( + "Device %s already defined in fstab: %s", + sanitized, + fstab_devs[sanitized], + ) continue defmnt[0] = sanitized @@ -474,8 +512,7 @@ def handle(_name, cfg, cloud, log, _args): break if cfgmnt_has: - log.debug(("Not including %s, already" - " previously included"), start) + log.debug("Not including %s, already previously included", start) continue cfgmnt.append(defmnt) @@ -488,7 +525,7 @@ def handle(_name, cfg, cloud, log, _args): else: actlist.append(x) - swapret = handle_swapcfg(cfg.get('swap', {})) + swapret = handle_swapcfg(cfg.get("swap", {})) if swapret: actlist.append([swapret, "none", "swap", "sw", "0", "0"]) @@ -507,10 +544,11 @@ def handle(_name, cfg, cloud, log, _args): needswap = True if line[1].startswith("/"): dirs.append(line[1]) - cc_lines.append('\t'.join(line)) + cc_lines.append("\t".join(line)) - mount_points = [v['mountpoint'] for k, v in util.mounts().items() - if 'mountpoint' in v] + mount_points = [ + v["mountpoint"] for k, v in util.mounts().items() if "mountpoint" in v + ] for d in dirs: try: util.ensure_dir(d) @@ -525,11 +563,12 @@ def handle(_name, cfg, cloud, log, _args): sadds = [WS.sub(" ", n) for n in cc_lines] sdrops = [WS.sub(" ", n) for n in fstab_removed] - sops = (["- " + drop for drop in sdrops if drop not in sadds] + - ["+ " + add for add in sadds if add not in sdrops]) + sops = ["- " + drop for drop in sdrops if drop not in sadds] + [ + "+ " + add for add in sadds if add not in sdrops + ] fstab_lines.extend(cc_lines) - contents = "%s\n" % ('\n'.join(fstab_lines)) + contents = "%s\n" % "\n".join(fstab_lines) util.write_file(FSTAB_PATH, contents) activate_cmds = [] @@ -549,7 +588,7 @@ def handle(_name, cfg, cloud, log, _args): fmt = "Activating swap and mounts with: %s" for cmd in activate_cmds: - fmt = "Activate mounts: %s:" + ' '.join(cmd) + fmt = "Activate mounts: %s:" + " ".join(cmd) try: subp.subp(cmd) log.debug(fmt, "PASS") @@ -557,4 +596,5 @@ def handle(_name, cfg, cloud, log, _args): log.warning(fmt, "FAIL") util.logexc(log, fmt, "FAIL") + # vi: ts=4 expandtab diff --git a/cloudinit/config/cc_ntp.py b/cloudinit/config/cc_ntp.py index c55d5d86..a31da9bb 100644 --- a/cloudinit/config/cc_ntp.py +++ b/cloudinit/config/cc_ntp.py @@ -11,124 +11,132 @@ import os from textwrap import dedent from cloudinit import log as logging -from cloudinit import temp_utils -from cloudinit import templater -from cloudinit import type_utils -from cloudinit import subp -from cloudinit import util +from cloudinit import subp, temp_utils, templater, type_utils, util from cloudinit.config.schema import get_meta_doc, validate_cloudconfig_schema from cloudinit.settings import PER_INSTANCE LOG = logging.getLogger(__name__) frequency = PER_INSTANCE -NTP_CONF = '/etc/ntp.conf' +NTP_CONF = "/etc/ntp.conf" NR_POOL_SERVERS = 4 -distros = ['almalinux', 'alpine', 'centos', 'cloudlinux', 'debian', - 'eurolinux', 'fedora', 'miraclelinux', 'openEuler', 'opensuse', - 'photon', 'rhel', 'rocky', 'sles', 'ubuntu', 'virtuozzo'] +distros = [ + "almalinux", + "alpine", + "centos", + "cloudlinux", + "debian", + "eurolinux", + "fedora", + "miraclelinux", + "openEuler", + "opensuse", + "photon", + "rhel", + "rocky", + "sles", + "ubuntu", + "virtuozzo", +] NTP_CLIENT_CONFIG = { - 'chrony': { - 'check_exe': 'chronyd', - 'confpath': '/etc/chrony.conf', - 'packages': ['chrony'], - 'service_name': 'chrony', - 'template_name': 'chrony.conf.{distro}', - 'template': None, + "chrony": { + "check_exe": "chronyd", + "confpath": "/etc/chrony.conf", + "packages": ["chrony"], + "service_name": "chrony", + "template_name": "chrony.conf.{distro}", + "template": None, }, - 'ntp': { - 'check_exe': 'ntpd', - 'confpath': NTP_CONF, - 'packages': ['ntp'], - 'service_name': 'ntp', - 'template_name': 'ntp.conf.{distro}', - 'template': None, + "ntp": { + "check_exe": "ntpd", + "confpath": NTP_CONF, + "packages": ["ntp"], + "service_name": "ntp", + "template_name": "ntp.conf.{distro}", + "template": None, }, - 'ntpdate': { - 'check_exe': 'ntpdate', - 'confpath': NTP_CONF, - 'packages': ['ntpdate'], - 'service_name': 'ntpdate', - 'template_name': 'ntp.conf.{distro}', - 'template': None, + "ntpdate": { + "check_exe": "ntpdate", + "confpath": NTP_CONF, + "packages": ["ntpdate"], + "service_name": "ntpdate", + "template_name": "ntp.conf.{distro}", + "template": None, }, - 'systemd-timesyncd': { - 'check_exe': '/lib/systemd/systemd-timesyncd', - 'confpath': '/etc/systemd/timesyncd.conf.d/cloud-init.conf', - 'packages': [], - 'service_name': 'systemd-timesyncd', - 'template_name': 'timesyncd.conf', - 'template': None, + "systemd-timesyncd": { + "check_exe": "/lib/systemd/systemd-timesyncd", + "confpath": "/etc/systemd/timesyncd.conf.d/cloud-init.conf", + "packages": [], + "service_name": "systemd-timesyncd", + "template_name": "timesyncd.conf", + "template": None, }, } # This is Distro-specific configuration overrides of the base config DISTRO_CLIENT_CONFIG = { - 'alpine': { - 'chrony': { - 'confpath': '/etc/chrony/chrony.conf', - 'service_name': 'chronyd', + "alpine": { + "chrony": { + "confpath": "/etc/chrony/chrony.conf", + "service_name": "chronyd", }, - 'ntp': { - 'confpath': '/etc/ntp.conf', - 'packages': [], - 'service_name': 'ntpd', + "ntp": { + "confpath": "/etc/ntp.conf", + "packages": [], + "service_name": "ntpd", }, }, - 'debian': { - 'chrony': { - 'confpath': '/etc/chrony/chrony.conf', + "debian": { + "chrony": { + "confpath": "/etc/chrony/chrony.conf", }, }, - 'opensuse': { - 'chrony': { - 'service_name': 'chronyd', + "opensuse": { + "chrony": { + "service_name": "chronyd", }, - 'ntp': { - 'confpath': '/etc/ntp.conf', - 'service_name': 'ntpd', + "ntp": { + "confpath": "/etc/ntp.conf", + "service_name": "ntpd", }, - 'systemd-timesyncd': { - 'check_exe': '/usr/lib/systemd/systemd-timesyncd', + "systemd-timesyncd": { + "check_exe": "/usr/lib/systemd/systemd-timesyncd", }, }, - 'photon': { - 'chrony': { - 'service_name': 'chronyd', + "photon": { + "chrony": { + "service_name": "chronyd", }, - 'ntp': { - 'service_name': 'ntpd', - 'confpath': '/etc/ntp.conf' - }, - 'systemd-timesyncd': { - 'check_exe': '/usr/lib/systemd/systemd-timesyncd', - 'confpath': '/etc/systemd/timesyncd.conf', + "ntp": {"service_name": "ntpd", "confpath": "/etc/ntp.conf"}, + "systemd-timesyncd": { + "check_exe": "/usr/lib/systemd/systemd-timesyncd", + "confpath": "/etc/systemd/timesyncd.conf", }, }, - 'rhel': { - 'ntp': { - 'service_name': 'ntpd', + "rhel": { + "ntp": { + "service_name": "ntpd", }, - 'chrony': { - 'service_name': 'chronyd', + "chrony": { + "service_name": "chronyd", }, }, - 'sles': { - 'chrony': { - 'service_name': 'chronyd', + "sles": { + "chrony": { + "service_name": "chronyd", }, - 'ntp': { - 'confpath': '/etc/ntp.conf', - 'service_name': 'ntpd', + "ntp": { + "confpath": "/etc/ntp.conf", + "service_name": "ntpd", }, - 'systemd-timesyncd': { - 'check_exe': '/usr/lib/systemd/systemd-timesyncd', + "systemd-timesyncd": { + "check_exe": "/usr/lib/systemd/systemd-timesyncd", }, }, - 'ubuntu': { - 'chrony': { - 'confpath': '/etc/chrony/chrony.conf', + "ubuntu": { + "chrony": { + "confpath": "/etc/chrony/chrony.conf", }, }, } @@ -141,10 +149,11 @@ DISTRO_CLIENT_CONFIG = { # configuration. meta = { - 'id': 'cc_ntp', - 'name': 'NTP', - 'title': 'enable and configure ntp', - 'description': dedent("""\ + "id": "cc_ntp", + "name": "NTP", + "title": "enable and configure ntp", + "description": dedent( + """\ Handle ntp configuration. If ntp is not installed on the system and ntp configuration is specified, ntp will be installed. If there is a default ntp config file in the image or one is present in the @@ -152,16 +161,20 @@ meta = { appended to the filename before any changes are made. A list of ntp pools and ntp servers can be provided under the ``ntp`` config key. If no ntp ``servers`` or ``pools`` are provided, 4 pools will be used - in the format ``{0-3}.{distro}.pool.ntp.org``."""), - 'distros': distros, - 'examples': [ - dedent("""\ + in the format ``{0-3}.{distro}.pool.ntp.org``.""" + ), + "distros": distros, + "examples": [ + dedent( + """\ # Override ntp with chrony configuration on Ubuntu ntp: enabled: true ntp_client: chrony # Uses cloud-init default chrony configuration - """), - dedent("""\ + """ + ), + dedent( + """\ # Provide a custom ntp client configuration ntp: enabled: true @@ -188,120 +201,137 @@ meta = { servers: - ntp.server.local - ntp.ubuntu.com - - 192.168.23.2""")], - 'frequency': PER_INSTANCE, + - 192.168.23.2""" + ), + ], + "frequency": PER_INSTANCE, } schema = { - 'type': 'object', - 'properties': { - 'ntp': { - 'type': ['object', 'null'], - 'properties': { - 'pools': { - 'type': 'array', - 'items': { - 'type': 'string', - 'format': 'hostname' - }, - 'uniqueItems': True, - 'description': dedent("""\ + "type": "object", + "properties": { + "ntp": { + "type": ["object", "null"], + "properties": { + "pools": { + "type": "array", + "items": {"type": "string", "format": "hostname"}, + "uniqueItems": True, + "description": dedent( + """\ List of ntp pools. If both pools and servers are empty, 4 default pool servers will be provided of the format ``{0-3}.{distro}.pool.ntp.org``. NOTE: for Alpine Linux when using the Busybox NTP client this setting will be ignored due to the limited - functionality of Busybox's ntpd.""") + functionality of Busybox's ntpd.""" + ), }, - 'servers': { - 'type': 'array', - 'items': { - 'type': 'string', - 'format': 'hostname' - }, - 'uniqueItems': True, - 'description': dedent("""\ + "servers": { + "type": "array", + "items": {"type": "string", "format": "hostname"}, + "uniqueItems": True, + "description": dedent( + """\ List of ntp servers. If both pools and servers are empty, 4 default pool servers will be provided with - the format ``{0-3}.{distro}.pool.ntp.org``.""") + the format ``{0-3}.{distro}.pool.ntp.org``.""" + ), }, - 'ntp_client': { - 'type': 'string', - 'default': 'auto', - 'description': dedent("""\ + "ntp_client": { + "type": "string", + "default": "auto", + "description": dedent( + """\ Name of an NTP client to use to configure system NTP. When unprovided or 'auto' the default client preferred by the distribution will be used. The following built-in client names can be used to override existing configuration defaults: chrony, ntp, ntpdate, - systemd-timesyncd."""), + systemd-timesyncd.""" + ), }, - 'enabled': { - 'type': 'boolean', - 'default': True, - 'description': dedent("""\ + "enabled": { + "type": "boolean", + "default": True, + "description": dedent( + """\ Attempt to enable ntp clients if set to True. If set to False, ntp client will not be configured or - installed"""), + installed""" + ), }, - 'config': { - 'description': dedent("""\ + "config": { + "description": dedent( + """\ Configuration settings or overrides for the - ``ntp_client`` specified."""), - 'type': ['object'], - 'properties': { - 'confpath': { - 'type': 'string', - 'description': dedent("""\ + ``ntp_client`` specified.""" + ), + "type": ["object"], + "properties": { + "confpath": { + "type": "string", + "description": dedent( + """\ The path to where the ``ntp_client`` - configuration is written."""), + configuration is written.""" + ), }, - 'check_exe': { - 'type': 'string', - 'description': dedent("""\ + "check_exe": { + "type": "string", + "description": dedent( + """\ The executable name for the ``ntp_client``. For example, ntp service ``check_exe`` is - 'ntpd' because it runs the ntpd binary."""), + 'ntpd' because it runs the ntpd binary.""" + ), }, - 'packages': { - 'type': 'array', - 'items': { - 'type': 'string', + "packages": { + "type": "array", + "items": { + "type": "string", }, - 'uniqueItems': True, - 'description': dedent("""\ + "uniqueItems": True, + "description": dedent( + """\ List of packages needed to be installed for the - selected ``ntp_client``."""), + selected ``ntp_client``.""" + ), }, - 'service_name': { - 'type': 'string', - 'description': dedent("""\ + "service_name": { + "type": "string", + "description": dedent( + """\ The systemd or sysvinit service name used to start and stop the ``ntp_client`` - service."""), + service.""" + ), }, - 'template': { - 'type': 'string', - 'description': dedent("""\ + "template": { + "type": "string", + "description": dedent( + """\ Inline template allowing users to define their own ``ntp_client`` configuration template. The value must start with '## template:jinja' to enable use of templating support. - """), + """ + ), }, }, # Don't use REQUIRED_NTP_CONFIG_KEYS to allow for override # of builtin client values. - 'minProperties': 1, # If we have config, define something - 'additionalProperties': False + "minProperties": 1, # If we have config, define something + "additionalProperties": False, }, }, - 'additionalProperties': False + "additionalProperties": False, } - } + }, } -REQUIRED_NTP_CONFIG_KEYS = frozenset([ - 'check_exe', 'confpath', 'packages', 'service_name']) +REQUIRED_NTP_CONFIG_KEYS = frozenset( + ["check_exe", "confpath", "packages", "service_name"] +) __doc__ = get_meta_doc(meta, schema) # Supplement python help() @@ -334,21 +364,23 @@ def select_ntp_client(ntp_client, distro): distro_cfg = distro_ntp_client_configs(distro.name) # user specified client, return its config - if ntp_client and ntp_client != 'auto': - LOG.debug('Selected NTP client "%s" via user-data configuration', - ntp_client) + if ntp_client and ntp_client != "auto": + LOG.debug( + 'Selected NTP client "%s" via user-data configuration', ntp_client + ) return distro_cfg.get(ntp_client, {}) # default to auto if unset in distro - distro_ntp_client = distro.get_option('ntp_client', 'auto') + distro_ntp_client = distro.get_option("ntp_client", "auto") clientcfg = {} if distro_ntp_client == "auto": for client in distro.preferred_ntp_clients: cfg = distro_cfg.get(client) - if subp.which(cfg.get('check_exe')): - LOG.debug('Selected NTP client "%s", already installed', - client) + if subp.which(cfg.get("check_exe")): + LOG.debug( + 'Selected NTP client "%s", already installed', client + ) clientcfg = cfg break @@ -356,11 +388,14 @@ def select_ntp_client(ntp_client, distro): client = distro.preferred_ntp_clients[0] LOG.debug( 'Selected distro preferred NTP client "%s", not yet installed', - client) + client, + ) clientcfg = distro_cfg.get(client) else: - LOG.debug('Selected NTP client "%s" via distro system config', - distro_ntp_client) + LOG.debug( + 'Selected NTP client "%s" via distro system config', + distro_ntp_client, + ) clientcfg = distro_cfg.get(distro_ntp_client, {}) return clientcfg @@ -378,7 +413,7 @@ def install_ntp_client(install_func, packages=None, check_exe="ntpd"): if subp.which(check_exe): return if packages is None: - packages = ['ntp'] + packages = ["ntp"] install_func(packages) @@ -403,25 +438,34 @@ def generate_server_names(distro): names = [] pool_distro = distro - if distro == 'sles': + if distro == "sles": # For legal reasons x.pool.sles.ntp.org does not exist, # use the opensuse pool - pool_distro = 'opensuse' - elif distro == 'alpine' or distro == 'eurolinux': + pool_distro = "opensuse" + elif distro == "alpine" or distro == "eurolinux": # Alpine-specific pool (i.e. x.alpine.pool.ntp.org) does not exist # so use general x.pool.ntp.org instead. The same applies to EuroLinux - pool_distro = '' + pool_distro = "" for x in range(0, NR_POOL_SERVERS): - names.append(".".join( - [n for n in [str(x)] + [pool_distro] + ['pool.ntp.org'] if n])) + names.append( + ".".join( + [n for n in [str(x)] + [pool_distro] + ["pool.ntp.org"] if n] + ) + ) return names -def write_ntp_config_template(distro_name, service_name=None, servers=None, - pools=None, path=None, template_fn=None, - template=None): +def write_ntp_config_template( + distro_name, + service_name=None, + servers=None, + pools=None, + path=None, + template_fn=None, + template=None, +): """Render a ntp client configuration for the specified client. @param distro_name: string. The distro class name. @@ -444,27 +488,30 @@ def write_ntp_config_template(distro_name, service_name=None, servers=None, if not pools: pools = [] - if (len(servers) == 0 and distro_name == 'alpine' and - service_name == 'ntpd'): + if ( + len(servers) == 0 + and distro_name == "alpine" + and service_name == "ntpd" + ): # Alpine's Busybox ntpd only understands "servers" configuration # and not "pool" configuration. servers = generate_server_names(distro_name) - LOG.debug( - 'Adding distro default ntp servers: %s', ','.join(servers)) + LOG.debug("Adding distro default ntp servers: %s", ",".join(servers)) elif len(servers) == 0 and len(pools) == 0: pools = generate_server_names(distro_name) LOG.debug( - 'Adding distro default ntp pool servers: %s', ','.join(pools)) + "Adding distro default ntp pool servers: %s", ",".join(pools) + ) if not path: - raise ValueError('Invalid value for path parameter') + raise ValueError("Invalid value for path parameter") if not template_fn and not template: - raise ValueError('Not template_fn or template provided') + raise ValueError("Not template_fn or template provided") - params = {'servers': servers, 'pools': pools} + params = {"servers": servers, "pools": pools} if template: - tfile = temp_utils.mkstemp(prefix='template_name-', suffix=".tmpl") + tfile = temp_utils.mkstemp(prefix="template_name-", suffix=".tmpl") template_fn = tfile[1] # filepath is second item in tuple util.write_file(template_fn, content=template) @@ -487,50 +534,62 @@ def supplemental_schema_validation(ntp_config): errors = [] missing = REQUIRED_NTP_CONFIG_KEYS.difference(set(ntp_config.keys())) if missing: - keys = ', '.join(sorted(missing)) + keys = ", ".join(sorted(missing)) errors.append( - 'Missing required ntp:config keys: {keys}'.format(keys=keys)) - elif not any([ntp_config.get('template'), - ntp_config.get('template_name')]): + "Missing required ntp:config keys: {keys}".format(keys=keys) + ) + elif not any( + [ntp_config.get("template"), ntp_config.get("template_name")] + ): errors.append( - 'Either ntp:config:template or ntp:config:template_name values' - ' are required') + "Either ntp:config:template or ntp:config:template_name values" + " are required" + ) for key, value in sorted(ntp_config.items()): - keypath = 'ntp:config:' + key - if key == 'confpath': + keypath = "ntp:config:" + key + if key == "confpath": if not all([value, isinstance(value, str)]): errors.append( - 'Expected a config file path {keypath}.' - ' Found ({value})'.format(keypath=keypath, value=value)) - elif key == 'packages': + "Expected a config file path {keypath}." + " Found ({value})".format(keypath=keypath, value=value) + ) + elif key == "packages": if not isinstance(value, list): errors.append( - 'Expected a list of required package names for {keypath}.' - ' Found ({value})'.format(keypath=keypath, value=value)) - elif key in ('template', 'template_name'): + "Expected a list of required package names for {keypath}." + " Found ({value})".format(keypath=keypath, value=value) + ) + elif key in ("template", "template_name"): if value is None: # Either template or template_name can be none continue if not isinstance(value, str): errors.append( - 'Expected a string type for {keypath}.' - ' Found ({value})'.format(keypath=keypath, value=value)) + "Expected a string type for {keypath}." + " Found ({value})".format(keypath=keypath, value=value) + ) elif not isinstance(value, str): errors.append( - 'Expected a string type for {keypath}.' - ' Found ({value})'.format(keypath=keypath, value=value)) + "Expected a string type for {keypath}. Found ({value})".format( + keypath=keypath, value=value + ) + ) if errors: - raise ValueError(r'Invalid ntp configuration:\n{errors}'.format( - errors='\n'.join(errors))) + raise ValueError( + r"Invalid ntp configuration:\n{errors}".format( + errors="\n".join(errors) + ) + ) def handle(name, cfg, cloud, log, _args): """Enable and configure ntp.""" - if 'ntp' not in cfg: + if "ntp" not in cfg: LOG.debug( - "Skipping module named %s, not present or disabled by cfg", name) + "Skipping module named %s, not present or disabled by cfg", name + ) return - ntp_cfg = cfg['ntp'] + ntp_cfg = cfg["ntp"] if ntp_cfg is None: ntp_cfg = {} # Allow empty config which will install the package @@ -538,52 +597,61 @@ def handle(name, cfg, cloud, log, _args): if not isinstance(ntp_cfg, (dict)): raise RuntimeError( "'ntp' key existed in config, but not a dictionary type," - " is a {_type} instead".format(_type=type_utils.obj_name(ntp_cfg))) + " is a {_type} instead".format(_type=type_utils.obj_name(ntp_cfg)) + ) validate_cloudconfig_schema(cfg, schema) # Allow users to explicitly enable/disable - enabled = ntp_cfg.get('enabled', True) + enabled = ntp_cfg.get("enabled", True) if util.is_false(enabled): LOG.debug("Skipping module named %s, disabled by cfg", name) return # Select which client is going to be used and get the configuration - ntp_client_config = select_ntp_client(ntp_cfg.get('ntp_client'), - cloud.distro) + ntp_client_config = select_ntp_client( + ntp_cfg.get("ntp_client"), cloud.distro + ) # Allow user ntp config to override distro configurations ntp_client_config = util.mergemanydict( - [ntp_client_config, ntp_cfg.get('config', {})], reverse=True) + [ntp_client_config, ntp_cfg.get("config", {})], reverse=True + ) supplemental_schema_validation(ntp_client_config) - rename_ntp_conf(confpath=ntp_client_config.get('confpath')) + rename_ntp_conf(confpath=ntp_client_config.get("confpath")) template_fn = None - if not ntp_client_config.get('template'): - template_name = ( - ntp_client_config.get('template_name').replace('{distro}', - cloud.distro.name)) + if not ntp_client_config.get("template"): + template_name = ntp_client_config.get("template_name").replace( + "{distro}", cloud.distro.name + ) template_fn = cloud.get_template_filename(template_name) if not template_fn: - msg = ('No template found, not rendering %s' % - ntp_client_config.get('template_name')) + msg = ( + "No template found, not rendering %s" + % ntp_client_config.get("template_name") + ) raise RuntimeError(msg) - write_ntp_config_template(cloud.distro.name, - service_name=ntp_client_config.get( - 'service_name'), - servers=ntp_cfg.get('servers', []), - pools=ntp_cfg.get('pools', []), - path=ntp_client_config.get('confpath'), - template_fn=template_fn, - template=ntp_client_config.get('template')) - - install_ntp_client(cloud.distro.install_packages, - packages=ntp_client_config['packages'], - check_exe=ntp_client_config['check_exe']) + write_ntp_config_template( + cloud.distro.name, + service_name=ntp_client_config.get("service_name"), + servers=ntp_cfg.get("servers", []), + pools=ntp_cfg.get("pools", []), + path=ntp_client_config.get("confpath"), + template_fn=template_fn, + template=ntp_client_config.get("template"), + ) + + install_ntp_client( + cloud.distro.install_packages, + packages=ntp_client_config["packages"], + check_exe=ntp_client_config["check_exe"], + ) try: - cloud.distro.manage_service('reload', - ntp_client_config.get('service_name')) + cloud.distro.manage_service( + "reload", ntp_client_config.get("service_name") + ) except subp.ProcessExecutionError as e: LOG.exception("Failed to reload/start ntp service: %s", e) raise diff --git a/cloudinit/config/cc_package_update_upgrade_install.py b/cloudinit/config/cc_package_update_upgrade_install.py index 036baf85..14cdfab8 100644 --- a/cloudinit/config/cc_package_update_upgrade_install.py +++ b/cloudinit/config/cc_package_update_upgrade_install.py @@ -43,8 +43,7 @@ import os import time from cloudinit import log as logging -from cloudinit import subp -from cloudinit import util +from cloudinit import subp, util REBOOT_FILE = "/var/run/reboot-required" REBOOT_CMD = ["/sbin/reboot"] @@ -68,17 +67,19 @@ def _fire_reboot(log, wait_attempts=6, initial_sleep=1, backoff=2): log.debug("Rebooted, but still running after %s seconds", int(elapsed)) # If we got here, not good elapsed = time.time() - start - raise RuntimeError(("Reboot did not happen" - " after %s seconds!") % (int(elapsed))) + raise RuntimeError( + "Reboot did not happen after %s seconds!" % (int(elapsed)) + ) def handle(_name, cfg, cloud, log, _args): # Handle the old style + new config names - update = _multi_cfg_bool_get(cfg, 'apt_update', 'package_update') - upgrade = _multi_cfg_bool_get(cfg, 'package_upgrade', 'apt_upgrade') - reboot_if_required = _multi_cfg_bool_get(cfg, 'apt_reboot_if_required', - 'package_reboot_if_required') - pkglist = util.get_cfg_option_list(cfg, 'packages', []) + update = _multi_cfg_bool_get(cfg, "apt_update", "package_update") + upgrade = _multi_cfg_bool_get(cfg, "package_upgrade", "apt_upgrade") + reboot_if_required = _multi_cfg_bool_get( + cfg, "apt_reboot_if_required", "package_reboot_if_required" + ) + pkglist = util.get_cfg_option_list(cfg, "packages", []) errors = [] if update or len(pkglist) or upgrade: @@ -109,8 +110,9 @@ def handle(_name, cfg, cloud, log, _args): reboot_fn_exists = os.path.isfile(REBOOT_FILE) if (upgrade or pkglist) and reboot_if_required and reboot_fn_exists: try: - log.warning("Rebooting after upgrade or install per " - "%s", REBOOT_FILE) + log.warning( + "Rebooting after upgrade or install per %s", REBOOT_FILE + ) # Flush the above warning + anything else out... logging.flushLoggers(log) _fire_reboot(log) @@ -119,8 +121,10 @@ def handle(_name, cfg, cloud, log, _args): errors.append(e) if len(errors): - log.warning("%s failed with exceptions, re-raising the last one", - len(errors)) + log.warning( + "%s failed with exceptions, re-raising the last one", len(errors) + ) raise errors[-1] + # vi: ts=4 expandtab diff --git a/cloudinit/config/cc_phone_home.py b/cloudinit/config/cc_phone_home.py index 733c3910..cc1fe53e 100644 --- a/cloudinit/config/cc_phone_home.py +++ b/cloudinit/config/cc_phone_home.py @@ -41,22 +41,19 @@ keys to post. Available keys are: tries: 10 """ -from cloudinit import templater -from cloudinit import url_helper -from cloudinit import util - +from cloudinit import templater, url_helper, util from cloudinit.settings import PER_INSTANCE frequency = PER_INSTANCE POST_LIST_ALL = [ - 'pub_key_dsa', - 'pub_key_rsa', - 'pub_key_ecdsa', - 'pub_key_ed25519', - 'instance_id', - 'hostname', - 'fqdn' + "pub_key_dsa", + "pub_key_rsa", + "pub_key_ecdsa", + "pub_key_ed25519", + "instance_id", + "hostname", + "fqdn", ] @@ -74,48 +71,58 @@ def handle(name, cfg, cloud, log, args): if len(args) != 0: ph_cfg = util.read_conf(args[0]) else: - if 'phone_home' not in cfg: - log.debug(("Skipping module named %s, " - "no 'phone_home' configuration found"), name) + if "phone_home" not in cfg: + log.debug( + "Skipping module named %s, " + "no 'phone_home' configuration found", + name, + ) return - ph_cfg = cfg['phone_home'] - - if 'url' not in ph_cfg: - log.warning(("Skipping module named %s, " - "no 'url' found in 'phone_home' configuration"), name) + ph_cfg = cfg["phone_home"] + + if "url" not in ph_cfg: + log.warning( + "Skipping module named %s, " + "no 'url' found in 'phone_home' configuration", + name, + ) return - url = ph_cfg['url'] - post_list = ph_cfg.get('post', 'all') - tries = ph_cfg.get('tries') + url = ph_cfg["url"] + post_list = ph_cfg.get("post", "all") + tries = ph_cfg.get("tries") try: tries = int(tries) except Exception: tries = 10 - util.logexc(log, "Configuration entry 'tries' is not an integer, " - "using %s instead", tries) + util.logexc( + log, + "Configuration entry 'tries' is not an integer, using %s instead", + tries, + ) if post_list == "all": post_list = POST_LIST_ALL all_keys = {} - all_keys['instance_id'] = cloud.get_instance_id() - all_keys['hostname'] = cloud.get_hostname() - all_keys['fqdn'] = cloud.get_hostname(fqdn=True) + all_keys["instance_id"] = cloud.get_instance_id() + all_keys["hostname"] = cloud.get_hostname() + all_keys["fqdn"] = cloud.get_hostname(fqdn=True) pubkeys = { - 'pub_key_dsa': '/etc/ssh/ssh_host_dsa_key.pub', - 'pub_key_rsa': '/etc/ssh/ssh_host_rsa_key.pub', - 'pub_key_ecdsa': '/etc/ssh/ssh_host_ecdsa_key.pub', - 'pub_key_ed25519': '/etc/ssh/ssh_host_ed25519_key.pub', + "pub_key_dsa": "/etc/ssh/ssh_host_dsa_key.pub", + "pub_key_rsa": "/etc/ssh/ssh_host_rsa_key.pub", + "pub_key_ecdsa": "/etc/ssh/ssh_host_ecdsa_key.pub", + "pub_key_ed25519": "/etc/ssh/ssh_host_ed25519_key.pub", } for (n, path) in pubkeys.items(): try: all_keys[n] = util.load_file(path) except Exception: - util.logexc(log, "%s: failed to open, can not phone home that " - "data!", path) + util.logexc( + log, "%s: failed to open, can not phone home that data!", path + ) submit_keys = {} for k in post_list: @@ -123,28 +130,37 @@ def handle(name, cfg, cloud, log, args): submit_keys[k] = all_keys[k] else: submit_keys[k] = None - log.warning(("Requested key %s from 'post'" - " configuration list not available"), k) + log.warning( + "Requested key %s from 'post'" + " configuration list not available", + k, + ) # Get them read to be posted real_submit_keys = {} for (k, v) in submit_keys.items(): if v is None: - real_submit_keys[k] = 'N/A' + real_submit_keys[k] = "N/A" else: real_submit_keys[k] = str(v) # Incase the url is parameterized url_params = { - 'INSTANCE_ID': all_keys['instance_id'], + "INSTANCE_ID": all_keys["instance_id"], } url = templater.render_string(url, url_params) try: url_helper.read_file_or_url( - url, data=real_submit_keys, retries=tries, sec_between=3, - ssl_details=util.fetch_ssl_details(cloud.paths)) + url, + data=real_submit_keys, + retries=tries, + sec_between=3, + ssl_details=util.fetch_ssl_details(cloud.paths), + ) except Exception: - util.logexc(log, "Failed to post phone home data to %s in %s tries", - url, tries) + util.logexc( + log, "Failed to post phone home data to %s in %s tries", url, tries + ) + # vi: ts=4 expandtab diff --git a/cloudinit/config/cc_power_state_change.py b/cloudinit/config/cc_power_state_change.py index 5780a7e9..d4eb68c0 100644 --- a/cloudinit/config/cc_power_state_change.py +++ b/cloudinit/config/cc_power_state_change.py @@ -58,9 +58,8 @@ import re import subprocess import time +from cloudinit import subp, util from cloudinit.settings import PER_INSTANCE -from cloudinit import subp -from cloudinit import util frequency = PER_INSTANCE @@ -75,9 +74,9 @@ def givecmdline(pid): # PID COMM ARGS # 1 init /bin/init -- if util.is_FreeBSD(): - (output, _err) = subp.subp(['procstat', '-c', str(pid)]) + (output, _err) = subp.subp(["procstat", "-c", str(pid)]) line = output.splitlines()[1] - m = re.search(r'\d+ (\w|\.|-)+\s+(/\w.+)', line) + m = re.search(r"\d+ (\w|\.|-)+\s+(/\w.+)", line) return m.group(2) else: return util.load_file("/proc/%s/cmdline" % pid) @@ -106,8 +105,9 @@ def check_condition(cond, log=None): return False else: if log: - log.warning(pre + "unexpected exit %s. " % ret + - "do not apply change.") + log.warning( + pre + "unexpected exit %s. " % ret + "do not apply change." + ) return False except Exception as e: if log: @@ -138,16 +138,24 @@ def handle(_name, cfg, cloud, log, _args): devnull_fp = open(os.devnull, "w") - log.debug("After pid %s ends, will execute: %s" % (mypid, ' '.join(args))) + log.debug("After pid %s ends, will execute: %s" % (mypid, " ".join(args))) - util.fork_cb(run_after_pid_gone, mypid, cmdline, timeout, log, - condition, execmd, [args, devnull_fp]) + util.fork_cb( + run_after_pid_gone, + mypid, + cmdline, + timeout, + log, + condition, + execmd, + [args, devnull_fp], + ) def load_power_state(cfg, distro): # returns a tuple of shutdown_command, timeout # shutdown_command is None if no config found - pstate = cfg.get('power_state') + pstate = cfg.get("power_state") if pstate is None: return (None, None, None) @@ -155,22 +163,25 @@ def load_power_state(cfg, distro): if not isinstance(pstate, dict): raise TypeError("power_state is not a dict.") - modes_ok = ['halt', 'poweroff', 'reboot'] + modes_ok = ["halt", "poweroff", "reboot"] mode = pstate.get("mode") if mode not in distro.shutdown_options_map: raise TypeError( - "power_state[mode] required, must be one of: %s. found: '%s'." % - (','.join(modes_ok), mode)) + "power_state[mode] required, must be one of: %s. found: '%s'." + % (",".join(modes_ok), mode) + ) - args = distro.shutdown_command(mode=mode, - delay=pstate.get("delay", "now"), - message=pstate.get("message")) + args = distro.shutdown_command( + mode=mode, + delay=pstate.get("delay", "now"), + message=pstate.get("message"), + ) try: - timeout = float(pstate.get('timeout', 30.0)) + timeout = float(pstate.get("timeout", 30.0)) except ValueError as e: raise ValueError( - "failed to convert timeout '%s' to float." % pstate['timeout'] + "failed to convert timeout '%s' to float." % pstate["timeout"] ) from e condition = pstate.get("condition", True) @@ -186,8 +197,12 @@ def doexit(sysexit): def execmd(exe_args, output=None, data_in=None): ret = 1 try: - proc = subprocess.Popen(exe_args, stdin=subprocess.PIPE, - stdout=output, stderr=subprocess.STDOUT) + proc = subprocess.Popen( + exe_args, + stdin=subprocess.PIPE, + stdout=output, + stderr=subprocess.STDOUT, + ) proc.communicate(data_in) ret = proc.returncode except Exception: @@ -230,7 +245,7 @@ def run_after_pid_gone(pid, pidcmdline, timeout, log, condition, func, args): except Exception as e: fatal("Unexpected Exception: %s" % e) - time.sleep(.25) + time.sleep(0.25) if not msg: fatal("Unexpected error in run_after_pid_gone") @@ -246,4 +261,5 @@ def run_after_pid_gone(pid, pidcmdline, timeout, log, condition, func, args): func(*args) + # vi: ts=4 expandtab diff --git a/cloudinit/config/cc_puppet.py b/cloudinit/config/cc_puppet.py index dc20fc44..f51f49bc 100644 --- a/cloudinit/config/cc_puppet.py +++ b/cloudinit/config/cc_puppet.py @@ -108,23 +108,20 @@ key (by default the agent will execute with the ``--test`` flag). import os import socket -import yaml from io import StringIO -from cloudinit import helpers -from cloudinit import subp -from cloudinit import temp_utils -from cloudinit import util -from cloudinit import url_helper +import yaml -AIO_INSTALL_URL = 'https://raw.githubusercontent.com/puppetlabs/install-puppet/main/install.sh' # noqa: E501 -PUPPET_AGENT_DEFAULT_ARGS = ['--test'] +from cloudinit import helpers, subp, temp_utils, url_helper, util +AIO_INSTALL_URL = "https://raw.githubusercontent.com/puppetlabs/install-puppet/main/install.sh" # noqa: E501 +PUPPET_AGENT_DEFAULT_ARGS = ["--test"] -class PuppetConstants(object): - def __init__(self, puppet_conf_file, puppet_ssl_dir, - csr_attributes_path, log): +class PuppetConstants(object): + def __init__( + self, puppet_conf_file, puppet_ssl_dir, csr_attributes_path, log + ): self.conf_path = puppet_conf_file self.ssl_dir = puppet_ssl_dir self.ssl_cert_dir = os.path.join(puppet_ssl_dir, "certs") @@ -134,18 +131,27 @@ class PuppetConstants(object): def _autostart_puppet(log): # Set puppet to automatically start - if os.path.exists('/etc/default/puppet'): - subp.subp(['sed', '-i', - '-e', 's/^START=.*/START=yes/', - '/etc/default/puppet'], capture=False) - elif os.path.exists('/bin/systemctl'): - subp.subp(['/bin/systemctl', 'enable', 'puppet.service'], - capture=False) - elif os.path.exists('/sbin/chkconfig'): - subp.subp(['/sbin/chkconfig', 'puppet', 'on'], capture=False) + if os.path.exists("/etc/default/puppet"): + subp.subp( + [ + "sed", + "-i", + "-e", + "s/^START=.*/START=yes/", + "/etc/default/puppet", + ], + capture=False, + ) + elif os.path.exists("/bin/systemctl"): + subp.subp( + ["/bin/systemctl", "enable", "puppet.service"], capture=False + ) + elif os.path.exists("/sbin/chkconfig"): + subp.subp(["/sbin/chkconfig", "puppet", "on"], capture=False) else: - log.warning(("Sorry we do not know how to enable" - " puppet services on this system")) + log.warning( + "Sorry we do not know how to enable puppet services on this system" + ) def get_config_value(puppet_bin, setting): @@ -153,12 +159,13 @@ def get_config_value(puppet_bin, setting): :param puppet_bin: path to puppet binary :param setting: setting to query """ - out, _ = subp.subp([puppet_bin, 'config', 'print', setting]) + out, _ = subp.subp([puppet_bin, "config", "print", setting]) return out.rstrip() -def install_puppet_aio(url=AIO_INSTALL_URL, version=None, - collection=None, cleanup=True): +def install_puppet_aio( + url=AIO_INSTALL_URL, version=None, collection=None, cleanup=True +): """Install puppet-agent from the puppetlabs repositories using the one-shot shell script @@ -169,62 +176,70 @@ def install_puppet_aio(url=AIO_INSTALL_URL, version=None, """ args = [] if version is not None: - args = ['-v', version] + args = ["-v", version] if collection is not None: - args += ['-c', collection] + args += ["-c", collection] # Purge puppetlabs repos after installation if cleanup: - args += ['--cleanup'] + args += ["--cleanup"] content = url_helper.readurl(url=url, retries=5).contents # Use tmpdir over tmpfile to avoid 'text file busy' on execute with temp_utils.tempdir(needs_exe=True) as tmpd: - tmpf = os.path.join(tmpd, 'puppet-install') + tmpf = os.path.join(tmpd, "puppet-install") util.write_file(tmpf, content, mode=0o700) return subp.subp([tmpf] + args, capture=False) def handle(name, cfg, cloud, log, _args): # If there isn't a puppet key in the configuration don't do anything - if 'puppet' not in cfg: - log.debug(("Skipping module named %s," - " no 'puppet' configuration found"), name) + if "puppet" not in cfg: + log.debug( + "Skipping module named %s, no 'puppet' configuration found", name + ) return - puppet_cfg = cfg['puppet'] + puppet_cfg = cfg["puppet"] # Start by installing the puppet package if necessary... - install = util.get_cfg_option_bool(puppet_cfg, 'install', True) - version = util.get_cfg_option_str(puppet_cfg, 'version', None) - collection = util.get_cfg_option_str(puppet_cfg, 'collection', None) + install = util.get_cfg_option_bool(puppet_cfg, "install", True) + version = util.get_cfg_option_str(puppet_cfg, "version", None) + collection = util.get_cfg_option_str(puppet_cfg, "collection", None) install_type = util.get_cfg_option_str( - puppet_cfg, 'install_type', 'packages') - cleanup = util.get_cfg_option_bool(puppet_cfg, 'cleanup', True) - run = util.get_cfg_option_bool(puppet_cfg, 'exec', default=False) - start_puppetd = util.get_cfg_option_bool(puppet_cfg, - 'start_service', - default=True) + puppet_cfg, "install_type", "packages" + ) + cleanup = util.get_cfg_option_bool(puppet_cfg, "cleanup", True) + run = util.get_cfg_option_bool(puppet_cfg, "exec", default=False) + start_puppetd = util.get_cfg_option_bool( + puppet_cfg, "start_service", default=True + ) aio_install_url = util.get_cfg_option_str( - puppet_cfg, 'aio_install_url', default=AIO_INSTALL_URL) + puppet_cfg, "aio_install_url", default=AIO_INSTALL_URL + ) # AIO and distro packages use different paths - if install_type == 'aio': - puppet_user = 'root' - puppet_bin = '/opt/puppetlabs/bin/puppet' - puppet_package = 'puppet-agent' + if install_type == "aio": + puppet_user = "root" + puppet_bin = "/opt/puppetlabs/bin/puppet" + puppet_package = "puppet-agent" else: # default to 'packages' - puppet_user = 'puppet' - puppet_bin = 'puppet' - puppet_package = 'puppet' + puppet_user = "puppet" + puppet_bin = "puppet" + puppet_package = "puppet" package_name = util.get_cfg_option_str( - puppet_cfg, 'package_name', puppet_package) + puppet_cfg, "package_name", puppet_package + ) if not install and version: - log.warning(("Puppet install set to false but version supplied," - " doing nothing.")) + log.warning( + "Puppet install set to false but version supplied, doing nothing." + ) elif install: - log.debug(("Attempting to install puppet %s from %s"), - version if version else 'latest', install_type) + log.debug( + "Attempting to install puppet %s from %s", + version if version else "latest", + install_type, + ) if install_type == "packages": cloud.distro.install_packages((package_name, version)) @@ -235,17 +250,21 @@ def handle(name, cfg, cloud, log, _args): run = False conf_file = util.get_cfg_option_str( - puppet_cfg, 'conf_file', get_config_value(puppet_bin, 'config')) + puppet_cfg, "conf_file", get_config_value(puppet_bin, "config") + ) ssl_dir = util.get_cfg_option_str( - puppet_cfg, 'ssl_dir', get_config_value(puppet_bin, 'ssldir')) + puppet_cfg, "ssl_dir", get_config_value(puppet_bin, "ssldir") + ) csr_attributes_path = util.get_cfg_option_str( - puppet_cfg, 'csr_attributes_path', - get_config_value(puppet_bin, 'csr_attributes')) + puppet_cfg, + "csr_attributes_path", + get_config_value(puppet_bin, "csr_attributes"), + ) p_constants = PuppetConstants(conf_file, ssl_dir, csr_attributes_path, log) # ... and then update the puppet configuration - if 'conf' in puppet_cfg: + if "conf" in puppet_cfg: # Add all sections from the conf object to puppet.conf contents = util.load_file(p_constants.conf_path) # Create object for reading puppet.conf values @@ -254,30 +273,31 @@ def handle(name, cfg, cloud, log, _args): # mix the rest up. First clean them up # (TODO(harlowja) is this really needed??) cleaned_lines = [i.lstrip() for i in contents.splitlines()] - cleaned_contents = '\n'.join(cleaned_lines) + cleaned_contents = "\n".join(cleaned_lines) # Move to puppet_config.read_file when dropping py2.7 puppet_config.read_file( - StringIO(cleaned_contents), - source=p_constants.conf_path) - for (cfg_name, cfg) in puppet_cfg['conf'].items(): + StringIO(cleaned_contents), source=p_constants.conf_path + ) + for (cfg_name, cfg) in puppet_cfg["conf"].items(): # Cert configuration is a special case # Dump the puppetserver ca certificate in the correct place - if cfg_name == 'ca_cert': + if cfg_name == "ca_cert": # Puppet ssl sub-directory isn't created yet # Create it with the proper permissions and ownership util.ensure_dir(p_constants.ssl_dir, 0o771) - util.chownbyname(p_constants.ssl_dir, puppet_user, 'root') + util.chownbyname(p_constants.ssl_dir, puppet_user, "root") util.ensure_dir(p_constants.ssl_cert_dir) - util.chownbyname(p_constants.ssl_cert_dir, puppet_user, 'root') + util.chownbyname(p_constants.ssl_cert_dir, puppet_user, "root") util.write_file(p_constants.ssl_cert_path, cfg) - util.chownbyname(p_constants.ssl_cert_path, - puppet_user, 'root') + util.chownbyname( + p_constants.ssl_cert_path, puppet_user, "root" + ) else: # Iterate through the config items, we'll use ConfigParser.set # to overwrite or create new items as needed for (o, v) in cfg.items(): - if o == 'certname': + if o == "certname": # Expand %f as the fqdn # TODO(harlowja) should this use the cloud fqdn?? v = v.replace("%f", socket.getfqdn()) @@ -288,14 +308,16 @@ def handle(name, cfg, cloud, log, _args): puppet_config.set(cfg_name, o, v) # We got all our config as wanted we'll rename # the previous puppet.conf and create our new one - util.rename(p_constants.conf_path, "%s.old" - % (p_constants.conf_path)) + util.rename( + p_constants.conf_path, "%s.old" % (p_constants.conf_path) + ) util.write_file(p_constants.conf_path, puppet_config.stringify()) - if 'csr_attributes' in puppet_cfg: - util.write_file(p_constants.csr_attributes_path, - yaml.dump(puppet_cfg['csr_attributes'], - default_flow_style=False)) + if "csr_attributes" in puppet_cfg: + util.write_file( + p_constants.csr_attributes_path, + yaml.dump(puppet_cfg["csr_attributes"], default_flow_style=False), + ) # Set it up so it autostarts if start_puppetd: @@ -303,18 +325,21 @@ def handle(name, cfg, cloud, log, _args): # Run the agent if needed if run: - log.debug('Running puppet-agent') - cmd = [puppet_bin, 'agent'] - if 'exec_args' in puppet_cfg: - cmd_args = puppet_cfg['exec_args'] + log.debug("Running puppet-agent") + cmd = [puppet_bin, "agent"] + if "exec_args" in puppet_cfg: + cmd_args = puppet_cfg["exec_args"] if isinstance(cmd_args, (list, tuple)): cmd.extend(cmd_args) elif isinstance(cmd_args, str): cmd.extend(cmd_args.split()) else: - log.warning("Unknown type %s provided for puppet" - " 'exec_args' expected list, tuple," - " or string", type(cmd_args)) + log.warning( + "Unknown type %s provided for puppet" + " 'exec_args' expected list, tuple," + " or string", + type(cmd_args), + ) cmd.extend(PUPPET_AGENT_DEFAULT_ARGS) else: cmd.extend(PUPPET_AGENT_DEFAULT_ARGS) @@ -322,6 +347,7 @@ def handle(name, cfg, cloud, log, _args): if start_puppetd: # Start puppetd - subp.subp(['service', 'puppet', 'start'], capture=False) + subp.subp(["service", "puppet", "start"], capture=False) + # vi: ts=4 expandtab diff --git a/cloudinit/config/cc_refresh_rmc_and_interface.py b/cloudinit/config/cc_refresh_rmc_and_interface.py index d5e0ecb2..87be5348 100644 --- a/cloudinit/config/cc_refresh_rmc_and_interface.py +++ b/cloudinit/config/cc_refresh_rmc_and_interface.py @@ -34,20 +34,18 @@ This module handles """ +import errno + from cloudinit import log as logging +from cloudinit import netinfo, subp, util from cloudinit.settings import PER_ALWAYS -from cloudinit import util -from cloudinit import subp -from cloudinit import netinfo - -import errno frequency = PER_ALWAYS LOG = logging.getLogger(__name__) # Ensure that /opt/rsct/bin has been added to standard PATH of the # distro. The symlink to rmcctrl is /usr/sbin/rsct/bin/rmcctrl . -RMCCTRL = 'rmcctrl' +RMCCTRL = "rmcctrl" def handle(name, _cfg, _cloud, _log, _args): @@ -56,10 +54,11 @@ def handle(name, _cfg, _cloud, _log, _args): return LOG.debug( - 'Making the IPv6 up explicitly. ' - 'Ensuring IPv6 interface is not being handled by NetworkManager ' - 'and it is restarted to re-establish the communication with ' - 'the hypervisor') + "Making the IPv6 up explicitly. " + "Ensuring IPv6 interface is not being handled by NetworkManager " + "and it is restarted to re-establish the communication with " + "the hypervisor" + ) ifaces = find_ipv6_ifaces() @@ -80,7 +79,7 @@ def find_ipv6_ifaces(): ifaces = [] for iface, data in info.items(): if iface == "lo": - LOG.debug('Skipping localhost interface') + LOG.debug("Skipping localhost interface") if len(data.get("ipv4", [])) != 0: # skip this interface, as it has ipv4 addrs continue @@ -92,16 +91,16 @@ def refresh_ipv6(interface): # IPv6 interface is explicitly brought up, subsequent to which the # RMC services are restarted to re-establish the communication with # the hypervisor. - subp.subp(['ip', 'link', 'set', interface, 'down']) - subp.subp(['ip', 'link', 'set', interface, 'up']) + subp.subp(["ip", "link", "set", interface, "down"]) + subp.subp(["ip", "link", "set", interface, "up"]) def sysconfig_path(iface): - return '/etc/sysconfig/network-scripts/ifcfg-' + iface + return "/etc/sysconfig/network-scripts/ifcfg-" + iface def restart_network_manager(): - subp.subp(['systemctl', 'restart', 'NetworkManager']) + subp.subp(["systemctl", "restart", "NetworkManager"]) def disable_ipv6(iface_file): @@ -113,12 +112,11 @@ def disable_ipv6(iface_file): contents = util.load_file(iface_file) except IOError as e: if e.errno == errno.ENOENT: - LOG.debug("IPv6 interface file %s does not exist\n", - iface_file) + LOG.debug("IPv6 interface file %s does not exist\n", iface_file) else: raise e - if 'IPV6INIT' not in contents: + if "IPV6INIT" not in contents: LOG.debug("Interface file %s did not have IPV6INIT", iface_file) return @@ -135,11 +133,12 @@ def disable_ipv6(iface_file): def search(contents): # Search for any NM_CONTROLLED or IPV6 lines in IPv6 interface file. - return( - contents.startswith("IPV6ADDR") or - contents.startswith("IPADDR6") or - contents.startswith("IPV6INIT") or - contents.startswith("NM_CONTROLLED")) + return ( + contents.startswith("IPV6ADDR") + or contents.startswith("IPADDR6") + or contents.startswith("IPV6INIT") + or contents.startswith("NM_CONTROLLED") + ) def refresh_rmc(): @@ -152,8 +151,8 @@ def refresh_rmc(): # until the subsystem and all resource managers are stopped. # -s : start Resource Monitoring & Control subsystem. try: - subp.subp([RMCCTRL, '-z']) - subp.subp([RMCCTRL, '-s']) + subp.subp([RMCCTRL, "-z"]) + subp.subp([RMCCTRL, "-s"]) except Exception: - util.logexc(LOG, 'Failed to refresh the RMC subsystem.') + util.logexc(LOG, "Failed to refresh the RMC subsystem.") raise diff --git a/cloudinit/config/cc_reset_rmc.py b/cloudinit/config/cc_reset_rmc.py index 1cd72774..3b929903 100644 --- a/cloudinit/config/cc_reset_rmc.py +++ b/cloudinit/config/cc_reset_rmc.py @@ -39,9 +39,8 @@ Prerequisite of using this module is to install RSCT packages. import os from cloudinit import log as logging +from cloudinit import subp, util from cloudinit.settings import PER_INSTANCE -from cloudinit import util -from cloudinit import subp frequency = PER_INSTANCE @@ -49,34 +48,34 @@ frequency = PER_INSTANCE # The symlink for RMCCTRL and RECFGCT are # /usr/sbin/rsct/bin/rmcctrl and # /usr/sbin/rsct/install/bin/recfgct respectively. -RSCT_PATH = '/opt/rsct/install/bin' -RMCCTRL = 'rmcctrl' -RECFGCT = 'recfgct' +RSCT_PATH = "/opt/rsct/install/bin" +RMCCTRL = "rmcctrl" +RECFGCT = "recfgct" LOG = logging.getLogger(__name__) -NODE_ID_FILE = '/etc/ct_node_id' +NODE_ID_FILE = "/etc/ct_node_id" def handle(name, _cfg, cloud, _log, _args): # Ensuring node id has to be generated only once during first boot - if cloud.datasource.platform_type == 'none': - LOG.debug('Skipping creation of new ct_node_id node') + if cloud.datasource.platform_type == "none": + LOG.debug("Skipping creation of new ct_node_id node") return if not os.path.isdir(RSCT_PATH): LOG.debug("module disabled, RSCT_PATH not present") return - orig_path = os.environ.get('PATH') + orig_path = os.environ.get("PATH") try: add_path(orig_path) reset_rmc() finally: if orig_path: - os.environ['PATH'] = orig_path + os.environ["PATH"] = orig_path else: - del os.environ['PATH'] + del os.environ["PATH"] def reconfigure_rsct_subsystems(): @@ -88,17 +87,17 @@ def reconfigure_rsct_subsystems(): LOG.debug(out.strip()) return out except subp.ProcessExecutionError: - util.logexc(LOG, 'Failed to reconfigure the RSCT subsystems.') + util.logexc(LOG, "Failed to reconfigure the RSCT subsystems.") raise def get_node_id(): try: fp = util.load_file(NODE_ID_FILE) - node_id = fp.split('\n')[0] + node_id = fp.split("\n")[0] return node_id except Exception: - util.logexc(LOG, 'Failed to get node ID from file %s.' % NODE_ID_FILE) + util.logexc(LOG, "Failed to get node ID from file %s." % NODE_ID_FILE) raise @@ -107,25 +106,25 @@ def add_path(orig_path): # So thet cloud init automatically find and # run RECFGCT to create new node_id. suff = ":" + orig_path if orig_path else "" - os.environ['PATH'] = RSCT_PATH + suff - return os.environ['PATH'] + os.environ["PATH"] = RSCT_PATH + suff + return os.environ["PATH"] def rmcctrl(): # Stop the RMC subsystem and all resource managers so that we can make # some changes to it try: - return subp.subp([RMCCTRL, '-z']) + return subp.subp([RMCCTRL, "-z"]) except Exception: - util.logexc(LOG, 'Failed to stop the RMC subsystem.') + util.logexc(LOG, "Failed to stop the RMC subsystem.") raise def reset_rmc(): - LOG.debug('Attempting to reset RMC.') + LOG.debug("Attempting to reset RMC.") node_id_before = get_node_id() - LOG.debug('Node ID at beginning of module: %s', node_id_before) + LOG.debug("Node ID at beginning of module: %s", node_id_before) # Stop the RMC subsystem and all resource managers so that we can make # some changes to it @@ -133,11 +132,11 @@ def reset_rmc(): reconfigure_rsct_subsystems() node_id_after = get_node_id() - LOG.debug('Node ID at end of module: %s', node_id_after) + LOG.debug("Node ID at end of module: %s", node_id_after) # Check if new node ID is generated or not # by comparing old and new node ID if node_id_after == node_id_before: - msg = 'New node ID did not get generated.' + msg = "New node ID did not get generated." LOG.error(msg) raise Exception(msg) diff --git a/cloudinit/config/cc_resizefs.py b/cloudinit/config/cc_resizefs.py index 00bb7ae7..b009c392 100644 --- a/cloudinit/config/cc_resizefs.py +++ b/cloudinit/config/cc_resizefs.py @@ -13,21 +13,21 @@ import os import stat from textwrap import dedent +from cloudinit import subp, util from cloudinit.config.schema import get_meta_doc, validate_cloudconfig_schema from cloudinit.settings import PER_ALWAYS -from cloudinit import subp -from cloudinit import util NOBLOCK = "noblock" frequency = PER_ALWAYS -distros = ['all'] +distros = ["all"] meta = { - 'id': 'cc_resizefs', - 'name': 'Resizefs', - 'title': 'Resize filesystem', - 'description': dedent("""\ + "id": "cc_resizefs", + "name": "Resizefs", + "title": "Resize filesystem", + "description": dedent( + """\ Resize a filesystem to use all avaliable space on partition. This module is useful along with ``cc_growpart`` and will ensure that if the root partition has been resized the root filesystem will be resized @@ -36,22 +36,26 @@ meta = { running. Optionally, the resize operation can be performed in the background while cloud-init continues running modules. This can be enabled by setting ``resize_rootfs`` to ``true``. This module can be - disabled altogether by setting ``resize_rootfs`` to ``false``."""), - 'distros': distros, - 'examples': [ - 'resize_rootfs: false # disable root filesystem resize operation'], - 'frequency': PER_ALWAYS, + disabled altogether by setting ``resize_rootfs`` to ``false``.""" + ), + "distros": distros, + "examples": [ + "resize_rootfs: false # disable root filesystem resize operation" + ], + "frequency": PER_ALWAYS, } schema = { - 'type': 'object', - 'properties': { - 'resize_rootfs': { - 'enum': [True, False, NOBLOCK], - 'description': dedent("""\ - Whether to resize the root partition. Default: 'true'""") + "type": "object", + "properties": { + "resize_rootfs": { + "enum": [True, False, NOBLOCK], + "description": dedent( + """\ + Whether to resize the root partition. Default: 'true'""" + ), } - } + }, } __doc__ = get_meta_doc(meta, schema) # Supplement python help() @@ -63,32 +67,38 @@ def _resize_btrfs(mount_point, devpth): # Use a subvolume that is not ro to trick the resize operation to do the # "right" thing. The use of ".snapshot" is specific to "snapper" a generic # solution would be walk the subvolumes and find a rw mounted subvolume. - if (not util.mount_is_read_write(mount_point) and - os.path.isdir("%s/.snapshots" % mount_point)): - return ('btrfs', 'filesystem', 'resize', 'max', - '%s/.snapshots' % mount_point) + if not util.mount_is_read_write(mount_point) and os.path.isdir( + "%s/.snapshots" % mount_point + ): + return ( + "btrfs", + "filesystem", + "resize", + "max", + "%s/.snapshots" % mount_point, + ) else: - return ('btrfs', 'filesystem', 'resize', 'max', mount_point) + return ("btrfs", "filesystem", "resize", "max", mount_point) def _resize_ext(mount_point, devpth): - return ('resize2fs', devpth) + return ("resize2fs", devpth) def _resize_xfs(mount_point, devpth): - return ('xfs_growfs', mount_point) + return ("xfs_growfs", mount_point) def _resize_ufs(mount_point, devpth): - return ('growfs', '-y', mount_point) + return ("growfs", "-y", mount_point) def _resize_zfs(mount_point, devpth): - return ('zpool', 'online', '-e', mount_point, devpth) + return ("zpool", "online", "-e", mount_point, devpth) def _resize_hammer2(mount_point, devpth): - return ('hammer2', 'growfs', mount_point) + return ("hammer2", "growfs", mount_point) def _can_skip_resize_ufs(mount_point, devpth): @@ -100,7 +110,7 @@ def _can_skip_resize_ufs(mount_point, devpth): # growfs exits with 1 for almost all cases up to this one. # This means we can't just use rcs=[0, 1] as subp parameter: try: - subp.subp(['growfs', '-N', devpth]) + subp.subp(["growfs", "-N", devpth]) except subp.ProcessExecutionError as e: if e.stderr.startswith(skip_start) and skip_contain in e.stderr: # This FS is already at the desired size @@ -114,17 +124,15 @@ def _can_skip_resize_ufs(mount_point, devpth): # for multiple filesystem types if possible, e.g. one command for # ext2, ext3 and ext4. RESIZE_FS_PREFIXES_CMDS = [ - ('btrfs', _resize_btrfs), - ('ext', _resize_ext), - ('xfs', _resize_xfs), - ('ufs', _resize_ufs), - ('zfs', _resize_zfs), - ('hammer2', _resize_hammer2), + ("btrfs", _resize_btrfs), + ("ext", _resize_ext), + ("xfs", _resize_xfs), + ("ufs", _resize_ufs), + ("zfs", _resize_zfs), + ("hammer2", _resize_hammer2), ] -RESIZE_FS_PRECHECK_CMDS = { - 'ufs': _can_skip_resize_ufs -} +RESIZE_FS_PRECHECK_CMDS = {"ufs": _can_skip_resize_ufs} def can_skip_resize(fs_type, resize_what, devpth): @@ -148,52 +156,66 @@ def maybe_get_writable_device_path(devpath, info, log): container = util.is_container() # Ensure the path is a block device. - if (devpath == "/dev/root" and not os.path.exists(devpath) and - not container): + if ( + devpath == "/dev/root" + and not os.path.exists(devpath) + and not container + ): devpath = util.rootdev_from_cmdline(util.get_cmdline()) if devpath is None: log.warning("Unable to find device '/dev/root'") return None log.debug("Converted /dev/root to '%s' per kernel cmdline", devpath) - if devpath == 'overlayroot': + if devpath == "overlayroot": log.debug("Not attempting to resize devpath '%s': %s", devpath, info) return None # FreeBSD zpool can also just use gpt/<label> # with that in mind we can not do an os.stat on "gpt/whatever" # therefore return the devpath already here. - if devpath.startswith('gpt/'): - log.debug('We have a gpt label - just go ahead') + if devpath.startswith("gpt/"): + log.debug("We have a gpt label - just go ahead") return devpath # Alternatively, our device could simply be a name as returned by gpart, # such as da0p3 - if not devpath.startswith('/dev/') and not os.path.exists(devpath): - fulldevpath = '/dev/' + devpath.lstrip('/') - log.debug("'%s' doesn't appear to be a valid device path. Trying '%s'", - devpath, fulldevpath) + if not devpath.startswith("/dev/") and not os.path.exists(devpath): + fulldevpath = "/dev/" + devpath.lstrip("/") + log.debug( + "'%s' doesn't appear to be a valid device path. Trying '%s'", + devpath, + fulldevpath, + ) devpath = fulldevpath try: statret = os.stat(devpath) except OSError as exc: if container and exc.errno == errno.ENOENT: - log.debug("Device '%s' did not exist in container. " - "cannot resize: %s", devpath, info) + log.debug( + "Device '%s' did not exist in container. cannot resize: %s", + devpath, + info, + ) elif exc.errno == errno.ENOENT: - log.warning("Device '%s' did not exist. cannot resize: %s", - devpath, info) + log.warning( + "Device '%s' did not exist. cannot resize: %s", devpath, info + ) else: raise exc return None if not stat.S_ISBLK(statret.st_mode) and not stat.S_ISCHR(statret.st_mode): if container: - log.debug("device '%s' not a block device in container." - " cannot resize: %s" % (devpath, info)) + log.debug( + "device '%s' not a block device in container." + " cannot resize: %s" % (devpath, info) + ) else: - log.warning("device '%s' not a block device. cannot resize: %s" % - (devpath, info)) + log.warning( + "device '%s' not a block device. cannot resize: %s" + % (devpath, info) + ) return None return devpath # The writable block devpath @@ -222,8 +244,8 @@ def handle(name, cfg, _cloud, log, args): # we will have to get the zpool name out of this # and set the resize_what variable to the zpool # so the _resize_zfs function gets the right attribute. - if fs_type == 'zfs': - zpool = devpth.split('/')[0] + if fs_type == "zfs": + zpool = devpth.split("/")[0] devpth = util.get_device_info_from_zpool(zpool) if not devpth: return # could not find device from zpool @@ -238,8 +260,9 @@ def handle(name, cfg, _cloud, log, args): resizer = None if can_skip_resize(fs_type, resize_what, devpth): - log.debug("Skip resize filesystem type %s for %s", - fs_type, resize_what) + log.debug( + "Skip resize filesystem type %s for %s", fs_type, resize_what + ) return fstype_lc = fs_type.lower() @@ -249,29 +272,42 @@ def handle(name, cfg, _cloud, log, args): break if not resizer: - log.warning("Not resizing unknown filesystem type %s for %s", - fs_type, resize_what) + log.warning( + "Not resizing unknown filesystem type %s for %s", + fs_type, + resize_what, + ) return resize_cmd = resizer(resize_what, devpth) - log.debug("Resizing %s (%s) using %s", resize_what, fs_type, - ' '.join(resize_cmd)) + log.debug( + "Resizing %s (%s) using %s", resize_what, fs_type, " ".join(resize_cmd) + ) if resize_root == NOBLOCK: # Fork to a child that will run # the resize command util.fork_cb( - util.log_time, logfunc=log.debug, msg="backgrounded Resizing", - func=do_resize, args=(resize_cmd, log)) + util.log_time, + logfunc=log.debug, + msg="backgrounded Resizing", + func=do_resize, + args=(resize_cmd, log), + ) else: - util.log_time(logfunc=log.debug, msg="Resizing", - func=do_resize, args=(resize_cmd, log)) - - action = 'Resized' + util.log_time( + logfunc=log.debug, + msg="Resizing", + func=do_resize, + args=(resize_cmd, log), + ) + + action = "Resized" if resize_root == NOBLOCK: - action = 'Resizing (via forking)' - log.debug("%s root filesystem (type=%s, val=%s)", action, fs_type, - resize_root) + action = "Resizing (via forking)" + log.debug( + "%s root filesystem (type=%s, val=%s)", action, fs_type, resize_root + ) def do_resize(resize_cmd, log): @@ -283,4 +319,5 @@ def do_resize(resize_cmd, log): # TODO(harlowja): Should we add a fsck check after this to make # sure we didn't corrupt anything? + # vi: ts=4 expandtab diff --git a/cloudinit/config/cc_resolv_conf.py b/cloudinit/config/cc_resolv_conf.py index 648935e4..b2970d51 100644 --- a/cloudinit/config/cc_resolv_conf.py +++ b/cloudinit/config/cc_resolv_conf.py @@ -47,19 +47,18 @@ are configured correctly. """ from cloudinit import log as logging -from cloudinit import templater +from cloudinit import templater, util from cloudinit.settings import PER_INSTANCE -from cloudinit import util LOG = logging.getLogger(__name__) frequency = PER_INSTANCE -distros = ['alpine', 'fedora', 'opensuse', 'photon', 'rhel', 'sles'] +distros = ["alpine", "fedora", "opensuse", "photon", "rhel", "sles"] RESOLVE_CONFIG_TEMPLATE_MAP = { - '/etc/resolv.conf': 'resolv.conf', - '/etc/systemd/resolved.conf': 'systemd.resolved.conf', + "/etc/resolv.conf": "resolv.conf", + "/etc/systemd/resolved.conf": "systemd.resolved.conf", } @@ -67,8 +66,8 @@ def generate_resolv_conf(template_fn, params, target_fname): flags = [] false_flags = [] - if 'options' in params: - for key, val in params['options'].items(): + if "options" in params: + for key, val in params["options"].items(): if isinstance(val, bool): if val: flags.append(key) @@ -76,12 +75,12 @@ def generate_resolv_conf(template_fn, params, target_fname): false_flags.append(key) for flag in flags + false_flags: - del params['options'][flag] + del params["options"][flag] - if not params.get('options'): - params['options'] = {} + if not params.get("options"): + params["options"] = {} - params['flags'] = flags + params["flags"] = flags LOG.debug("Writing resolv.conf from template %s", template_fn) templater.render_to_file(template_fn, target_fname, params) @@ -97,13 +96,19 @@ def handle(name, cfg, cloud, log, _args): @param args: Any module arguments from cloud.cfg """ if "manage_resolv_conf" not in cfg: - log.debug(("Skipping module named %s," - " no 'manage_resolv_conf' key in configuration"), name) + log.debug( + "Skipping module named %s," + " no 'manage_resolv_conf' key in configuration", + name, + ) return if not util.get_cfg_option_bool(cfg, "manage_resolv_conf", False): - log.debug(("Skipping module named %s," - " 'manage_resolv_conf' present but set to False"), name) + log.debug( + "Skipping module named %s," + " 'manage_resolv_conf' present but set to False", + name, + ) return if "resolv_conf" not in cfg: @@ -112,7 +117,8 @@ def handle(name, cfg, cloud, log, _args): try: template_fn = cloud.get_template_filename( - RESOLVE_CONFIG_TEMPLATE_MAP[cloud.distro.resolve_conf_fn]) + RESOLVE_CONFIG_TEMPLATE_MAP[cloud.distro.resolve_conf_fn] + ) except KeyError: log.warning("No template found, not rendering resolve configs") return @@ -120,8 +126,9 @@ def handle(name, cfg, cloud, log, _args): generate_resolv_conf( template_fn=template_fn, params=cfg["resolv_conf"], - target_fname=cloud.distro.resolve_conf_fn + target_fname=cloud.distro.resolve_conf_fn, ) return + # vi: ts=4 expandtab diff --git a/cloudinit/config/cc_rh_subscription.py b/cloudinit/config/cc_rh_subscription.py index 693317c2..b81a7a9b 100644 --- a/cloudinit/config/cc_rh_subscription.py +++ b/cloudinit/config/cc_rh_subscription.py @@ -39,12 +39,11 @@ Subscription`` example config. """ from cloudinit import log as logging -from cloudinit import subp -from cloudinit import util +from cloudinit import subp, util LOG = logging.getLogger(__name__) -distros = ['fedora', 'rhel'] +distros = ["fedora", "rhel"] def handle(name, cfg, _cloud, log, _args): @@ -60,8 +59,9 @@ def handle(name, cfg, _cloud, log, _args): raise SubscriptionError(verify_msg) cont = sm.rhn_register() if not cont: - raise SubscriptionError("Registration failed or did not " - "run completely") + raise SubscriptionError( + "Registration failed or did not run completely" + ) # Splitting up the registration, auto-attach, and servicelevel # commands because the error codes, messages from subman are not @@ -70,8 +70,7 @@ def handle(name, cfg, _cloud, log, _args): # Attempt to change the service level if sm.auto_attach and sm.servicelevel is not None: if not sm._set_service_level(): - raise SubscriptionError("Setting of service-level " - "failed") + raise SubscriptionError("Setting of service-level failed") else: sm.log.debug("Completed auto-attach with service level") elif sm.auto_attach: @@ -87,8 +86,9 @@ def handle(name, cfg, _cloud, log, _args): return_stat = sm.addPool(sm.pools) if not return_stat: - raise SubscriptionError("Unable to attach pools {0}" - .format(sm.pools)) + raise SubscriptionError( + "Unable to attach pools {0}".format(sm.pools) + ) return_stat = sm.update_repos() if not return_stat: raise SubscriptionError("Unable to add or remove repos") @@ -105,72 +105,87 @@ class SubscriptionError(Exception): class SubscriptionManager(object): - valid_rh_keys = ['org', 'activation-key', 'username', 'password', - 'disable-repo', 'enable-repo', 'add-pool', - 'rhsm-baseurl', 'server-hostname', - 'auto-attach', 'service-level'] + valid_rh_keys = [ + "org", + "activation-key", + "username", + "password", + "disable-repo", + "enable-repo", + "add-pool", + "rhsm-baseurl", + "server-hostname", + "auto-attach", + "service-level", + ] def __init__(self, cfg, log=None): if log is None: log = LOG self.log = log self.cfg = cfg - self.rhel_cfg = self.cfg.get('rh_subscription', {}) - self.rhsm_baseurl = self.rhel_cfg.get('rhsm-baseurl') - self.server_hostname = self.rhel_cfg.get('server-hostname') - self.pools = self.rhel_cfg.get('add-pool') - self.activation_key = self.rhel_cfg.get('activation-key') - self.org = self.rhel_cfg.get('org') - self.userid = self.rhel_cfg.get('username') - self.password = self.rhel_cfg.get('password') - self.auto_attach = self.rhel_cfg.get('auto-attach') - self.enable_repo = self.rhel_cfg.get('enable-repo') - self.disable_repo = self.rhel_cfg.get('disable-repo') - self.servicelevel = self.rhel_cfg.get('service-level') + self.rhel_cfg = self.cfg.get("rh_subscription", {}) + self.rhsm_baseurl = self.rhel_cfg.get("rhsm-baseurl") + self.server_hostname = self.rhel_cfg.get("server-hostname") + self.pools = self.rhel_cfg.get("add-pool") + self.activation_key = self.rhel_cfg.get("activation-key") + self.org = self.rhel_cfg.get("org") + self.userid = self.rhel_cfg.get("username") + self.password = self.rhel_cfg.get("password") + self.auto_attach = self.rhel_cfg.get("auto-attach") + self.enable_repo = self.rhel_cfg.get("enable-repo") + self.disable_repo = self.rhel_cfg.get("disable-repo") + self.servicelevel = self.rhel_cfg.get("service-level") def log_success(self, msg): - '''Simple wrapper for logging info messages. Useful for unittests''' + """Simple wrapper for logging info messages. Useful for unittests""" self.log.info(msg) def log_warn(self, msg): - '''Simple wrapper for logging warning messages. Useful for unittests''' + """Simple wrapper for logging warning messages. Useful for unittests""" self.log.warning(msg) def _verify_keys(self): - ''' + """ Checks that the keys in the rh_subscription dict from the user-data are what we expect. - ''' + """ for k in self.rhel_cfg: if k not in self.valid_rh_keys: - bad_key = "{0} is not a valid key for rh_subscription. "\ - "Valid keys are: "\ - "{1}".format(k, ', '.join(self.valid_rh_keys)) + bad_key = ( + "{0} is not a valid key for rh_subscription. " + "Valid keys are: " + "{1}".format(k, ", ".join(self.valid_rh_keys)) + ) return False, bad_key # Check for bad auto-attach value - if (self.auto_attach is not None) and \ - not (util.is_true(self.auto_attach) or - util.is_false(self.auto_attach)): - not_bool = "The key auto-attach must be a boolean value "\ - "(True/False " + if (self.auto_attach is not None) and not ( + util.is_true(self.auto_attach) or util.is_false(self.auto_attach) + ): + not_bool = ( + "The key auto-attach must be a boolean value (True/False " + ) return False, not_bool - if (self.servicelevel is not None) and ((not self.auto_attach) or - (util.is_false(str(self.auto_attach)))): - no_auto = ("The service-level key must be used in conjunction " - "with the auto-attach key. Please re-run with " - "auto-attach: True") + if (self.servicelevel is not None) and ( + (not self.auto_attach) or (util.is_false(str(self.auto_attach))) + ): + no_auto = ( + "The service-level key must be used in conjunction " + "with the auto-attach key. Please re-run with " + "auto-attach: True" + ) return False, no_auto return True, None def is_registered(self): - ''' + """ Checks if the system is already registered and returns True if so, else False - ''' - cmd = ['identity'] + """ + cmd = ["identity"] try: _sub_man_cli(cmd) @@ -180,15 +195,18 @@ class SubscriptionManager(object): return True def rhn_register(self): - ''' + """ Registers the system by userid and password or activation key and org. Returns True when successful False when not. - ''' + """ if (self.activation_key is not None) and (self.org is not None): # register by activation key - cmd = ['register', '--activationkey={0}'. - format(self.activation_key), '--org={0}'.format(self.org)] + cmd = [ + "register", + "--activationkey={0}".format(self.activation_key), + "--org={0}".format(self.org), + ] # If the baseurl and/or server url are passed in, we register # with them. @@ -203,14 +221,18 @@ class SubscriptionManager(object): return_out = _sub_man_cli(cmd, logstring_val=True)[0] except subp.ProcessExecutionError as e: if e.stdout == "": - self.log_warn("Registration failed due " - "to: {0}".format(e.stderr)) + self.log_warn( + "Registration failed due to: {0}".format(e.stderr) + ) return False elif (self.userid is not None) and (self.password is not None): # register by username and password - cmd = ['register', '--username={0}'.format(self.userid), - '--password={0}'.format(self.password)] + cmd = [ + "register", + "--username={0}".format(self.userid), + "--password={0}".format(self.password), + ] # If the baseurl and/or server url are passed in, we register # with them. @@ -226,15 +248,18 @@ class SubscriptionManager(object): return_out = _sub_man_cli(cmd, logstring_val=True)[0] except subp.ProcessExecutionError as e: if e.stdout == "": - self.log_warn("Registration failed due " - "to: {0}".format(e.stderr)) + self.log_warn( + "Registration failed due to: {0}".format(e.stderr) + ) return False else: - self.log_warn("Unable to register system due to incomplete " - "information.") - self.log_warn("Use either activationkey and org *or* userid " - "and password") + self.log_warn( + "Unable to register system due to incomplete information." + ) + self.log_warn( + "Use either activationkey and org *or* userid and password" + ) return False reg_id = return_out.split("ID: ")[1].rstrip() @@ -242,19 +267,25 @@ class SubscriptionManager(object): return True def _set_service_level(self): - cmd = ['attach', '--auto', '--servicelevel={0}' - .format(self.servicelevel)] + cmd = [ + "attach", + "--auto", + "--servicelevel={0}".format(self.servicelevel), + ] try: return_out = _sub_man_cli(cmd)[0] except subp.ProcessExecutionError as e: - if e.stdout.rstrip() != '': + if e.stdout.rstrip() != "": for line in e.stdout.split("\n"): - if line != '': + if line != "": self.log_warn(line) else: - self.log_warn("Setting the service level failed with: " - "{0}".format(e.stderr.strip())) + self.log_warn( + "Setting the service level failed with: {0}".format( + e.stderr.strip() + ) + ) return False for line in return_out.split("\n"): if line != "": @@ -262,7 +293,7 @@ class SubscriptionManager(object): return True def _set_auto_attach(self): - cmd = ['attach', '--auto'] + cmd = ["attach", "--auto"] try: return_out = _sub_man_cli(cmd)[0] except subp.ProcessExecutionError as e: @@ -274,52 +305,52 @@ class SubscriptionManager(object): return True def _getPools(self): - ''' + """ Gets the list pools for the active subscription and returns them in list form. - ''' + """ available = [] consumed = [] # Get all available pools - cmd = ['list', '--available', '--pool-only'] + cmd = ["list", "--available", "--pool-only"] results = _sub_man_cli(cmd)[0] available = (results.rstrip()).split("\n") # Get all consumed pools - cmd = ['list', '--consumed', '--pool-only'] + cmd = ["list", "--consumed", "--pool-only"] results = _sub_man_cli(cmd)[0] consumed = (results.rstrip()).split("\n") return available, consumed def _getRepos(self): - ''' + """ Obtains the current list of active yum repositories and returns them in list form. - ''' + """ - cmd = ['repos', '--list-enabled'] + cmd = ["repos", "--list-enabled"] return_out = _sub_man_cli(cmd)[0] active_repos = [] for repo in return_out.split("\n"): if "Repo ID:" in repo: - active_repos.append((repo.split(':')[1]).strip()) + active_repos.append((repo.split(":")[1]).strip()) - cmd = ['repos', '--list-disabled'] + cmd = ["repos", "--list-disabled"] return_out = _sub_man_cli(cmd)[0] inactive_repos = [] for repo in return_out.split("\n"): if "Repo ID:" in repo: - inactive_repos.append((repo.split(':')[1]).strip()) + inactive_repos.append((repo.split(":")[1]).strip()) return active_repos, inactive_repos def addPool(self, pools): - ''' + """ Takes a list of subscription pools and "attaches" them to the current subscription - ''' + """ # An empty list was passed if len(pools) == 0: @@ -328,31 +359,33 @@ class SubscriptionManager(object): pool_available, pool_consumed = self._getPools() pool_list = [] - cmd = ['attach'] + cmd = ["attach"] for pool in pools: if (pool not in pool_consumed) and (pool in pool_available): - pool_list.append('--pool={0}'.format(pool)) + pool_list.append("--pool={0}".format(pool)) else: self.log_warn("Pool {0} is not available".format(pool)) if len(pool_list) > 0: cmd.extend(pool_list) try: _sub_man_cli(cmd) - self.log.debug("Attached the following pools to your " - "system: %s", (", ".join(pool_list)) - .replace('--pool=', '')) + self.log.debug( + "Attached the following pools to your system: %s", + (", ".join(pool_list)).replace("--pool=", ""), + ) return True except subp.ProcessExecutionError as e: - self.log_warn("Unable to attach pool {0} " - "due to {1}".format(pool, e)) + self.log_warn( + "Unable to attach pool {0} due to {1}".format(pool, e) + ) return False def update_repos(self): - ''' + """ Takes a list of yum repo ids that need to be disabled or enabled; then it verifies if they are already enabled or disabled and finally executes the action to disable or enable - ''' + """ erepos = self.enable_repo drepos = self.disable_repo @@ -378,7 +411,7 @@ class SubscriptionManager(object): enable_list = [] enable_list_fail = [] for repoid in erepos: - if (repoid in inactive_repos): + if repoid in inactive_repos: enable_list.append("--enable={0}".format(repoid)) else: enable_list_fail.append(repoid) @@ -399,14 +432,16 @@ class SubscriptionManager(object): if fail in active_repos: self.log.debug("Repo %s is already enabled", fail) else: - self.log_warn("Repo {0} does not appear to " - "exist".format(fail)) + self.log_warn( + "Repo {0} does not appear to exist".format(fail) + ) if len(disable_list_fail) > 0: for fail in disable_list_fail: - self.log.debug("Repo %s not disabled " - "because it is not enabled", fail) + self.log.debug( + "Repo %s not disabled because it is not enabled", fail + ) - cmd = ['repos'] + cmd = ["repos"] if len(disable_list) > 0: cmd.extend(disable_list) @@ -420,11 +455,15 @@ class SubscriptionManager(object): return False if len(enable_list) > 0: - self.log.debug("Enabled the following repos: %s", - (", ".join(enable_list)).replace('--enable=', '')) + self.log.debug( + "Enabled the following repos: %s", + (", ".join(enable_list)).replace("--enable=", ""), + ) if len(disable_list) > 0: - self.log.debug("Disabled the following repos: %s", - (", ".join(disable_list)).replace('--disable=', '')) + self.log.debug( + "Disabled the following repos: %s", + (", ".join(disable_list)).replace("--disable=", ""), + ) return True def is_configured(self): @@ -432,13 +471,12 @@ class SubscriptionManager(object): def _sub_man_cli(cmd, logstring_val=False): - ''' + """ Uses the prefered cloud-init subprocess def of subp.subp and runs subscription-manager. Breaking this to a separate function for later use in mocking and unittests - ''' - return subp.subp(['subscription-manager'] + cmd, - logstring=logstring_val) + """ + return subp.subp(["subscription-manager"] + cmd, logstring=logstring_val) # vi: ts=4 expandtab diff --git a/cloudinit/config/cc_rightscale_userdata.py b/cloudinit/config/cc_rightscale_userdata.py index c75dc57d..36a009a2 100644 --- a/cloudinit/config/cc_rightscale_userdata.py +++ b/cloudinit/config/cc_rightscale_userdata.py @@ -52,14 +52,14 @@ user scripts configuration directory, to be run later by ``cc_scripts_user``. import os from urllib.parse import parse_qs -from cloudinit.settings import PER_INSTANCE from cloudinit import url_helper as uhelp from cloudinit import util +from cloudinit.settings import PER_INSTANCE frequency = PER_INSTANCE MY_NAME = "cc_rightscale_userdata" -MY_HOOKNAME = 'CLOUD_INIT_REMOTE_HOOK' +MY_HOOKNAME = "CLOUD_INIT_REMOTE_HOOK" def handle(name, _cfg, cloud, log, _args): @@ -72,13 +72,16 @@ def handle(name, _cfg, cloud, log, _args): try: mdict = parse_qs(ud) if not mdict or MY_HOOKNAME not in mdict: - log.debug(("Skipping module %s, " - "did not find %s in parsed" - " raw userdata"), name, MY_HOOKNAME) + log.debug( + "Skipping module %s, did not find %s in parsed raw userdata", + name, + MY_HOOKNAME, + ) return except Exception: - util.logexc(log, "Failed to parse query string %s into a dictionary", - ud) + util.logexc( + log, "Failed to parse query string %s into a dictionary", ud + ) raise wrote_fns = [] @@ -87,7 +90,7 @@ def handle(name, _cfg, cloud, log, _args): # These will eventually be then ran by the cc_scripts_user # TODO(harlowja): maybe this should just be a new user data handler?? # Instead of a late module that acts like a user data handler? - scripts_d = cloud.get_ipath_cur('scripts') + scripts_d = cloud.get_ipath_cur("scripts") urls = mdict[MY_HOOKNAME] for (i, url) in enumerate(urls): fname = os.path.join(scripts_d, "rightscale-%02i" % (i)) @@ -99,8 +102,9 @@ def handle(name, _cfg, cloud, log, _args): wrote_fns.append(fname) except Exception as e: captured_excps.append(e) - util.logexc(log, "%s failed to read %s and write %s", MY_NAME, url, - fname) + util.logexc( + log, "%s failed to read %s and write %s", MY_NAME, url, fname + ) if wrote_fns: log.debug("Wrote out rightscale userdata to %s files", len(wrote_fns)) @@ -110,8 +114,11 @@ def handle(name, _cfg, cloud, log, _args): log.debug("%s urls were skipped or failed", skipped) if captured_excps: - log.warning("%s failed with exceptions, re-raising the last one", - len(captured_excps)) + log.warning( + "%s failed with exceptions, re-raising the last one", + len(captured_excps), + ) raise captured_excps[-1] + # vi: ts=4 expandtab diff --git a/cloudinit/config/cc_rsyslog.py b/cloudinit/config/cc_rsyslog.py index dd2bbd00..8dd4950f 100644 --- a/cloudinit/config/cc_rsyslog.py +++ b/cloudinit/config/cc_rsyslog.py @@ -182,45 +182,45 @@ import os import re from cloudinit import log as logging -from cloudinit import subp -from cloudinit import util +from cloudinit import subp, util DEF_FILENAME = "20-cloud-config.conf" DEF_DIR = "/etc/rsyslog.d" DEF_RELOAD = "auto" DEF_REMOTES = {} -KEYNAME_CONFIGS = 'configs' -KEYNAME_FILENAME = 'config_filename' -KEYNAME_DIR = 'config_dir' -KEYNAME_RELOAD = 'service_reload_command' -KEYNAME_LEGACY_FILENAME = 'rsyslog_filename' -KEYNAME_LEGACY_DIR = 'rsyslog_dir' -KEYNAME_REMOTES = 'remotes' +KEYNAME_CONFIGS = "configs" +KEYNAME_FILENAME = "config_filename" +KEYNAME_DIR = "config_dir" +KEYNAME_RELOAD = "service_reload_command" +KEYNAME_LEGACY_FILENAME = "rsyslog_filename" +KEYNAME_LEGACY_DIR = "rsyslog_dir" +KEYNAME_REMOTES = "remotes" LOG = logging.getLogger(__name__) -COMMENT_RE = re.compile(r'[ ]*[#]+[ ]*') +COMMENT_RE = re.compile(r"[ ]*[#]+[ ]*") HOST_PORT_RE = re.compile( - r'^(?P<proto>[@]{0,2})' - r'(([\[](?P<bracket_addr>[^\]]*)[\]])|(?P<addr>[^:]*))' - r'([:](?P<port>[0-9]+))?$') + r"^(?P<proto>[@]{0,2})" + r"(([\[](?P<bracket_addr>[^\]]*)[\]])|(?P<addr>[^:]*))" + r"([:](?P<port>[0-9]+))?$" +) def reload_syslog(distro, command=DEF_RELOAD): if command == DEF_RELOAD: - service = distro.get_option('rsyslog_svcname', 'rsyslog') - return distro.manage_service('try-reload', service) + service = distro.get_option("rsyslog_svcname", "rsyslog") + return distro.manage_service("try-reload", service) return subp.subp(command, capture=True) def load_config(cfg): # return an updated config with entries of the correct type # support converting the old top level format into new format - mycfg = cfg.get('rsyslog', {}) + mycfg = cfg.get("rsyslog", {}) - if isinstance(cfg.get('rsyslog'), list): - mycfg = {KEYNAME_CONFIGS: cfg.get('rsyslog')} + if isinstance(cfg.get("rsyslog"), list): + mycfg = {KEYNAME_CONFIGS: cfg.get("rsyslog")} if KEYNAME_LEGACY_FILENAME in cfg: mycfg[KEYNAME_FILENAME] = cfg[KEYNAME_LEGACY_FILENAME] if KEYNAME_LEGACY_DIR in cfg: @@ -231,7 +231,8 @@ def load_config(cfg): (KEYNAME_DIR, DEF_DIR, str), (KEYNAME_FILENAME, DEF_FILENAME, str), (KEYNAME_RELOAD, DEF_RELOAD, (str, list)), - (KEYNAME_REMOTES, DEF_REMOTES, dict)) + (KEYNAME_REMOTES, DEF_REMOTES, dict), + ) for key, default, vtypes in fillup: if key not in mycfg or not isinstance(mycfg[key], vtypes): @@ -247,10 +248,11 @@ def apply_rsyslog_changes(configs, def_fname, cfg_dir): for cur_pos, ent in enumerate(configs): if isinstance(ent, dict): if "content" not in ent: - LOG.warning("No 'content' entry in config entry %s", - cur_pos + 1) + LOG.warning( + "No 'content' entry in config entry %s", cur_pos + 1 + ) continue - content = ent['content'] + content = ent["content"] filename = ent.get("filename", def_fname) else: content = ent @@ -301,9 +303,9 @@ def parse_remotes_line(line, name=None): if not toks: raise ValueError("Invalid host specification '%s'" % host_port) - proto = toks.group('proto') - addr = toks.group('addr') or toks.group('bracket_addr') - port = toks.group('port') + proto = toks.group("proto") + addr = toks.group("addr") or toks.group("bracket_addr") + port = toks.group("port") if addr.startswith("[") and not addr.endswith("]"): raise ValueError("host spec had invalid brackets: %s" % addr) @@ -311,15 +313,17 @@ def parse_remotes_line(line, name=None): if comment and not name: name = comment - t = SyslogRemotesLine(name=name, match=match, proto=proto, - addr=addr, port=port) + t = SyslogRemotesLine( + name=name, match=match, proto=proto, addr=addr, port=port + ) t.validate() return t class SyslogRemotesLine(object): - def __init__(self, name=None, match=None, proto=None, addr=None, - port=None): + def __init__( + self, name=None, match=None, proto=None, addr=None, port=None + ): if not match: match = "*.*" self.name = name @@ -352,7 +356,11 @@ class SyslogRemotesLine(object): def __repr__(self): return "[name=%s match=%s proto=%s address=%s port=%s]" % ( - self.name, self.match, self.proto, self.addr, self.port + self.name, + self.match, + self.proto, + self.addr, + self.port, ) def __str__(self): @@ -390,13 +398,14 @@ def remotes_to_rsyslog_cfg(remotes, header=None, footer=None): LOG.warning("failed loading remote %s: %s [%s]", name, line, e) if footer is not None: lines.append(footer) - return '\n'.join(lines) + "\n" + return "\n".join(lines) + "\n" def handle(name, cfg, cloud, log, _args): - if 'rsyslog' not in cfg: - log.debug(("Skipping module named %s," - " no 'rsyslog' key in configuration"), name) + if "rsyslog" not in cfg: + log.debug( + "Skipping module named %s, no 'rsyslog' key in configuration", name + ) return mycfg = load_config(cfg) @@ -408,16 +417,18 @@ def handle(name, cfg, cloud, log, _args): mycfg[KEYNAME_REMOTES], header="# begin remotes", footer="# end remotes", - )) + ) + ) - if not mycfg['configs']: + if not mycfg["configs"]: log.debug("Empty config rsyslog['configs'], nothing to do") return changes = apply_rsyslog_changes( configs=mycfg[KEYNAME_CONFIGS], def_fname=mycfg[KEYNAME_FILENAME], - cfg_dir=mycfg[KEYNAME_DIR]) + cfg_dir=mycfg[KEYNAME_DIR], + ) if not changes: log.debug("restart of syslog not necessary, no changes made") @@ -437,4 +448,5 @@ def handle(name, cfg, cloud, log, _args): # the logging was setup to use it... log.debug("%s configured %s files", name, changes) + # vi: ts=4 expandtab syntax=python diff --git a/cloudinit/config/cc_runcmd.py b/cloudinit/config/cc_runcmd.py index 2f5e02cb..15cbaf1a 100644 --- a/cloudinit/config/cc_runcmd.py +++ b/cloudinit/config/cc_runcmd.py @@ -8,14 +8,13 @@ """Runcmd: run arbitrary commands at rc.local with output to the console""" -from cloudinit.config.schema import get_meta_doc, validate_cloudconfig_schema -from cloudinit.distros import ALL_DISTROS -from cloudinit.settings import PER_INSTANCE -from cloudinit import util - import os from textwrap import dedent +from cloudinit import util +from cloudinit.config.schema import get_meta_doc, validate_cloudconfig_schema +from cloudinit.distros import ALL_DISTROS +from cloudinit.settings import PER_INSTANCE # The schema definition for each cloud-config module is a strict contract for # describing supported configuration parameters for each cloud-config section. @@ -26,10 +25,11 @@ from textwrap import dedent distros = [ALL_DISTROS] meta = { - 'id': 'cc_runcmd', - 'name': 'Runcmd', - 'title': 'Run arbitrary commands', - 'description': dedent("""\ + "id": "cc_runcmd", + "name": "Runcmd", + "title": "Run arbitrary commands", + "description": dedent( + """\ Run arbitrary commands at a rc.local like level with output to the console. Each item can be either a list or a string. If the item is a list, it will be properly executed as if passed to ``execve()`` (with @@ -46,35 +46,41 @@ meta = { when writing files, do not use /tmp dir as it races with systemd-tmpfiles-clean LP: #1707222. Use /run/somedir instead. - """), - 'distros': distros, - 'examples': [dedent("""\ + """ + ), + "distros": distros, + "examples": [ + dedent( + """\ runcmd: - [ ls, -l, / ] - [ sh, -xc, "echo $(date) ': hello world!'" ] - [ sh, -c, echo "=========hello world'=========" ] - ls -l /root - [ wget, "http://example.org", -O, /tmp/index.html ] - """)], - 'frequency': PER_INSTANCE, + """ + ) + ], + "frequency": PER_INSTANCE, } schema = { - 'type': 'object', - 'properties': { - 'runcmd': { - 'type': 'array', - 'items': { - 'oneOf': [ - {'type': 'array', 'items': {'type': 'string'}}, - {'type': 'string'}, - {'type': 'null'}] + "type": "object", + "properties": { + "runcmd": { + "type": "array", + "items": { + "oneOf": [ + {"type": "array", "items": {"type": "string"}}, + {"type": "string"}, + {"type": "null"}, + ] }, - 'additionalItems': False, # Reject items of non-string non-list - 'additionalProperties': False, - 'minItems': 1, + "additionalItems": False, # Reject items of non-string non-list + "additionalProperties": False, + "minItems": 1, } - } + }, } __doc__ = get_meta_doc(meta, schema) # Supplement python help() @@ -82,17 +88,19 @@ __doc__ = get_meta_doc(meta, schema) # Supplement python help() def handle(name, cfg, cloud, log, _args): if "runcmd" not in cfg: - log.debug(("Skipping module named %s," - " no 'runcmd' key in configuration"), name) + log.debug( + "Skipping module named %s, no 'runcmd' key in configuration", name + ) return validate_cloudconfig_schema(cfg, schema) - out_fn = os.path.join(cloud.get_ipath('scripts'), "runcmd") + out_fn = os.path.join(cloud.get_ipath("scripts"), "runcmd") cmd = cfg["runcmd"] try: content = util.shellify(cmd) util.write_file(out_fn, content, 0o700) except Exception as e: - raise type(e)('Failed to shellify {} into file {}'.format(cmd, out_fn)) + raise type(e)("Failed to shellify {} into file {}".format(cmd, out_fn)) + # vi: ts=4 expandtab diff --git a/cloudinit/config/cc_salt_minion.py b/cloudinit/config/cc_salt_minion.py index b61876aa..b2e5eefd 100644 --- a/cloudinit/config/cc_salt_minion.py +++ b/cloudinit/config/cc_salt_minion.py @@ -48,7 +48,6 @@ import os from cloudinit import safeyaml, subp, util from cloudinit.distros import rhel_util - # Note: see https://docs.saltstack.com/en/latest/topics/installation/ # Note: see https://docs.saltstack.com/en/latest/ref/configuration/ @@ -57,36 +56,40 @@ class SaltConstants(object): """ defines default distribution specific salt variables """ + def __init__(self, cfg): # constants tailored for FreeBSD if util.is_FreeBSD(): - self.pkg_name = 'py36-salt' - self.srv_name = 'salt_minion' - self.conf_dir = '/usr/local/etc/salt' + self.pkg_name = "py36-salt" + self.srv_name = "salt_minion" + self.conf_dir = "/usr/local/etc/salt" # constants for any other OS else: - self.pkg_name = 'salt-minion' - self.srv_name = 'salt-minion' - self.conf_dir = '/etc/salt' + self.pkg_name = "salt-minion" + self.srv_name = "salt-minion" + self.conf_dir = "/etc/salt" # if there are constants given in cloud config use those - self.pkg_name = util.get_cfg_option_str(cfg, 'pkg_name', - self.pkg_name) - self.conf_dir = util.get_cfg_option_str(cfg, 'config_dir', - self.conf_dir) - self.srv_name = util.get_cfg_option_str(cfg, 'service_name', - self.srv_name) + self.pkg_name = util.get_cfg_option_str(cfg, "pkg_name", self.pkg_name) + self.conf_dir = util.get_cfg_option_str( + cfg, "config_dir", self.conf_dir + ) + self.srv_name = util.get_cfg_option_str( + cfg, "service_name", self.srv_name + ) def handle(name, cfg, cloud, log, _args): # If there isn't a salt key in the configuration don't do anything - if 'salt_minion' not in cfg: - log.debug(("Skipping module named %s," - " no 'salt_minion' key in configuration"), name) + if "salt_minion" not in cfg: + log.debug( + "Skipping module named %s, no 'salt_minion' key in configuration", + name, + ) return - s_cfg = cfg['salt_minion'] + s_cfg = cfg["salt_minion"] const = SaltConstants(cfg=s_cfg) # Start by installing the salt package ... @@ -96,40 +99,42 @@ def handle(name, cfg, cloud, log, _args): util.ensure_dir(const.conf_dir) # ... and then update the salt configuration - if 'conf' in s_cfg: + if "conf" in s_cfg: # Add all sections from the conf object to minion config file - minion_config = os.path.join(const.conf_dir, 'minion') - minion_data = safeyaml.dumps(s_cfg.get('conf')) + minion_config = os.path.join(const.conf_dir, "minion") + minion_data = safeyaml.dumps(s_cfg.get("conf")) util.write_file(minion_config, minion_data) - if 'grains' in s_cfg: + if "grains" in s_cfg: # add grains to /etc/salt/grains - grains_config = os.path.join(const.conf_dir, 'grains') - grains_data = safeyaml.dumps(s_cfg.get('grains')) + grains_config = os.path.join(const.conf_dir, "grains") + grains_data = safeyaml.dumps(s_cfg.get("grains")) util.write_file(grains_config, grains_data) # ... copy the key pair if specified - if 'public_key' in s_cfg and 'private_key' in s_cfg: + if "public_key" in s_cfg and "private_key" in s_cfg: pki_dir_default = os.path.join(const.conf_dir, "pki/minion") if not os.path.isdir(pki_dir_default): pki_dir_default = os.path.join(const.conf_dir, "pki") - pki_dir = s_cfg.get('pki_dir', pki_dir_default) + pki_dir = s_cfg.get("pki_dir", pki_dir_default) with util.umask(0o77): util.ensure_dir(pki_dir) - pub_name = os.path.join(pki_dir, 'minion.pub') - pem_name = os.path.join(pki_dir, 'minion.pem') - util.write_file(pub_name, s_cfg['public_key']) - util.write_file(pem_name, s_cfg['private_key']) + pub_name = os.path.join(pki_dir, "minion.pub") + pem_name = os.path.join(pki_dir, "minion.pem") + util.write_file(pub_name, s_cfg["public_key"]) + util.write_file(pem_name, s_cfg["private_key"]) # we need to have the salt minion service enabled in rc in order to be # able to start the service. this does only apply on FreeBSD servers. - if cloud.distro.osfamily == 'freebsd': + if cloud.distro.osfamily == "freebsd": rhel_util.update_sysconfig_file( - '/etc/rc.conf', {'salt_minion_enable': 'YES'}) + "/etc/rc.conf", {"salt_minion_enable": "YES"} + ) # restart salt-minion. 'service' will start even if not started. if it # was started, it needs to be restarted for config change. - subp.subp(['service', const.srv_name, 'restart'], capture=False) + subp.subp(["service", const.srv_name, "restart"], capture=False) + # vi: ts=4 expandtab diff --git a/cloudinit/config/cc_scripts_per_boot.py b/cloudinit/config/cc_scripts_per_boot.py index 1bf3f508..b7bfb7aa 100644 --- a/cloudinit/config/cc_scripts_per_boot.py +++ b/cloudinit/config/cc_scripts_per_boot.py @@ -25,23 +25,27 @@ module does not accept any config keys. import os from cloudinit import subp - from cloudinit.settings import PER_ALWAYS frequency = PER_ALWAYS -SCRIPT_SUBDIR = 'per-boot' +SCRIPT_SUBDIR = "per-boot" def handle(name, _cfg, cloud, log, _args): # Comes from the following: # https://forums.aws.amazon.com/thread.jspa?threadID=96918 - runparts_path = os.path.join(cloud.get_cpath(), 'scripts', SCRIPT_SUBDIR) + runparts_path = os.path.join(cloud.get_cpath(), "scripts", SCRIPT_SUBDIR) try: subp.runparts(runparts_path) except Exception: - log.warning("Failed to run module %s (%s in %s)", - name, SCRIPT_SUBDIR, runparts_path) + log.warning( + "Failed to run module %s (%s in %s)", + name, + SCRIPT_SUBDIR, + runparts_path, + ) raise + # vi: ts=4 expandtab diff --git a/cloudinit/config/cc_scripts_per_instance.py b/cloudinit/config/cc_scripts_per_instance.py index 5966fb9a..ef102b1c 100644 --- a/cloudinit/config/cc_scripts_per_instance.py +++ b/cloudinit/config/cc_scripts_per_instance.py @@ -28,23 +28,27 @@ the system. As a result per-instance scripts will run again. import os from cloudinit import subp - from cloudinit.settings import PER_INSTANCE frequency = PER_INSTANCE -SCRIPT_SUBDIR = 'per-instance' +SCRIPT_SUBDIR = "per-instance" def handle(name, _cfg, cloud, log, _args): # Comes from the following: # https://forums.aws.amazon.com/thread.jspa?threadID=96918 - runparts_path = os.path.join(cloud.get_cpath(), 'scripts', SCRIPT_SUBDIR) + runparts_path = os.path.join(cloud.get_cpath(), "scripts", SCRIPT_SUBDIR) try: subp.runparts(runparts_path) except Exception: - log.warning("Failed to run module %s (%s in %s)", - name, SCRIPT_SUBDIR, runparts_path) + log.warning( + "Failed to run module %s (%s in %s)", + name, + SCRIPT_SUBDIR, + runparts_path, + ) raise + # vi: ts=4 expandtab diff --git a/cloudinit/config/cc_scripts_per_once.py b/cloudinit/config/cc_scripts_per_once.py index bcca859e..bf4231e7 100644 --- a/cloudinit/config/cc_scripts_per_once.py +++ b/cloudinit/config/cc_scripts_per_once.py @@ -26,23 +26,27 @@ be run in alphabetical order. This module does not accept any config keys. import os from cloudinit import subp - from cloudinit.settings import PER_ONCE frequency = PER_ONCE -SCRIPT_SUBDIR = 'per-once' +SCRIPT_SUBDIR = "per-once" def handle(name, _cfg, cloud, log, _args): # Comes from the following: # https://forums.aws.amazon.com/thread.jspa?threadID=96918 - runparts_path = os.path.join(cloud.get_cpath(), 'scripts', SCRIPT_SUBDIR) + runparts_path = os.path.join(cloud.get_cpath(), "scripts", SCRIPT_SUBDIR) try: subp.runparts(runparts_path) except Exception: - log.warning("Failed to run module %s (%s in %s)", - name, SCRIPT_SUBDIR, runparts_path) + log.warning( + "Failed to run module %s (%s in %s)", + name, + SCRIPT_SUBDIR, + runparts_path, + ) raise + # vi: ts=4 expandtab diff --git a/cloudinit/config/cc_scripts_user.py b/cloudinit/config/cc_scripts_user.py index 215703ef..e0d6c560 100644 --- a/cloudinit/config/cc_scripts_user.py +++ b/cloudinit/config/cc_scripts_user.py @@ -28,12 +28,11 @@ This module does not accept any config keys. import os from cloudinit import subp - from cloudinit.settings import PER_INSTANCE frequency = PER_INSTANCE -SCRIPT_SUBDIR = 'scripts' +SCRIPT_SUBDIR = "scripts" def handle(name, _cfg, cloud, log, _args): @@ -44,8 +43,13 @@ def handle(name, _cfg, cloud, log, _args): try: subp.runparts(runparts_path) except Exception: - log.warning("Failed to run module %s (%s in %s)", - name, SCRIPT_SUBDIR, runparts_path) + log.warning( + "Failed to run module %s (%s in %s)", + name, + SCRIPT_SUBDIR, + runparts_path, + ) raise + # vi: ts=4 expandtab diff --git a/cloudinit/config/cc_scripts_vendor.py b/cloudinit/config/cc_scripts_vendor.py index e0a4bfff..1b30fa1b 100644 --- a/cloudinit/config/cc_scripts_vendor.py +++ b/cloudinit/config/cc_scripts_vendor.py @@ -28,29 +28,33 @@ entry under the ``vendor_data`` config key. import os -from cloudinit import subp -from cloudinit import util - +from cloudinit import subp, util from cloudinit.settings import PER_INSTANCE frequency = PER_INSTANCE -SCRIPT_SUBDIR = 'vendor' +SCRIPT_SUBDIR = "vendor" def handle(name, cfg, cloud, log, _args): # This is written to by the vendor data handlers # any vendor data shell scripts get placed in runparts_path - runparts_path = os.path.join(cloud.get_ipath_cur(), 'scripts', - SCRIPT_SUBDIR) + runparts_path = os.path.join( + cloud.get_ipath_cur(), "scripts", SCRIPT_SUBDIR + ) - prefix = util.get_cfg_by_path(cfg, ('vendor_data', 'prefix'), []) + prefix = util.get_cfg_by_path(cfg, ("vendor_data", "prefix"), []) try: subp.runparts(runparts_path, exe_prefix=prefix) except Exception: - log.warning("Failed to run module %s (%s in %s)", - name, SCRIPT_SUBDIR, runparts_path) + log.warning( + "Failed to run module %s (%s in %s)", + name, + SCRIPT_SUBDIR, + runparts_path, + ) raise + # vi: ts=4 expandtab diff --git a/cloudinit/config/cc_seed_random.py b/cloudinit/config/cc_seed_random.py index 911789c7..67ba8ef5 100644 --- a/cloudinit/config/cc_seed_random.py +++ b/cloudinit/config/cc_seed_random.py @@ -68,9 +68,8 @@ import os from io import BytesIO from cloudinit import log as logging +from cloudinit import subp, util from cloudinit.settings import PER_INSTANCE -from cloudinit import subp -from cloudinit import util frequency = PER_INSTANCE LOG = logging.getLogger(__name__) @@ -78,12 +77,12 @@ LOG = logging.getLogger(__name__) def _decode(data, encoding=None): if not data: - return b'' - if not encoding or encoding.lower() in ['raw']: + return b"" + if not encoding or encoding.lower() in ["raw"]: return util.encode_text(data) - elif encoding.lower() in ['base64', 'b64']: + elif encoding.lower() in ["base64", "b64"]: return base64.b64decode(data) - elif encoding.lower() in ['gzip', 'gz']: + elif encoding.lower() in ["gzip", "gz"]: return util.decomp_gzip(data, quiet=False, decode=None) else: raise IOError("Unknown random_seed encoding: %s" % (encoding)) @@ -100,7 +99,8 @@ def handle_random_seed_command(command, required, env=None): if not subp.which(cmd): if required: raise ValueError( - "command '{cmd}' not found but required=true".format(cmd=cmd)) + "command '{cmd}' not found but required=true".format(cmd=cmd) + ) else: LOG.debug("command '%s' not found for seed_command", cmd) return @@ -108,34 +108,39 @@ def handle_random_seed_command(command, required, env=None): def handle(name, cfg, cloud, log, _args): - mycfg = cfg.get('random_seed', {}) - seed_path = mycfg.get('file', '/dev/urandom') - seed_data = mycfg.get('data', b'') + mycfg = cfg.get("random_seed", {}) + seed_path = mycfg.get("file", "/dev/urandom") + seed_data = mycfg.get("data", b"") seed_buf = BytesIO() if seed_data: - seed_buf.write(_decode(seed_data, encoding=mycfg.get('encoding'))) + seed_buf.write(_decode(seed_data, encoding=mycfg.get("encoding"))) # 'random_seed' is set up by Azure datasource, and comes already in # openstack meta_data.json metadata = cloud.datasource.metadata - if metadata and 'random_seed' in metadata: - seed_buf.write(util.encode_text(metadata['random_seed'])) + if metadata and "random_seed" in metadata: + seed_buf.write(util.encode_text(metadata["random_seed"])) seed_data = seed_buf.getvalue() if len(seed_data): - log.debug("%s: adding %s bytes of random seed entropy to %s", name, - len(seed_data), seed_path) + log.debug( + "%s: adding %s bytes of random seed entropy to %s", + name, + len(seed_data), + seed_path, + ) util.append_file(seed_path, seed_data) - command = mycfg.get('command', None) - req = mycfg.get('command_required', False) + command = mycfg.get("command", None) + req = mycfg.get("command_required", False) try: env = os.environ.copy() - env['RANDOM_SEED_FILE'] = seed_path + env["RANDOM_SEED_FILE"] = seed_path handle_random_seed_command(command=command, required=req, env=env) except ValueError as e: log.warning("handling random command [%s] failed: %s", command, e) raise e + # vi: ts=4 expandtab diff --git a/cloudinit/config/cc_set_hostname.py b/cloudinit/config/cc_set_hostname.py index a96bcc18..eb0ca328 100644 --- a/cloudinit/config/cc_set_hostname.py +++ b/cloudinit/config/cc_set_hostname.py @@ -48,9 +48,8 @@ based on initial hostname. import os - -from cloudinit.atomic_helper import write_json from cloudinit import util +from cloudinit.atomic_helper import write_json class SetHostnameError(Exception): @@ -63,16 +62,19 @@ class SetHostnameError(Exception): def handle(name, cfg, cloud, log, _args): if util.get_cfg_option_bool(cfg, "preserve_hostname", False): - log.debug(("Configuration option 'preserve_hostname' is set," - " not setting the hostname in module %s"), name) + log.debug( + "Configuration option 'preserve_hostname' is set," + " not setting the hostname in module %s", + name, + ) return # Set prefer_fqdn_over_hostname value in distro - hostname_fqdn = util.get_cfg_option_bool(cfg, - "prefer_fqdn_over_hostname", - None) + hostname_fqdn = util.get_cfg_option_bool( + cfg, "prefer_fqdn_over_hostname", None + ) if hostname_fqdn is not None: - cloud.distro.set_option('prefer_fqdn_over_hostname', hostname_fqdn) + cloud.distro.set_option("prefer_fqdn_over_hostname", hostname_fqdn) (hostname, fqdn) = util.get_hostname_fqdn(cfg, cloud) # Check for previous successful invocation of set-hostname @@ -82,14 +84,15 @@ def handle(name, cfg, cloud, log, _args): # previous-hostname file which only contains the base hostname. # TODO consolidate previous-hostname and set-hostname artifact files and # distro._read_hostname implementation so we only validate one artifact. - prev_fn = os.path.join(cloud.get_cpath('data'), "set-hostname") + prev_fn = os.path.join(cloud.get_cpath("data"), "set-hostname") prev_hostname = {} if os.path.exists(prev_fn): prev_hostname = util.load_json(util.load_file(prev_fn)) - hostname_changed = (hostname != prev_hostname.get('hostname') or - fqdn != prev_hostname.get('fqdn')) + hostname_changed = hostname != prev_hostname.get( + "hostname" + ) or fqdn != prev_hostname.get("fqdn") if not hostname_changed: - log.debug('No hostname changes. Skipping set-hostname') + log.debug("No hostname changes. Skipping set-hostname") return log.debug("Setting the hostname to %s (%s)", fqdn, hostname) try: @@ -98,6 +101,7 @@ def handle(name, cfg, cloud, log, _args): msg = "Failed to set the hostname to %s (%s)" % (fqdn, hostname) util.logexc(log, msg) raise SetHostnameError("%s: %s" % (msg, e)) from e - write_json(prev_fn, {'hostname': hostname, 'fqdn': fqdn}) + write_json(prev_fn, {"hostname": hostname, "fqdn": fqdn}) + # vi: ts=4 expandtab diff --git a/cloudinit/config/cc_set_passwords.py b/cloudinit/config/cc_set_passwords.py index 3843aaf7..d8df8e23 100755 --- a/cloudinit/config/cc_set_passwords.py +++ b/cloudinit/config/cc_set_passwords.py @@ -78,20 +78,17 @@ password. """ import re +from string import ascii_letters, digits -from cloudinit.distros import ug_util from cloudinit import log as logging +from cloudinit import subp, util +from cloudinit.distros import ug_util from cloudinit.ssh_util import update_ssh_config -from cloudinit import subp -from cloudinit import util - -from string import ascii_letters, digits LOG = logging.getLogger(__name__) # We are removing certain 'painful' letters/numbers -PW_SET = (''.join([x for x in ascii_letters + digits - if x not in 'loLOI01'])) +PW_SET = "".join([x for x in ascii_letters + digits if x not in "loLOI01"]) def handle_ssh_pwauth(pw_auth, distro): @@ -105,12 +102,12 @@ def handle_ssh_pwauth(pw_auth, distro): cfg_name = "PasswordAuthentication" if util.is_true(pw_auth): - cfg_val = 'yes' + cfg_val = "yes" elif util.is_false(pw_auth): - cfg_val = 'no' + cfg_val = "no" else: bmsg = "Leaving SSH config '%s' unchanged." % cfg_name - if pw_auth is None or pw_auth.lower() == 'unchanged': + if pw_auth is None or pw_auth.lower() == "unchanged": LOG.debug("%s ssh_pwauth=%s", bmsg, pw_auth) else: LOG.warning("%s Unrecognized value: ssh_pwauth=%s", bmsg, pw_auth) @@ -121,7 +118,7 @@ def handle_ssh_pwauth(pw_auth, distro): LOG.debug("No need to restart SSH service, %s not updated.", cfg_name) return - distro.manage_service('restart', distro.get_option('ssh_svcname', 'ssh')) + distro.manage_service("restart", distro.get_option("ssh_svcname", "ssh")) LOG.debug("Restarted the SSH daemon.") @@ -129,27 +126,27 @@ def handle(_name, cfg, cloud, log, args): if args: # if run from command line, and give args, wipe the chpasswd['list'] password = args[0] - if 'chpasswd' in cfg and 'list' in cfg['chpasswd']: - del cfg['chpasswd']['list'] + if "chpasswd" in cfg and "list" in cfg["chpasswd"]: + del cfg["chpasswd"]["list"] else: password = util.get_cfg_option_str(cfg, "password", None) expire = True plist = None - if 'chpasswd' in cfg: - chfg = cfg['chpasswd'] - if 'list' in chfg and chfg['list']: - if isinstance(chfg['list'], list): + if "chpasswd" in cfg: + chfg = cfg["chpasswd"] + if "list" in chfg and chfg["list"]: + if isinstance(chfg["list"], list): log.debug("Handling input for chpasswd as list.") - plist = util.get_cfg_option_list(chfg, 'list', plist) + plist = util.get_cfg_option_list(chfg, "list", plist) else: log.debug("Handling input for chpasswd as multiline string.") - plist = util.get_cfg_option_str(chfg, 'list', plist) + plist = util.get_cfg_option_str(chfg, "list", plist) if plist: plist = plist.splitlines() - expire = util.get_cfg_option_bool(chfg, 'expire', expire) + expire = util.get_cfg_option_bool(chfg, "expire", expire) if not plist and password: (users, _groups) = ug_util.normalize_users_groups(cfg, cloud.distro) @@ -168,9 +165,9 @@ def handle(_name, cfg, cloud, log, args): users = [] # N.B. This regex is included in the documentation (i.e. the module # docstring), so any changes to it should be reflected there. - prog = re.compile(r'\$(1|2a|2y|5|6)(\$.+){2}') + prog = re.compile(r"\$(1|2a|2y|5|6)(\$.+){2}") for line in plist: - u, p = line.split(':', 1) + u, p = line.split(":", 1) if prog.match(p) is not None and ":" not in p: hashed_plist_in.append(line) hashed_users.append(u) @@ -182,7 +179,7 @@ def handle(_name, cfg, cloud, log, args): randlist.append("%s:%s" % (u, p)) plist_in.append("%s:%s" % (u, p)) users.append(u) - ch_in = '\n'.join(plist_in) + '\n' + ch_in = "\n".join(plist_in) + "\n" if users: try: log.debug("Changing password for %s:", users) @@ -190,9 +187,10 @@ def handle(_name, cfg, cloud, log, args): except Exception as e: errors.append(e) util.logexc( - log, "Failed to set passwords with chpasswd for %s", users) + log, "Failed to set passwords with chpasswd for %s", users + ) - hashed_ch_in = '\n'.join(hashed_plist_in) + '\n' + hashed_ch_in = "\n".join(hashed_plist_in) + "\n" if hashed_users: try: log.debug("Setting hashed password for %s:", hashed_users) @@ -200,12 +198,16 @@ def handle(_name, cfg, cloud, log, args): except Exception as e: errors.append(e) util.logexc( - log, "Failed to set hashed passwords with chpasswd for %s", - hashed_users) + log, + "Failed to set hashed passwords with chpasswd for %s", + hashed_users, + ) if len(randlist): - blurb = ("Set the following 'random' passwords\n", - '\n'.join(randlist)) + blurb = ( + "Set the following 'random' passwords\n", + "\n".join(randlist), + ) util.multi_log( "%s\n%s\n" % blurb, stderr=False, fallback_to_stdout=False ) @@ -222,7 +224,7 @@ def handle(_name, cfg, cloud, log, args): if expired_users: log.debug("Expired passwords for: %s users", expired_users) - handle_ssh_pwauth(cfg.get('ssh_pwauth'), cloud.distro) + handle_ssh_pwauth(cfg.get("ssh_pwauth"), cloud.distro) if len(errors): log.debug("%s errors occured, re-raising the last one", len(errors)) @@ -239,7 +241,8 @@ def chpasswd(distro, plist_in, hashed=False): u, p = pentry.split(":") distro.set_passwd(u, p, hashed=hashed) else: - cmd = ['chpasswd'] + (['-e'] if hashed else []) + cmd = ["chpasswd"] + (["-e"] if hashed else []) subp.subp(cmd, plist_in) + # vi: ts=4 expandtab diff --git a/cloudinit/config/cc_snap.py b/cloudinit/config/cc_snap.py index 21f30b57..9c38046c 100644 --- a/cloudinit/config/cc_snap.py +++ b/cloudinit/config/cc_snap.py @@ -8,23 +8,22 @@ import sys from textwrap import dedent from cloudinit import log as logging +from cloudinit import subp, util from cloudinit.config.schema import get_meta_doc, validate_cloudconfig_schema from cloudinit.settings import PER_INSTANCE from cloudinit.subp import prepend_base_command -from cloudinit import subp -from cloudinit import util - -distros = ['ubuntu'] +distros = ["ubuntu"] frequency = PER_INSTANCE LOG = logging.getLogger(__name__) meta = { - 'id': 'cc_snap', - 'name': 'Snap', - 'title': 'Install, configure and manage snapd and snap packages', - 'description': dedent("""\ + "id": "cc_snap", + "name": "Snap", + "title": "Install, configure and manage snapd and snap packages", + "description": dedent( + """\ This module provides a simple configuration namespace in cloud-init to both setup snapd and install snaps. @@ -55,9 +54,12 @@ meta = { **Development only**: The ``squashfuse_in_container`` boolean can be set true to install squashfuse package when in a container to enable snap installs. Default is false. - """), - 'distros': distros, - 'examples': [dedent("""\ + """ + ), + "distros": distros, + "examples": [ + dedent( + """\ snap: assertions: 00: | @@ -68,14 +70,20 @@ meta = { 00: snap create-user --sudoer --known <snap-user>@mydomain.com 01: snap install canonical-livepatch 02: canonical-livepatch enable <AUTH_TOKEN> - """), dedent("""\ + """ + ), + dedent( + """\ # LXC-based containers require squashfuse before snaps can be installed snap: commands: 00: apt-get install squashfuse -y 11: snap install emoj - """), dedent("""\ + """ + ), + dedent( + """\ # Convenience: the snap command can be omitted when specifying commands # as a list and 'snap' will automatically be prepended. # The following commands are equivalent: @@ -85,7 +93,10 @@ meta = { 01: ['snap', 'install', 'vlc'] 02: snap install vlc 03: 'snap install vlc' - """), dedent("""\ + """ + ), + dedent( + """\ # You can use a list of commands snap: commands: @@ -93,57 +104,61 @@ meta = { - ['snap', 'install', 'vlc'] - snap install vlc - 'snap install vlc' - """), dedent("""\ + """ + ), + dedent( + """\ # You can use a list of assertions snap: assertions: - signed_assertion_blob_here - | signed_assertion_blob_here - """)], - 'frequency': PER_INSTANCE, + """ + ), + ], + "frequency": PER_INSTANCE, } schema = { - 'type': 'object', - 'properties': { - 'snap': { - 'type': 'object', - 'properties': { - 'assertions': { - 'type': ['object', 'array'], # Array of strings or dict - 'items': {'type': 'string'}, - 'additionalItems': False, # Reject items non-string - 'minItems': 1, - 'minProperties': 1, - 'uniqueItems': True, - 'additionalProperties': {'type': 'string'}, + "type": "object", + "properties": { + "snap": { + "type": "object", + "properties": { + "assertions": { + "type": ["object", "array"], # Array of strings or dict + "items": {"type": "string"}, + "additionalItems": False, # Reject items non-string + "minItems": 1, + "minProperties": 1, + "uniqueItems": True, + "additionalProperties": {"type": "string"}, }, - 'commands': { - 'type': ['object', 'array'], # Array of strings or dict - 'items': { - 'oneOf': [ - {'type': 'array', 'items': {'type': 'string'}}, - {'type': 'string'}] + "commands": { + "type": ["object", "array"], # Array of strings or dict + "items": { + "oneOf": [ + {"type": "array", "items": {"type": "string"}}, + {"type": "string"}, + ] }, - 'additionalItems': False, # Reject non-string & non-list - 'minItems': 1, - 'minProperties': 1, - 'additionalProperties': { - 'oneOf': [ - {'type': 'string'}, - {'type': 'array', 'items': {'type': 'string'}}, + "additionalItems": False, # Reject non-string & non-list + "minItems": 1, + "minProperties": 1, + "additionalProperties": { + "oneOf": [ + {"type": "string"}, + {"type": "array", "items": {"type": "string"}}, ], }, }, - 'squashfuse_in_container': { - 'type': 'boolean' - } + "squashfuse_in_container": {"type": "boolean"}, }, - 'additionalProperties': False, # Reject keys not in schema - 'minProperties': 1 + "additionalProperties": False, # Reject keys not in schema + "minProperties": 1, } - } + }, } __doc__ = get_meta_doc(meta, schema) # Supplement python help() @@ -162,45 +177,49 @@ def add_assertions(assertions): """ if not assertions: return - LOG.debug('Importing user-provided snap assertions') + LOG.debug("Importing user-provided snap assertions") if isinstance(assertions, dict): assertions = assertions.values() elif not isinstance(assertions, list): raise TypeError( - 'assertion parameter was not a list or dict: {assertions}'.format( - assertions=assertions)) + "assertion parameter was not a list or dict: {assertions}".format( + assertions=assertions + ) + ) - snap_cmd = [SNAP_CMD, 'ack'] + snap_cmd = [SNAP_CMD, "ack"] combined = "\n".join(assertions) for asrt in assertions: - LOG.debug('Snap acking: %s', asrt.split('\n')[0:2]) + LOG.debug("Snap acking: %s", asrt.split("\n")[0:2]) - util.write_file(ASSERTIONS_FILE, combined.encode('utf-8')) + util.write_file(ASSERTIONS_FILE, combined.encode("utf-8")) subp.subp(snap_cmd + [ASSERTIONS_FILE], capture=True) def run_commands(commands): """Run the provided commands provided in snap:commands configuration. - Commands are run individually. Any errors are collected and reported - after attempting all commands. + Commands are run individually. Any errors are collected and reported + after attempting all commands. - @param commands: A list or dict containing commands to run. Keys of a - dict will be used to order the commands provided as dict values. - """ + @param commands: A list or dict containing commands to run. Keys of a + dict will be used to order the commands provided as dict values. + """ if not commands: return - LOG.debug('Running user-provided snap commands') + LOG.debug("Running user-provided snap commands") if isinstance(commands, dict): # Sort commands based on dictionary key commands = [v for _, v in sorted(commands.items())] elif not isinstance(commands, list): raise TypeError( - 'commands parameter was not a list or dict: {commands}'.format( - commands=commands)) + "commands parameter was not a list or dict: {commands}".format( + commands=commands + ) + ) - fixed_snap_commands = prepend_base_command('snap', commands) + fixed_snap_commands = prepend_base_command("snap", commands) cmd_failures = [] for command in fixed_snap_commands: @@ -210,8 +229,9 @@ def run_commands(commands): except subp.ProcessExecutionError as e: cmd_failures.append(str(e)) if cmd_failures: - msg = 'Failures running snap commands:\n{cmd_failures}'.format( - cmd_failures=cmd_failures) + msg = "Failures running snap commands:\n{cmd_failures}".format( + cmd_failures=cmd_failures + ) util.logexc(LOG, msg) raise RuntimeError(msg) @@ -227,23 +247,25 @@ def maybe_install_squashfuse(cloud): util.logexc(LOG, "Package update failed") raise try: - cloud.distro.install_packages(['squashfuse']) + cloud.distro.install_packages(["squashfuse"]) except Exception: util.logexc(LOG, "Failed to install squashfuse") raise def handle(name, cfg, cloud, log, args): - cfgin = cfg.get('snap', {}) + cfgin = cfg.get("snap", {}) if not cfgin: - LOG.debug(("Skipping module named %s," - " no 'snap' key in configuration"), name) + LOG.debug( + "Skipping module named %s, no 'snap' key in configuration", name + ) return validate_cloudconfig_schema(cfg, schema) - if util.is_true(cfgin.get('squashfuse_in_container', False)): + if util.is_true(cfgin.get("squashfuse_in_container", False)): maybe_install_squashfuse(cloud) - add_assertions(cfgin.get('assertions', [])) - run_commands(cfgin.get('commands', [])) + add_assertions(cfgin.get("assertions", [])) + run_commands(cfgin.get("commands", [])) + # vi: ts=4 expandtab diff --git a/cloudinit/config/cc_spacewalk.py b/cloudinit/config/cc_spacewalk.py index 95083607..3fa6c388 100644 --- a/cloudinit/config/cc_spacewalk.py +++ b/cloudinit/config/cc_spacewalk.py @@ -29,9 +29,8 @@ For more information about spacewalk see: https://fedorahosted.org/spacewalk/ from cloudinit import subp - -distros = ['redhat', 'fedora'] -required_packages = ['rhn-setup'] +distros = ["redhat", "fedora"] +required_packages = ["rhn-setup"] def_ca_cert_path = "/usr/share/rhn/RHN-ORG-TRUSTED-SSL-CERT" @@ -41,7 +40,7 @@ def is_registered(): # assume we aren't registered; which is sorta ghetto... already_registered = False try: - subp.subp(['rhn-profile-sync', '--verbose'], capture=False) + subp.subp(["rhn-profile-sync", "--verbose"], capture=False) already_registered = True except subp.ProcessExecutionError as e: if e.exit_code != 1: @@ -49,42 +48,58 @@ def is_registered(): return already_registered -def do_register(server, profile_name, - ca_cert_path=def_ca_cert_path, - proxy=None, log=None, - activation_key=None): +def do_register( + server, + profile_name, + ca_cert_path=def_ca_cert_path, + proxy=None, + log=None, + activation_key=None, +): if log is not None: - log.info("Registering using `rhnreg_ks` profile '%s'" - " into server '%s'", profile_name, server) - cmd = ['rhnreg_ks'] - cmd.extend(['--serverUrl', 'https://%s/XMLRPC' % server]) - cmd.extend(['--profilename', str(profile_name)]) + log.info( + "Registering using `rhnreg_ks` profile '%s' into server '%s'", + profile_name, + server, + ) + cmd = ["rhnreg_ks"] + cmd.extend(["--serverUrl", "https://%s/XMLRPC" % server]) + cmd.extend(["--profilename", str(profile_name)]) if proxy: cmd.extend(["--proxy", str(proxy)]) if ca_cert_path: - cmd.extend(['--sslCACert', str(ca_cert_path)]) + cmd.extend(["--sslCACert", str(ca_cert_path)]) if activation_key: - cmd.extend(['--activationkey', str(activation_key)]) + cmd.extend(["--activationkey", str(activation_key)]) subp.subp(cmd, capture=False) def handle(name, cfg, cloud, log, _args): - if 'spacewalk' not in cfg: - log.debug(("Skipping module named %s," - " no 'spacewalk' key in configuration"), name) + if "spacewalk" not in cfg: + log.debug( + "Skipping module named %s, no 'spacewalk' key in configuration", + name, + ) return - cfg = cfg['spacewalk'] - spacewalk_server = cfg.get('server') + cfg = cfg["spacewalk"] + spacewalk_server = cfg.get("server") if spacewalk_server: # Need to have this installed before further things will work. cloud.distro.install_packages(required_packages) if not is_registered(): - do_register(spacewalk_server, - cloud.datasource.get_hostname(fqdn=True), - proxy=cfg.get("proxy"), log=log, - activation_key=cfg.get('activation_key')) + do_register( + spacewalk_server, + cloud.datasource.get_hostname(fqdn=True), + proxy=cfg.get("proxy"), + log=log, + activation_key=cfg.get("activation_key"), + ) else: - log.debug("Skipping module named %s, 'spacewalk/server' key" - " was not found in configuration", name) + log.debug( + "Skipping module named %s, 'spacewalk/server' key" + " was not found in configuration", + name, + ) + # vi: ts=4 expandtab diff --git a/cloudinit/config/cc_ssh.py b/cloudinit/config/cc_ssh.py index 1053ab67..a1f05176 100755 --- a/cloudinit/config/cc_ssh.py +++ b/cloudinit/config/cc_ssh.py @@ -162,27 +162,26 @@ import glob import os import sys +from cloudinit import ssh_util, subp, util from cloudinit.distros import ug_util -from cloudinit import ssh_util -from cloudinit import subp -from cloudinit import util - -GENERATE_KEY_NAMES = ['rsa', 'dsa', 'ecdsa', 'ed25519'] -KEY_FILE_TPL = '/etc/ssh/ssh_host_%s_key' +GENERATE_KEY_NAMES = ["rsa", "dsa", "ecdsa", "ed25519"] +KEY_FILE_TPL = "/etc/ssh/ssh_host_%s_key" PUBLISH_HOST_KEYS = True # Don't publish the dsa hostkey by default since OpenSSH recommends not using # it. -HOST_KEY_PUBLISH_BLACKLIST = ['dsa'] +HOST_KEY_PUBLISH_BLACKLIST = ["dsa"] CONFIG_KEY_TO_FILE = {} PRIV_TO_PUB = {} for k in GENERATE_KEY_NAMES: CONFIG_KEY_TO_FILE.update({"%s_private" % k: (KEY_FILE_TPL % k, 0o600)}) CONFIG_KEY_TO_FILE.update( - {"%s_public" % k: (KEY_FILE_TPL % k + ".pub", 0o600)}) + {"%s_public" % k: (KEY_FILE_TPL % k + ".pub", 0o600)} + ) CONFIG_KEY_TO_FILE.update( - {"%s_certificate" % k: (KEY_FILE_TPL % k + "-cert.pub", 0o600)}) + {"%s_certificate" % k: (KEY_FILE_TPL % k + "-cert.pub", 0o600)} + ) PRIV_TO_PUB["%s_private" % k] = "%s_public" % k KEY_GEN_TPL = 'o=$(ssh-keygen -yf "%s") && echo "$o" root@localhost > "%s"' @@ -209,43 +208,48 @@ def handle(_name, cfg, cloud, log, _args): tgt_perms = CONFIG_KEY_TO_FILE[key][1] util.write_file(tgt_fn, val, tgt_perms) # set server to present the most recently identified certificate - if '_certificate' in key: - cert_config = {'HostCertificate': tgt_fn} + if "_certificate" in key: + cert_config = {"HostCertificate": tgt_fn} ssh_util.update_ssh_config(cert_config) for (priv, pub) in PRIV_TO_PUB.items(): - if pub in cfg['ssh_keys'] or priv not in cfg['ssh_keys']: + if pub in cfg["ssh_keys"] or priv not in cfg["ssh_keys"]: continue pair = (CONFIG_KEY_TO_FILE[priv][0], CONFIG_KEY_TO_FILE[pub][0]) - cmd = ['sh', '-xc', KEY_GEN_TPL % pair] + cmd = ["sh", "-xc", KEY_GEN_TPL % pair] try: # TODO(harlowja): Is this guard needed? with util.SeLinuxGuard("/etc/ssh", recursive=True): subp.subp(cmd, capture=False) log.debug("Generated a key for %s from %s", pair[0], pair[1]) except Exception: - util.logexc(log, "Failed generated a key for %s from %s", - pair[0], pair[1]) + util.logexc( + log, + "Failed generated a key for %s from %s", + pair[0], + pair[1], + ) else: # if not, generate them - genkeys = util.get_cfg_option_list(cfg, - 'ssh_genkeytypes', - GENERATE_KEY_NAMES) + genkeys = util.get_cfg_option_list( + cfg, "ssh_genkeytypes", GENERATE_KEY_NAMES + ) lang_c = os.environ.copy() - lang_c['LANG'] = 'C' + lang_c["LANG"] = "C" for keytype in genkeys: keyfile = KEY_FILE_TPL % (keytype) if os.path.exists(keyfile): continue util.ensure_dir(os.path.dirname(keyfile)) - cmd = ['ssh-keygen', '-t', keytype, '-N', '', '-f', keyfile] + cmd = ["ssh-keygen", "-t", keytype, "-N", "", "-f", keyfile] # TODO(harlowja): Is this guard needed? with util.SeLinuxGuard("/etc/ssh", recursive=True): try: out, err = subp.subp(cmd, capture=True, env=lang_c) - if not util.get_cfg_option_bool(cfg, 'ssh_quiet_keygen', - False): + if not util.get_cfg_option_bool( + cfg, "ssh_quiet_keygen", False + ): sys.stdout.write(util.decode_binary(out)) gid = util.get_group_id("ssh_keys") @@ -256,19 +260,27 @@ def handle(_name, cfg, cloud, log, _args): os.chmod(keyfile + ".pub", 0o644) except subp.ProcessExecutionError as e: err = util.decode_binary(e.stderr).lower() - if (e.exit_code == 1 and - err.lower().startswith("unknown key")): + if e.exit_code == 1 and err.lower().startswith( + "unknown key" + ): log.debug("ssh-keygen: unknown key type '%s'", keytype) else: - util.logexc(log, "Failed generating key type %s to " - "file %s", keytype, keyfile) + util.logexc( + log, + "Failed generating key type %s to file %s", + keytype, + keyfile, + ) if "ssh_publish_hostkeys" in cfg: host_key_blacklist = util.get_cfg_option_list( - cfg["ssh_publish_hostkeys"], "blacklist", - HOST_KEY_PUBLISH_BLACKLIST) + cfg["ssh_publish_hostkeys"], + "blacklist", + HOST_KEY_PUBLISH_BLACKLIST, + ) publish_hostkeys = util.get_cfg_option_bool( - cfg["ssh_publish_hostkeys"], "enabled", PUBLISH_HOST_KEYS) + cfg["ssh_publish_hostkeys"], "enabled", PUBLISH_HOST_KEYS + ) else: host_key_blacklist = HOST_KEY_PUBLISH_BLACKLIST publish_hostkeys = PUBLISH_HOST_KEYS @@ -284,15 +296,18 @@ def handle(_name, cfg, cloud, log, _args): (users, _groups) = ug_util.normalize_users_groups(cfg, cloud.distro) (user, _user_config) = ug_util.extract_default(users) disable_root = util.get_cfg_option_bool(cfg, "disable_root", True) - disable_root_opts = util.get_cfg_option_str(cfg, "disable_root_opts", - ssh_util.DISABLE_USER_OPTS) + disable_root_opts = util.get_cfg_option_str( + cfg, "disable_root_opts", ssh_util.DISABLE_USER_OPTS + ) keys = [] - if util.get_cfg_option_bool(cfg, 'allow_public_ssh_keys', True): + if util.get_cfg_option_bool(cfg, "allow_public_ssh_keys", True): keys = cloud.get_public_ssh_keys() or [] else: - log.debug('Skipping import of publish SSH keys per ' - 'config setting: allow_public_ssh_keys=False') + log.debug( + "Skipping import of publish SSH keys per " + "config setting: allow_public_ssh_keys=False" + ) if "ssh_authorized_keys" in cfg: cfgkeys = cfg["ssh_authorized_keys"] @@ -312,12 +327,12 @@ def apply_credentials(keys, user, disable_root, disable_root_opts): if disable_root: if not user: user = "NONE" - key_prefix = disable_root_opts.replace('$USER', user) - key_prefix = key_prefix.replace('$DISABLE_USER', 'root') + key_prefix = disable_root_opts.replace("$USER", user) + key_prefix = key_prefix.replace("$DISABLE_USER", "root") else: - key_prefix = '' + key_prefix = "" - ssh_util.setup_user_keys(keys, 'root', options=key_prefix) + ssh_util.setup_user_keys(keys, "root", options=key_prefix) def get_public_host_keys(blacklist=None): @@ -327,18 +342,21 @@ def get_public_host_keys(blacklist=None): @returns: List of keys, each formatted as a two-element tuple. e.g. [('ssh-rsa', 'AAAAB3Nz...'), ('ssh-ed25519', 'AAAAC3Nx...')] """ - public_key_file_tmpl = '%s.pub' % (KEY_FILE_TPL,) + public_key_file_tmpl = "%s.pub" % (KEY_FILE_TPL,) key_list = [] blacklist_files = [] if blacklist: # Convert blacklist to filenames: # 'dsa' -> '/etc/ssh/ssh_host_dsa_key.pub' - blacklist_files = [public_key_file_tmpl % (key_type,) - for key_type in blacklist] + blacklist_files = [ + public_key_file_tmpl % (key_type,) for key_type in blacklist + ] # Get list of public key files and filter out blacklisted files. - file_list = [hostfile for hostfile - in glob.glob(public_key_file_tmpl % ('*',)) - if hostfile not in blacklist_files] + file_list = [ + hostfile + for hostfile in glob.glob(public_key_file_tmpl % ("*",)) + if hostfile not in blacklist_files + ] # Read host key files, retrieve first two fields as a tuple and # append that tuple to key_list. diff --git a/cloudinit/config/cc_ssh_authkey_fingerprints.py b/cloudinit/config/cc_ssh_authkey_fingerprints.py index 5323522c..020c3469 100755 --- a/cloudinit/config/cc_ssh_authkey_fingerprints.py +++ b/cloudinit/config/cc_ssh_authkey_fingerprints.py @@ -28,23 +28,21 @@ the keys can be specified, but defaults to ``sha256``. import base64 import hashlib -from cloudinit.simpletable import SimpleTable - +from cloudinit import ssh_util, util from cloudinit.distros import ug_util -from cloudinit import ssh_util -from cloudinit import util +from cloudinit.simpletable import SimpleTable def _split_hash(bin_hash): split_up = [] for i in range(0, len(bin_hash), 2): - split_up.append(bin_hash[i:i + 2]) + split_up.append(bin_hash[i : i + 2]) return split_up -def _gen_fingerprint(b64_text, hash_meth='sha256'): +def _gen_fingerprint(b64_text, hash_meth="sha256"): if not b64_text: - return '' + return "" # TBD(harlowja): Maybe we should feed this into 'ssh -lf'? try: hasher = hashlib.new(hash_meth) @@ -54,58 +52,75 @@ def _gen_fingerprint(b64_text, hash_meth='sha256'): # Raised when b64 not really b64... # or when the hash type is not really # a known/supported hash type... - return '?' + return "?" def _is_printable_key(entry): if any([entry.keytype, entry.base64, entry.comment, entry.options]): - if (entry.keytype and entry.keytype.lower().strip() - in ssh_util.VALID_KEY_TYPES): + if ( + entry.keytype + and entry.keytype.lower().strip() in ssh_util.VALID_KEY_TYPES + ): return True return False -def _pprint_key_entries(user, key_fn, key_entries, hash_meth='sha256', - prefix='ci-info: '): +def _pprint_key_entries( + user, key_fn, key_entries, hash_meth="sha256", prefix="ci-info: " +): if not key_entries: - message = ("%sno authorized SSH keys fingerprints found for user %s.\n" - % (prefix, user)) + message = ( + "%sno authorized SSH keys fingerprints found for user %s.\n" + % (prefix, user) + ) util.multi_log(message, console=True, stderr=False) return - tbl_fields = ['Keytype', 'Fingerprint (%s)' % (hash_meth), 'Options', - 'Comment'] + tbl_fields = [ + "Keytype", + "Fingerprint (%s)" % (hash_meth), + "Options", + "Comment", + ] tbl = SimpleTable(tbl_fields) for entry in key_entries: if _is_printable_key(entry): - row = [entry.keytype or '-', - _gen_fingerprint(entry.base64, hash_meth) or '-', - entry.options or '-', - entry.comment or '-'] + row = [ + entry.keytype or "-", + _gen_fingerprint(entry.base64, hash_meth) or "-", + entry.options or "-", + entry.comment or "-", + ] tbl.add_row(row) authtbl_s = tbl.get_string() authtbl_lines = authtbl_s.splitlines() max_len = len(max(authtbl_lines, key=len)) lines = [ - util.center("Authorized keys from %s for user %s" % - (key_fn, user), "+", max_len), + util.center( + "Authorized keys from %s for user %s" % (key_fn, user), + "+", + max_len, + ), ] lines.extend(authtbl_lines) for line in lines: - util.multi_log(text="%s%s\n" % (prefix, line), - stderr=False, console=True) + util.multi_log( + text="%s%s\n" % (prefix, line), stderr=False, console=True + ) def handle(name, cfg, cloud, log, _args): - if util.is_true(cfg.get('no_ssh_fingerprints', False)): - log.debug(("Skipping module named %s, " - "logging of SSH fingerprints disabled"), name) + if util.is_true(cfg.get("no_ssh_fingerprints", False)): + log.debug( + "Skipping module named %s, logging of SSH fingerprints disabled", + name, + ) return hash_meth = util.get_cfg_option_str(cfg, "authkey_hash", "sha256") (users, _groups) = ug_util.normalize_users_groups(cfg, cloud.distro) for (user_name, _cfg) in users.items(): (key_fn, key_entries) = ssh_util.extract_authorized_keys(user_name) - _pprint_key_entries(user_name, key_fn, - key_entries, hash_meth) + _pprint_key_entries(user_name, key_fn, key_entries, hash_meth) + # vi: ts=4 expandtab diff --git a/cloudinit/config/cc_ssh_import_id.py b/cloudinit/config/cc_ssh_import_id.py index 856e5a9e..a9575c59 100755 --- a/cloudinit/config/cc_ssh_import_id.py +++ b/cloudinit/config/cc_ssh_import_id.py @@ -30,13 +30,13 @@ either ``lp:`` for launchpad or ``gh:`` for github to the username. - lp:user """ -from cloudinit.distros import ug_util -from cloudinit import subp -from cloudinit import util import pwd +from cloudinit import subp, util +from cloudinit.distros import ug_util + # https://launchpad.net/ssh-import-id -distros = ['ubuntu', 'debian'] +distros = ["ubuntu", "debian"] def handle(_name, cfg, cloud, log, args): @@ -56,11 +56,11 @@ def handle(_name, cfg, cloud, log, args): elist = [] for (user, user_cfg) in users.items(): import_ids = [] - if user_cfg['default']: + if user_cfg["default"]: import_ids = util.get_cfg_option_list(cfg, "ssh_import_id", []) else: try: - import_ids = user_cfg['ssh_import_id'] + import_ids = user_cfg["ssh_import_id"] except Exception: log.debug("User %s is not configured for ssh_import_id", user) continue @@ -69,8 +69,9 @@ def handle(_name, cfg, cloud, log, args): import_ids = util.uniq_merge(import_ids) import_ids = [str(i) for i in import_ids] except Exception: - log.debug("User %s is not correctly configured for ssh_import_id", - user) + log.debug( + "User %s is not correctly configured for ssh_import_id", user + ) continue if not len(import_ids): @@ -79,8 +80,9 @@ def handle(_name, cfg, cloud, log, args): try: import_ssh_ids(import_ids, user, log) except Exception as exc: - util.logexc(log, "ssh-import-id failed for: %s %s", user, - import_ids) + util.logexc( + log, "ssh-import-id failed for: %s %s", user, import_ids + ) elist.append(exc) if len(elist): @@ -107,4 +109,5 @@ def import_ssh_ids(ids, user, log): util.logexc(log, "Failed to run command to import %s SSH ids", user) raise exc + # vi: ts=4 expandtab diff --git a/cloudinit/config/cc_timezone.py b/cloudinit/config/cc_timezone.py index a9de8fac..24e6099e 100644 --- a/cloudinit/config/cc_timezone.py +++ b/cloudinit/config/cc_timezone.py @@ -27,7 +27,6 @@ the timezone from cloud config. """ from cloudinit import util - from cloudinit.settings import PER_INSTANCE frequency = PER_INSTANCE @@ -46,4 +45,5 @@ def handle(name, cfg, cloud, log, args): # Let the distro handle settings its timezone cloud.distro.set_timezone(timezone) + # vi: ts=4 expandtab diff --git a/cloudinit/config/cc_ubuntu_advantage.py b/cloudinit/config/cc_ubuntu_advantage.py index 831a92a2..413fd3c1 100644 --- a/cloudinit/config/cc_ubuntu_advantage.py +++ b/cloudinit/config/cc_ubuntu_advantage.py @@ -4,22 +4,21 @@ from textwrap import dedent -from cloudinit.config.schema import get_meta_doc, validate_cloudconfig_schema from cloudinit import log as logging +from cloudinit import subp, util +from cloudinit.config.schema import get_meta_doc, validate_cloudconfig_schema from cloudinit.settings import PER_INSTANCE -from cloudinit import subp -from cloudinit import util - -UA_URL = 'https://ubuntu.com/advantage' +UA_URL = "https://ubuntu.com/advantage" -distros = ['ubuntu'] +distros = ["ubuntu"] meta = { - 'id': 'cc_ubuntu_advantage', - 'name': 'Ubuntu Advantage', - 'title': 'Configure Ubuntu Advantage support services', - 'description': dedent("""\ + "id": "cc_ubuntu_advantage", + "name": "Ubuntu Advantage", + "title": "Configure Ubuntu Advantage support services", + "description": dedent( + """\ Attach machine to an existing Ubuntu Advantage support contract and enable or disable support services such as Livepatch, ESM, FIPS and FIPS Updates. When attaching a machine to Ubuntu Advantage, @@ -31,14 +30,21 @@ meta = { a reboot to ensure the machine is running the FIPS-compliant kernel. See :ref:`Power State Change` for information on how to configure cloud-init to perform this reboot. - """), - 'distros': distros, - 'examples': [dedent("""\ + """ + ), + "distros": distros, + "examples": [ + dedent( + """\ # Attach the machine to an Ubuntu Advantage support contract with a # UA contract token obtained from %s. ubuntu_advantage: token: <ua_contract_token> - """ % UA_URL), dedent("""\ + """ + % UA_URL + ), + dedent( + """\ # Attach the machine to an Ubuntu Advantage support contract enabling # only fips and esm services. Services will only be enabled if # the environment supports said service. Otherwise warnings will @@ -48,7 +54,10 @@ meta = { enable: - fips - esm - """), dedent("""\ + """ + ), + dedent( + """\ # Attach the machine to an Ubuntu Advantage support contract and enable # the FIPS service. Perform a reboot once cloud-init has # completed. @@ -58,30 +67,32 @@ meta = { token: <ua_contract_token> enable: - fips - """)], - 'frequency': PER_INSTANCE, + """ + ), + ], + "frequency": PER_INSTANCE, } schema = { - 'type': 'object', - 'properties': { - 'ubuntu_advantage': { - 'type': 'object', - 'properties': { - 'enable': { - 'type': 'array', - 'items': {'type': 'string'}, + "type": "object", + "properties": { + "ubuntu_advantage": { + "type": "object", + "properties": { + "enable": { + "type": "array", + "items": {"type": "string"}, + }, + "token": { + "type": "string", + "description": "A contract token obtained from %s." + % UA_URL, }, - 'token': { - 'type': 'string', - 'description': ( - 'A contract token obtained from %s.' % UA_URL) - } }, - 'required': ['token'], - 'additionalProperties': False + "required": ["token"], + "additionalProperties": False, } - } + }, } __doc__ = get_meta_doc(meta, schema) # Supplement python help() @@ -93,52 +104,61 @@ def configure_ua(token=None, enable=None): """Call ua commandline client to attach or enable services.""" error = None if not token: - error = ('ubuntu_advantage: token must be provided') + error = "ubuntu_advantage: token must be provided" LOG.error(error) raise RuntimeError(error) if enable is None: enable = [] elif isinstance(enable, str): - LOG.warning('ubuntu_advantage: enable should be a list, not' - ' a string; treating as a single enable') + LOG.warning( + "ubuntu_advantage: enable should be a list, not" + " a string; treating as a single enable" + ) enable = [enable] elif not isinstance(enable, list): - LOG.warning('ubuntu_advantage: enable should be a list, not' - ' a %s; skipping enabling services', - type(enable).__name__) + LOG.warning( + "ubuntu_advantage: enable should be a list, not" + " a %s; skipping enabling services", + type(enable).__name__, + ) enable = [] - attach_cmd = ['ua', 'attach', token] - LOG.debug('Attaching to Ubuntu Advantage. %s', ' '.join(attach_cmd)) + attach_cmd = ["ua", "attach", token] + LOG.debug("Attaching to Ubuntu Advantage. %s", " ".join(attach_cmd)) try: subp.subp(attach_cmd) except subp.ProcessExecutionError as e: - msg = 'Failure attaching Ubuntu Advantage:\n{error}'.format( - error=str(e)) + msg = "Failure attaching Ubuntu Advantage:\n{error}".format( + error=str(e) + ) util.logexc(LOG, msg) raise RuntimeError(msg) from e enable_errors = [] for service in enable: try: - cmd = ['ua', 'enable', service] + cmd = ["ua", "enable", service] subp.subp(cmd, capture=True) except subp.ProcessExecutionError as e: enable_errors.append((service, e)) if enable_errors: for service, error in enable_errors: msg = 'Failure enabling "{service}":\n{error}'.format( - service=service, error=str(error)) + service=service, error=str(error) + ) util.logexc(LOG, msg) raise RuntimeError( - 'Failure enabling Ubuntu Advantage service(s): {}'.format( - ', '.join('"{}"'.format(service) - for service, _ in enable_errors))) + "Failure enabling Ubuntu Advantage service(s): {}".format( + ", ".join( + '"{}"'.format(service) for service, _ in enable_errors + ) + ) + ) def maybe_install_ua_tools(cloud): """Install ubuntu-advantage-tools if not present.""" - if subp.which('ua'): + if subp.which("ua"): return try: cloud.distro.update_package_sources() @@ -146,7 +166,7 @@ def maybe_install_ua_tools(cloud): util.logexc(LOG, "Package update failed") raise try: - cloud.distro.install_packages(['ubuntu-advantage-tools']) + cloud.distro.install_packages(["ubuntu-advantage-tools"]) except Exception: util.logexc(LOG, "Failed to install ubuntu-advantage-tools") raise @@ -154,27 +174,35 @@ def maybe_install_ua_tools(cloud): def handle(name, cfg, cloud, log, args): ua_section = None - if 'ubuntu-advantage' in cfg: - LOG.warning('Deprecated configuration key "ubuntu-advantage" provided.' - ' Expected underscore delimited "ubuntu_advantage"; will' - ' attempt to continue.') - ua_section = cfg['ubuntu-advantage'] - if 'ubuntu_advantage' in cfg: - ua_section = cfg['ubuntu_advantage'] + if "ubuntu-advantage" in cfg: + LOG.warning( + 'Deprecated configuration key "ubuntu-advantage" provided.' + ' Expected underscore delimited "ubuntu_advantage"; will' + " attempt to continue." + ) + ua_section = cfg["ubuntu-advantage"] + if "ubuntu_advantage" in cfg: + ua_section = cfg["ubuntu_advantage"] if ua_section is None: - LOG.debug("Skipping module named %s," - " no 'ubuntu_advantage' configuration found", name) + LOG.debug( + "Skipping module named %s," + " no 'ubuntu_advantage' configuration found", + name, + ) return validate_cloudconfig_schema(cfg, schema) - if 'commands' in ua_section: + if "commands" in ua_section: msg = ( 'Deprecated configuration "ubuntu-advantage: commands" provided.' - ' Expected "token"') + ' Expected "token"' + ) LOG.error(msg) raise RuntimeError(msg) maybe_install_ua_tools(cloud) - configure_ua(token=ua_section.get('token'), - enable=ua_section.get('enable')) + configure_ua( + token=ua_section.get("token"), enable=ua_section.get("enable") + ) + # vi: ts=4 expandtab diff --git a/cloudinit/config/cc_ubuntu_drivers.py b/cloudinit/config/cc_ubuntu_drivers.py index 7f617efe..6c8494c8 100644 --- a/cloudinit/config/cc_ubuntu_drivers.py +++ b/cloudinit/config/cc_ubuntu_drivers.py @@ -5,57 +5,62 @@ import os from textwrap import dedent -from cloudinit.config.schema import get_meta_doc, validate_cloudconfig_schema from cloudinit import log as logging +from cloudinit import subp, temp_utils, type_utils, util +from cloudinit.config.schema import get_meta_doc, validate_cloudconfig_schema from cloudinit.settings import PER_INSTANCE -from cloudinit import subp -from cloudinit import temp_utils -from cloudinit import type_utils -from cloudinit import util LOG = logging.getLogger(__name__) frequency = PER_INSTANCE -distros = ['ubuntu'] +distros = ["ubuntu"] meta = { - 'id': 'cc_ubuntu_drivers', - 'name': 'Ubuntu Drivers', - 'title': 'Interact with third party drivers in Ubuntu.', - 'description': dedent("""\ + "id": "cc_ubuntu_drivers", + "name": "Ubuntu Drivers", + "title": "Interact with third party drivers in Ubuntu.", + "description": dedent( + """\ This module interacts with the 'ubuntu-drivers' command to install - third party driver packages."""), - 'distros': distros, - 'examples': [dedent("""\ + third party driver packages.""" + ), + "distros": distros, + "examples": [ + dedent( + """\ drivers: nvidia: license-accepted: true - """)], - 'frequency': frequency, + """ + ) + ], + "frequency": frequency, } schema = { - 'type': 'object', - 'properties': { - 'drivers': { - 'type': 'object', - 'additionalProperties': False, - 'properties': { - 'nvidia': { - 'type': 'object', - 'additionalProperties': False, - 'required': ['license-accepted'], - 'properties': { - 'license-accepted': { - 'type': 'boolean', - 'description': ("Do you accept the NVIDIA driver" - " license?"), + "type": "object", + "properties": { + "drivers": { + "type": "object", + "additionalProperties": False, + "properties": { + "nvidia": { + "type": "object", + "additionalProperties": False, + "required": ["license-accepted"], + "properties": { + "license-accepted": { + "type": "boolean", + "description": ( + "Do you accept the NVIDIA driver license?" + ), }, - 'version': { - 'type': 'string', - 'description': ( - 'The version of the driver to install (e.g.' + "version": { + "type": "string", + "description": ( + "The version of the driver to install (e.g." ' "390", "410"). Defaults to the latest' - ' version.'), + " version." + ), }, }, }, @@ -64,7 +69,8 @@ schema = { }, } OLD_UBUNTU_DRIVERS_STDERR_NEEDLE = ( - "ubuntu-drivers: error: argument <command>: invalid choice: 'install'") + "ubuntu-drivers: error: argument <command>: invalid choice: 'install'" +) __doc__ = get_meta_doc(meta, schema) # Supplement python help() @@ -99,10 +105,11 @@ db_x_loadtemplatefile "$1" cloud-init def install_drivers(cfg, pkg_install_func): if not isinstance(cfg, dict): raise TypeError( - "'drivers' config expected dict, found '%s': %s" % - (type_utils.obj_name(cfg), cfg)) + "'drivers' config expected dict, found '%s': %s" + % (type_utils.obj_name(cfg), cfg) + ) - cfgpath = 'nvidia/license-accepted' + cfgpath = "nvidia/license-accepted" # Call translate_bool to ensure that we treat string values like "yes" as # acceptance and _don't_ treat string values like "nah" as acceptance # because they're True-ish @@ -111,46 +118,56 @@ def install_drivers(cfg, pkg_install_func): LOG.debug("Not installing NVIDIA drivers. %s=%s", cfgpath, nv_acc) return - if not subp.which('ubuntu-drivers'): - LOG.debug("'ubuntu-drivers' command not available. " - "Installing ubuntu-drivers-common") - pkg_install_func(['ubuntu-drivers-common']) + if not subp.which("ubuntu-drivers"): + LOG.debug( + "'ubuntu-drivers' command not available. " + "Installing ubuntu-drivers-common" + ) + pkg_install_func(["ubuntu-drivers-common"]) - driver_arg = 'nvidia' - version_cfg = util.get_cfg_by_path(cfg, 'nvidia/version') + driver_arg = "nvidia" + version_cfg = util.get_cfg_by_path(cfg, "nvidia/version") if version_cfg: - driver_arg += ':{}'.format(version_cfg) + driver_arg += ":{}".format(version_cfg) - LOG.debug("Installing and activating NVIDIA drivers (%s=%s, version=%s)", - cfgpath, nv_acc, version_cfg if version_cfg else 'latest') + LOG.debug( + "Installing and activating NVIDIA drivers (%s=%s, version=%s)", + cfgpath, + nv_acc, + version_cfg if version_cfg else "latest", + ) # Register and set debconf selection linux/nvidia/latelink = true tdir = temp_utils.mkdtemp(needs_exe=True) - debconf_file = os.path.join(tdir, 'nvidia.template') - debconf_script = os.path.join(tdir, 'nvidia-debconf.sh') + debconf_file = os.path.join(tdir, "nvidia.template") + debconf_script = os.path.join(tdir, "nvidia-debconf.sh") try: util.write_file(debconf_file, NVIDIA_DEBCONF_CONTENT) util.write_file( debconf_script, util.encode_text(NVIDIA_DRIVER_LATELINK_DEBCONF_SCRIPT), - mode=0o755) + mode=0o755, + ) subp.subp([debconf_script, debconf_file]) except Exception as e: util.logexc( - LOG, "Failed to register NVIDIA debconf template: %s", str(e)) + LOG, "Failed to register NVIDIA debconf template: %s", str(e) + ) raise finally: if os.path.isdir(tdir): util.del_dir(tdir) try: - subp.subp(['ubuntu-drivers', 'install', '--gpgpu', driver_arg]) + subp.subp(["ubuntu-drivers", "install", "--gpgpu", driver_arg]) except subp.ProcessExecutionError as exc: if OLD_UBUNTU_DRIVERS_STDERR_NEEDLE in exc.stderr: - LOG.warning('the available version of ubuntu-drivers is' - ' too old to perform requested driver installation') - elif 'No drivers found for installation.' in exc.stdout: - LOG.warning('ubuntu-drivers found no drivers for installation') + LOG.warning( + "the available version of ubuntu-drivers is" + " too old to perform requested driver installation" + ) + elif "No drivers found for installation." in exc.stdout: + LOG.warning("ubuntu-drivers found no drivers for installation") raise @@ -160,4 +177,4 @@ def handle(name, cfg, cloud, log, _args): return validate_cloudconfig_schema(cfg, schema) - install_drivers(cfg['drivers'], cloud.distro.install_packages) + install_drivers(cfg["drivers"], cloud.distro.install_packages) diff --git a/cloudinit/config/cc_update_etc_hosts.py b/cloudinit/config/cc_update_etc_hosts.py index 32368bbb..f0aa9b0f 100644 --- a/cloudinit/config/cc_update_etc_hosts.py +++ b/cloudinit/config/cc_update_etc_hosts.py @@ -50,9 +50,7 @@ ping ``127.0.0.1`` or ``127.0.1.1`` or other ip). hostname: <fqdn/hostname> """ -from cloudinit import templater -from cloudinit import util - +from cloudinit import templater, util from cloudinit.settings import PER_ALWAYS frequency = PER_ALWAYS @@ -63,35 +61,45 @@ def handle(name, cfg, cloud, log, _args): hosts_fn = cloud.distro.hosts_fn - if util.translate_bool(manage_hosts, addons=['template']): + if util.translate_bool(manage_hosts, addons=["template"]): (hostname, fqdn) = util.get_hostname_fqdn(cfg, cloud) if not hostname: - log.warning(("Option 'manage_etc_hosts' was set," - " but no hostname was found")) + log.warning( + "Option 'manage_etc_hosts' was set, but no hostname was found" + ) return # Render from a template file - tpl_fn_name = cloud.get_template_filename("hosts.%s" % - (cloud.distro.osfamily)) + tpl_fn_name = cloud.get_template_filename( + "hosts.%s" % (cloud.distro.osfamily) + ) if not tpl_fn_name: - raise RuntimeError(("No hosts template could be" - " found for distro %s") % - (cloud.distro.osfamily)) + raise RuntimeError( + "No hosts template could be found for distro %s" + % (cloud.distro.osfamily) + ) - templater.render_to_file(tpl_fn_name, hosts_fn, - {'hostname': hostname, 'fqdn': fqdn}) + templater.render_to_file( + tpl_fn_name, hosts_fn, {"hostname": hostname, "fqdn": fqdn} + ) elif manage_hosts == "localhost": (hostname, fqdn) = util.get_hostname_fqdn(cfg, cloud) if not hostname: - log.warning(("Option 'manage_etc_hosts' was set," - " but no hostname was found")) + log.warning( + "Option 'manage_etc_hosts' was set, but no hostname was found" + ) return log.debug("Managing localhost in %s", hosts_fn) cloud.distro.update_etc_hosts(hostname, fqdn) else: - log.debug(("Configuration option 'manage_etc_hosts' is not set," - " not managing %s in module %s"), hosts_fn, name) + log.debug( + "Configuration option 'manage_etc_hosts' is not set," + " not managing %s in module %s", + hosts_fn, + name, + ) + # vi: ts=4 expandtab diff --git a/cloudinit/config/cc_update_hostname.py b/cloudinit/config/cc_update_hostname.py index 370de73a..09f6f6da 100644 --- a/cloudinit/config/cc_update_hostname.py +++ b/cloudinit/config/cc_update_hostname.py @@ -34,33 +34,38 @@ is set, then the hostname will not be altered. import os -from cloudinit.settings import PER_ALWAYS from cloudinit import util +from cloudinit.settings import PER_ALWAYS frequency = PER_ALWAYS def handle(name, cfg, cloud, log, _args): if util.get_cfg_option_bool(cfg, "preserve_hostname", False): - log.debug(("Configuration option 'preserve_hostname' is set," - " not updating the hostname in module %s"), name) + log.debug( + "Configuration option 'preserve_hostname' is set," + " not updating the hostname in module %s", + name, + ) return # Set prefer_fqdn_over_hostname value in distro - hostname_fqdn = util.get_cfg_option_bool(cfg, - "prefer_fqdn_over_hostname", - None) + hostname_fqdn = util.get_cfg_option_bool( + cfg, "prefer_fqdn_over_hostname", None + ) if hostname_fqdn is not None: - cloud.distro.set_option('prefer_fqdn_over_hostname', hostname_fqdn) + cloud.distro.set_option("prefer_fqdn_over_hostname", hostname_fqdn) (hostname, fqdn) = util.get_hostname_fqdn(cfg, cloud) try: - prev_fn = os.path.join(cloud.get_cpath('data'), "previous-hostname") + prev_fn = os.path.join(cloud.get_cpath("data"), "previous-hostname") log.debug("Updating hostname to %s (%s)", fqdn, hostname) cloud.distro.update_hostname(hostname, fqdn, prev_fn) except Exception: - util.logexc(log, "Failed to update the hostname to %s (%s)", fqdn, - hostname) + util.logexc( + log, "Failed to update the hostname to %s (%s)", fqdn, hostname + ) raise + # vi: ts=4 expandtab diff --git a/cloudinit/config/cc_users_groups.py b/cloudinit/config/cc_users_groups.py index ac4a4410..ef77a799 100644 --- a/cloudinit/config/cc_users_groups.py +++ b/cloudinit/config/cc_users_groups.py @@ -127,12 +127,12 @@ config keys for an entry in ``users`` are as follows: uid: <user id> """ +from cloudinit import log as logging + # Ensure this is aliased to a name not 'distros' # since the module attribute 'distros' # is a list of distros that are supported, not a sub-module from cloudinit.distros import ug_util -from cloudinit import log as logging - from cloudinit.settings import PER_INSTANCE LOG = logging.getLogger(__name__) @@ -149,26 +149,31 @@ def handle(name, cfg, cloud, _log, _args): for (user, config) in users.items(): ssh_redirect_user = config.pop("ssh_redirect_user", False) if ssh_redirect_user: - if 'ssh_authorized_keys' in config or 'ssh_import_id' in config: + if "ssh_authorized_keys" in config or "ssh_import_id" in config: raise ValueError( - 'Not creating user %s. ssh_redirect_user cannot be' - ' provided with ssh_import_id or ssh_authorized_keys' % - user) - if ssh_redirect_user not in (True, 'default'): + "Not creating user %s. ssh_redirect_user cannot be" + " provided with ssh_import_id or ssh_authorized_keys" + % user + ) + if ssh_redirect_user not in (True, "default"): raise ValueError( - 'Not creating user %s. Invalid value of' - ' ssh_redirect_user: %s. Expected values: true, default' - ' or false.' % (user, ssh_redirect_user)) + "Not creating user %s. Invalid value of" + " ssh_redirect_user: %s. Expected values: true, default" + " or false." % (user, ssh_redirect_user) + ) if default_user is None: LOG.warning( - 'Ignoring ssh_redirect_user: %s for %s.' - ' No default_user defined.' - ' Perhaps missing cloud configuration users: ' - ' [default, ..].', - ssh_redirect_user, user) + "Ignoring ssh_redirect_user: %s for %s." + " No default_user defined." + " Perhaps missing cloud configuration users: " + " [default, ..].", + ssh_redirect_user, + user, + ) else: - config['ssh_redirect_user'] = default_user - config['cloud_public_ssh_keys'] = cloud_keys + config["ssh_redirect_user"] = default_user + config["cloud_public_ssh_keys"] = cloud_keys cloud.distro.create_user(user, **config) + # vi: ts=4 expandtab diff --git a/cloudinit/config/cc_write_files.py b/cloudinit/config/cc_write_files.py index 55f8c684..2c580328 100644 --- a/cloudinit/config/cc_write_files.py +++ b/cloudinit/config/cc_write_files.py @@ -10,22 +10,21 @@ import base64 import os from textwrap import dedent -from cloudinit.config.schema import get_meta_doc, validate_cloudconfig_schema from cloudinit import log as logging -from cloudinit.settings import PER_INSTANCE from cloudinit import util - +from cloudinit.config.schema import get_meta_doc, validate_cloudconfig_schema +from cloudinit.settings import PER_INSTANCE frequency = PER_INSTANCE DEFAULT_OWNER = "root:root" DEFAULT_PERMS = 0o644 DEFAULT_DEFER = False -UNKNOWN_ENC = 'text/plain' +UNKNOWN_ENC = "text/plain" LOG = logging.getLogger(__name__) -distros = ['all'] +distros = ["all"] # The schema definition for each cloud-config module is a strict contract for # describing supported configuration parameters for each cloud-config section. @@ -34,14 +33,22 @@ distros = ['all'] # configuration. supported_encoding_types = [ - 'gz', 'gzip', 'gz+base64', 'gzip+base64', 'gz+b64', 'gzip+b64', 'b64', - 'base64'] + "gz", + "gzip", + "gz+base64", + "gzip+base64", + "gz+b64", + "gzip+b64", + "b64", + "base64", +] meta = { - 'id': 'cc_write_files', - 'name': 'Write Files', - 'title': 'write arbitrary files', - 'description': dedent("""\ + "id": "cc_write_files", + "name": "Write Files", + "title": "write arbitrary files", + "description": dedent( + """\ Write out arbitrary content to files, optionally setting permissions. Parent folders in the path are created if absent. Content can be specified in plain text or binary. Data encoded with @@ -57,10 +64,12 @@ meta = { Do not write files under /tmp during boot because of a race with systemd-tmpfiles-clean that can cause temp files to get cleaned during the early boot process. Use /run/somedir instead to avoid race - LP:1707222."""), - 'distros': distros, - 'examples': [ - dedent("""\ + LP:1707222.""" + ), + "distros": distros, + "examples": [ + dedent( + """\ # Write out base64 encoded content to /etc/sysconfig/selinux write_files: - encoding: b64 @@ -68,16 +77,20 @@ meta = { owner: root:root path: /etc/sysconfig/selinux permissions: '0644' - """), - dedent("""\ + """ + ), + dedent( + """\ # Appending content to an existing file write_files: - content: | 15 * * * * root ship_logs path: /etc/crontab append: true - """), - dedent("""\ + """ + ), + dedent( + """\ # Provide gziped binary content write_files: - encoding: gzip @@ -85,13 +98,17 @@ meta = { H4sIAIDb/U8C/1NW1E/KzNMvzuBKTc7IV8hIzcnJVyjPL8pJ4QIA6N+MVxsAAAA= path: /usr/bin/hello permissions: '0755' - """), - dedent("""\ + """ + ), + dedent( + """\ # Create an empty file on the system write_files: - path: /root/CLOUD_INIT_WAS_HERE - """), - dedent("""\ + """ + ), + dedent( + """\ # Defer writing the file until after the package (Nginx) is # installed and its user is created alongside write_files: @@ -108,85 +125,109 @@ meta = { owner: 'nginx:nginx' permissions: '0640' defer: true - """)], - 'frequency': frequency, + """ + ), + ], + "frequency": frequency, } schema = { - 'type': 'object', - 'properties': { - 'write_files': { - 'type': 'array', - 'items': { - 'type': 'object', - 'properties': { - 'path': { - 'type': 'string', - 'description': dedent("""\ + "type": "object", + "properties": { + "write_files": { + "type": "array", + "items": { + "type": "object", + "properties": { + "path": { + "type": "string", + "description": dedent( + """\ Path of the file to which ``content`` is decoded and written - """), + """ + ), }, - 'content': { - 'type': 'string', - 'default': '', - 'description': dedent("""\ + "content": { + "type": "string", + "default": "", + "description": dedent( + """\ Optional content to write to the provided ``path``. When content is present and encoding is not '%s', decode the content prior to writing. Default: **''** - """ % UNKNOWN_ENC), + """ + % UNKNOWN_ENC + ), }, - 'owner': { - 'type': 'string', - 'default': DEFAULT_OWNER, - 'description': dedent("""\ + "owner": { + "type": "string", + "default": DEFAULT_OWNER, + "description": dedent( + """\ Optional owner:group to chown on the file. Default: **{owner}** - """.format(owner=DEFAULT_OWNER)), + """.format( + owner=DEFAULT_OWNER + ) + ), }, - 'permissions': { - 'type': 'string', - 'default': oct(DEFAULT_PERMS).replace('o', ''), - 'description': dedent("""\ + "permissions": { + "type": "string", + "default": oct(DEFAULT_PERMS).replace("o", ""), + "description": dedent( + """\ Optional file permissions to set on ``path`` represented as an octal string '0###'. Default: **'{perms}'** - """.format(perms=oct(DEFAULT_PERMS).replace('o', ''))), + """.format( + perms=oct(DEFAULT_PERMS).replace("o", "") + ) + ), }, - 'encoding': { - 'type': 'string', - 'default': UNKNOWN_ENC, - 'enum': supported_encoding_types, - 'description': dedent("""\ + "encoding": { + "type": "string", + "default": UNKNOWN_ENC, + "enum": supported_encoding_types, + "description": dedent( + """\ Optional encoding type of the content. Default is **text/plain** and no content decoding is performed. Supported encoding types are: - %s.""" % ", ".join(supported_encoding_types)), + %s.""" + % ", ".join(supported_encoding_types) + ), }, - 'append': { - 'type': 'boolean', - 'default': False, - 'description': dedent("""\ + "append": { + "type": "boolean", + "default": False, + "description": dedent( + """\ Whether to append ``content`` to existing file if ``path`` exists. Default: **false**. - """), + """ + ), }, - 'defer': { - 'type': 'boolean', - 'default': DEFAULT_DEFER, - 'description': dedent("""\ + "defer": { + "type": "boolean", + "default": DEFAULT_DEFER, + "description": dedent( + """\ Defer writing the file until 'final' stage, after users were created, and packages were installed. Default: **{defer}**. - """.format(defer=DEFAULT_DEFER)), + """.format( + defer=DEFAULT_DEFER + ) + ), }, }, - 'required': ['path'], - 'additionalProperties': False + "required": ["path"], + "additionalProperties": False, }, } - } + }, } __doc__ = get_meta_doc(meta, schema) # Supplement python help() @@ -194,36 +235,40 @@ __doc__ = get_meta_doc(meta, schema) # Supplement python help() def handle(name, cfg, _cloud, log, _args): validate_cloudconfig_schema(cfg, schema) - file_list = cfg.get('write_files', []) + file_list = cfg.get("write_files", []) filtered_files = [ - f for f in file_list if not util.get_cfg_option_bool(f, - 'defer', - DEFAULT_DEFER) + f + for f in file_list + if not util.get_cfg_option_bool(f, "defer", DEFAULT_DEFER) ] if not filtered_files: - log.debug(("Skipping module named %s," - " no/empty 'write_files' key in configuration"), name) + log.debug( + "Skipping module named %s," + " no/empty 'write_files' key in configuration", + name, + ) return write_files(name, filtered_files) def canonicalize_extraction(encoding_type): if not encoding_type: - encoding_type = '' + encoding_type = "" encoding_type = encoding_type.lower().strip() - if encoding_type in ['gz', 'gzip']: - return ['application/x-gzip'] - if encoding_type in ['gz+base64', 'gzip+base64', 'gz+b64', 'gzip+b64']: - return ['application/base64', 'application/x-gzip'] + if encoding_type in ["gz", "gzip"]: + return ["application/x-gzip"] + if encoding_type in ["gz+base64", "gzip+base64", "gz+b64", "gzip+b64"]: + return ["application/base64", "application/x-gzip"] # Yaml already encodes binary data as base64 if it is given to the # yaml file as binary, so those will be automatically decoded for you. # But the above b64 is just for people that are more 'comfortable' # specifing it manually (which might be a possiblity) - if encoding_type in ['b64', 'base64']: - return ['application/base64'] + if encoding_type in ["b64", "base64"]: + return ["application/base64"] if encoding_type: - LOG.warning("Unknown encoding type %s, assuming %s", - encoding_type, UNKNOWN_ENC) + LOG.warning( + "Unknown encoding type %s, assuming %s", encoding_type, UNKNOWN_ENC + ) return [UNKNOWN_ENC] @@ -232,17 +277,20 @@ def write_files(name, files): return for (i, f_info) in enumerate(files): - path = f_info.get('path') + path = f_info.get("path") if not path: - LOG.warning("No path provided to write for entry %s in module %s", - i + 1, name) + LOG.warning( + "No path provided to write for entry %s in module %s", + i + 1, + name, + ) continue path = os.path.abspath(path) - extractions = canonicalize_extraction(f_info.get('encoding')) - contents = extract_contents(f_info.get('content', ''), extractions) - (u, g) = util.extract_usergroup(f_info.get('owner', DEFAULT_OWNER)) - perms = decode_perms(f_info.get('permissions'), DEFAULT_PERMS) - omode = 'ab' if util.get_cfg_option_bool(f_info, 'append') else 'wb' + extractions = canonicalize_extraction(f_info.get("encoding")) + contents = extract_contents(f_info.get("content", ""), extractions) + (u, g) = util.extract_usergroup(f_info.get("owner", DEFAULT_OWNER)) + perms = decode_perms(f_info.get("permissions"), DEFAULT_PERMS) + omode = "ab" if util.get_cfg_option_bool(f_info, "append") else "wb" util.write_file(path, contents, omode=omode, mode=perms) util.chownbyname(path, u, g) @@ -264,20 +312,20 @@ def decode_perms(perm, default): reps.append("%o" % r) except TypeError: reps.append("%r" % r) - LOG.warning( - "Undecodable permissions %s, returning default %s", *reps) + LOG.warning("Undecodable permissions %s, returning default %s", *reps) return default def extract_contents(contents, extraction_types): result = contents for t in extraction_types: - if t == 'application/x-gzip': + if t == "application/x-gzip": result = util.decomp_gzip(result, quiet=False, decode=False) - elif t == 'application/base64': + elif t == "application/base64": result = base64.b64decode(result) elif t == UNKNOWN_ENC: pass return result + # vi: ts=4 expandtab diff --git a/cloudinit/config/cc_write_files_deferred.py b/cloudinit/config/cc_write_files_deferred.py index 4fc8659c..1294628c 100644 --- a/cloudinit/config/cc_write_files_deferred.py +++ b/cloudinit/config/cc_write_files_deferred.py @@ -4,10 +4,11 @@ """Defer writing certain files""" -from cloudinit.config.schema import validate_cloudconfig_schema from cloudinit import util -from cloudinit.config.cc_write_files import ( - schema as write_files_schema, write_files, DEFAULT_DEFER) +from cloudinit.config.cc_write_files import DEFAULT_DEFER +from cloudinit.config.cc_write_files import schema as write_files_schema +from cloudinit.config.cc_write_files import write_files +from cloudinit.config.schema import validate_cloudconfig_schema # meta is not used in this module, but it remains as code documentation # @@ -36,15 +37,18 @@ __doc__ = None def handle(name, cfg, _cloud, log, _args): validate_cloudconfig_schema(cfg, schema) - file_list = cfg.get('write_files', []) + file_list = cfg.get("write_files", []) filtered_files = [ - f for f in file_list if util.get_cfg_option_bool(f, - 'defer', - DEFAULT_DEFER) + f + for f in file_list + if util.get_cfg_option_bool(f, "defer", DEFAULT_DEFER) ] if not filtered_files: - log.debug(("Skipping module named %s," - " no deferred file defined in configuration"), name) + log.debug( + "Skipping module named %s," + " no deferred file defined in configuration", + name, + ) return write_files(name, filtered_files) diff --git a/cloudinit/config/cc_yum_add_repo.py b/cloudinit/config/cc_yum_add_repo.py index 046a2852..7a232689 100644 --- a/cloudinit/config/cc_yum_add_repo.py +++ b/cloudinit/config/cc_yum_add_repo.py @@ -37,8 +37,18 @@ from configparser import ConfigParser from cloudinit import util -distros = ['almalinux', 'centos', 'cloudlinux', 'eurolinux', 'fedora', - 'openEuler', 'photon', 'rhel', 'rocky', 'virtuozzo'] +distros = [ + "almalinux", + "centos", + "cloudlinux", + "eurolinux", + "fedora", + "openEuler", + "photon", + "rhel", + "rocky", + "virtuozzo", +] def _canonicalize_id(repo_id): @@ -79,25 +89,34 @@ def _format_repository_config(repo_id, repo_config): def handle(name, cfg, _cloud, log, _args): - repos = cfg.get('yum_repos') + repos = cfg.get("yum_repos") if not repos: - log.debug(("Skipping module named %s," - " no 'yum_repos' configuration found"), name) + log.debug( + "Skipping module named %s, no 'yum_repos' configuration found", + name, + ) return - repo_base_path = util.get_cfg_option_str(cfg, 'yum_repo_dir', - '/etc/yum.repos.d/') + repo_base_path = util.get_cfg_option_str( + cfg, "yum_repo_dir", "/etc/yum.repos.d/" + ) repo_locations = {} repo_configs = {} for (repo_id, repo_config) in repos.items(): canon_repo_id = _canonicalize_id(repo_id) repo_fn_pth = os.path.join(repo_base_path, "%s.repo" % (canon_repo_id)) if os.path.exists(repo_fn_pth): - log.info("Skipping repo %s, file %s already exists!", - repo_id, repo_fn_pth) + log.info( + "Skipping repo %s, file %s already exists!", + repo_id, + repo_fn_pth, + ) continue elif canon_repo_id in repo_locations: - log.info("Skipping repo %s, file %s already pending!", - repo_id, repo_fn_pth) + log.info( + "Skipping repo %s, file %s already pending!", + repo_id, + repo_fn_pth, + ) continue if not repo_config: repo_config = {} @@ -109,21 +128,29 @@ def handle(name, cfg, _cloud, log, _args): n_repo_config[k] = v repo_config = n_repo_config missing_required = 0 - for req_field in ['baseurl']: + for req_field in ["baseurl"]: if req_field not in repo_config: - log.warning(("Repository %s does not contain a %s" - " configuration 'required' entry"), - repo_id, req_field) + log.warning( + "Repository %s does not contain a %s" + " configuration 'required' entry", + repo_id, + req_field, + ) missing_required += 1 if not missing_required: repo_configs[canon_repo_id] = repo_config repo_locations[canon_repo_id] = repo_fn_pth else: - log.warning("Repository %s is missing %s required fields, " - "skipping!", repo_id, missing_required) + log.warning( + "Repository %s is missing %s required fields, skipping!", + repo_id, + missing_required, + ) for (c_repo_id, path) in repo_locations.items(): - repo_blob = _format_repository_config(c_repo_id, - repo_configs.get(c_repo_id)) + repo_blob = _format_repository_config( + c_repo_id, repo_configs.get(c_repo_id) + ) util.write_file(path, repo_blob) + # vi: ts=4 expandtab diff --git a/cloudinit/config/cc_zypper_add_repo.py b/cloudinit/config/cc_zypper_add_repo.py index bf1638fb..41605b97 100644 --- a/cloudinit/config/cc_zypper_add_repo.py +++ b/cloudinit/config/cc_zypper_add_repo.py @@ -5,22 +5,24 @@ """zypper_add_repo: Add zyper repositories to the system""" -import configobj import os from textwrap import dedent -from cloudinit.config.schema import get_meta_doc +import configobj + from cloudinit import log as logging -from cloudinit.settings import PER_ALWAYS from cloudinit import util +from cloudinit.config.schema import get_meta_doc +from cloudinit.settings import PER_ALWAYS -distros = ['opensuse', 'sles'] +distros = ["opensuse", "sles"] meta = { - 'id': 'cc_zypper_add_repo', - 'name': 'ZypperAddRepo', - 'title': 'Configure zypper behavior and add zypper repositories', - 'description': dedent("""\ + "id": "cc_zypper_add_repo", + "name": "ZypperAddRepo", + "title": "Configure zypper behavior and add zypper repositories", + "description": dedent( + """\ Configure zypper behavior by modifying /etc/zypp/zypp.conf. The configuration writer is "dumb" and will simply append the provided configuration options to the configuration file. Option settings @@ -28,9 +30,12 @@ meta = { is parsed. The file is in INI format. Add repositories to the system. No validation is performed on the repository file entries, it is assumed the user is familiar with - the zypper repository file format."""), - 'distros': distros, - 'examples': [dedent("""\ + the zypper repository file format.""" + ), + "distros": distros, + "examples": [ + dedent( + """\ zypper: repos: - id: opensuse-oss @@ -49,50 +54,56 @@ meta = { servicesdir: /etc/zypp/services.d download.use_deltarpm: true # any setting in /etc/zypp/zypp.conf - """)], - 'frequency': PER_ALWAYS, + """ + ) + ], + "frequency": PER_ALWAYS, } schema = { - 'type': 'object', - 'properties': { - 'zypper': { - 'type': 'object', - 'properties': { - 'repos': { - 'type': 'array', - 'items': { - 'type': 'object', - 'properties': { - 'id': { - 'type': 'string', - 'description': dedent("""\ + "type": "object", + "properties": { + "zypper": { + "type": "object", + "properties": { + "repos": { + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "type": "string", + "description": dedent( + """\ The unique id of the repo, used when writing - /etc/zypp/repos.d/<id>.repo.""") + /etc/zypp/repos.d/<id>.repo.""" + ), + }, + "baseurl": { + "type": "string", + "format": "uri", # built-in format type + "description": "The base repositoy URL", }, - 'baseurl': { - 'type': 'string', - 'format': 'uri', # built-in format type - 'description': 'The base repositoy URL' - } }, - 'required': ['id', 'baseurl'], - 'additionalProperties': True + "required": ["id", "baseurl"], + "additionalProperties": True, }, - 'minItems': 1 + "minItems": 1, }, - 'config': { - 'type': 'object', - 'description': dedent("""\ + "config": { + "type": "object", + "description": dedent( + """\ Any supported zypo.conf key is written to - /etc/zypp/zypp.conf'""") - } + /etc/zypp/zypp.conf'""" + ), + }, }, - 'minProperties': 1, # Either config or repo must be provided - 'additionalProperties': False, # only repos and config allowed + "minProperties": 1, # Either config or repo must be provided + "additionalProperties": False, # only repos and config allowed } - } + }, } __doc__ = get_meta_doc(meta, schema) # Supplement python help() @@ -141,34 +152,43 @@ def _write_repos(repos, repo_base_path): valid_repos = {} for index, user_repo_config in enumerate(repos): # Skip on absent required keys - missing_keys = set(['id', 'baseurl']).difference(set(user_repo_config)) + missing_keys = set(["id", "baseurl"]).difference(set(user_repo_config)) if missing_keys: LOG.warning( "Repo config at index %d is missing required config keys: %s", - index, ",".join(missing_keys)) + index, + ",".join(missing_keys), + ) continue - repo_id = user_repo_config.get('id') + repo_id = user_repo_config.get("id") canon_repo_id = _canonicalize_id(repo_id) repo_fn_pth = os.path.join(repo_base_path, "%s.repo" % (canon_repo_id)) if os.path.exists(repo_fn_pth): - LOG.info("Skipping repo %s, file %s already exists!", - repo_id, repo_fn_pth) + LOG.info( + "Skipping repo %s, file %s already exists!", + repo_id, + repo_fn_pth, + ) continue elif repo_id in valid_repos: - LOG.info("Skipping repo %s, file %s already pending!", - repo_id, repo_fn_pth) + LOG.info( + "Skipping repo %s, file %s already pending!", + repo_id, + repo_fn_pth, + ) continue # Do some basic key formatting repo_config = dict( (k.lower().strip().replace("-", "_"), v) for k, v in user_repo_config.items() - if k and k != 'id') + if k and k != "id" + ) # Set defaults if not present - for field in ['enabled', 'autorefresh']: + for field in ["enabled", "autorefresh"]: if field not in repo_config: - repo_config[field] = '1' + repo_config[field] = "1" valid_repos[repo_id] = (repo_fn_pth, repo_config) @@ -181,39 +201,44 @@ def _write_zypp_config(zypper_config): """Write to the default zypp configuration file /etc/zypp/zypp.conf""" if not zypper_config: return - zypp_config = '/etc/zypp/zypp.conf' + zypp_config = "/etc/zypp/zypp.conf" zypp_conf_content = util.load_file(zypp_config) - new_settings = ['# Added via cloud.cfg'] + new_settings = ["# Added via cloud.cfg"] for setting, value in zypper_config.items(): - if setting == 'configdir': - msg = 'Changing the location of the zypper configuration is ' + if setting == "configdir": + msg = "Changing the location of the zypper configuration is " msg += 'not supported, skipping "configdir" setting' LOG.warning(msg) continue if value: - new_settings.append('%s=%s' % (setting, value)) + new_settings.append("%s=%s" % (setting, value)) if len(new_settings) > 1: - new_config = zypp_conf_content + '\n'.join(new_settings) + new_config = zypp_conf_content + "\n".join(new_settings) else: new_config = zypp_conf_content util.write_file(zypp_config, new_config) def handle(name, cfg, _cloud, log, _args): - zypper_section = cfg.get('zypper') + zypper_section = cfg.get("zypper") if not zypper_section: - LOG.debug(("Skipping module named %s," - " no 'zypper' relevant configuration found"), name) + LOG.debug( + "Skipping module named %s," + " no 'zypper' relevant configuration found", + name, + ) return - repos = zypper_section.get('repos') + repos = zypper_section.get("repos") if not repos: - LOG.debug(("Skipping module named %s," - " no 'repos' configuration found"), name) + LOG.debug( + "Skipping module named %s, no 'repos' configuration found", name + ) return - zypper_config = zypper_section.get('config', {}) - repo_base_path = zypper_config.get('reposdir', '/etc/zypp/repos.d/') + zypper_config = zypper_section.get("config", {}) + repo_base_path = zypper_config.get("reposdir", "/etc/zypp/repos.d/") _write_zypp_config(zypper_config) _write_repos(repos, repo_base_path) + # vi: ts=4 expandtab diff --git a/cloudinit/config/schema.py b/cloudinit/config/schema.py index d772b4f9..8ec4ab6a 100644 --- a/cloudinit/config/schema.py +++ b/cloudinit/config/schema.py @@ -1,26 +1,27 @@ # This file is part of cloud-init. See LICENSE file for license information. """schema.py: Set of module functions for processing cloud-config schema.""" -from cloudinit.cmd.devel import read_cfg_paths -from cloudinit import importer -from cloudinit.importer import MetaSchema -from cloudinit.util import find_modules, load_file, error - import argparse -from collections import defaultdict -from copy import deepcopy -from functools import partial import logging import os import re import sys +from collections import defaultdict +from copy import deepcopy +from functools import partial + import yaml +from cloudinit import importer +from cloudinit.cmd.devel import read_cfg_paths +from cloudinit.importer import MetaSchema +from cloudinit.util import error, find_modules, load_file + error = partial(error, sys_exit=True) LOG = logging.getLogger(__name__) -_YAML_MAP = {True: 'true', False: 'false', None: 'null'} -CLOUD_CONFIG_HEADER = b'#cloud-config' +_YAML_MAP = {True: "true", False: "false", None: "null"} +CLOUD_CONFIG_HEADER = b"#cloud-config" SCHEMA_DOC_TMPL = """ {name} {title_underbar} @@ -40,9 +41,10 @@ SCHEMA_DOC_TMPL = """ """ SCHEMA_PROPERTY_TMPL = "{prefix}**{prop_name}:** ({prop_type}) {description}" SCHEMA_LIST_ITEM_TMPL = ( - '{prefix}Each item in **{prop_name}** list supports the following keys:') -SCHEMA_EXAMPLES_HEADER = '\n**Examples**::\n\n' -SCHEMA_EXAMPLES_SPACER_TEMPLATE = '\n # --- Example{0} ---' + "{prefix}Each item in **{prop_name}** list supports the following keys:" +) +SCHEMA_EXAMPLES_HEADER = "\n**Examples**::\n\n" +SCHEMA_EXAMPLES_SPACER_TEMPLATE = "\n # --- Example{0} ---" class SchemaValidationError(ValueError): @@ -56,10 +58,12 @@ class SchemaValidationError(ValueError): """ self.schema_errors = schema_errors error_messages = [ - '{0}: {1}'.format(config_key, message) - for config_key, message in schema_errors] + "{0}: {1}".format(config_key, message) + for config_key, message in schema_errors + ] message = "Cloud config schema errors: {0}".format( - ', '.join(error_messages)) + ", ".join(error_messages) + ) super(SchemaValidationError, self).__init__(message) @@ -72,8 +76,9 @@ def is_schema_byte_string(checker, instance): from jsonschema import Draft4Validator except ImportError: return False - return (Draft4Validator.TYPE_CHECKER.is_type(instance, "string") or - isinstance(instance, (bytes,))) + return Draft4Validator.TYPE_CHECKER.is_type( + instance, "string" + ) or isinstance(instance, (bytes,)) def get_jsonschema_validator(): @@ -102,25 +107,28 @@ def get_jsonschema_validator(): # http://json-schema.org/understanding-json-schema/reference/object.html#pattern-properties strict_metaschema["properties"]["label"] = {"type": "string"} - if hasattr(Draft4Validator, 'TYPE_CHECKER'): # jsonschema 3.0+ + if hasattr(Draft4Validator, "TYPE_CHECKER"): # jsonschema 3.0+ type_checker = Draft4Validator.TYPE_CHECKER.redefine( - 'string', is_schema_byte_string) + "string", is_schema_byte_string + ) cloudinitValidator = create( meta_schema=strict_metaschema, validators=Draft4Validator.VALIDATORS, version="draft4", - type_checker=type_checker) + type_checker=type_checker, + ) else: # jsonschema 2.6 workaround types = Draft4Validator.DEFAULT_TYPES # Allow bytes as well as string (and disable a spurious unsupported # assignment-operation pylint warning which appears because this # code path isn't written against the latest jsonschema). - types['string'] = (str, bytes) # pylint: disable=E1137 + types["string"] = (str, bytes) # pylint: disable=E1137 cloudinitValidator = create( meta_schema=strict_metaschema, validators=Draft4Validator.VALIDATORS, version="draft4", - default_types=types) + default_types=types, + ) return (cloudinitValidator, FormatChecker) @@ -147,12 +155,14 @@ def validate_cloudconfig_metaschema(validator, schema: dict, throw=True): if throw: raise SchemaValidationError( schema_errors=( - ('.'.join([str(p) for p in err.path]), err.message), + (".".join([str(p) for p in err.path]), err.message), ) ) from err LOG.warning( "Meta-schema validation failed, attempting to validate config " - "anyway: %s", err) + "anyway: %s", + err, + ) def validate_cloudconfig_schema( @@ -176,7 +186,8 @@ def validate_cloudconfig_schema( (cloudinitValidator, FormatChecker) = get_jsonschema_validator() if strict_metaschema: validate_cloudconfig_metaschema( - cloudinitValidator, schema, throw=False) + cloudinitValidator, schema, throw=False + ) except ImportError: LOG.debug("Ignoring schema validation. jsonschema is not present") return @@ -184,7 +195,7 @@ def validate_cloudconfig_schema( validator = cloudinitValidator(schema, format_checker=FormatChecker()) errors = () for error in sorted(validator.iter_errors(config), key=lambda e: e.path): - path = '.'.join([str(p) for p in error.path]) + path = ".".join([str(p) for p in error.path]) errors += ((path, error.message),) if errors: if strict: @@ -208,12 +219,13 @@ def annotated_cloudconfig_file(cloudconfig, original_content, schema_errors): schemapaths = {} if cloudconfig: schemapaths = _schemapath_for_cloudconfig( - cloudconfig, original_content) + cloudconfig, original_content + ) errors_by_line = defaultdict(list) error_footer = [] annotated_content = [] for path, msg in schema_errors: - match = re.match(r'format-l(?P<line>\d+)\.c(?P<col>\d+).*', path) + match = re.match(r"format-l(?P<line>\d+)\.c(?P<col>\d+).*", path) if match: line, col = match.groups() errors_by_line[int(line)].append(msg) @@ -221,24 +233,26 @@ def annotated_cloudconfig_file(cloudconfig, original_content, schema_errors): col = None errors_by_line[schemapaths[path]].append(msg) if col is not None: - msg = 'Line {line} column {col}: {msg}'.format( - line=line, col=col, msg=msg) - lines = original_content.decode().split('\n') + msg = "Line {line} column {col}: {msg}".format( + line=line, col=col, msg=msg + ) + lines = original_content.decode().split("\n") error_index = 1 for line_number, line in enumerate(lines, 1): errors = errors_by_line[line_number] if errors: error_label = [] for error in errors: - error_label.append('E{0}'.format(error_index)) - error_footer.append('# E{0}: {1}'.format(error_index, error)) + error_label.append("E{0}".format(error_index)) + error_footer.append("# E{0}: {1}".format(error_index, error)) error_index += 1 - annotated_content.append(line + '\t\t# ' + ','.join(error_label)) + annotated_content.append(line + "\t\t# " + ",".join(error_label)) else: annotated_content.append(line) annotated_content.append( - '# Errors: -------------\n{0}\n\n'.format('\n'.join(error_footer))) - return '\n'.join(annotated_content) + "# Errors: -------------\n{0}\n\n".format("\n".join(error_footer)) + ) + return "\n".join(annotated_content) def validate_cloudconfig_file(config_path, schema, annotate=False): @@ -266,15 +280,18 @@ def validate_cloudconfig_file(config_path, schema, annotate=False): else: if not os.path.exists(config_path): raise RuntimeError( - 'Configfile {0} does not exist'.format( - config_path - ) + "Configfile {0} does not exist".format(config_path) ) content = load_file(config_path, decode=False) if not content.startswith(CLOUD_CONFIG_HEADER): errors = ( - ('format-l1.c1', 'File {0} needs to begin with "{1}"'.format( - config_path, CLOUD_CONFIG_HEADER.decode())),) + ( + "format-l1.c1", + 'File {0} needs to begin with "{1}"'.format( + config_path, CLOUD_CONFIG_HEADER.decode() + ), + ), + ) error = SchemaValidationError(errors) if annotate: print(annotated_cloudconfig_file({}, content, error.schema_errors)) @@ -284,27 +301,32 @@ def validate_cloudconfig_file(config_path, schema, annotate=False): except (yaml.YAMLError) as e: line = column = 1 mark = None - if hasattr(e, 'context_mark') and getattr(e, 'context_mark'): - mark = getattr(e, 'context_mark') - elif hasattr(e, 'problem_mark') and getattr(e, 'problem_mark'): - mark = getattr(e, 'problem_mark') + if hasattr(e, "context_mark") and getattr(e, "context_mark"): + mark = getattr(e, "context_mark") + elif hasattr(e, "problem_mark") and getattr(e, "problem_mark"): + mark = getattr(e, "problem_mark") if mark: line = mark.line + 1 column = mark.column + 1 - errors = (('format-l{line}.c{col}'.format(line=line, col=column), - 'File {0} is not valid yaml. {1}'.format( - config_path, str(e))),) + errors = ( + ( + "format-l{line}.c{col}".format(line=line, col=column), + "File {0} is not valid yaml. {1}".format(config_path, str(e)), + ), + ) error = SchemaValidationError(errors) if annotate: print(annotated_cloudconfig_file({}, content, error.schema_errors)) raise error from e try: - validate_cloudconfig_schema( - cloudconfig, schema, strict=True) + validate_cloudconfig_schema(cloudconfig, schema, strict=True) except SchemaValidationError as e: if annotate: - print(annotated_cloudconfig_file( - cloudconfig, content, e.schema_errors)) + print( + annotated_cloudconfig_file( + cloudconfig, content, e.schema_errors + ) + ) raise @@ -315,26 +337,26 @@ def _schemapath_for_cloudconfig(config, original_content): @param original_content: The simple file content of the cloud-config file """ # FIXME Doesn't handle multi-line lists or multi-line strings - content_lines = original_content.decode().split('\n') + content_lines = original_content.decode().split("\n") schema_line_numbers = {} list_index = 0 - RE_YAML_INDENT = r'^(\s*)' + RE_YAML_INDENT = r"^(\s*)" scopes = [] for line_number, line in enumerate(content_lines, 1): indent_depth = len(re.match(RE_YAML_INDENT, line).groups()[0]) line = line.strip() - if not line or line.startswith('#'): + if not line or line.startswith("#"): continue if scopes: previous_depth, path_prefix = scopes[-1] else: previous_depth = -1 - path_prefix = '' - if line.startswith('- '): + path_prefix = "" + if line.startswith("- "): # Process list items adding a list_index to the path prefix - previous_list_idx = '.%d' % (list_index - 1) + previous_list_idx = ".%d" % (list_index - 1) if path_prefix and path_prefix.endswith(previous_list_idx): - path_prefix = path_prefix[:-len(previous_list_idx)] + path_prefix = path_prefix[: -len(previous_list_idx)] key = str(list_index) schema_line_numbers[key] = line_number item_indent = len(re.match(RE_YAML_INDENT, line[1:]).groups()[0]) @@ -346,26 +368,26 @@ def _schemapath_for_cloudconfig(config, original_content): else: # Process non-list lines setting value if present list_index = 0 - key, value = line.split(':', 1) + key, value = line.split(":", 1) if path_prefix: # Append any existing path_prefix for a fully-pathed key - key = path_prefix + '.' + key + key = path_prefix + "." + key while indent_depth <= previous_depth: if scopes: previous_depth, path_prefix = scopes.pop() if list_index > 0 and indent_depth == previous_depth: - path_prefix = '.'.join(path_prefix.split('.')[:-1]) + path_prefix = ".".join(path_prefix.split(".")[:-1]) break else: previous_depth = -1 - path_prefix = '' + path_prefix = "" scopes.append((indent_depth, key)) if value: value = value.strip() - if value.startswith('['): - scopes.append((indent_depth + 2, key + '.0')) + if value.startswith("["): + scopes.append((indent_depth + 2, key + ".0")) for inner_list_index in range(0, len(yaml.safe_load(value))): - list_key = key + '.' + str(inner_list_index) + list_key = key + "." + str(inner_list_index) schema_line_numbers[list_key] = line_number schema_line_numbers[key] = line_number return schema_line_numbers @@ -381,14 +403,14 @@ def _get_property_type(property_dict: dict) -> str: str(_YAML_MAP.get(k, k)) for k in property_dict["enum"] ] if isinstance(property_type, list): - property_type = '/'.join(property_type) - items = property_dict.get('items', {}) - sub_property_type = items.get('type', '') + property_type = "/".join(property_type) + items = property_dict.get("items", {}) + sub_property_type = items.get("type", "") # Collect each item type - for sub_item in items.get('oneOf', {}): + for sub_item in items.get("oneOf", {}): if sub_property_type: - sub_property_type += '/' - sub_property_type += '(' + _get_property_type(sub_item) + ')' + sub_property_type += "/" + sub_property_type += "(" + _get_property_type(sub_item) + ")" if sub_property_type: return "{0} of {1}".format(property_type, sub_property_type) return property_type or "UNDEFINED" @@ -408,17 +430,17 @@ def _parse_description(description, prefix) -> str: """ list_paragraph = prefix * 3 description = re.sub(r"(\S)\n(\S)", r"\1 \2", description) + description = re.sub(r"\n\n", r"\n\n{}".format(prefix), description) description = re.sub( - r"\n\n", r"\n\n{}".format(prefix), description) - description = re.sub( - r"\n( +)-", r"\n{}-".format(list_paragraph), description) + r"\n( +)-", r"\n{}-".format(list_paragraph), description + ) return description def _get_property_doc(schema: dict, prefix=" ") -> str: """Return restructured text describing the supported schema properties.""" - new_prefix = prefix + ' ' + new_prefix = prefix + " " properties = [] property_keys = [ schema.get("properties", {}), @@ -473,16 +495,17 @@ def _get_examples(meta: MetaSchema) -> str: """Return restructured text describing the meta examples if present.""" examples = meta.get("examples") if not examples: - return '' + return "" rst_content = SCHEMA_EXAMPLES_HEADER for count, example in enumerate(examples): # Python2.6 is missing textwrapper.indent - lines = example.split('\n') - indented_lines = [' {0}'.format(line) for line in lines] + lines = example.split("\n") + indented_lines = [" {0}".format(line) for line in lines] if rst_content != SCHEMA_EXAMPLES_HEADER: indented_lines.insert( - 0, SCHEMA_EXAMPLES_SPACER_TEMPLATE.format(count + 1)) - rst_content += '\n'.join(indented_lines) + 0, SCHEMA_EXAMPLES_SPACER_TEMPLATE.format(count + 1) + ) + rst_content += "\n".join(indented_lines) return rst_content @@ -552,7 +575,8 @@ def load_doc(requested_modules: list) -> str: if invalid_docs: error( "Invalid --docs value {}. Must be one of: {}".format( - list(invalid_docs), ", ".join(all_modules), + list(invalid_docs), + ", ".join(all_modules), ) ) for mod_name in all_modules: @@ -601,17 +625,35 @@ def get_parser(parser=None): """Return a parser for supported cmdline arguments.""" if not parser: parser = argparse.ArgumentParser( - prog='cloudconfig-schema', - description='Validate cloud-config files or document schema') - parser.add_argument('-c', '--config-file', - help='Path of the cloud-config yaml file to validate') - parser.add_argument('--system', action='store_true', default=False, - help='Validate the system cloud-config userdata') - parser.add_argument('-d', '--docs', nargs='+', - help=('Print schema module docs. Choices: all or' - ' space-delimited cc_names.')) - parser.add_argument('--annotate', action="store_true", default=False, - help='Annotate existing cloud-config file with errors') + prog="cloudconfig-schema", + description="Validate cloud-config files or document schema", + ) + parser.add_argument( + "-c", + "--config-file", + help="Path of the cloud-config yaml file to validate", + ) + parser.add_argument( + "--system", + action="store_true", + default=False, + help="Validate the system cloud-config userdata", + ) + parser.add_argument( + "-d", + "--docs", + nargs="+", + help=( + "Print schema module docs. Choices: all or" + " space-delimited cc_names." + ), + ) + parser.add_argument( + "--annotate", + action="store_true", + default=False, + help="Annotate existing cloud-config file with errors", + ) return parser @@ -619,12 +661,13 @@ def handle_schema_args(name, args): """Handle provided schema args and perform the appropriate actions.""" exclusive_args = [args.config_file, args.docs, args.system] if len([arg for arg in exclusive_args if arg]) != 1: - error('Expected one of --config-file, --system or --docs arguments') + error("Expected one of --config-file, --system or --docs arguments") full_schema = get_schema() if args.config_file or args.system: try: validate_cloudconfig_file( - args.config_file, full_schema, args.annotate) + args.config_file, full_schema, args.annotate + ) except SchemaValidationError as e: if not args.annotate: error(str(e)) @@ -643,11 +686,11 @@ def handle_schema_args(name, args): def main(): """Tool to validate schema of a cloud-config file or print schema docs.""" parser = get_parser() - handle_schema_args('cloudconfig-schema', parser.parse_args()) + handle_schema_args("cloudconfig-schema", parser.parse_args()) return 0 -if __name__ == '__main__': +if __name__ == "__main__": sys.exit(main()) # vi: ts=4 expandtab |