From 0a448dd034883c07f85091dbfc9117de7227eb8d Mon Sep 17 00:00:00 2001 From: Chad Smith Date: Thu, 25 May 2017 11:04:55 -0600 Subject: ntp: Add schema definition and passive schema validation. cloud-config files are very flexible and permissive. This adds a jsonsschema definition to the cc_ntp module and validation functions in cloudinit/config/schema which will log warnings about invalid configuration values in the ntp section. A cmdline tools/cloudconfig-schema is added which can be used in our dev environments to quickly attempt to exercise the ntp schema. It is also exposed as a main in cloudinit.config.schema. (python3 -m cloudinit.config.schema) LP: #1692916 --- cloudinit/config/cc_ntp.py | 69 +++++++++++++- cloudinit/config/schema.py | 222 +++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 289 insertions(+), 2 deletions(-) create mode 100644 cloudinit/config/schema.py (limited to 'cloudinit/config') diff --git a/cloudinit/config/cc_ntp.py b/cloudinit/config/cc_ntp.py index 5cc54536..31ed64e3 100644 --- a/cloudinit/config/cc_ntp.py +++ b/cloudinit/config/cc_ntp.py @@ -36,6 +36,7 @@ servers or pools are provided, 4 pools will be used in the format - 192.168.23.2 """ +from cloudinit.config.schema import validate_cloudconfig_schema from cloudinit import log as logging from cloudinit.settings import PER_INSTANCE from cloudinit import templater @@ -43,6 +44,7 @@ from cloudinit import type_utils from cloudinit import util import os +from textwrap import dedent LOG = logging.getLogger(__name__) @@ -52,21 +54,84 @@ NR_POOL_SERVERS = 4 distros = ['centos', 'debian', 'fedora', 'opensuse', 'ubuntu'] +# The schema definition for each cloud-config module is a strict contract for +# describing supported configuration parameters for each cloud-config section. +# It allows cloud-config to validate and alert users to invalid or ignored +# configuration options before actually attempting to deploy with said +# configuration. + +schema = { + 'id': 'cc_ntp', + 'name': 'NTP', + 'title': 'enable and configure ntp', + 'description': dedent("""\ + Handle ntp configuration. If ntp is not installed on the system and + ntp configuration is specified, ntp will be installed. If there is a + default ntp config file in the image or one is present in the + distro's ntp package, it will be copied to ``/etc/ntp.conf.dist`` + before any changes are made. A list of ntp pools and ntp servers can + be provided under the ``ntp`` config key. If no ntp ``servers`` or + ``pools`` are provided, 4 pools will be used in the format + ``{0-3}.{distro}.pool.ntp.org``."""), + 'distros': distros, + 'examples': [ + {'ntp': {'pools': ['0.company.pool.ntp.org', '1.company.pool.ntp.org', + 'ntp.myorg.org'], + 'servers': ['my.ntp.server.local', 'ntp.ubuntu.com', + '192.168.23.2']}}], + 'frequency': PER_INSTANCE, + 'type': 'object', + 'properties': { + 'ntp': { + 'type': ['object', 'null'], + 'properties': { + 'pools': { + 'type': 'array', + 'items': { + 'type': 'string', + 'format': 'hostname' + }, + 'uniqueItems': True, + 'description': dedent("""\ + List of ntp pools. If both pools and servers are + empty, 4 default pool servers will be provided of + the format ``{0-3}.{distro}.pool.ntp.org``.""") + }, + 'servers': { + 'type': 'array', + 'items': { + 'type': 'string', + 'format': 'hostname' + }, + 'uniqueItems': True, + 'description': dedent("""\ + List of ntp servers. If both pools and servers are + empty, 4 default pool servers will be provided with + the format ``{0-3}.{distro}.pool.ntp.org``.""") + } + }, + 'required': [], + 'additionalProperties': False + } + } +} + + def handle(name, cfg, cloud, log, _args): """Enable and configure ntp.""" - if 'ntp' not in cfg: LOG.debug( "Skipping module named %s, not present or disabled by cfg", name) return - ntp_cfg = cfg.get('ntp', {}) + # TODO drop this when validate_cloudconfig_schema is strict=True if not isinstance(ntp_cfg, (dict)): raise RuntimeError(("'ntp' key existed in config," " but not a dictionary type," " is a %s %instead"), type_utils.obj_name(ntp_cfg)) + validate_cloudconfig_schema(cfg, schema) rename_ntp_conf() # ensure when ntp is installed it has a configuration file # to use instead of starting up with packaged defaults diff --git a/cloudinit/config/schema.py b/cloudinit/config/schema.py new file mode 100644 index 00000000..6400f005 --- /dev/null +++ b/cloudinit/config/schema.py @@ -0,0 +1,222 @@ +# This file is part of cloud-init. See LICENSE file for license information. +"""schema.py: Set of module functions for processing cloud-config schema.""" + +from __future__ import print_function + +from cloudinit.util import read_file_or_url + +import argparse +import logging +import os +import sys +import yaml + +SCHEMA_UNDEFINED = b'UNDEFINED' +CLOUD_CONFIG_HEADER = b'#cloud-config' +SCHEMA_DOC_TMPL = """ +{name} +--- +**Summary:** {title} + +{description} + +**Internal name:** ``{id}`` + +**Module frequency:** {frequency} + +**Supported distros:** {distros} + +**Config schema**: +{property_doc} +{examples} +""" +SCHEMA_PROPERTY_TMPL = '{prefix}**{prop_name}:** ({type}) {description}' + + +class SchemaValidationError(ValueError): + """Raised when validating a cloud-config file against a schema.""" + + def __init__(self, schema_errors=()): + """Init the exception an n-tuple of schema errors. + + @param schema_errors: An n-tuple of the format: + ((flat.config.key, msg),) + """ + self.schema_errors = schema_errors + error_messages = [ + '{0}: {1}'.format(config_key, message) + for config_key, message in schema_errors] + message = "Cloud config schema errors: {0}".format( + ', '.join(error_messages)) + super(SchemaValidationError, self).__init__(message) + + +def validate_cloudconfig_schema(config, schema, strict=False): + """Validate provided config meets the schema definition. + + @param config: Dict of cloud configuration settings validated against + schema. + @param schema: jsonschema dict describing the supported schema definition + for the cloud config module (config.cc_*). + @param strict: Boolean, when True raise SchemaValidationErrors instead of + logging warnings. + + @raises: SchemaValidationError when provided config does not validate + against the provided schema. + """ + try: + from jsonschema import Draft4Validator, FormatChecker + except ImportError: + logging.warning( + 'Ignoring schema validation. python-jsonschema is not present') + return + validator = Draft4Validator(schema, format_checker=FormatChecker()) + errors = () + for error in sorted(validator.iter_errors(config), key=lambda e: e.path): + path = '.'.join([str(p) for p in error.path]) + errors += ((path, error.message),) + if errors: + if strict: + raise SchemaValidationError(errors) + else: + messages = ['{0}: {1}'.format(k, msg) for k, msg in errors] + logging.warning('Invalid config:\n%s', '\n'.join(messages)) + + +def validate_cloudconfig_file(config_path, schema): + """Validate cloudconfig file adheres to a specific jsonschema. + + @param config_path: Path to the yaml cloud-config file to parse. + @param schema: Dict describing a valid jsonschema to validate against. + + @raises SchemaValidationError containing any of schema_errors encountered. + @raises RuntimeError when config_path does not exist. + """ + if not os.path.exists(config_path): + raise RuntimeError('Configfile {0} does not exist'.format(config_path)) + content = read_file_or_url('file://{0}'.format(config_path)).contents + if not content.startswith(CLOUD_CONFIG_HEADER): + errors = ( + ('header', 'File {0} needs to begin with "{1}"'.format( + config_path, CLOUD_CONFIG_HEADER.decode())),) + raise SchemaValidationError(errors) + + try: + cloudconfig = yaml.safe_load(content) + except yaml.parser.ParserError as e: + errors = ( + ('format', 'File {0} is not valid yaml. {1}'.format( + config_path, str(e))),) + raise SchemaValidationError(errors) + validate_cloudconfig_schema( + cloudconfig, schema, strict=True) + + +def _get_property_type(property_dict): + """Return a string representing a property type from a given jsonschema.""" + property_type = property_dict.get('type', SCHEMA_UNDEFINED) + if isinstance(property_type, list): + property_type = '/'.join(property_type) + item_type = property_dict.get('items', {}).get('type') + if item_type: + property_type = '{0} of {1}'.format(property_type, item_type) + return property_type + + +def _get_property_doc(schema, prefix=' '): + """Return restructured text describing the supported schema properties.""" + new_prefix = prefix + ' ' + properties = [] + for prop_key, prop_config in schema.get('properties', {}).items(): + # Define prop_name and dscription for SCHEMA_PROPERTY_TMPL + description = prop_config.get('description', '') + properties.append(SCHEMA_PROPERTY_TMPL.format( + prefix=prefix, + prop_name=prop_key, + type=_get_property_type(prop_config), + description=description.replace('\n', ''))) + if 'properties' in prop_config: + properties.append( + _get_property_doc(prop_config, prefix=new_prefix)) + return '\n\n'.join(properties) + + +def _get_schema_examples(schema, prefix=''): + """Return restructured text describing the schema examples if present.""" + examples = schema.get('examples') + if not examples: + return '' + rst_content = '\n**Examples**::\n\n' + for example in examples: + example_yaml = yaml.dump(example, default_flow_style=False) + # Python2.6 is missing textwrapper.indent + lines = example_yaml.split('\n') + indented_lines = [' {0}'.format(line) for line in lines] + rst_content += '\n'.join(indented_lines) + return rst_content + + +def get_schema_doc(schema): + """Return reStructured text rendering the provided jsonschema. + + @param schema: Dict of jsonschema to render. + @raise KeyError: If schema lacks an expected key. + """ + schema['property_doc'] = _get_property_doc(schema) + schema['examples'] = _get_schema_examples(schema) + schema['distros'] = ', '.join(schema['distros']) + return SCHEMA_DOC_TMPL.format(**schema) + + +def get_schema(section_key=None): + """Return a dict of jsonschema defined in any cc_* module. + + @param: section_key: Optionally limit schema to a specific top-level key. + """ + # TODO use util.find_modules in subsequent branch + from cloudinit.config.cc_ntp import schema + return schema + + +def error(message): + print(message, file=sys.stderr) + return 1 + + +def get_parser(): + """Return a parser for supported cmdline arguments.""" + parser = argparse.ArgumentParser() + parser.add_argument('-c', '--config-file', + help='Path of the cloud-config yaml file to validate') + parser.add_argument('-d', '--doc', action="store_true", default=False, + help='Print schema documentation') + parser.add_argument('-k', '--key', + help='Limit validation or docs to a section key') + return parser + + +def main(): + """Tool to validate schema of a cloud-config file or print schema docs.""" + parser = get_parser() + args = parser.parse_args() + exclusive_args = [args.config_file, args.doc] + if not any(exclusive_args) or all(exclusive_args): + return error('Expected either --config-file argument or --doc') + + schema = get_schema() + if args.config_file: + try: + validate_cloudconfig_file(args.config_file, schema) + except (SchemaValidationError, RuntimeError) as e: + return error(str(e)) + print("Valid cloud-config file {0}".format(args.config_file)) + if args.doc: + print(get_schema_doc(schema)) + return 0 + + +if __name__ == '__main__': + sys.exit(main()) + + +# vi: ts=4 expandtab -- cgit v1.2.3 From ad2680a689ab78847ccce7766d6591797d99e219 Mon Sep 17 00:00:00 2001 From: JJ Asghar Date: Mon, 5 Jun 2017 20:36:12 -0500 Subject: Chef: Update omnibus url to chef.io, minor doc changes. - Updated to standard chef.io url - Removed the port 4000, due to that has been deprecated - Added Note about the run_list not being required Signed-off-by: JJ Asghar --- cloudinit/config/cc_chef.py | 2 +- doc/examples/cloud-config-chef.txt | 12 ++++++------ .../configs/examples/install_run_chef_recipes.yaml | 6 +++--- 3 files changed, 10 insertions(+), 10 deletions(-) (limited to 'cloudinit/config') diff --git a/cloudinit/config/cc_chef.py b/cloudinit/config/cc_chef.py index 2be2532c..02c70b10 100644 --- a/cloudinit/config/cc_chef.py +++ b/cloudinit/config/cc_chef.py @@ -92,7 +92,7 @@ REQUIRED_CHEF_DIRS = tuple([ ]) # Used if fetching chef from a omnibus style package -OMNIBUS_URL = "https://www.getchef.com/chef/install.sh" +OMNIBUS_URL = "https://www.chef.io/chef/install.sh" OMNIBUS_URL_RETRIES = 5 CHEF_VALIDATION_PEM_PATH = '/etc/chef/validation.pem' diff --git a/doc/examples/cloud-config-chef.txt b/doc/examples/cloud-config-chef.txt index 3cb62006..9d235817 100644 --- a/doc/examples/cloud-config-chef.txt +++ b/doc/examples/cloud-config-chef.txt @@ -1,6 +1,6 @@ #cloud-config # -# This is an example file to automatically install chef-client and run a +# This is an example file to automatically install chef-client and run a # list of recipes when the instance boots for the first time. # Make sure that this file is valid yaml before starting instances. # It should be passed as user-data when starting the instance. @@ -8,7 +8,7 @@ # This example assumes the instance is 16.04 (xenial) -# The default is to install from packages. +# The default is to install from packages. # Key from https://packages.chef.io/chef.asc apt: @@ -60,7 +60,7 @@ chef: force_install: false # Chef settings - server_url: "https://chef.yourorg.com:4000" + server_url: "https://chef.yourorg.com" # Node Name # Defaults to the instance-id if not present @@ -78,8 +78,8 @@ chef: -----BEGIN RSA PRIVATE KEY----- YOUR-ORGS-VALIDATION-KEY-HERE -----END RSA PRIVATE KEY----- - - # A run list for a first boot json + + # A run list for a first boot json, an example (not required) run_list: - "recipe[apache2]" - "role[db]" @@ -92,7 +92,7 @@ chef: keepalive: "off" # if install_type is 'omnibus', change the url to download - omnibus_url: "https://www.opscode.com/chef/install.sh" + omnibus_url: "https://www.chef.io/chef/install.sh" # Capture all subprocess output into a logfile diff --git a/tests/cloud_tests/configs/examples/install_run_chef_recipes.yaml b/tests/cloud_tests/configs/examples/install_run_chef_recipes.yaml index 0bec305e..66b635a8 100644 --- a/tests/cloud_tests/configs/examples/install_run_chef_recipes.yaml +++ b/tests/cloud_tests/configs/examples/install_run_chef_recipes.yaml @@ -56,7 +56,7 @@ cloud_config: | force_install: false # Chef settings - server_url: "https://chef.yourorg.com:4000" + server_url: "https://chef.yourorg.com" # Node Name # Defaults to the instance-id if not present @@ -75,7 +75,7 @@ cloud_config: | YOUR-ORGS-VALIDATION-KEY-HERE -----END RSA PRIVATE KEY----- - # A run list for a first boot json + # A run list for a first boot json, this is an example (not required) run_list: - "recipe[apache2]" - "role[db]" @@ -88,7 +88,7 @@ cloud_config: | keepalive: "off" # if install_type is 'omnibus', change the url to download - omnibus_url: "https://www.opscode.com/chef/install.sh" + omnibus_url: "https://www.chef.io/chef/install.sh" # Capture all subprocess output into a logfile -- cgit v1.2.3 From 0fe6a0607408d387f4b0d4482b95afbc5d3f3909 Mon Sep 17 00:00:00 2001 From: Andrew Jorgensen Date: Fri, 5 Dec 2014 14:34:10 -0800 Subject: write_file(s): Print permissions as octal, not decimal Unix file modes are usually represented as octal, but they were being interpreted as decimal, for example 0o644 would be printed as '420'. Reviewed-by: Tom Kirchner --- cloudinit/config/cc_write_files.py | 10 ++++++- cloudinit/util.py | 6 ++++- tests/unittests/helpers.py | 8 +++--- tests/unittests/test_datasource/test_azure.py | 3 ++- tests/unittests/test_handler/test_handler_ntp.py | 3 ++- .../test_handler/test_handler_write_files.py | 31 ++++++++++++++++++++-- 6 files changed, 52 insertions(+), 9 deletions(-) (limited to 'cloudinit/config') diff --git a/cloudinit/config/cc_write_files.py b/cloudinit/config/cc_write_files.py index 72e1cdd6..1835a31b 100644 --- a/cloudinit/config/cc_write_files.py +++ b/cloudinit/config/cc_write_files.py @@ -53,6 +53,7 @@ import six from cloudinit.settings import PER_INSTANCE from cloudinit import util + frequency = PER_INSTANCE DEFAULT_OWNER = "root:root" @@ -119,7 +120,14 @@ def decode_perms(perm, default, log): # Force to string and try octal conversion return int(str(perm), 8) except (TypeError, ValueError): - log.warn("Undecodable permissions %s, assuming %s", perm, default) + reps = [] + for r in (perm, default): + try: + reps.append("%o" % r) + except TypeError: + reps.append("%r" % r) + log.warning( + "Undecodable permissions {0}, returning default {1}".format(*reps)) return default diff --git a/cloudinit/util.py b/cloudinit/util.py index 415ca374..ec68925e 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -1751,8 +1751,12 @@ def write_file(filename, content, mode=0o644, omode="wb", copy_mode=False): else: content = decode_binary(content) write_type = 'characters' + try: + mode_r = "%o" % mode + except TypeError: + mode_r = "%r" % mode LOG.debug("Writing to %s - %s: [%s] %s %s", - filename, omode, mode, len(content), write_type) + filename, omode, mode_r, len(content), write_type) with SeLinuxGuard(path=filename): with open(filename, omode) as fh: fh.write(content) diff --git a/tests/unittests/helpers.py b/tests/unittests/helpers.py index e78abce2..569f1aef 100644 --- a/tests/unittests/helpers.py +++ b/tests/unittests/helpers.py @@ -97,11 +97,13 @@ class CiTestCase(TestCase): super(CiTestCase, self).setUp() if self.with_logs: # Create a log handler so unit tests can search expected logs. - logger = logging.getLogger() + self.logger = logging.getLogger() self.logs = six.StringIO() + formatter = logging.Formatter('%(levelname)s: %(message)s') handler = logging.StreamHandler(self.logs) - self.old_handlers = logger.handlers - logger.handlers = [handler] + handler.setFormatter(formatter) + self.old_handlers = self.logger.handlers + self.logger.handlers = [handler] def tearDown(self): if self.with_logs: diff --git a/tests/unittests/test_datasource/test_azure.py b/tests/unittests/test_datasource/test_azure.py index 42f49e06..b17f389c 100644 --- a/tests/unittests/test_datasource/test_azure.py +++ b/tests/unittests/test_datasource/test_azure.py @@ -263,7 +263,8 @@ fdescfs /dev/fd fdescfs rw 0 0 {}, distro=None, paths=self.paths) self.assertFalse(dsrc.get_data()) self.assertEqual( - "Non-Azure DMI asset tag '{0}' discovered.\n".format(nonazure_tag), + "DEBUG: Non-Azure DMI asset tag '{0}' discovered.\n".format( + nonazure_tag), self.logs.getvalue()) def test_basic_seed_dir(self): diff --git a/tests/unittests/test_handler/test_handler_ntp.py b/tests/unittests/test_handler/test_handler_ntp.py index 3a9f7f7e..c4299d94 100644 --- a/tests/unittests/test_handler/test_handler_ntp.py +++ b/tests/unittests/test_handler/test_handler_ntp.py @@ -216,7 +216,8 @@ class TestNtp(FilesystemMockingTestCase): """When no ntp section is defined handler logs a warning and noops.""" cc_ntp.handle('cc_ntp', {}, None, None, []) self.assertEqual( - 'Skipping module named cc_ntp, not present or disabled by cfg\n', + 'DEBUG: Skipping module named cc_ntp, ' + 'not present or disabled by cfg\n', self.logs.getvalue()) def test_ntp_handler_schema_validation_allows_empty_ntp_config(self): diff --git a/tests/unittests/test_handler/test_handler_write_files.py b/tests/unittests/test_handler/test_handler_write_files.py index fb252d1d..88a4742b 100644 --- a/tests/unittests/test_handler/test_handler_write_files.py +++ b/tests/unittests/test_handler/test_handler_write_files.py @@ -1,10 +1,10 @@ # This file is part of cloud-init. See LICENSE file for license information. -from cloudinit.config.cc_write_files import write_files +from cloudinit.config.cc_write_files import write_files, decode_perms from cloudinit import log as logging from cloudinit import util -from ..helpers import FilesystemMockingTestCase +from ..helpers import CiTestCase, FilesystemMockingTestCase import base64 import gzip @@ -98,6 +98,33 @@ class TestWriteFiles(FilesystemMockingTestCase): self.assertEqual(len(expected), flen_expected) +class TestDecodePerms(CiTestCase): + + with_logs = True + + def test_none_returns_default(self): + """If None is passed as perms, then default should be returned.""" + default = object() + found = decode_perms(None, default, self.logger) + self.assertEqual(default, found) + + def test_integer(self): + """A valid integer should return itself.""" + found = decode_perms(0o755, None, self.logger) + self.assertEqual(0o755, found) + + def test_valid_octal_string(self): + """A string should be read as octal.""" + found = decode_perms("644", None, self.logger) + self.assertEqual(0o644, found) + + def test_invalid_octal_string_returns_default_and_warns(self): + """A string with invalid octal should warn and return default.""" + found = decode_perms("999", None, self.logger) + self.assertIsNone(found) + self.assertIn("WARNING: Undecodable", self.logs.getvalue()) + + def _gzip_bytes(data): buf = six.BytesIO() fp = None -- cgit v1.2.3 From ecb408afa1104fe49ce6eb1dc5708be56abd5cb2 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Thu, 15 Jun 2017 10:03:45 -0400 Subject: FreeBSD: Make freebsd a variant, fix unittests and tools/build-on-freebsd. - Simplify the logic of 'variant' in util.system_info much of the data from https://github.com/hpcugent/easybuild/wiki/OS_flavor_name_version - fix get_resource_disk_on_freebsd when running on a system without an Azure resource disk. - fix tools/build-on-freebsd to replace oauth with oauthlib and add bash which is a dependency for tests. - update a fiew places that were checking for freebsd but not using the util.is_FreeBSD() --- cloudinit/config/cc_growpart.py | 2 +- cloudinit/config/cc_power_state_change.py | 2 +- cloudinit/sources/DataSourceAzure.py | 2 +- cloudinit/util.py | 46 ++++++++++-------------- config/cloud.cfg.tmpl | 20 +++++------ tests/unittests/test_handler/test_handler_ntp.py | 2 +- tests/unittests/test_util.py | 9 +++-- tools/build-on-freebsd | 6 ++-- 8 files changed, 40 insertions(+), 49 deletions(-) (limited to 'cloudinit/config') diff --git a/cloudinit/config/cc_growpart.py b/cloudinit/config/cc_growpart.py index d2bc6e6c..bafca9d8 100644 --- a/cloudinit/config/cc_growpart.py +++ b/cloudinit/config/cc_growpart.py @@ -214,7 +214,7 @@ def device_part_info(devpath): # FreeBSD doesn't know of sysfs so just get everything we need from # the device, like /dev/vtbd0p2. - if util.system_info()["platform"].startswith('FreeBSD'): + if util.is_FreeBSD(): m = re.search('^(/dev/.+)p([0-9])$', devpath) return (m.group(1), m.group(2)) diff --git a/cloudinit/config/cc_power_state_change.py b/cloudinit/config/cc_power_state_change.py index c1c6fe7e..eba58b02 100644 --- a/cloudinit/config/cc_power_state_change.py +++ b/cloudinit/config/cc_power_state_change.py @@ -71,7 +71,7 @@ def givecmdline(pid): # Example output from procstat -c 1 # PID COMM ARGS # 1 init /bin/init -- - if util.system_info()["platform"].startswith('FreeBSD'): + if util.is_FreeBSD(): (output, _err) = util.subp(['procstat', '-c', str(pid)]) line = output.splitlines()[1] m = re.search('\d+ (\w|\.|-)+\s+(/\w.+)', line) diff --git a/cloudinit/sources/DataSourceAzure.py b/cloudinit/sources/DataSourceAzure.py index 71e7c55c..4fe0d635 100644 --- a/cloudinit/sources/DataSourceAzure.py +++ b/cloudinit/sources/DataSourceAzure.py @@ -101,7 +101,7 @@ def get_dev_storvsc_sysctl(): sysctl_out, err = util.subp(['sysctl', 'dev.storvsc']) except util.ProcessExecutionError: LOG.debug("Fail to execute sysctl dev.storvsc") - return None + sysctl_out = "" return sysctl_out diff --git a/cloudinit/util.py b/cloudinit/util.py index ec68925e..c93b6d7e 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -573,7 +573,7 @@ def is_ipv4(instr): def is_FreeBSD(): - return system_info()['platform'].startswith('FreeBSD') + return system_info()['variant'] == "freebsd" def get_cfg_option_bool(yobj, key, default=False): @@ -598,37 +598,29 @@ def get_cfg_option_int(yobj, key, default=0): def system_info(): info = { 'platform': platform.platform(), + 'system': platform.system(), 'release': platform.release(), 'python': platform.python_version(), 'uname': platform.uname(), - 'dist': platform.linux_distribution(), # pylint: disable=W1505 + 'dist': platform.dist(), # pylint: disable=W1505 } - plat = info['platform'].lower() - # Try to get more info about what it actually is, in a format - # that we can easily use across linux and variants... - if plat.startswith('darwin'): - info['variant'] = 'darwin' - elif plat.endswith("bsd"): - info['variant'] = 'bsd' - elif plat.startswith('win'): - info['variant'] = 'windows' - elif 'linux' in plat: - # Try to get a single string out of these... - linux_dist, _version, _id = info['dist'] - linux_dist = linux_dist.lower() - if linux_dist in ('ubuntu', 'linuxmint', 'mint'): - info['variant'] = 'ubuntu' + system = info['system'].lower() + var = 'unknown' + if system == "linux": + linux_dist = info['dist'][0].lower() + if linux_dist in ('centos', 'fedora', 'debian'): + var = linux_dist + elif linux_dist in ('ubuntu', 'linuxmint', 'mint'): + var = 'ubuntu' + elif linux_dist == 'redhat': + var = 'rhel' else: - for prefix, variant in [('redhat', 'rhel'), - ('centos', 'centos'), - ('fedora', 'fedora'), - ('debian', 'debian')]: - if linux_dist.startswith(prefix): - info['variant'] = variant - if 'variant' not in info: - info['variant'] = 'linux' - if 'variant' not in info: - info['variant'] = 'unknown' + var = 'linux' + elif system in ('windows', 'darwin', "freebsd"): + var = system + + info['variant'] = var + return info diff --git a/config/cloud.cfg.tmpl b/config/cloud.cfg.tmpl index 5af2a88f..f4b9069b 100644 --- a/config/cloud.cfg.tmpl +++ b/config/cloud.cfg.tmpl @@ -2,7 +2,7 @@ # The top level settings are used as module # and system configuration. -{% if variant in ["bsd"] %} +{% if variant in ["freebsd"] %} syslog_fix_perms: root:wheel {% endif %} # A set of users which may be applied and/or used by various modules @@ -13,7 +13,7 @@ users: # If this is set, 'root' will not be able to ssh in and they # will get a message to login instead as the default $user -{% if variant in ["bsd"] %} +{% if variant in ["freebsd"] %} disable_root: false {% else %} disable_root: true @@ -30,7 +30,7 @@ ssh_pwauth: 0 # This will cause the set+update hostname module to not operate (if true) preserve_hostname: false -{% if variant in ["bsd"] %} +{% if variant in ["freebsd"] %} # This should not be required, but leave it in place until the real cause of # not beeing able to find -any- datasources is resolved. datasource_list: ['ConfigDrive', 'Azure', 'OpenStack', 'Ec2'] @@ -53,13 +53,13 @@ cloud_init_modules: - write-files - growpart - resizefs -{% if variant not in ["bsd"] %} +{% if variant not in ["freebsd"] %} - disk_setup - mounts {% endif %} - set_hostname - update_hostname -{% if variant not in ["bsd"] %} +{% if variant not in ["freebsd"] %} - update_etc_hosts - ca-certs - rsyslog @@ -87,7 +87,7 @@ cloud_config_modules: - apt-pipelining - apt-configure {% endif %} -{% if variant not in ["bsd"] %} +{% if variant not in ["freebsd"] %} - ntp {% endif %} - timezone @@ -108,7 +108,7 @@ cloud_final_modules: - landscape - lxd {% endif %} -{% if variant not in ["bsd"] %} +{% if variant not in ["freebsd"] %} - puppet - chef - salt-minion @@ -130,10 +130,8 @@ cloud_final_modules: # (not accessible to handlers/transforms) system_info: # This will affect which distro class gets used -{% if variant in ["centos", "debian", "fedora", "rhel", "ubuntu"] %} +{% if variant in ["centos", "debian", "fedora", "rhel", "ubuntu", "freebsd"] %} distro: {{ variant }} -{% elif variant in ["bsd"] %} - distro: freebsd {% else %} # Unknown/fallback distro. distro: ubuntu @@ -182,7 +180,7 @@ system_info: cloud_dir: /var/lib/cloud/ templates_dir: /etc/cloud/templates/ ssh_svcname: sshd -{% elif variant in ["bsd"] %} +{% elif variant in ["freebsd"] %} # Default user name + that default users groups (if added/used) default_user: name: freebsd diff --git a/tests/unittests/test_handler/test_handler_ntp.py b/tests/unittests/test_handler/test_handler_ntp.py index c4299d94..7f278646 100644 --- a/tests/unittests/test_handler/test_handler_ntp.py +++ b/tests/unittests/test_handler/test_handler_ntp.py @@ -62,7 +62,7 @@ class TestNtp(FilesystemMockingTestCase): def test_ntp_rename_ntp_conf(self): """When NTP_CONF exists, rename_ntp moves it.""" ntpconf = self.tmp_path("ntp.conf", self.new_root) - os.mknod(ntpconf) + util.write_file(ntpconf, "") with mock.patch("cloudinit.config.cc_ntp.NTP_CONF", ntpconf): cc_ntp.rename_ntp_conf() self.assertFalse(os.path.exists(ntpconf)) diff --git a/tests/unittests/test_util.py b/tests/unittests/test_util.py index 014aa6a3..a73fd26a 100644 --- a/tests/unittests/test_util.py +++ b/tests/unittests/test_util.py @@ -20,6 +20,9 @@ except ImportError: import mock +BASH = util.which('bash') + + class FakeSelinux(object): def __init__(self, match_what): @@ -544,17 +547,17 @@ class TestReadSeeded(helpers.TestCase): class TestSubp(helpers.TestCase): - stdin2err = ['bash', '-c', 'cat >&2'] + stdin2err = [BASH, '-c', 'cat >&2'] stdin2out = ['cat'] utf8_invalid = b'ab\xaadef' utf8_valid = b'start \xc3\xa9 end' utf8_valid_2 = b'd\xc3\xa9j\xc8\xa7' - printenv = ['bash', '-c', 'for n in "$@"; do echo "$n=${!n}"; done', '--'] + printenv = [BASH, '-c', 'for n in "$@"; do echo "$n=${!n}"; done', '--'] def printf_cmd(self, *args): # bash's printf supports \xaa. So does /usr/bin/printf # but by using bash, we remove dependency on another program. - return(['bash', '-c', 'printf "$@"', 'printf'] + list(args)) + return([BASH, '-c', 'printf "$@"', 'printf'] + list(args)) def test_subp_handles_utf8(self): # The given bytes contain utf-8 accented characters as seen in e.g. diff --git a/tools/build-on-freebsd b/tools/build-on-freebsd index ccc10b40..ff9153ad 100755 --- a/tools/build-on-freebsd +++ b/tools/build-on-freebsd @@ -8,6 +8,7 @@ fail() { echo "FAILED:" "$@" 1>&2; exit 1; } # Check dependencies: depschecked=/tmp/c-i.dependencieschecked pkgs=" + bash dmidecode e2fsprogs py27-Jinja2 @@ -16,7 +17,7 @@ pkgs=" py27-configobj py27-jsonpatch py27-jsonpointer - py27-oauth + py27-oauthlib py27-prettytable py27-requests py27-serial @@ -35,9 +36,6 @@ touch $depschecked python setup.py build python setup.py install -O1 --skip-build --prefix /usr/local/ --init-system sysvinit_freebsd -# Install the correct config file: -cp config/cloud.cfg-freebsd /etc/cloud/cloud.cfg - # Enable cloud-init in /etc/rc.conf: sed -i.bak -e "/cloudinit_enable=.*/d" /etc/rc.conf echo 'cloudinit_enable="YES"' >> /etc/rc.conf -- cgit v1.2.3 From 4330a98161a5c514c7f85a76a58d057c85b00174 Mon Sep 17 00:00:00 2001 From: Andrew Jorgensen Date: Fri, 16 Jun 2017 18:17:36 +0000 Subject: write_files: Remove log from helper function signatures. Instead of passing around a 'log' reference to functions, just import logging and use that. This is the pattern that is now more common in cloud-init. --- cloudinit/config/cc_write_files.py | 27 ++++++++++++---------- .../test_handler/test_handler_write_files.py | 14 +++++------ 2 files changed, 22 insertions(+), 19 deletions(-) (limited to 'cloudinit/config') diff --git a/cloudinit/config/cc_write_files.py b/cloudinit/config/cc_write_files.py index 1835a31b..54ae3a68 100644 --- a/cloudinit/config/cc_write_files.py +++ b/cloudinit/config/cc_write_files.py @@ -50,6 +50,7 @@ import base64 import os import six +from cloudinit import log as logging from cloudinit.settings import PER_INSTANCE from cloudinit import util @@ -60,6 +61,8 @@ DEFAULT_OWNER = "root:root" DEFAULT_PERMS = 0o644 UNKNOWN_ENC = 'text/plain' +LOG = logging.getLogger(__name__) + def handle(name, cfg, _cloud, log, _args): files = cfg.get('write_files') @@ -67,10 +70,10 @@ def handle(name, cfg, _cloud, log, _args): log.debug(("Skipping module named %s," " no/empty 'write_files' key in configuration"), name) return - write_files(name, files, log) + write_files(name, files) -def canonicalize_extraction(encoding_type, log): +def canonicalize_extraction(encoding_type): if not encoding_type: encoding_type = '' encoding_type = encoding_type.lower().strip() @@ -85,31 +88,31 @@ def canonicalize_extraction(encoding_type, log): if encoding_type in ['b64', 'base64']: return ['application/base64'] if encoding_type: - log.warn("Unknown encoding type %s, assuming %s", - encoding_type, UNKNOWN_ENC) + LOG.warning("Unknown encoding type %s, assuming %s", + encoding_type, UNKNOWN_ENC) return [UNKNOWN_ENC] -def write_files(name, files, log): +def write_files(name, files): if not files: return for (i, f_info) in enumerate(files): path = f_info.get('path') if not path: - log.warn("No path provided to write for entry %s in module %s", - i + 1, name) + LOG.warning("No path provided to write for entry %s in module %s", + i + 1, name) continue path = os.path.abspath(path) - extractions = canonicalize_extraction(f_info.get('encoding'), log) + extractions = canonicalize_extraction(f_info.get('encoding')) contents = extract_contents(f_info.get('content', ''), extractions) (u, g) = util.extract_usergroup(f_info.get('owner', DEFAULT_OWNER)) - perms = decode_perms(f_info.get('permissions'), DEFAULT_PERMS, log) + perms = decode_perms(f_info.get('permissions'), DEFAULT_PERMS) util.write_file(path, contents, mode=perms) util.chownbyname(path, u, g) -def decode_perms(perm, default, log): +def decode_perms(perm, default): if perm is None: return default try: @@ -126,8 +129,8 @@ def decode_perms(perm, default, log): reps.append("%o" % r) except TypeError: reps.append("%r" % r) - log.warning( - "Undecodable permissions {0}, returning default {1}".format(*reps)) + LOG.warning( + "Undecodable permissions %s, returning default %s", *reps) return default diff --git a/tests/unittests/test_handler/test_handler_write_files.py b/tests/unittests/test_handler/test_handler_write_files.py index 88a4742b..1129e77d 100644 --- a/tests/unittests/test_handler/test_handler_write_files.py +++ b/tests/unittests/test_handler/test_handler_write_files.py @@ -49,13 +49,13 @@ class TestWriteFiles(FilesystemMockingTestCase): expected = "hello world\n" filename = "/tmp/my.file" write_files( - "test_simple", [{"content": expected, "path": filename}], LOG) + "test_simple", [{"content": expected, "path": filename}]) self.assertEqual(util.load_file(filename), expected) def test_yaml_binary(self): self.patchUtils(self.tmp) data = util.load_yaml(YAML_TEXT) - write_files("testname", data['write_files'], LOG) + write_files("testname", data['write_files']) for path, content in YAML_CONTENT_EXPECTED.items(): self.assertEqual(util.load_file(path), content) @@ -87,7 +87,7 @@ class TestWriteFiles(FilesystemMockingTestCase): files.append(cur) expected.append((cur['path'], data)) - write_files("test_decoding", files, LOG) + write_files("test_decoding", files) for path, content in expected: self.assertEqual(util.load_file(path, decode=False), content) @@ -105,22 +105,22 @@ class TestDecodePerms(CiTestCase): def test_none_returns_default(self): """If None is passed as perms, then default should be returned.""" default = object() - found = decode_perms(None, default, self.logger) + found = decode_perms(None, default) self.assertEqual(default, found) def test_integer(self): """A valid integer should return itself.""" - found = decode_perms(0o755, None, self.logger) + found = decode_perms(0o755, None) self.assertEqual(0o755, found) def test_valid_octal_string(self): """A string should be read as octal.""" - found = decode_perms("644", None, self.logger) + found = decode_perms("644", None) self.assertEqual(0o644, found) def test_invalid_octal_string_returns_default_and_warns(self): """A string with invalid octal should warn and return default.""" - found = decode_perms("999", None, self.logger) + found = decode_perms("999", None) self.assertIsNone(found) self.assertIn("WARNING: Undecodable", self.logs.getvalue()) -- cgit v1.2.3