From bae9b11da9ed7dd0b16fe5adeaf4774b7cc628cf Mon Sep 17 00:00:00 2001 From: James Falcon Date: Wed, 15 Dec 2021 20:16:38 -0600 Subject: Adopt Black and isort (SC-700) (#1157) Applied Black and isort, fixed any linting issues, updated tox.ini and CI. --- cloudinit/handlers/cloud_config.py | 29 ++++++++++++++--------------- 1 file changed, 14 insertions(+), 15 deletions(-) (limited to 'cloudinit/handlers/cloud_config.py') diff --git a/cloudinit/handlers/cloud_config.py b/cloudinit/handlers/cloud_config.py index 2a307364..2e694671 100644 --- a/cloudinit/handlers/cloud_config.py +++ b/cloudinit/handlers/cloud_config.py @@ -12,15 +12,12 @@ import jsonpatch from cloudinit import handlers from cloudinit import log as logging -from cloudinit import mergers -from cloudinit import util -from cloudinit import safeyaml - -from cloudinit.settings import (PER_ALWAYS) +from cloudinit import mergers, safeyaml, util +from cloudinit.settings import PER_ALWAYS LOG = logging.getLogger(__name__) -MERGE_HEADER = 'Merge-Type' +MERGE_HEADER = "Merge-Type" # Due to the way the loading of yaml configuration was done previously, # where previously each cloud config part was appended to a larger yaml @@ -39,7 +36,7 @@ MERGE_HEADER = 'Merge-Type' # a: 22 # # This gets loaded into yaml with final result {'a': 22} -DEF_MERGERS = mergers.string_extract_mergers('dict(replace)+list()+str()') +DEF_MERGERS = mergers.string_extract_mergers("dict(replace)+list()+str()") CLOUD_PREFIX = "#cloud-config" JSONP_PREFIX = "#cloud-config-jsonp" @@ -53,7 +50,7 @@ class CloudConfigPartHandler(handlers.Handler): handlers.Handler.__init__(self, PER_ALWAYS, version=3) self.cloud_buf = None self.cloud_fn = paths.get_ipath("cloud_config") - if 'cloud_config_path' in _kwargs: + if "cloud_config_path" in _kwargs: self.cloud_fn = paths.get_ipath(_kwargs["cloud_config_path"]) self.file_names = [] @@ -66,14 +63,14 @@ class CloudConfigPartHandler(handlers.Handler): file_lines.append("# from %s files" % (len(self.file_names))) for fn in self.file_names: if not fn: - fn = '?' + fn = "?" file_lines.append("# %s" % (fn)) file_lines.append("") if self.cloud_buf is not None: # Something was actually gathered.... lines = [ CLOUD_PREFIX, - '', + "", ] lines.extend(file_lines) lines.append(safeyaml.dumps(self.cloud_buf)) @@ -82,9 +79,9 @@ class CloudConfigPartHandler(handlers.Handler): util.write_file(self.cloud_fn, "\n".join(lines), 0o600) def _extract_mergers(self, payload, headers): - merge_header_headers = '' - for h in [MERGE_HEADER, 'X-%s' % (MERGE_HEADER)]: - tmp_h = headers.get(h, '') + merge_header_headers = "" + for h in [MERGE_HEADER, "X-%s" % (MERGE_HEADER)]: + tmp_h = headers.get(h, "") if tmp_h: merge_header_headers = tmp_h break @@ -143,7 +140,9 @@ class CloudConfigPartHandler(handlers.Handler): filename = filename.replace(i, " ") self.file_names.append(filename.strip()) except Exception: - util.logexc(LOG, "Failed at merging in cloud config part from %s", - filename) + util.logexc( + LOG, "Failed at merging in cloud config part from %s", filename + ) + # vi: ts=4 expandtab -- cgit v1.2.3 From 3e64acda13c826074a502d37b9e11e07d4238bc6 Mon Sep 17 00:00:00 2001 From: Brett Holman Date: Thu, 6 Jan 2022 15:33:18 -0700 Subject: Don't throw exceptions for empty cloud config (#1130) Warn during boot when an empty config is provided. Likewise, `cloud-init devel schema --annotate` should not throw exception, return something meaningful instead. --- cloudinit/config/schema.py | 27 ++++++++++++++++++------- cloudinit/handlers/cloud_config.py | 9 +++++++++ tests/unittests/config/test_schema.py | 38 +++++++++++++++++++++++++++++++++++ 3 files changed, 67 insertions(+), 7 deletions(-) (limited to 'cloudinit/handlers/cloud_config.py') diff --git a/cloudinit/config/schema.py b/cloudinit/config/schema.py index 8ec4ab6a..3a77ca00 100644 --- a/cloudinit/config/schema.py +++ b/cloudinit/config/schema.py @@ -181,6 +181,7 @@ def validate_cloudconfig_schema( @raises: SchemaValidationError when provided config does not validate against the provided schema. + @raises: RuntimeError when provided config sourced from YAML is not a dict. """ try: (cloudinitValidator, FormatChecker) = get_jsonschema_validator() @@ -217,13 +218,21 @@ def annotated_cloudconfig_file(cloudconfig, original_content, schema_errors): if not schema_errors: return original_content schemapaths = {} + errors_by_line = defaultdict(list) + error_footer = [] + error_header = "# Errors: -------------\n{0}\n\n" + annotated_content = [] + lines = original_content.decode().split("\n") + if not isinstance(cloudconfig, dict): + # Return a meaningful message on empty cloud-config + return "\n".join( + lines + + [error_header.format("# E1: Cloud-config is not a YAML dict.")] + ) if cloudconfig: schemapaths = _schemapath_for_cloudconfig( cloudconfig, original_content ) - errors_by_line = defaultdict(list) - error_footer = [] - annotated_content = [] for path, msg in schema_errors: match = re.match(r"format-l(?P\d+)\.c(?P\d+).*", path) if match: @@ -236,7 +245,6 @@ def annotated_cloudconfig_file(cloudconfig, original_content, schema_errors): msg = "Line {line} column {col}: {msg}".format( line=line, col=col, msg=msg ) - lines = original_content.decode().split("\n") error_index = 1 for line_number, line in enumerate(lines, 1): errors = errors_by_line[line_number] @@ -247,11 +255,10 @@ def annotated_cloudconfig_file(cloudconfig, original_content, schema_errors): error_footer.append("# E{0}: {1}".format(error_index, error)) error_index += 1 annotated_content.append(line + "\t\t# " + ",".join(error_label)) + else: annotated_content.append(line) - annotated_content.append( - "# Errors: -------------\n{0}\n\n".format("\n".join(error_footer)) - ) + annotated_content.append(error_header.format("\n".join(error_footer))) return "\n".join(annotated_content) @@ -318,6 +325,10 @@ def validate_cloudconfig_file(config_path, schema, annotate=False): if annotate: print(annotated_cloudconfig_file({}, content, error.schema_errors)) raise error from e + if not isinstance(cloudconfig, dict): + # Return a meaningful message on empty cloud-config + if not annotate: + raise RuntimeError("Cloud-config is not a YAML dict.") try: validate_cloudconfig_schema(cloudconfig, schema, strict=True) except SchemaValidationError as e: @@ -662,6 +673,8 @@ def handle_schema_args(name, args): exclusive_args = [args.config_file, args.docs, args.system] if len([arg for arg in exclusive_args if arg]) != 1: error("Expected one of --config-file, --system or --docs arguments") + if args.annotate and args.docs: + error("Invalid flag combination. Cannot use --annotate with --docs") full_schema = get_schema() if args.config_file or args.system: try: diff --git a/cloudinit/handlers/cloud_config.py b/cloudinit/handlers/cloud_config.py index 2e694671..8070c6cb 100644 --- a/cloudinit/handlers/cloud_config.py +++ b/cloudinit/handlers/cloud_config.py @@ -89,6 +89,9 @@ class CloudConfigPartHandler(handlers.Handler): # or the merge type from the headers or default to our own set # if neither exists (or is empty) from the later. payload_yaml = util.load_yaml(payload) + if payload_yaml is None: + raise ValueError("empty cloud config") + mergers_yaml = mergers.dict_extract_mergers(payload_yaml) mergers_header = mergers.string_extract_mergers(merge_header_headers) all_mergers = [] @@ -139,6 +142,12 @@ class CloudConfigPartHandler(handlers.Handler): for i in ("\n", "\r", "\t"): filename = filename.replace(i, " ") self.file_names.append(filename.strip()) + except ValueError as err: + LOG.warning( + "Failed at merging in cloud config part from %s: %s", + filename, + err, + ) except Exception: util.logexc( LOG, "Failed at merging in cloud config part from %s", filename diff --git a/tests/unittests/config/test_schema.py b/tests/unittests/config/test_schema.py index fb5b891d..822efe5a 100644 --- a/tests/unittests/config/test_schema.py +++ b/tests/unittests/config/test_schema.py @@ -546,6 +546,31 @@ class AnnotatedCloudconfigFileTest(CiTestCase): content, annotated_cloudconfig_file({}, content, schema_errors=[]) ) + def test_annotated_cloudconfig_file_with_non_dict_cloud_config(self): + """Error when empty non-dict cloud-config is provided. + + OurJSON validation when user-data is None type generates a bunch + schema validation errors of the format: + ('', "None is not of type 'object'"). Ignore those symptoms and + report the general problem instead. + """ + content = b"\n\n\n" + expected = "\n".join( + [ + content.decode(), + "# Errors: -------------", + "# E1: Cloud-config is not a YAML dict.\n\n", + ] + ) + self.assertEqual( + expected, + annotated_cloudconfig_file( + None, + content, + schema_errors=[("", "None is not of type 'object'")], + ), + ) + def test_annotated_cloudconfig_file_schema_annotates_and_adds_footer(self): """With schema_errors, error lines are annotated and a footer added.""" content = dedent( @@ -658,6 +683,19 @@ class TestMain: _out, err = capsys.readouterr() assert "Error:\nConfigfile NOT_A_FILE does not exist\n" == err + def test_main_invalid_flag_combo(self, capsys): + """Main exits non-zero when invalid flag combo used.""" + myargs = ["mycmd", "--annotate", "--docs", "DOES_NOT_MATTER"] + with mock.patch("sys.argv", myargs): + with pytest.raises(SystemExit) as context_manager: + main() + assert 1 == context_manager.value.code + _, err = capsys.readouterr() + assert ( + "Error:\nInvalid flag combination. " + "Cannot use --annotate with --docs\n" == err + ) + def test_main_prints_docs(self, capsys): """When --docs parameter is provided, main generates documentation.""" myargs = ["mycmd", "--docs", "all"] -- cgit v1.2.3