From bae9b11da9ed7dd0b16fe5adeaf4774b7cc628cf Mon Sep 17 00:00:00 2001 From: James Falcon Date: Wed, 15 Dec 2021 20:16:38 -0600 Subject: Adopt Black and isort (SC-700) (#1157) Applied Black and isort, fixed any linting issues, updated tox.ini and CI. --- cloudinit/handlers/__init__.py | 152 +++++++++++++++++++++++++---------------- 1 file changed, 92 insertions(+), 60 deletions(-) (limited to 'cloudinit/handlers/__init__.py') diff --git a/cloudinit/handlers/__init__.py b/cloudinit/handlers/__init__.py index a409ff8a..62c1fd26 100644 --- a/cloudinit/handlers/__init__.py +++ b/cloudinit/handlers/__init__.py @@ -13,9 +13,8 @@ import os from cloudinit import importer from cloudinit import log as logging -from cloudinit import type_utils -from cloudinit import util -from cloudinit.settings import (PER_ALWAYS, PER_INSTANCE, FREQUENCIES) +from cloudinit import type_utils, util +from cloudinit.settings import FREQUENCIES, PER_ALWAYS, PER_INSTANCE LOG = logging.getLogger(__name__) @@ -24,7 +23,7 @@ LOG = logging.getLogger(__name__) NOT_MULTIPART_TYPE = "text/x-not-multipart" # When none is assigned this gets used -OCTET_TYPE = 'application/octet-stream' +OCTET_TYPE = "application/octet-stream" # Special content types that signal the start and end of processing CONTENT_END = "__end__" @@ -34,32 +33,32 @@ CONTENT_SIGNALS = [CONTENT_START, CONTENT_END] # Used when a part-handler type is encountered # to allow for registration of new types. PART_CONTENT_TYPES = ["text/part-handler"] -PART_HANDLER_FN_TMPL = 'part-handler-%03d' +PART_HANDLER_FN_TMPL = "part-handler-%03d" # For parts without filenames -PART_FN_TPL = 'part-%03d' +PART_FN_TPL = "part-%03d" # Different file beginnings to their content type INCLUSION_TYPES_MAP = { - '#include': 'text/x-include-url', - '#include-once': 'text/x-include-once-url', - '#!': 'text/x-shellscript', - '#cloud-config': 'text/cloud-config', - '#upstart-job': 'text/upstart-job', - '#part-handler': 'text/part-handler', - '#cloud-boothook': 'text/cloud-boothook', - '#cloud-config-archive': 'text/cloud-config-archive', - '#cloud-config-jsonp': 'text/cloud-config-jsonp', - '## template: jinja': 'text/jinja2', + "#include": "text/x-include-url", + "#include-once": "text/x-include-once-url", + "#!": "text/x-shellscript", + "#cloud-config": "text/cloud-config", + "#upstart-job": "text/upstart-job", + "#part-handler": "text/part-handler", + "#cloud-boothook": "text/cloud-boothook", + "#cloud-config-archive": "text/cloud-config-archive", + "#cloud-config-jsonp": "text/cloud-config-jsonp", + "## template: jinja": "text/jinja2", } # Sorted longest first -INCLUSION_SRCH = sorted(list(INCLUSION_TYPES_MAP.keys()), - key=(lambda e: 0 - len(e))) +INCLUSION_SRCH = sorted( + list(INCLUSION_TYPES_MAP.keys()), key=(lambda e: 0 - len(e)) +) class Handler(metaclass=abc.ABCMeta): - def __init__(self, frequency, version=2): self.handler_version = version self.frequency = frequency @@ -69,11 +68,13 @@ class Handler(metaclass=abc.ABCMeta): def list_types(self): # Each subclass must define the supported content prefixes it handles. - if not hasattr(self, 'prefixes'): - raise NotImplementedError('Missing prefixes subclass attribute') + if not hasattr(self, "prefixes"): + raise NotImplementedError("Missing prefixes subclass attribute") else: - return [INCLUSION_TYPES_MAP[prefix] - for prefix in getattr(self, 'prefixes')] + return [ + INCLUSION_TYPES_MAP[prefix] + for prefix in getattr(self, "prefixes") + ] @abc.abstractmethod def handle_part(self, *args, **kwargs): @@ -82,8 +83,10 @@ class Handler(metaclass=abc.ABCMeta): def run_part(mod, data, filename, payload, frequency, headers): mod_freq = mod.frequency - if not (mod_freq == PER_ALWAYS or - (frequency == PER_INSTANCE and mod_freq == PER_INSTANCE)): + if not ( + mod_freq == PER_ALWAYS + or (frequency == PER_INSTANCE and mod_freq == PER_INSTANCE) + ): return # Sanity checks on version (should be an int convertable) try: @@ -91,33 +94,45 @@ def run_part(mod, data, filename, payload, frequency, headers): mod_ver = int(mod_ver) except (TypeError, ValueError, AttributeError): mod_ver = 1 - content_type = headers['Content-Type'] + content_type = headers["Content-Type"] try: - LOG.debug("Calling handler %s (%s, %s, %s) with frequency %s", - mod, content_type, filename, mod_ver, frequency) + LOG.debug( + "Calling handler %s (%s, %s, %s) with frequency %s", + mod, + content_type, + filename, + mod_ver, + frequency, + ) if mod_ver == 3: # Treat as v. 3 which does get a frequency + headers - mod.handle_part(data, content_type, filename, - payload, frequency, headers) + mod.handle_part( + data, content_type, filename, payload, frequency, headers + ) elif mod_ver == 2: # Treat as v. 2 which does get a frequency - mod.handle_part(data, content_type, filename, - payload, frequency) + mod.handle_part(data, content_type, filename, payload, frequency) elif mod_ver == 1: # Treat as v. 1 which gets no frequency mod.handle_part(data, content_type, filename, payload) else: raise ValueError("Unknown module version %s" % (mod_ver)) except Exception: - util.logexc(LOG, "Failed calling handler %s (%s, %s, %s) with " - "frequency %s", mod, content_type, filename, mod_ver, - frequency) + util.logexc( + LOG, + "Failed calling handler %s (%s, %s, %s) with frequency %s", + mod, + content_type, + filename, + mod_ver, + frequency, + ) def call_begin(mod, data, frequency): # Create a fake header set headers = { - 'Content-Type': CONTENT_START, + "Content-Type": CONTENT_START, } run_part(mod, data, None, None, frequency, headers) @@ -125,31 +140,35 @@ def call_begin(mod, data, frequency): def call_end(mod, data, frequency): # Create a fake header set headers = { - 'Content-Type': CONTENT_END, + "Content-Type": CONTENT_END, } run_part(mod, data, None, None, frequency, headers) def walker_handle_handler(pdata, _ctype, _filename, payload): - curcount = pdata['handlercount'] + curcount = pdata["handlercount"] modname = PART_HANDLER_FN_TMPL % (curcount) - frequency = pdata['frequency'] - modfname = os.path.join(pdata['handlerdir'], "%s" % (modname)) + frequency = pdata["frequency"] + modfname = os.path.join(pdata["handlerdir"], "%s" % (modname)) if not modfname.endswith(".py"): modfname = "%s.py" % (modfname) # TODO(harlowja): Check if path exists?? util.write_file(modfname, payload, 0o600) - handlers = pdata['handlers'] + handlers = pdata["handlers"] try: mod = fixup_handler(importer.import_module(modname)) - call_begin(mod, pdata['data'], frequency) + call_begin(mod, pdata["data"], frequency) # Only register and increment after the above have worked, so we don't # register if it fails starting. handlers.register(mod, initialized=True) - pdata['handlercount'] = curcount + 1 + pdata["handlercount"] = curcount + 1 except Exception: - util.logexc(LOG, "Failed at registering python file: %s (part " - "handler %s)", modfname, curcount) + util.logexc( + LOG, + "Failed at registering python file: %s (part handler %s)", + modfname, + curcount, + ) def _extract_first_or_bytes(blob, size): @@ -161,7 +180,7 @@ def _extract_first_or_bytes(blob, size): else: # We want to avoid decoding the whole blob (it might be huge) # By taking 4*size bytes we guarantee to decode size utf8 chars - start = blob[:4 * size].decode(errors='ignore').split("\n", 1)[0] + start = blob[: 4 * size].decode(errors="ignore").split("\n", 1)[0] if len(start) >= size: start = start[:size] except UnicodeDecodeError: @@ -176,7 +195,7 @@ def _escape_string(text): except (LookupError, TypeError): try: # Unicode (and Python 3's str) doesn't support string_escape... - return text.encode('unicode_escape') + return text.encode("unicode_escape") except TypeError: # Give up... pass @@ -189,28 +208,40 @@ def _escape_string(text): def walker_callback(data, filename, payload, headers): - content_type = headers['Content-Type'] - if content_type in data.get('excluded'): + content_type = headers["Content-Type"] + if content_type in data.get("excluded"): LOG.debug('content_type "%s" is excluded', content_type) return if content_type in PART_CONTENT_TYPES: walker_handle_handler(data, content_type, filename, payload) return - handlers = data['handlers'] + handlers = data["handlers"] if content_type in handlers: - run_part(handlers[content_type], data['data'], filename, - payload, data['frequency'], headers) + run_part( + handlers[content_type], + data["data"], + filename, + payload, + data["frequency"], + headers, + ) elif payload: # Extract the first line or 24 bytes for displaying in the log start = _extract_first_or_bytes(payload, 24) details = "'%s...'" % (_escape_string(start)) if content_type == NOT_MULTIPART_TYPE: - LOG.warning("Unhandled non-multipart (%s) userdata: %s", - content_type, details) + LOG.warning( + "Unhandled non-multipart (%s) userdata: %s", + content_type, + details, + ) else: - LOG.warning("Unhandled unknown content-type (%s) userdata: %s", - content_type, details) + LOG.warning( + "Unhandled unknown content-type (%s) userdata: %s", + content_type, + details, + ) else: LOG.debug("Empty payload of type %s", content_type) @@ -221,7 +252,7 @@ def walk(msg, callback, data): partnum = 0 for part in msg.walk(): # multipart/* are just containers - if part.get_content_maintype() == 'multipart': + if part.get_content_maintype() == "multipart": continue ctype = part.get_content_type() @@ -234,7 +265,7 @@ def walk(msg, callback, data): headers = dict(part) LOG.debug(headers) - headers['Content-Type'] = ctype + headers["Content-Type"] = ctype payload = util.fully_decoded_payload(part) callback(data, filename, payload, headers) partnum = partnum + 1 @@ -243,8 +274,8 @@ def walk(msg, callback, data): def fixup_handler(mod, def_freq=PER_INSTANCE): if not hasattr(mod, "handler_version"): setattr(mod, "handler_version", 1) - if not hasattr(mod, 'frequency'): - setattr(mod, 'frequency', def_freq) + if not hasattr(mod, "frequency"): + setattr(mod, "frequency", def_freq) else: freq = mod.frequency if freq and freq not in FREQUENCIES: @@ -263,4 +294,5 @@ def type_from_starts_with(payload, default=None): return INCLUSION_TYPES_MAP[text] return default + # vi: ts=4 expandtab -- cgit v1.2.3 From 600b870b399feb9e072748e07ea223556261fbe7 Mon Sep 17 00:00:00 2001 From: Chris Lalos Date: Thu, 10 Feb 2022 15:49:38 -0600 Subject: Shell script handlers by freq (#1166) Handlers for per-boot/per-instance/per-once multipart MIME Add handlers for adding scripts to userdata that can be run at various frequencies. Scripts of type x-shellscript-per-boot, x-shellscript-per-instance, or x-shellscript-per-once can be added to a multipart MIME userdata message as part of instance userdata. These scripts will then be added to the appropriate per-boot, per-instance, or per-once directory in /var/lib/cloud/scripts/ during processing of userdata. --- cloudinit/cmd/devel/make_mime.py | 51 ++++++++++-------- cloudinit/handlers/__init__.py | 7 +++ cloudinit/handlers/shell_script_by_frequency.py | 62 ++++++++++++++++++++++ cloudinit/stages.py | 7 +++ .../test_shell_script_by_frequency.py | 48 +++++++++++++++++ tests/unittests/test_builtin_handlers.py | 25 ++++++++- tools/.github-cla-signers | 1 + 7 files changed, 178 insertions(+), 23 deletions(-) create mode 100644 cloudinit/handlers/shell_script_by_frequency.py create mode 100644 tests/integration_tests/test_shell_script_by_frequency.py (limited to 'cloudinit/handlers/__init__.py') diff --git a/cloudinit/cmd/devel/make_mime.py b/cloudinit/cmd/devel/make_mime.py index a7493c74..c7671a93 100755 --- a/cloudinit/cmd/devel/make_mime.py +++ b/cloudinit/cmd/devel/make_mime.py @@ -20,6 +20,28 @@ EPILOG = ( ) +def create_mime_message(files): + sub_messages = [] + errors = [] + for i, (fh, filename, format_type) in enumerate(files): + contents = fh.read() + sub_message = MIMEText(contents, format_type, sys.getdefaultencoding()) + sub_message.add_header( + "Content-Disposition", 'attachment; filename="%s"' % (filename) + ) + content_type = sub_message.get_content_type().lower() + if content_type not in get_content_types(): + msg = ( + "content type %r for attachment %s " "may be incorrect!" + ) % (content_type, i + 1) + errors.append(msg) + sub_messages.append(sub_message) + combined_message = MIMEMultipart() + for msg in sub_messages: + combined_message.attach(msg) + return (combined_message, errors) + + def file_content_type(text): """Return file content type by reading the first line of the input.""" try: @@ -97,29 +119,14 @@ def handle_args(name, args): print("\n".join(get_content_types(strip_prefix=True))) return 0 - sub_messages = [] - errors = [] - for i, (fh, filename, format_type) in enumerate(args.files): - contents = fh.read() - sub_message = MIMEText(contents, format_type, sys.getdefaultencoding()) - sub_message.add_header( - "Content-Disposition", 'attachment; filename="%s"' % (filename) - ) - content_type = sub_message.get_content_type().lower() - if content_type not in get_content_types(): - level = "WARNING" if args.force else "ERROR" - msg = ( - level + ": content type %r for attachment %s may be incorrect!" - ) % (content_type, i + 1) - sys.stderr.write(msg + "\n") - errors.append(msg) - sub_messages.append(sub_message) - if len(errors) and not args.force: + combined_message, errors = create_mime_message(args.files) + if errors: + level = "WARNING" if args.force else "ERROR" + for error in errors: + sys.stderr.write(f"{level}: {error}\n") sys.stderr.write("Invalid content-types, override with --force\n") - return 1 - combined_message = MIMEMultipart() - for msg in sub_messages: - combined_message.attach(msg) + if not args.force: + return 1 print(combined_message) return 0 diff --git a/cloudinit/handlers/__init__.py b/cloudinit/handlers/__init__.py index 62c1fd26..7d8a9208 100644 --- a/cloudinit/handlers/__init__.py +++ b/cloudinit/handlers/__init__.py @@ -50,6 +50,13 @@ INCLUSION_TYPES_MAP = { "#cloud-config-archive": "text/cloud-config-archive", "#cloud-config-jsonp": "text/cloud-config-jsonp", "## template: jinja": "text/jinja2", + # Note: for the next 3 entries, the prefix doesn't matter because these + # are for types that can only be used as part of a MIME message. However, + # including these entries supresses warnings during `cloudinit devel + # make-mime`, which otherwise would require `--force`. + "text/x-shellscript-per-boot": "text/x-shellscript-per-boot", + "text/x-shellscript-per-instance": "text/x-shellscript-per-instance", + "text/x-shellscript-per-once": "text/x-shellscript-per-once", } # Sorted longest first diff --git a/cloudinit/handlers/shell_script_by_frequency.py b/cloudinit/handlers/shell_script_by_frequency.py new file mode 100644 index 00000000..923cca57 --- /dev/null +++ b/cloudinit/handlers/shell_script_by_frequency.py @@ -0,0 +1,62 @@ +import os + +from cloudinit import log, util +from cloudinit.handlers import Handler +from cloudinit.settings import PER_ALWAYS, PER_INSTANCE, PER_ONCE + +LOG = log.getLogger(__name__) + +# cloudinit/settings.py defines PER_*** frequency constants. It makes sense to +# use them here, instead of hardcodes, and map them to the 'per-***' frequency- +# specific folders in /v/l/c/scripts. It might make sense to expose this at a +# higher level or in a more general module -- eg maybe in cloudinit/settings.py +# itself -- but for now it's here. +path_map = { + PER_ALWAYS: "per-boot", + PER_INSTANCE: "per-instance", + PER_ONCE: "per-once", +} + + +def get_mime_type_by_frequency(freq): + mime_type = f"text/x-shellscript-{path_map[freq]}" + return mime_type + + +def get_script_folder_by_frequency(freq, scripts_dir): + """Return the frequency-specific subfolder for a given frequency constant + and parent folder.""" + freqPath = path_map[freq] + folder = os.path.join(scripts_dir, freqPath) + return folder + + +def write_script_by_frequency(script_path, payload, frequency, scripts_dir): + """Given a filename, a payload, a frequency, and a scripts folder, write + the payload to the correct frequency-specific path""" + filename = os.path.basename(script_path) + filename = util.clean_filename(filename) + folder = get_script_folder_by_frequency(frequency, scripts_dir) + path = os.path.join(folder, filename) + payload = util.dos2unix(payload) + util.write_file(path, payload, 0o700) + + +class ShellScriptByFreqPartHandler(Handler): + """Common base class for the frequency-specific script handlers.""" + + def __init__(self, script_frequency, paths, **_kwargs): + Handler.__init__(self, PER_ALWAYS) + self.prefixes = [get_mime_type_by_frequency(script_frequency)] + self.script_frequency = script_frequency + self.scripts_dir = paths.get_cpath("scripts") + if "script_path" in _kwargs: + self.scripts_dir = paths.get_cpath(_kwargs["script_path"]) + + def handle_part(self, data, ctype, script_path, payload, frequency): + if script_path is not None: + filename = os.path.basename(script_path) + filename = util.clean_filename(filename) + write_script_by_frequency( + script_path, payload, self.script_frequency, self.scripts_dir + ) diff --git a/cloudinit/stages.py b/cloudinit/stages.py index b1a6bc49..3f17294b 100644 --- a/cloudinit/stages.py +++ b/cloudinit/stages.py @@ -21,12 +21,16 @@ from cloudinit.handlers.boot_hook import BootHookPartHandler from cloudinit.handlers.cloud_config import CloudConfigPartHandler from cloudinit.handlers.jinja_template import JinjaTemplatePartHandler from cloudinit.handlers.shell_script import ShellScriptPartHandler +from cloudinit.handlers.shell_script_by_frequency import ( + ShellScriptByFreqPartHandler, +) from cloudinit.handlers.upstart_job import UpstartJobPartHandler from cloudinit.net import cmdline from cloudinit.reporting import events from cloudinit.settings import ( CLOUD_CONFIG, FREQUENCIES, + PER_ALWAYS, PER_INSTANCE, PER_ONCE, RUN_CLOUD_CONFIG, @@ -519,6 +523,9 @@ class Init(object): def_handlers = [ cloudconfig_handler, shellscript_handler, + ShellScriptByFreqPartHandler(PER_ALWAYS, **opts), + ShellScriptByFreqPartHandler(PER_INSTANCE, **opts), + ShellScriptByFreqPartHandler(PER_ONCE, **opts), BootHookPartHandler(**opts), UpstartJobPartHandler(**opts), ] diff --git a/tests/integration_tests/test_shell_script_by_frequency.py b/tests/integration_tests/test_shell_script_by_frequency.py new file mode 100644 index 00000000..25157722 --- /dev/null +++ b/tests/integration_tests/test_shell_script_by_frequency.py @@ -0,0 +1,48 @@ +"""Integration tests for various handlers.""" + +from io import StringIO + +import pytest + +from cloudinit.cmd.devel.make_mime import create_mime_message +from tests.integration_tests.instances import IntegrationInstance + +PER_FREQ_TEMPLATE = """\ +#!/bin/bash +touch /tmp/test_per_freq_{} +""" + +PER_ALWAYS_FILE = StringIO(PER_FREQ_TEMPLATE.format("always")) +PER_INSTANCE_FILE = StringIO(PER_FREQ_TEMPLATE.format("instance")) +PER_ONCE_FILE = StringIO(PER_FREQ_TEMPLATE.format("once")) + +FILES = [ + (PER_ALWAYS_FILE, "always.sh", "x-shellscript-per-boot"), + (PER_INSTANCE_FILE, "instance.sh", "x-shellscript-per-instance"), + (PER_ONCE_FILE, "once.sh", "x-shellscript-per-once"), +] + +USER_DATA, errors = create_mime_message(FILES) + + +@pytest.mark.ci +@pytest.mark.user_data(USER_DATA) +def test_per_freq(client: IntegrationInstance): + # Sanity test for scripts folder + cmd = "test -d /var/lib/cloud/scripts" + assert client.execute(cmd).ok + # Test per-boot + cmd = "test -f /var/lib/cloud/scripts/per-boot/always.sh" + assert client.execute(cmd).ok + cmd = "test -f /tmp/test_per_freq_always" + assert client.execute(cmd).ok + # Test per-instance + cmd = "test -f /var/lib/cloud/scripts/per-instance/instance.sh" + assert client.execute(cmd).ok + cmd = "test -f /tmp/test_per_freq_instance" + assert client.execute(cmd).ok + # Test per-once + cmd = "test -f /var/lib/cloud/scripts/per-once/once.sh" + assert client.execute(cmd).ok + cmd = "test -f /tmp/test_per_freq_once" + assert client.execute(cmd).ok diff --git a/tests/unittests/test_builtin_handlers.py b/tests/unittests/test_builtin_handlers.py index a057be2a..0dae924d 100644 --- a/tests/unittests/test_builtin_handlers.py +++ b/tests/unittests/test_builtin_handlers.py @@ -12,6 +12,7 @@ from textwrap import dedent import pytest from cloudinit import handlers, helpers, subp, util +from cloudinit.cmd.devel import read_cfg_paths from cloudinit.handlers.cloud_config import CloudConfigPartHandler from cloudinit.handlers.jinja_template import ( JinjaTemplatePartHandler, @@ -19,8 +20,12 @@ from cloudinit.handlers.jinja_template import ( render_jinja_payload, ) from cloudinit.handlers.shell_script import ShellScriptPartHandler +from cloudinit.handlers.shell_script_by_frequency import ( + get_script_folder_by_frequency, + path_map, +) from cloudinit.handlers.upstart_job import UpstartJobPartHandler -from cloudinit.settings import PER_ALWAYS, PER_INSTANCE +from cloudinit.settings import PER_ALWAYS, PER_INSTANCE, PER_ONCE from tests.unittests.helpers import ( CiTestCase, FilesystemMockingTestCase, @@ -473,4 +478,22 @@ class TestRenderJinjaPayload(CiTestCase): self.assertIn(expected_log, self.logs.getvalue()) +class TestShellScriptByFrequencyHandlers: + def do_test_frequency(self, frequency): + ci_paths = read_cfg_paths() + scripts_dir = ci_paths.get_cpath("scripts") + testFolder = os.path.join(scripts_dir, path_map[frequency]) + folder = get_script_folder_by_frequency(frequency, scripts_dir) + assert testFolder == folder + + def test_get_script_folder_per_boot(self): + self.do_test_frequency(PER_ALWAYS) + + def test_get_script_folder_per_instance(self): + self.do_test_frequency(PER_INSTANCE) + + def test_get_script_folder_per_once(self): + self.do_test_frequency(PER_ONCE) + + # vi: ts=4 expandtab diff --git a/tools/.github-cla-signers b/tools/.github-cla-signers index 6d957d9b..ac157a2f 100644 --- a/tools/.github-cla-signers +++ b/tools/.github-cla-signers @@ -8,6 +8,7 @@ andrewbogott andrewlukoshko antonyc aswinrajamannar +beantaxi beezly bipinbachhao BirknerAlex -- cgit v1.2.3