util: read_cc_from_cmdline handle urlencoded yaml content (#275)

Add support for additional escaping of formatting characters
in the YAML content between the 'cc:' and 'end_cc' tokens.  On
s390x legacy terminals the use of square brackets [] are not
available limiting the ability to indicate lists of values in
yaml content. Using #5B and #5D, [ and ] respectively enables
s390x users to pass list yaml content into cloud-init via
command line interface.

3 files changed, 107 insertions, 7 deletions

diff --git a/cloudinit/tests/test_util.py b/cloudinit/tests/test_util.py
index 815da0fd..877ab5c5 100644
--- a/cloudinit/tests/test_util.py
+++ b/cloudinit/tests/test_util.py
@@ -6,6 +6,7 @@ import base64
 import logging
 import json
 import platform
+import pytest
 
 import cloudinit.util as util
 
@@ -605,4 +606,99 @@ class TestIsLXD(CiTestCase):
         self.assertFalse(util.is_lxd())
         m_exists.assert_called_once_with('/dev/lxd/sock')
 
+
+class TestReadCcFromCmdline:
+
+    @pytest.mark.parametrize(
+        "cmdline,expected_cfg",
+        [
+            # Return None if cmdline has no cc:<YAML>end_cc content.
+            (CiTestCase.random_string(), None),
+            # Return None if YAML content is empty string.
+            ('foo cc: end_cc bar', None),
+            # Return expected dictionary without trailing end_cc marker.
+            ('foo cc: ssh_pwauth: true', {'ssh_pwauth': True}),
+            # Return expected dictionary w escaped newline and no end_cc.
+            ('foo cc: ssh_pwauth: true\\n', {'ssh_pwauth': True}),
+            # Return expected dictionary of yaml between cc: and end_cc.
+            ('foo cc: ssh_pwauth: true end_cc bar', {'ssh_pwauth': True}),
+            # Return dict with list value w escaped newline, no end_cc.
+            (
+                'cc: ssh_import_id: [smoser, kirkland]\\n',
+                {'ssh_import_id': ['smoser', 'kirkland']}
+            ),
+            # Parse urlencoded brackets in yaml content.
+            (
+                'cc: ssh_import_id: %5Bsmoser, kirkland%5D end_cc',
+                {'ssh_import_id': ['smoser', 'kirkland']}
+            ),
+            # Parse complete urlencoded yaml content.
+            (
+                'cc: ssh_import_id%3A%20%5Buser1%2C%20user2%5D end_cc',
+                {'ssh_import_id': ['user1', 'user2']}
+            ),
+            # Parse nested dictionary in yaml content.
+            (
+                'cc: ntp: {enabled: true, ntp_client: myclient} end_cc',
+                {'ntp': {'enabled': True, 'ntp_client': 'myclient'}}
+            ),
+            # Parse single mapping value in yaml content.
+            ('cc: ssh_import_id: smoser end_cc', {'ssh_import_id': 'smoser'}),
+            # Parse multiline content with multiple mapping and nested lists.
+            (
+                ('cc: ssh_import_id: [smoser, bob]\\n'
+                 'runcmd: [ [ ls, -l ], echo hi ] end_cc'),
+                {'ssh_import_id': ['smoser', 'bob'],
+                 'runcmd': [['ls', '-l'], 'echo hi']}
+            ),
+            # Parse multiline encoded content w/ mappings and nested lists.
+            (
+                ('cc: ssh_import_id: %5Bsmoser, bob%5D\\n'
+                 'runcmd: [ [ ls, -l ], echo hi ] end_cc'),
+                {'ssh_import_id': ['smoser', 'bob'],
+                 'runcmd': [['ls', '-l'], 'echo hi']}
+            ),
+            # test encoded escaped newlines work.
+            #
+            # unquote(encoded_content)
+            # 'ssh_import_id: [smoser, bob]\\nruncmd: [ [ ls, -l ], echo hi ]'
+            (
+                ('cc: ' +
+                 ('ssh_import_id%3A%20%5Bsmoser%2C%20bob%5D%5Cn'
+                  'runcmd%3A%20%5B%20%5B%20ls%2C%20-l%20%5D%2C'
+                  '%20echo%20hi%20%5D') + ' end_cc'),
+                {'ssh_import_id': ['smoser', 'bob'],
+                 'runcmd': [['ls', '-l'], 'echo hi']}
+            ),
+            # test encoded newlines work.
+            #
+            # unquote(encoded_content)
+            # 'ssh_import_id: [smoser, bob]\nruncmd: [ [ ls, -l ], echo hi ]'
+            (
+                ("cc: " +
+                    ('ssh_import_id%3A%20%5Bsmoser%2C%20bob%5D%0A'
+                     'runcmd%3A%20%5B%20%5B%20ls%2C%20-l%20%5D%2C'
+                     '%20echo%20hi%20%5D') + ' end_cc'),
+                {'ssh_import_id': ['smoser', 'bob'],
+                 'runcmd': [['ls', '-l'], 'echo hi']}
+            ),
+            # Parse and merge multiple yaml content sections.
+            (
+                ('cc:ssh_import_id: [smoser, bob] end_cc '
+                 'cc: runcmd: [ [ ls, -l ] ] end_cc'),
+                {'ssh_import_id': ['smoser', 'bob'],
+                 'runcmd': [['ls', '-l']]}
+            ),
+            # Parse and merge multiple encoded yaml content sections.
+            (
+                ('cc:ssh_import_id%3A%20%5Bsmoser%5D end_cc '
+                 'cc:runcmd%3A%20%5B%20%5B%20ls%2C%20-l%20%5D%20%5D end_cc'),
+                {'ssh_import_id': ['smoser'], 'runcmd': [['ls', '-l']]}
+            ),
+        ]
+    )
+    def test_read_conf_from_cmdline_config(self, expected_cfg, cmdline):
+        assert expected_cfg == util.read_conf_from_cmdline(cmdline=cmdline)
+
+
 # vi: ts=4 expandtab
diff --git a/cloudinit/util.py b/cloudinit/util.py
index 9cc87d7d..89889459 100644
--- a/cloudinit/util.py
+++ b/cloudinit/util.py
@@ -1048,7 +1048,7 @@ def read_conf_with_confd(cfgfile):
 
 
 def read_conf_from_cmdline(cmdline=None):
-    # return a dictionary or config on the cmdline or None
+    # return a dictionary of config on the cmdline or None
     return load_yaml(read_cc_from_cmdline(cmdline=cmdline))
 
 
@@ -1056,11 +1056,12 @@ def read_cc_from_cmdline(cmdline=None):
     # this should support reading cloud-config information from
     # the kernel command line.  It is intended to support content of the
     # format:
-    #  cc: <yaml content here> [end_cc]
+    #  cc: <yaml content here|urlencoded yaml content> [end_cc]
     # this would include:
     # cc: ssh_import_id: [smoser, kirkland]\\n
     # cc: ssh_import_id: [smoser, bob]\\nruncmd: [ [ ls, -l ], echo hi ] end_cc
     # cc:ssh_import_id: [smoser] end_cc cc:runcmd: [ [ ls, -l ] ] end_cc
+    # cc:ssh_import_id: %5Bsmoser%5D end_cc
     if cmdline is None:
         cmdline = get_cmdline()
 
@@ -1075,9 +1076,9 @@ def read_cc_from_cmdline(cmdline=None):
         end = cmdline.find(tag_end, begin + begin_l)
         if end < 0:
             end = clen
-        tokens.append(cmdline[begin + begin_l:end].lstrip().replace("\\n",
-                                                                    "\n"))
-
+        tokens.append(
+            parse.unquote(
+                cmdline[begin + begin_l:end].lstrip()).replace("\\n", "\n"))
         begin = cmdline.find(tag_begin, end + end_l)
 
     return '\n'.join(tokens)
diff --git a/doc/examples/kernel-cmdline.txt b/doc/examples/kernel-cmdline.txt
index f043baef..04a1ad89 100644
--- a/doc/examples/kernel-cmdline.txt
+++ b/doc/examples/kernel-cmdline.txt
@@ -3,16 +3,19 @@ configuration that comes from the kernel command line has higher priority
 than configuration in /etc/cloud/cloud.cfg
 
 The format is:
-  cc: <yaml content here> [end_cc]
+  cc: <yaml content here|URL encoded yaml content> [end_cc]
 
 cloud-config will consider any content after 'cc:' to be cloud-config
 data.  If an 'end_cc' string is present, then it will stop reading there.
 otherwise it considers everthing after 'cc:' to be cloud-config content.
 
-In order to allow carriage returns, you must enter '\\n', literally, 
+In order to allow carriage returns, you must enter '\\n', literally,
 on the command line two backslashes followed by a letter 'n'.
 
+The yaml content may also be URL encoded (urllib.parse.quote()).
+
 Here are some examples:
  root=/dev/sda1 cc: ssh_import_id: [smoser, kirkland]\\n
  root=LABEL=uec-rootfs cc: ssh_import_id: [smoser, bob]\\nruncmd: [ [ ls, -l ], echo hi ] end_cc
  cc:ssh_import_id: [smoser] end_cc cc:runcmd: [ [ ls, -l ] ] end_cc root=/dev/sda1
+ cc:ssh_import_id: %5Bsmoser%5D end_cc cc:runcmd: %5B %5B ls, -l %5D %5D end_cc root=/dev/sda1