From ade47866aa2f81ab0f3baabbb1fc1e484abdb741 Mon Sep 17 00:00:00 2001 From: Daniel Watkins Date: Thu, 19 Mar 2020 12:01:15 -0400 Subject: cloudinit/tests: remove unneeded with_logs configuration (#263) These classes don't use `self.logs` anywhere in their body, so we can remove the `with_logs = True` setting from them. These instances were found using astpath[0], with the following invocation: astpath "//Name[@id='with_logs' and not(ancestor::ClassDef//Attribute[@attr='logs'])]" [0] https://github.com/hchasestevens/astpath --- cloudinit/cmd/tests/test_main.py | 2 -- 1 file changed, 2 deletions(-) (limited to 'cloudinit/cmd/tests') diff --git a/cloudinit/cmd/tests/test_main.py b/cloudinit/cmd/tests/test_main.py index 384fddc6..585b3b0e 100644 --- a/cloudinit/cmd/tests/test_main.py +++ b/cloudinit/cmd/tests/test_main.py @@ -18,8 +18,6 @@ myargs = namedtuple('MyArgs', 'debug files force local reporter subcommand') class TestMain(FilesystemMockingTestCase): - with_logs = True - def setUp(self): super(TestMain, self).setUp() self.new_root = self.tmp_dir() -- cgit v1.2.3 From b698b349c827f30fa60de55e2469a21ae4342669 Mon Sep 17 00:00:00 2001 From: Daniel Watkins Date: Tue, 31 Mar 2020 16:32:40 -0400 Subject: CiTestCase: stop using and remove sys_exit helper (#283) This shim was required to support Python 2.6, so we no longer need it. --- cloudinit/cmd/tests/test_clean.py | 1 - cloudinit/cmd/tests/test_status.py | 1 - cloudinit/tests/helpers.py | 10 ---------- tests/unittests/test_handler/test_schema.py | 20 ++++++++------------ 4 files changed, 8 insertions(+), 24 deletions(-) (limited to 'cloudinit/cmd/tests') diff --git a/cloudinit/cmd/tests/test_clean.py b/cloudinit/cmd/tests/test_clean.py index 13a69aa1..a848a810 100644 --- a/cloudinit/cmd/tests/test_clean.py +++ b/cloudinit/cmd/tests/test_clean.py @@ -167,7 +167,6 @@ class TestClean(CiTestCase): wrap_and_call( 'cloudinit.cmd.clean', {'Init': {'side_effect': self.init_class}, - 'sys.exit': {'side_effect': self.sys_exit}, 'sys.argv': {'new': ['clean', '--logs']}}, clean.main) diff --git a/cloudinit/cmd/tests/test_status.py b/cloudinit/cmd/tests/test_status.py index 1ed10896..1c9eec37 100644 --- a/cloudinit/cmd/tests/test_status.py +++ b/cloudinit/cmd/tests/test_status.py @@ -382,7 +382,6 @@ class TestStatus(CiTestCase): wrap_and_call( 'cloudinit.cmd.status', {'sys.argv': {'new': ['status']}, - 'sys.exit': {'side_effect': self.sys_exit}, '_is_cloudinit_disabled': (False, ''), 'Init': {'side_effect': self.init_class}}, status.main) diff --git a/cloudinit/tests/helpers.py b/cloudinit/tests/helpers.py index 8d3512db..f4db5827 100644 --- a/cloudinit/tests/helpers.py +++ b/cloudinit/tests/helpers.py @@ -197,16 +197,6 @@ class CiTestCase(TestCase): dir = self.tmp_dir() return os.path.normpath(os.path.abspath(os.path.join(dir, path))) - def sys_exit(self, code): - """Provide a wrapper around sys.exit for python 2.6 - - In 2.6, this code would produce 'cm.exception' with value int(2) - rather than the SystemExit that was raised by sys.exit(2). - with assertRaises(SystemExit) as cm: - sys.exit(2) - """ - raise SystemExit(code) - def tmp_cloud(self, distro, sys_cfg=None, metadata=None): """Create a cloud with tmp working directory paths. diff --git a/tests/unittests/test_handler/test_schema.py b/tests/unittests/test_handler/test_schema.py index 987a89c9..abde02d3 100644 --- a/tests/unittests/test_handler/test_schema.py +++ b/tests/unittests/test_handler/test_schema.py @@ -345,12 +345,10 @@ class MainTest(CiTestCase): def test_main_missing_args(self): """Main exits non-zero and reports an error on missing parameters.""" - with mock.patch('sys.exit', side_effect=self.sys_exit): - with mock.patch('sys.argv', ['mycmd']): - with mock.patch('sys.stderr', new_callable=StringIO) as \ - m_stderr: - with self.assertRaises(SystemExit) as context_manager: - main() + with mock.patch('sys.argv', ['mycmd']): + with mock.patch('sys.stderr', new_callable=StringIO) as m_stderr: + with self.assertRaises(SystemExit) as context_manager: + main() self.assertEqual(1, context_manager.exception.code) self.assertEqual( 'Expected either --config-file argument or --doc\n', @@ -359,12 +357,10 @@ class MainTest(CiTestCase): def test_main_absent_config_file(self): """Main exits non-zero when config file is absent.""" myargs = ['mycmd', '--annotate', '--config-file', 'NOT_A_FILE'] - with mock.patch('sys.exit', side_effect=self.sys_exit): - with mock.patch('sys.argv', myargs): - with mock.patch('sys.stderr', new_callable=StringIO) as \ - m_stderr: - with self.assertRaises(SystemExit) as context_manager: - main() + with mock.patch('sys.argv', myargs): + with mock.patch('sys.stderr', new_callable=StringIO) as m_stderr: + with self.assertRaises(SystemExit) as context_manager: + main() self.assertEqual(1, context_manager.exception.code) self.assertEqual( 'Configfile NOT_A_FILE does not exist\n', -- cgit v1.2.3 From 5f7825e22241423322dbe628de1b00289cf34114 Mon Sep 17 00:00:00 2001 From: Joshua Powers Date: Tue, 2 Jun 2020 08:24:29 -0700 Subject: test: fix all flake8 E241 (#403) Remove extra spaces after a ',' --- cloudinit/cmd/tests/test_query.py | 2 +- cloudinit/config/cc_apt_configure.py | 2 +- cloudinit/net/tests/test_dhcp.py | 2 +- cloudinit/sources/DataSourceGCE.py | 2 +- cloudinit/tests/test_url_helper.py | 2 +- tests/unittests/test_net.py | 4 ++-- tox.ini | 3 +-- 7 files changed, 8 insertions(+), 9 deletions(-) (limited to 'cloudinit/cmd/tests') diff --git a/cloudinit/cmd/tests/test_query.py b/cloudinit/cmd/tests/test_query.py index 6d36a4ea..cb15b2d2 100644 --- a/cloudinit/cmd/tests/test_query.py +++ b/cloudinit/cmd/tests/test_query.py @@ -261,7 +261,7 @@ class TestQuery(CiTestCase): args = self.args( debug=False, dump_all=False, format=None, instance_data=self.instance_data, list_keys=True, user_data='ud', - vendor_data='vd', varname='top') + vendor_data='vd', varname='top') with mock.patch('sys.stderr', new_callable=StringIO) as m_stderr: with mock.patch('sys.stdout', new_callable=StringIO) as m_stdout: with mock.patch('os.getuid') as m_getuid: diff --git a/cloudinit/config/cc_apt_configure.py b/cloudinit/config/cc_apt_configure.py index 9a33451d..b1c7b471 100644 --- a/cloudinit/config/cc_apt_configure.py +++ b/cloudinit/config/cc_apt_configure.py @@ -297,7 +297,7 @@ schema = { }, 'conf': { 'type': 'string', - 'description': dedent("""\ + 'description': dedent("""\ Specify configuration for apt, such as proxy configuration. This configuration is specified as a string. For multiline apt configuration, make sure diff --git a/cloudinit/net/tests/test_dhcp.py b/cloudinit/net/tests/test_dhcp.py index bc7bef45..7768da7c 100644 --- a/cloudinit/net/tests/test_dhcp.py +++ b/cloudinit/net/tests/test_dhcp.py @@ -211,7 +211,7 @@ class TestDHCPParseStaticRoutes(CiTestCase): "class_b": "16,172,16,10", "class_a": "8,10,10", "gateway": "0,0", - "netlen": "33,0", + "netlen": "33,0", } for rfc3442 in bad_rfc3442.values(): self.assertEqual([], parse_static_routes(rfc3442)) diff --git a/cloudinit/sources/DataSourceGCE.py b/cloudinit/sources/DataSourceGCE.py index 6cbfbbac..0ec5f6ec 100644 --- a/cloudinit/sources/DataSourceGCE.py +++ b/cloudinit/sources/DataSourceGCE.py @@ -116,7 +116,7 @@ def _write_host_key_to_guest_attributes(key_type, key_value): resp = url_helper.readurl(url=url, data=key_value, headers=HEADERS, request_method='PUT', check_status=False) if resp.ok(): - LOG.debug('Wrote %s host key to guest attributes.', key_type) + LOG.debug('Wrote %s host key to guest attributes.', key_type) else: LOG.debug('Unable to write %s host key to guest attributes.', key_type) diff --git a/cloudinit/tests/test_url_helper.py b/cloudinit/tests/test_url_helper.py index 29b39374..364ec822 100644 --- a/cloudinit/tests/test_url_helper.py +++ b/cloudinit/tests/test_url_helper.py @@ -85,7 +85,7 @@ class TestReadFileOrUrl(CiTestCase): read_file_or_url(url, headers=headers, headers_redact=['sensitive']) logs = self.logs.getvalue() for k in headers.keys(): - self.assertEqual(headers[k], httpretty.last_request().headers[k]) + self.assertEqual(headers[k], httpretty.last_request().headers[k]) self.assertIn(REDACTED, logs) self.assertNotIn('sekret', logs) diff --git a/tests/unittests/test_net.py b/tests/unittests/test_net.py index e075a64c..84d3a5f0 100644 --- a/tests/unittests/test_net.py +++ b/tests/unittests/test_net.py @@ -944,7 +944,7 @@ NETWORK_CONFIGS = { dhcp6: true """).rstrip(' '), 'expected_sysconfig_opensuse': { - 'ifcfg-iface0': textwrap.dedent("""\ + 'ifcfg-iface0': textwrap.dedent("""\ BOOTPROTO=dhcp DHCLIENT6_MODE=managed STARTMODE=auto""") @@ -1028,7 +1028,7 @@ NETWORK_CONFIGS = { }, 'v6_and_v4': { 'expected_sysconfig_opensuse': { - 'ifcfg-iface0': textwrap.dedent("""\ + 'ifcfg-iface0': textwrap.dedent("""\ BOOTPROTO=dhcp DHCLIENT6_MODE=managed STARTMODE=auto""") diff --git a/tox.ini b/tox.ini index 725a0ca4..5ee67368 100644 --- a/tox.ini +++ b/tox.ini @@ -47,10 +47,9 @@ deps = -r{toxinidir}/test-requirements.txt # E123: closing bracket does not match indentation of opening bracket’s line # E126: continuation line over-indented for hanging indent # E226: missing whitespace around arithmetic operator -# E241: multiple spaces after ‘,’ # W503: line break before binary operator # W504: line break after binary operator -ignore=E121,E123,E126,E226,E241,W503,W504 +ignore=E121,E123,E126,E226,W503,W504 exclude = .venv,.tox,dist,doc,*egg,.git,build,tools per-file-ignores = cloudinit/cmd/main.py:E402 -- cgit v1.2.3 From 747723a42c98fa13080ea31127e289e7b826046f Mon Sep 17 00:00:00 2001 From: Chad Smith Date: Thu, 20 Aug 2020 15:51:35 -0600 Subject: cmd: cloud-init query to handle compressed userdata (#516) cloud-init query tries to directly load and decode raw user-data from /var/lib/cloud/instance/user-data.txt. This results in UnicodeDecodeErrors on some platforms which provide compressed content. Avoid UnicodeDecoderErrors when parsing compressed user-data at /var/lib/cloud/instance/user-data.txt. LP: #1889938 --- cloudinit/cmd/query.py | 43 ++++- cloudinit/cmd/tests/test_query.py | 397 ++++++++++++++++++++++---------------- 2 files changed, 272 insertions(+), 168 deletions(-) (limited to 'cloudinit/cmd/tests') diff --git a/cloudinit/cmd/query.py b/cloudinit/cmd/query.py index 0fb48ebd..03b77c17 100644 --- a/cloudinit/cmd/query.py +++ b/cloudinit/cmd/query.py @@ -1,6 +1,17 @@ # This file is part of cloud-init. See LICENSE file for license information. -"""Query standardized instance metadata from the command line.""" +"""Query standardized instance metadata provided to machine, returning a JSON +structure. + +Some instance-data values may be binary on some platforms, such as userdata and +vendordata. Attempt to decompress and decode UTF-8 any binary values. + +Any binary values in the instance metadata will be base64-encoded and prefixed +with "ci-b64:" in the output. userdata and, where applicable, vendordata may +be provided to the machine gzip-compressed (and therefore as binary data). +query will attempt to decompress these to a string before emitting the JSON +output; if this fails, they are treated as binary. +""" import argparse from errno import EACCES @@ -30,7 +41,7 @@ def get_parser(parser=None): """ if not parser: parser = argparse.ArgumentParser( - prog=NAME, description='Query cloud-init instance data') + prog=NAME, description=__doc__) parser.add_argument( '-d', '--debug', action='store_true', default=False, help='Add verbose messages during template render') @@ -52,8 +63,10 @@ def get_parser(parser=None): ' /var/lib/cloud/instance/vendor-data.txt')) parser.add_argument( 'varname', type=str, nargs='?', - help=('A dot-delimited instance data variable to query from' - ' instance-data query. For example: v2.local_hostname')) + help=('A dot-delimited specific variable to query from' + ' instance-data. For example: v1.local_hostname. If the' + ' value is not JSON serializable, it will be base64-encoded and' + ' will contain the prefix "ci-b64:". ')) parser.add_argument( '-a', '--all', action='store_true', default=False, dest='dump_all', help='Dump all available instance-data') @@ -65,6 +78,24 @@ def get_parser(parser=None): return parser +def load_userdata(ud_file_path): + """Attempt to return a string of user-data from ud_file_path + + Attempt to decode or decompress if needed. + If unable to decode the content, raw bytes will be returned. + + @returns: String of uncompressed userdata if possible, otherwise bytes. + """ + try: + return util.load_file(ud_file_path) + except UnicodeDecodeError: + encoded_data = util.load_file(ud_file_path, decode=False) + try: + return util.decomp_gzip(encoded_data, quiet=False) + except util.DecompressionError: + return encoded_data + + def handle_args(name, args): """Handle calls to 'cloud-init query' as a subcommand.""" paths = None @@ -121,8 +152,8 @@ def handle_args(name, args): instance_data['vendordata'] = ( '<%s> file:%s' % (REDACT_SENSITIVE_VALUE, vendor_data_fn)) else: - instance_data['userdata'] = util.load_file(user_data_fn) - instance_data['vendordata'] = util.load_file(vendor_data_fn) + instance_data['userdata'] = load_userdata(user_data_fn) + instance_data['vendordata'] = load_userdata(vendor_data_fn) if args.format: payload = '## template: jinja\n{fmt}'.format(fmt=args.format) rendered_payload = render_jinja_payload( diff --git a/cloudinit/cmd/tests/test_query.py b/cloudinit/cmd/tests/test_query.py index cb15b2d2..421031aa 100644 --- a/cloudinit/cmd/tests/test_query.py +++ b/cloudinit/cmd/tests/test_query.py @@ -1,195 +1,265 @@ # This file is part of cloud-init. See LICENSE file for license information. import errno -from io import StringIO +import gzip +from io import BytesIO +import json from textwrap import dedent -import os + +import pytest from collections import namedtuple from cloudinit.cmd import query from cloudinit.helpers import Paths from cloudinit.sources import ( REDACT_SENSITIVE_VALUE, INSTANCE_JSON_FILE, INSTANCE_JSON_SENSITIVE_FILE) -from cloudinit.tests.helpers import CiTestCase, mock -from cloudinit.util import ensure_dir, write_file +from cloudinit.tests.helpers import mock + +from cloudinit.util import b64e, write_file + +def _gzip_data(data): + with BytesIO() as iobuf: + with gzip.GzipFile(mode="wb", fileobj=iobuf) as gzfp: + gzfp.write(data) + return iobuf.getvalue() -class TestQuery(CiTestCase): - with_logs = True +@mock.patch("cloudinit.cmd.query.addLogHandlerCLI", lambda *args: "") +class TestQuery: args = namedtuple( 'queryargs', ('debug dump_all format instance_data list_keys user_data vendor_data' ' varname')) - def setUp(self): - super(TestQuery, self).setUp() - self.tmp = self.tmp_dir() - self.instance_data = self.tmp_path('instance-data', dir=self.tmp) + def _setup_paths(self, tmpdir, ud_val=None, vd_val=None): + """Write userdata and vendordata into a tmpdir. - def test_handle_args_error_on_missing_param(self): + Return: + 4-tuple : (paths, run_dir_path, userdata_path, vendordata_path) + """ + if ud_val: + user_data = tmpdir.join('user-data') + write_file(user_data.strpath, ud_val) + else: + user_data = None + if vd_val: + vendor_data = tmpdir.join('vendor-data') + write_file(vendor_data.strpath, vd_val) + else: + vendor_data = None + run_dir = tmpdir.join('run_dir') + run_dir.ensure_dir() + return ( + Paths({'run_dir': run_dir.strpath}), + run_dir, + user_data, + vendor_data + ) + + def test_handle_args_error_on_missing_param(self, caplog, capsys): """Error when missing required parameters and print usage.""" args = self.args( debug=False, dump_all=False, format=None, instance_data=None, list_keys=False, user_data=None, vendor_data=None, varname=None) - with mock.patch('sys.stderr', new_callable=StringIO) as m_stderr: - with mock.patch('sys.stdout', new_callable=StringIO) as m_stdout: - self.assertEqual(1, query.handle_args('anyname', args)) + with mock.patch( + "cloudinit.cmd.query.addLogHandlerCLI", return_value="" + ) as m_cli_log: + assert 1 == query.handle_args('anyname', args) expected_error = ( - 'ERROR: Expected one of the options: --all, --format, --list-keys' + 'Expected one of the options: --all, --format, --list-keys' ' or varname\n') - self.assertIn(expected_error, self.logs.getvalue()) - self.assertIn('usage: query', m_stdout.getvalue()) - self.assertIn(expected_error, m_stderr.getvalue()) + assert expected_error in caplog.text + out, _err = capsys.readouterr() + assert 'usage: query' in out + assert 1 == m_cli_log.call_count - def test_handle_args_error_on_missing_instance_data(self): + def test_handle_args_error_on_missing_instance_data(self, caplog, tmpdir): """When instance_data file path does not exist, log an error.""" - absent_fn = self.tmp_path('absent', dir=self.tmp) + absent_fn = tmpdir.join('absent') args = self.args( - debug=False, dump_all=True, format=None, instance_data=absent_fn, + debug=False, dump_all=True, format=None, + instance_data=absent_fn.strpath, list_keys=False, user_data='ud', vendor_data='vd', varname=None) - with mock.patch('sys.stderr', new_callable=StringIO) as m_stderr: - self.assertEqual(1, query.handle_args('anyname', args)) - self.assertIn( - 'ERROR: Missing instance-data file: %s' % absent_fn, - self.logs.getvalue()) - self.assertIn( - 'ERROR: Missing instance-data file: %s' % absent_fn, - m_stderr.getvalue()) + assert 1 == query.handle_args('anyname', args) - def test_handle_args_error_when_no_read_permission_instance_data(self): + msg = 'Missing instance-data file: %s' % absent_fn + assert msg in caplog.text + + def test_handle_args_error_when_no_read_permission_instance_data( + self, caplog, tmpdir + ): """When instance_data file is unreadable, log an error.""" - noread_fn = self.tmp_path('unreadable', dir=self.tmp) - write_file(noread_fn, 'thou shall not pass') + noread_fn = tmpdir.join('unreadable') + noread_fn.write('thou shall not pass') args = self.args( - debug=False, dump_all=True, format=None, instance_data=noread_fn, + debug=False, dump_all=True, format=None, + instance_data=noread_fn.strpath, list_keys=False, user_data='ud', vendor_data='vd', varname=None) - with mock.patch('sys.stderr', new_callable=StringIO) as m_stderr: - with mock.patch('cloudinit.cmd.query.util.load_file') as m_load: - m_load.side_effect = OSError(errno.EACCES, 'Not allowed') - self.assertEqual(1, query.handle_args('anyname', args)) - self.assertIn( - "ERROR: No read permission on '%s'. Try sudo" % noread_fn, - self.logs.getvalue()) - self.assertIn( - "ERROR: No read permission on '%s'. Try sudo" % noread_fn, - m_stderr.getvalue()) + with mock.patch('cloudinit.cmd.query.util.load_file') as m_load: + m_load.side_effect = OSError(errno.EACCES, 'Not allowed') + assert 1 == query.handle_args('anyname', args) + msg = "No read permission on '%s'. Try sudo" % noread_fn + assert msg in caplog.text - def test_handle_args_defaults_instance_data(self): + def test_handle_args_defaults_instance_data(self, caplog, tmpdir): """When no instance_data argument, default to configured run_dir.""" args = self.args( debug=False, dump_all=True, format=None, instance_data=None, list_keys=False, user_data=None, vendor_data=None, varname=None) - run_dir = self.tmp_path('run_dir', dir=self.tmp) - ensure_dir(run_dir) - paths = Paths({'run_dir': run_dir}) - self.add_patch('cloudinit.cmd.query.read_cfg_paths', 'm_paths') - self.m_paths.return_value = paths - with mock.patch('sys.stderr', new_callable=StringIO) as m_stderr: - self.assertEqual(1, query.handle_args('anyname', args)) - json_file = os.path.join(run_dir, INSTANCE_JSON_FILE) - self.assertIn( - 'ERROR: Missing instance-data file: %s' % json_file, - self.logs.getvalue()) - self.assertIn( - 'ERROR: Missing instance-data file: %s' % json_file, - m_stderr.getvalue()) + paths, run_dir, _, _ = self._setup_paths(tmpdir) + with mock.patch('cloudinit.cmd.query.read_cfg_paths') as m_paths: + m_paths.return_value = paths + assert 1 == query.handle_args('anyname', args) + json_file = run_dir.join(INSTANCE_JSON_FILE) + msg = 'Missing instance-data file: %s' % json_file.strpath + assert msg in caplog.text - def test_handle_args_root_fallsback_to_instance_data(self): + def test_handle_args_root_fallsback_to_instance_data(self, caplog, tmpdir): """When no instance_data argument, root falls back to redacted json.""" args = self.args( debug=False, dump_all=True, format=None, instance_data=None, list_keys=False, user_data=None, vendor_data=None, varname=None) - run_dir = self.tmp_path('run_dir', dir=self.tmp) - ensure_dir(run_dir) - paths = Paths({'run_dir': run_dir}) - self.add_patch('cloudinit.cmd.query.read_cfg_paths', 'm_paths') - self.m_paths.return_value = paths - with mock.patch('sys.stderr', new_callable=StringIO) as m_stderr: + paths, run_dir, _, _ = self._setup_paths(tmpdir) + with mock.patch('cloudinit.cmd.query.read_cfg_paths') as m_paths: + m_paths.return_value = paths with mock.patch('os.getuid') as m_getuid: m_getuid.return_value = 0 - self.assertEqual(1, query.handle_args('anyname', args)) - json_file = os.path.join(run_dir, INSTANCE_JSON_FILE) - sensitive_file = os.path.join(run_dir, INSTANCE_JSON_SENSITIVE_FILE) - self.assertIn( - 'WARNING: Missing root-readable %s. Using redacted %s instead.' % ( - sensitive_file, json_file), - m_stderr.getvalue()) + assert 1 == query.handle_args('anyname', args) + json_file = run_dir.join(INSTANCE_JSON_FILE) + sensitive_file = run_dir.join(INSTANCE_JSON_SENSITIVE_FILE) + msg = ( + 'Missing root-readable %s. Using redacted %s instead.' % + ( + sensitive_file.strpath, json_file.strpath + ) + ) + assert msg in caplog.text - def test_handle_args_root_uses_instance_sensitive_data(self): - """When no instance_data argument, root uses semsitive json.""" - user_data = self.tmp_path('user-data', dir=self.tmp) - vendor_data = self.tmp_path('vendor-data', dir=self.tmp) - write_file(user_data, 'ud') - write_file(vendor_data, 'vd') - run_dir = self.tmp_path('run_dir', dir=self.tmp) - sensitive_file = os.path.join(run_dir, INSTANCE_JSON_SENSITIVE_FILE) - write_file(sensitive_file, '{"my-var": "it worked"}') - ensure_dir(run_dir) - paths = Paths({'run_dir': run_dir}) - self.add_patch('cloudinit.cmd.query.read_cfg_paths', 'm_paths') - self.m_paths.return_value = paths + @pytest.mark.parametrize( + 'ud_src,ud_expected,vd_src,vd_expected', + ( + ('hi mom', 'hi mom', 'hi pops', 'hi pops'), + ('ud'.encode('utf-8'), 'ud', 'vd'.encode('utf-8'), 'vd'), + (_gzip_data(b'ud'), 'ud', _gzip_data(b'vd'), 'vd'), + (_gzip_data('ud'.encode('utf-8')), 'ud', _gzip_data(b'vd'), 'vd'), + (_gzip_data(b'ud') + b'invalid', 'ci-b64:', + _gzip_data(b'vd') + b'invalid', 'ci-b64:'), + # non-utf-8 encodable content + ('hi mom'.encode('utf-16'), 'ci-b64://5oAGkAIABtAG8AbQA=', + 'hi pops'.encode('utf-16'), 'ci-b64://5oAGkAIABwAG8AcABzAA=='), + ) + ) + def test_handle_args_root_processes_user_data( + self, ud_src, ud_expected, vd_src, vd_expected, capsys, tmpdir + ): + """Support reading multiple user-data file content types""" + paths, run_dir, user_data, vendor_data = self._setup_paths( + tmpdir, ud_val=ud_src, vd_val=vd_src + ) + sensitive_file = run_dir.join(INSTANCE_JSON_SENSITIVE_FILE) + sensitive_file.write('{"my-var": "it worked"}') args = self.args( debug=False, dump_all=True, format=None, instance_data=None, - list_keys=False, user_data=vendor_data, vendor_data=vendor_data, - varname=None) - with mock.patch('sys.stdout', new_callable=StringIO) as m_stdout: + list_keys=False, user_data=user_data.strpath, + vendor_data=vendor_data.strpath, varname=None) + with mock.patch('cloudinit.cmd.query.read_cfg_paths') as m_paths: + m_paths.return_value = paths with mock.patch('os.getuid') as m_getuid: m_getuid.return_value = 0 - self.assertEqual(0, query.handle_args('anyname', args)) - self.assertEqual( - '{\n "my_var": "it worked",\n "userdata": "vd",\n ' - '"vendordata": "vd"\n}\n', m_stdout.getvalue()) + assert 0 == query.handle_args('anyname', args) + out, _err = capsys.readouterr() + cmd_output = json.loads(out) + assert "it worked" == cmd_output['my_var'] + if ud_expected == "ci-b64:": + ud_expected = "ci-b64:{}".format(b64e(ud_src)) + if vd_expected == "ci-b64:": + vd_expected = "ci-b64:{}".format(b64e(vd_src)) + assert ud_expected == cmd_output['userdata'] + assert vd_expected == cmd_output['vendordata'] - def test_handle_args_dumps_all_instance_data(self): + def test_handle_args_root_uses_instance_sensitive_data( + self, capsys, tmpdir + ): + """When no instance_data argument, root uses sensitive json.""" + paths, run_dir, user_data, vendor_data = self._setup_paths( + tmpdir, ud_val='ud', vd_val='vd' + ) + sensitive_file = run_dir.join(INSTANCE_JSON_SENSITIVE_FILE) + sensitive_file.write('{"my-var": "it worked"}') + args = self.args( + debug=False, dump_all=True, format=None, instance_data=None, + list_keys=False, user_data=user_data.strpath, + vendor_data=vendor_data.strpath, varname=None) + with mock.patch('cloudinit.cmd.query.read_cfg_paths') as m_paths: + m_paths.return_value = paths + with mock.patch('os.getuid') as m_getuid: + m_getuid.return_value = 0 + assert 0 == query.handle_args('anyname', args) + expected = ( + '{\n "my_var": "it worked",\n "userdata": "ud",\n ' + '"vendordata": "vd"\n}\n' + ) + out, _err = capsys.readouterr() + assert expected == out + + def test_handle_args_dumps_all_instance_data(self, capsys, tmpdir): """When --all is specified query will dump all instance data vars.""" - write_file(self.instance_data, '{"my-var": "it worked"}') + instance_data = tmpdir.join('instance-data') + instance_data.write('{"my-var": "it worked"}') args = self.args( debug=False, dump_all=True, format=None, - instance_data=self.instance_data, list_keys=False, + instance_data=instance_data.strpath, list_keys=False, user_data='ud', vendor_data='vd', varname=None) - with mock.patch('sys.stdout', new_callable=StringIO) as m_stdout: - with mock.patch('os.getuid') as m_getuid: - m_getuid.return_value = 100 - self.assertEqual(0, query.handle_args('anyname', args)) - self.assertEqual( + with mock.patch('os.getuid') as m_getuid: + m_getuid.return_value = 100 + assert 0 == query.handle_args('anyname', args) + expected = ( '{\n "my_var": "it worked",\n "userdata": "<%s> file:ud",\n' ' "vendordata": "<%s> file:vd"\n}\n' % ( - REDACT_SENSITIVE_VALUE, REDACT_SENSITIVE_VALUE), - m_stdout.getvalue()) + REDACT_SENSITIVE_VALUE, REDACT_SENSITIVE_VALUE + ) + ) + out, _err = capsys.readouterr() + assert expected == out - def test_handle_args_returns_top_level_varname(self): + def test_handle_args_returns_top_level_varname(self, capsys, tmpdir): """When the argument varname is passed, report its value.""" - write_file(self.instance_data, '{"my-var": "it worked"}') + instance_data = tmpdir.join('instance-data') + instance_data.write('{"my-var": "it worked"}') args = self.args( debug=False, dump_all=True, format=None, - instance_data=self.instance_data, list_keys=False, + instance_data=instance_data.strpath, list_keys=False, user_data='ud', vendor_data='vd', varname='my_var') - with mock.patch('sys.stdout', new_callable=StringIO) as m_stdout: - with mock.patch('os.getuid') as m_getuid: - m_getuid.return_value = 100 - self.assertEqual(0, query.handle_args('anyname', args)) - self.assertEqual('it worked\n', m_stdout.getvalue()) + with mock.patch('os.getuid') as m_getuid: + m_getuid.return_value = 100 + assert 0 == query.handle_args('anyname', args) + out, _err = capsys.readouterr() + assert 'it worked\n' == out - def test_handle_args_returns_nested_varname(self): + def test_handle_args_returns_nested_varname(self, capsys, tmpdir): """If user_data file is a jinja template render instance-data vars.""" - write_file(self.instance_data, - '{"v1": {"key-2": "value-2"}, "my-var": "it worked"}') + instance_data = tmpdir.join('instance-data') + instance_data.write( + '{"v1": {"key-2": "value-2"}, "my-var": "it worked"}' + ) args = self.args( debug=False, dump_all=False, format=None, - instance_data=self.instance_data, user_data='ud', vendor_data='vd', - list_keys=False, varname='v1.key_2') - with mock.patch('sys.stdout', new_callable=StringIO) as m_stdout: - with mock.patch('os.getuid') as m_getuid: - m_getuid.return_value = 100 - self.assertEqual(0, query.handle_args('anyname', args)) - self.assertEqual('value-2\n', m_stdout.getvalue()) + instance_data=instance_data.strpath, user_data='ud', + vendor_data='vd', list_keys=False, varname='v1.key_2') + with mock.patch('os.getuid') as m_getuid: + m_getuid.return_value = 100 + assert 0 == query.handle_args('anyname', args) + out, _err = capsys.readouterr() + assert 'value-2\n' == out - def test_handle_args_returns_standardized_vars_to_top_level_aliases(self): + def test_handle_args_returns_standardized_vars_to_top_level_aliases( + self, capsys, tmpdir + ): """Any standardized vars under v# are promoted as top-level aliases.""" - write_file( - self.instance_data, + instance_data = tmpdir.join('instance-data') + instance_data.write( '{"v1": {"v1_1": "val1.1"}, "v2": {"v2_2": "val2.2"},' ' "top": "gun"}') expected = dedent("""\ @@ -209,65 +279,68 @@ class TestQuery(CiTestCase): """) args = self.args( debug=False, dump_all=True, format=None, - instance_data=self.instance_data, user_data='ud', vendor_data='vd', - list_keys=False, varname=None) - with mock.patch('sys.stdout', new_callable=StringIO) as m_stdout: - with mock.patch('os.getuid') as m_getuid: - m_getuid.return_value = 100 - self.assertEqual(0, query.handle_args('anyname', args)) - self.assertEqual(expected, m_stdout.getvalue()) + instance_data=instance_data.strpath, user_data='ud', + vendor_data='vd', list_keys=False, varname=None) + with mock.patch('os.getuid') as m_getuid: + m_getuid.return_value = 100 + assert 0 == query.handle_args('anyname', args) + out, _err = capsys.readouterr() + assert expected == out - def test_handle_args_list_keys_sorts_top_level_keys_when_no_varname(self): + def test_handle_args_list_keys_sorts_top_level_keys_when_no_varname( + self, capsys, tmpdir + ): """Sort all top-level keys when only --list-keys provided.""" - write_file( - self.instance_data, + instance_data = tmpdir.join('instance-data') + instance_data.write( '{"v1": {"v1_1": "val1.1"}, "v2": {"v2_2": "val2.2"},' ' "top": "gun"}') expected = 'top\nuserdata\nv1\nv1_1\nv2\nv2_2\nvendordata\n' args = self.args( debug=False, dump_all=False, format=None, - instance_data=self.instance_data, list_keys=True, user_data='ud', - vendor_data='vd', varname=None) - with mock.patch('sys.stdout', new_callable=StringIO) as m_stdout: - with mock.patch('os.getuid') as m_getuid: - m_getuid.return_value = 100 - self.assertEqual(0, query.handle_args('anyname', args)) - self.assertEqual(expected, m_stdout.getvalue()) + instance_data=instance_data.strpath, list_keys=True, + user_data='ud', vendor_data='vd', varname=None) + with mock.patch('os.getuid') as m_getuid: + m_getuid.return_value = 100 + assert 0 == query.handle_args('anyname', args) + out, _err = capsys.readouterr() + assert expected == out - def test_handle_args_list_keys_sorts_nested_keys_when_varname(self): + def test_handle_args_list_keys_sorts_nested_keys_when_varname( + self, capsys, tmpdir + ): """Sort all nested keys of varname object when --list-keys provided.""" - write_file( - self.instance_data, + instance_data = tmpdir.join('instance-data') + instance_data.write( '{"v1": {"v1_1": "val1.1", "v1_2": "val1.2"}, "v2":' + ' {"v2_2": "val2.2"}, "top": "gun"}') expected = 'v1_1\nv1_2\n' args = self.args( debug=False, dump_all=False, format=None, - instance_data=self.instance_data, list_keys=True, + instance_data=instance_data.strpath, list_keys=True, user_data='ud', vendor_data='vd', varname='v1') - with mock.patch('sys.stdout', new_callable=StringIO) as m_stdout: - with mock.patch('os.getuid') as m_getuid: - m_getuid.return_value = 100 - self.assertEqual(0, query.handle_args('anyname', args)) - self.assertEqual(expected, m_stdout.getvalue()) + with mock.patch('os.getuid') as m_getuid: + m_getuid.return_value = 100 + assert 0 == query.handle_args('anyname', args) + out, _err = capsys.readouterr() + assert expected == out - def test_handle_args_list_keys_errors_when_varname_is_not_a_dict(self): + def test_handle_args_list_keys_errors_when_varname_is_not_a_dict( + self, caplog, tmpdir + ): """Raise an error when --list-keys and varname specify a non-list.""" - write_file( - self.instance_data, + instance_data = tmpdir.join('instance-data') + instance_data.write( '{"v1": {"v1_1": "val1.1", "v1_2": "val1.2"}, "v2": ' + '{"v2_2": "val2.2"}, "top": "gun"}') - expected_error = "ERROR: --list-keys provided but 'top' is not a dict" + expected_error = "--list-keys provided but 'top' is not a dict" args = self.args( debug=False, dump_all=False, format=None, - instance_data=self.instance_data, list_keys=True, user_data='ud', - vendor_data='vd', varname='top') - with mock.patch('sys.stderr', new_callable=StringIO) as m_stderr: - with mock.patch('sys.stdout', new_callable=StringIO) as m_stdout: - with mock.patch('os.getuid') as m_getuid: - m_getuid.return_value = 100 - self.assertEqual(1, query.handle_args('anyname', args)) - self.assertEqual('', m_stdout.getvalue()) - self.assertIn(expected_error, m_stderr.getvalue()) + instance_data=instance_data.strpath, list_keys=True, + user_data='ud', vendor_data='vd', varname='top') + with mock.patch('os.getuid') as m_getuid: + m_getuid.return_value = 100 + assert 1 == query.handle_args('anyname', args) + assert expected_error in caplog.text # vi: ts=4 expandtab -- cgit v1.2.3 From c0450c02b25fe037c702e9bb145fbbaa28a165b1 Mon Sep 17 00:00:00 2001 From: Chad Smith Date: Mon, 24 Aug 2020 10:13:52 -0600 Subject: query: do not handle non-decodable non-gzipped content (#543) --- cloudinit/cmd/query.py | 9 +++------ cloudinit/cmd/tests/test_query.py | 5 ----- 2 files changed, 3 insertions(+), 11 deletions(-) (limited to 'cloudinit/cmd/tests') diff --git a/cloudinit/cmd/query.py b/cloudinit/cmd/query.py index 03b77c17..07db9552 100644 --- a/cloudinit/cmd/query.py +++ b/cloudinit/cmd/query.py @@ -86,14 +86,11 @@ def load_userdata(ud_file_path): @returns: String of uncompressed userdata if possible, otherwise bytes. """ + bdata = util.load_file(ud_file_path, decode=False) try: - return util.load_file(ud_file_path) + return bdata.decode('utf-8') except UnicodeDecodeError: - encoded_data = util.load_file(ud_file_path, decode=False) - try: - return util.decomp_gzip(encoded_data, quiet=False) - except util.DecompressionError: - return encoded_data + return util.decomp_gzip(bdata, quiet=False, decode=True) def handle_args(name, args): diff --git a/cloudinit/cmd/tests/test_query.py b/cloudinit/cmd/tests/test_query.py index 421031aa..c258d321 100644 --- a/cloudinit/cmd/tests/test_query.py +++ b/cloudinit/cmd/tests/test_query.py @@ -144,11 +144,6 @@ class TestQuery: ('ud'.encode('utf-8'), 'ud', 'vd'.encode('utf-8'), 'vd'), (_gzip_data(b'ud'), 'ud', _gzip_data(b'vd'), 'vd'), (_gzip_data('ud'.encode('utf-8')), 'ud', _gzip_data(b'vd'), 'vd'), - (_gzip_data(b'ud') + b'invalid', 'ci-b64:', - _gzip_data(b'vd') + b'invalid', 'ci-b64:'), - # non-utf-8 encodable content - ('hi mom'.encode('utf-16'), 'ci-b64://5oAGkAIABtAG8AbQA=', - 'hi pops'.encode('utf-16'), 'ci-b64://5oAGkAIABwAG8AcABzAA=='), ) ) def test_handle_args_root_processes_user_data( -- cgit v1.2.3