cli: add cloud-init query subcommand to query instance metadata

Cloud-init caches any cloud metadata crawled during boot in the file
/run/cloud-init/instance-data.json. Cloud-init also standardizes some of
that metadata across all clouds. The command 'cloud-init query' surfaces a
simple CLI to query or format any cached instance metadata so that scripts
or end-users do not have to write tools to crawl metadata themselves.

Since 'cloud-init query' is runnable by non-root users, redact any
sensitive data from instance-data.json and provide a root-readable
unredacted instance-data-sensitive.json. Datasources can now define a
sensitive_metadata_keys tuple which will redact any matching keys
which could contain passwords or credentials from instance-data.json.

Also add the following standardized 'v1' instance-data.json keys:
  - user_data: The base64encoded user-data provided at instance launch
  - vendor_data: Any vendor_data provided to the instance at launch
  - underscore_delimited versions of existing hyphenated keys:
    instance_id, local_hostname, availability_zone, cloud_name

4 files changed, 359 insertions, 6 deletions

diff --git a/cloudinit/cmd/devel/render.py b/cloudinit/cmd/devel/render.py
index e85933db..2ba6b681 100755
--- a/cloudinit/cmd/devel/render.py
+++ b/cloudinit/cmd/devel/render.py
@@ -9,7 +9,6 @@ import sys
 from cloudinit.handlers.jinja_template import render_jinja_payload_from_file
 from cloudinit import log
 from cloudinit.sources import INSTANCE_JSON_FILE
-from cloudinit import util
 from . import addLogHandlerCLI, read_cfg_paths
 
 NAME = 'render'
@@ -54,11 +53,7 @@ def handle_args(name, args):
             paths.run_dir, INSTANCE_JSON_FILE)
     else:
         instance_data_fn = args.instance_data
-    try:
-        with open(instance_data_fn) as stream:
-            instance_data = stream.read()
-        instance_data = util.load_json(instance_data)
-    except IOError:
+    if not os.path.exists(instance_data_fn):
         LOG.error('Missing instance-data.json file: %s', instance_data_fn)
         return 1
     try:
diff --git a/cloudinit/cmd/main.py b/cloudinit/cmd/main.py
index 0eee583c..5a437020 100644
--- a/cloudinit/cmd/main.py
+++ b/cloudinit/cmd/main.py
@@ -791,6 +791,10 @@ def main(sysv_args=None):
                                      ' pass to this module'))
     parser_single.set_defaults(action=('single', main_single))
 
+    parser_query = subparsers.add_parser(
+        'query',
+        help='Query standardized instance metadata from the command line.')
+
     parser_dhclient = subparsers.add_parser('dhclient-hook',
                                             help=('run the dhclient hook'
                                                   'to record network info'))
@@ -842,6 +846,12 @@ def main(sysv_args=None):
             clean_parser(parser_clean)
             parser_clean.set_defaults(
                 action=('clean', handle_clean_args))
+        elif sysv_args[0] == 'query':
+            from cloudinit.cmd.query import (
+                get_parser as query_parser, handle_args as handle_query_args)
+            query_parser(parser_query)
+            parser_query.set_defaults(
+                action=('render', handle_query_args))
         elif sysv_args[0] == 'status':
             from cloudinit.cmd.status import (
                 get_parser as status_parser, handle_status_args)
diff --git a/cloudinit/cmd/query.py b/cloudinit/cmd/query.py
new file mode 100644
index 00000000..7d2d4fe4
--- /dev/null
+++ b/cloudinit/cmd/query.py
@@ -0,0 +1,155 @@
+# This file is part of cloud-init. See LICENSE file for license information.
+
+"""Query standardized instance metadata from the command line."""
+
+import argparse
+import os
+import six
+import sys
+
+from cloudinit.handlers.jinja_template import (
+    convert_jinja_instance_data, render_jinja_payload)
+from cloudinit.cmd.devel import addLogHandlerCLI, read_cfg_paths
+from cloudinit import log
+from cloudinit.sources import (
+    INSTANCE_JSON_FILE, INSTANCE_JSON_SENSITIVE_FILE, REDACT_SENSITIVE_VALUE)
+from cloudinit import util
+
+NAME = 'query'
+LOG = log.getLogger(NAME)
+
+
+def get_parser(parser=None):
+    """Build or extend an arg parser for query utility.
+
+    @param parser: Optional existing ArgumentParser instance representing the
+        query subcommand which will be extended to support the args of
+        this utility.
+
+    @returns: ArgumentParser with proper argument configuration.
+    """
+    if not parser:
+        parser = argparse.ArgumentParser(
+            prog=NAME, description='Query cloud-init instance data')
+    parser.add_argument(
+        '-d', '--debug', action='store_true', default=False,
+        help='Add verbose messages during template render')
+    parser.add_argument(
+        '-i', '--instance-data', type=str,
+        help=('Path to instance-data.json file. Default is /run/cloud-init/%s'
+              % INSTANCE_JSON_FILE))
+    parser.add_argument(
+        '-l', '--list-keys', action='store_true', default=False,
+        help=('List query keys available at the provided instance-data'
+              ' <varname>.'))
+    parser.add_argument(
+        '-u', '--user-data', type=str,
+        help=('Path to user-data file. Default is'
+              ' /var/lib/cloud/instance/user-data.txt'))
+    parser.add_argument(
+        '-v', '--vendor-data', type=str,
+        help=('Path to vendor-data file. Default is'
+              ' /var/lib/cloud/instance/vendor-data.txt'))
+    parser.add_argument(
+        'varname', type=str, nargs='?',
+        help=('A dot-delimited instance data variable to query from'
+              ' instance-data query. For example: v2.local_hostname'))
+    parser.add_argument(
+        '-a', '--all', action='store_true', default=False, dest='dump_all',
+        help='Dump all available instance-data')
+    parser.add_argument(
+        '-f', '--format', type=str, dest='format',
+        help=('Optionally specify a custom output format string. Any'
+              ' instance-data variable can be specified between double-curly'
+              ' braces. For example -f "{{ v2.cloud_name }}"'))
+    return parser
+
+
+def handle_args(name, args):
+    """Handle calls to 'cloud-init query' as a subcommand."""
+    paths = None
+    addLogHandlerCLI(LOG, log.DEBUG if args.debug else log.WARNING)
+    if not any([args.list_keys, args.varname, args.format, args.dump_all]):
+        LOG.error(
+            'Expected one of the options: --all, --format,'
+            ' --list-keys or varname')
+        get_parser().print_help()
+        return 1
+
+    uid = os.getuid()
+    if not all([args.instance_data, args.user_data, args.vendor_data]):
+        paths = read_cfg_paths()
+    if not args.instance_data:
+        if uid == 0:
+            default_json_fn = INSTANCE_JSON_SENSITIVE_FILE
+        else:
+            default_json_fn = INSTANCE_JSON_FILE  # World readable
+        instance_data_fn = os.path.join(paths.run_dir, default_json_fn)
+    else:
+        instance_data_fn = args.instance_data
+    if not args.user_data:
+        user_data_fn = os.path.join(paths.instance_link, 'user-data.txt')
+    else:
+        user_data_fn = args.user_data
+    if not args.vendor_data:
+        vendor_data_fn = os.path.join(paths.instance_link, 'vendor-data.txt')
+    else:
+        vendor_data_fn = args.vendor_data
+
+    try:
+        instance_json = util.load_file(instance_data_fn)
+    except IOError:
+        LOG.error('Missing instance-data.json file: %s', instance_data_fn)
+        return 1
+
+    instance_data = util.load_json(instance_json)
+    if uid != 0:
+        instance_data['userdata'] = (
+            '<%s> file:%s' % (REDACT_SENSITIVE_VALUE, user_data_fn))
+        instance_data['vendordata'] = (
+            '<%s> file:%s' % (REDACT_SENSITIVE_VALUE, vendor_data_fn))
+    else:
+        instance_data['userdata'] = util.load_file(user_data_fn)
+        instance_data['vendordata'] = util.load_file(vendor_data_fn)
+    if args.format:
+        payload = '## template: jinja\n{fmt}'.format(fmt=args.format)
+        rendered_payload = render_jinja_payload(
+            payload=payload, payload_fn='query commandline',
+            instance_data=instance_data,
+            debug=True if args.debug else False)
+        if rendered_payload:
+            print(rendered_payload)
+            return 0
+        return 1
+
+    response = convert_jinja_instance_data(instance_data)
+    if args.varname:
+        try:
+            for var in args.varname.split('.'):
+                response = response[var]
+        except KeyError:
+            LOG.error('Undefined instance-data key %s', args.varname)
+            return 1
+        if args.list_keys:
+            if not isinstance(response, dict):
+                LOG.error("--list-keys provided but '%s' is not a dict", var)
+                return 1
+            response = '\n'.join(sorted(response.keys()))
+    elif args.list_keys:
+        response = '\n'.join(sorted(response.keys()))
+    if not isinstance(response, six.string_types):
+        response = util.json_dumps(response)
+    print(response)
+    return 0
+
+
+def main():
+    """Tool to query specific instance-data values."""
+    parser = get_parser()
+    sys.exit(handle_args(NAME, parser.parse_args()))
+
+
+if __name__ == '__main__':
+    main()
+
+# vi: ts=4 expandtab
diff --git a/cloudinit/cmd/tests/test_query.py b/cloudinit/cmd/tests/test_query.py
new file mode 100644
index 00000000..fb87c6ab
--- /dev/null
+++ b/cloudinit/cmd/tests/test_query.py
@@ -0,0 +1,193 @@
+# This file is part of cloud-init. See LICENSE file for license information.
+
+from six import StringIO
+from textwrap import dedent
+import os
+
+from collections import namedtuple
+from cloudinit.cmd import query
+from cloudinit.helpers import Paths
+from cloudinit.sources import REDACT_SENSITIVE_VALUE, INSTANCE_JSON_FILE
+from cloudinit.tests.helpers import CiTestCase, mock
+from cloudinit.util import ensure_dir, write_file
+
+
+class TestQuery(CiTestCase):
+
+    with_logs = True
+
+    args = namedtuple(
+        'queryargs',
+        ('debug dump_all format instance_data list_keys user_data vendor_data'
+         ' varname'))
+
+    def setUp(self):
+        super(TestQuery, self).setUp()
+        self.tmp = self.tmp_dir()
+        self.instance_data = self.tmp_path('instance-data', dir=self.tmp)
+
+    def test_handle_args_error_on_missing_param(self):
+        """Error when missing required parameters and print usage."""
+        args = self.args(
+            debug=False, dump_all=False, format=None, instance_data=None,
+            list_keys=False, user_data=None, vendor_data=None, varname=None)
+        with mock.patch('sys.stderr', new_callable=StringIO) as m_stderr:
+            with mock.patch('sys.stdout', new_callable=StringIO) as m_stdout:
+                self.assertEqual(1, query.handle_args('anyname', args))
+        expected_error = (
+            'ERROR: Expected one of the options: --all, --format, --list-keys'
+            ' or varname\n')
+        self.assertIn(expected_error, self.logs.getvalue())
+        self.assertIn('usage: query', m_stdout.getvalue())
+        self.assertIn(expected_error, m_stderr.getvalue())
+
+    def test_handle_args_error_on_missing_instance_data(self):
+        """When instance_data file path does not exist, log an error."""
+        absent_fn = self.tmp_path('absent', dir=self.tmp)
+        args = self.args(
+            debug=False, dump_all=True, format=None, instance_data=absent_fn,
+            list_keys=False, user_data='ud', vendor_data='vd', varname=None)
+        with mock.patch('sys.stderr', new_callable=StringIO) as m_stderr:
+            self.assertEqual(1, query.handle_args('anyname', args))
+        self.assertIn(
+            'ERROR: Missing instance-data.json file: %s' % absent_fn,
+            self.logs.getvalue())
+        self.assertIn(
+            'ERROR: Missing instance-data.json file: %s' % absent_fn,
+            m_stderr.getvalue())
+
+    def test_handle_args_defaults_instance_data(self):
+        """When no instance_data argument, default to configured run_dir."""
+        args = self.args(
+            debug=False, dump_all=True, format=None, instance_data=None,
+            list_keys=False, user_data=None, vendor_data=None, varname=None)
+        run_dir = self.tmp_path('run_dir', dir=self.tmp)
+        ensure_dir(run_dir)
+        paths = Paths({'run_dir': run_dir})
+        self.add_patch('cloudinit.cmd.query.read_cfg_paths', 'm_paths')
+        self.m_paths.return_value = paths
+        with mock.patch('sys.stderr', new_callable=StringIO) as m_stderr:
+            self.assertEqual(1, query.handle_args('anyname', args))
+        json_file = os.path.join(run_dir, INSTANCE_JSON_FILE)
+        self.assertIn(
+            'ERROR: Missing instance-data.json file: %s' % json_file,
+            self.logs.getvalue())
+        self.assertIn(
+            'ERROR: Missing instance-data.json file: %s' % json_file,
+            m_stderr.getvalue())
+
+    def test_handle_args_dumps_all_instance_data(self):
+        """When --all is specified query will dump all instance data vars."""
+        write_file(self.instance_data, '{"my-var": "it worked"}')
+        args = self.args(
+            debug=False, dump_all=True, format=None,
+            instance_data=self.instance_data, list_keys=False,
+            user_data='ud', vendor_data='vd', varname=None)
+        with mock.patch('sys.stdout', new_callable=StringIO) as m_stdout:
+            self.assertEqual(0, query.handle_args('anyname', args))
+        self.assertEqual(
+            '{\n "my_var": "it worked",\n "userdata": "<%s> file:ud",\n'
+            ' "vendordata": "<%s> file:vd"\n}\n' % (
+                REDACT_SENSITIVE_VALUE, REDACT_SENSITIVE_VALUE),
+            m_stdout.getvalue())
+
+    def test_handle_args_returns_top_level_varname(self):
+        """When the argument varname is passed, report its value."""
+        write_file(self.instance_data, '{"my-var": "it worked"}')
+        args = self.args(
+            debug=False, dump_all=True, format=None,
+            instance_data=self.instance_data, list_keys=False,
+            user_data='ud', vendor_data='vd', varname='my_var')
+        with mock.patch('sys.stdout', new_callable=StringIO) as m_stdout:
+            self.assertEqual(0, query.handle_args('anyname', args))
+        self.assertEqual('it worked\n', m_stdout.getvalue())
+
+    def test_handle_args_returns_nested_varname(self):
+        """If user_data file is a jinja template render instance-data vars."""
+        write_file(self.instance_data,
+                   '{"v1": {"key-2": "value-2"}, "my-var": "it worked"}')
+        args = self.args(
+            debug=False, dump_all=False, format=None,
+            instance_data=self.instance_data, user_data='ud', vendor_data='vd',
+            list_keys=False, varname='v1.key_2')
+        with mock.patch('sys.stdout', new_callable=StringIO) as m_stdout:
+            self.assertEqual(0, query.handle_args('anyname', args))
+        self.assertEqual('value-2\n', m_stdout.getvalue())
+
+    def test_handle_args_returns_standardized_vars_to_top_level_aliases(self):
+        """Any standardized vars under v# are promoted as top-level aliases."""
+        write_file(
+            self.instance_data,
+            '{"v1": {"v1_1": "val1.1"}, "v2": {"v2_2": "val2.2"},'
+            ' "top": "gun"}')
+        expected = dedent("""\
+            {
+             "top": "gun",
+             "userdata": "<redacted for non-root user> file:ud",
+             "v1": {
+              "v1_1": "val1.1"
+             },
+             "v1_1": "val1.1",
+             "v2": {
+              "v2_2": "val2.2"
+             },
+             "v2_2": "val2.2",
+             "vendordata": "<redacted for non-root user> file:vd"
+            }
+        """)
+        args = self.args(
+            debug=False, dump_all=True, format=None,
+            instance_data=self.instance_data, user_data='ud', vendor_data='vd',
+            list_keys=False, varname=None)
+        with mock.patch('sys.stdout', new_callable=StringIO) as m_stdout:
+            self.assertEqual(0, query.handle_args('anyname', args))
+        self.assertEqual(expected, m_stdout.getvalue())
+
+    def test_handle_args_list_keys_sorts_top_level_keys_when_no_varname(self):
+        """Sort all top-level keys when only --list-keys provided."""
+        write_file(
+            self.instance_data,
+            '{"v1": {"v1_1": "val1.1"}, "v2": {"v2_2": "val2.2"},'
+            ' "top": "gun"}')
+        expected = 'top\nuserdata\nv1\nv1_1\nv2\nv2_2\nvendordata\n'
+        args = self.args(
+            debug=False, dump_all=False, format=None,
+            instance_data=self.instance_data, list_keys=True, user_data='ud',
+            vendor_data='vd', varname=None)
+        with mock.patch('sys.stdout', new_callable=StringIO) as m_stdout:
+            self.assertEqual(0, query.handle_args('anyname', args))
+        self.assertEqual(expected, m_stdout.getvalue())
+
+    def test_handle_args_list_keys_sorts_nested_keys_when_varname(self):
+        """Sort all nested keys of varname object when --list-keys provided."""
+        write_file(
+            self.instance_data,
+            '{"v1": {"v1_1": "val1.1", "v1_2": "val1.2"}, "v2":' +
+            ' {"v2_2": "val2.2"}, "top": "gun"}')
+        expected = 'v1_1\nv1_2\n'
+        args = self.args(
+            debug=False, dump_all=False, format=None,
+            instance_data=self.instance_data, list_keys=True,
+            user_data='ud', vendor_data='vd', varname='v1')
+        with mock.patch('sys.stdout', new_callable=StringIO) as m_stdout:
+            self.assertEqual(0, query.handle_args('anyname', args))
+        self.assertEqual(expected, m_stdout.getvalue())
+
+    def test_handle_args_list_keys_errors_when_varname_is_not_a_dict(self):
+        """Raise an error when --list-keys and varname specify a non-list."""
+        write_file(
+            self.instance_data,
+            '{"v1": {"v1_1": "val1.1", "v1_2": "val1.2"}, "v2": ' +
+            '{"v2_2": "val2.2"}, "top": "gun"}')
+        expected_error = "ERROR: --list-keys provided but 'top' is not a dict"
+        args = self.args(
+            debug=False, dump_all=False, format=None,
+            instance_data=self.instance_data, list_keys=True, user_data='ud',
+            vendor_data='vd',  varname='top')
+        with mock.patch('sys.stderr', new_callable=StringIO) as m_stderr:
+            with mock.patch('sys.stdout', new_callable=StringIO) as m_stdout:
+                self.assertEqual(1, query.handle_args('anyname', args))
+        self.assertEqual('', m_stdout.getvalue())
+        self.assertIn(expected_error, m_stderr.getvalue())
+
+# vi: ts=4 expandtab