Datasources: Formalize DataSource get_data and related properties.

Each DataSource subclass must define its own get_data method. This branch
formalizes our DataSource class to require that subclasses define an
explicit dsname for sourcing cloud-config datasource configuration.
Subclasses must also override the _get_data method or a
NotImplementedError is raised.

The branch also writes /run/cloud-init/instance-data.json. This file
contains all meta-data, user-data and vendor-data and a standardized set
of metadata keys in a json blob which other utilities with root-access
could make use of. Because some meta-data or user-data is potentially
sensitive the file is only readable by root.

Generally most metadata content types should be json serializable. If
specific keys or values are not serializable, those specific values will
be base64encoded and the key path will be listed under the top-level key
'base64-encoded-keys' in instance-data.json. If json writing fails due to
other TypeErrors or UnicodeDecodeErrors, a warning log will be emitted to
/var/log/cloud-init.log and no instance-data.json will be created.

1 files changed, 116 insertions, 13 deletions

diff --git a/cloudinit/sources/__init__.py b/cloudinit/sources/__init__.py
index 9a43fbee..4b819ce6 100644
--- a/cloudinit/sources/__init__.py
+++ b/cloudinit/sources/__init__.py
@@ -10,9 +10,11 @@
 
 import abc
 import copy
+import json
 import os
 import six
 
+from cloudinit.atomic_helper import write_json
 from cloudinit import importer
 from cloudinit import log as logging
 from cloudinit import type_utils
@@ -33,6 +35,12 @@ DEP_FILESYSTEM = "FILESYSTEM"
 DEP_NETWORK = "NETWORK"
 DS_PREFIX = 'DataSource'
 
+# File in which instance meta-data, user-data and vendor-data is written
+INSTANCE_JSON_FILE = 'instance-data.json'
+
+# Key which can be provide a cloud's official product name to cloud-init
+METADATA_CLOUD_NAME_KEY = 'cloud-name'
+
 LOG = logging.getLogger(__name__)
 
 
@@ -40,12 +48,39 @@ class DataSourceNotFoundException(Exception):
     pass
 
 
+def process_base64_metadata(metadata, key_path=''):
+    """Strip ci-b64 prefix and return metadata with base64-encoded-keys set."""
+    md_copy = copy.deepcopy(metadata)
+    md_copy['base64-encoded-keys'] = []
+    for key, val in metadata.items():
+        if key_path:
+            sub_key_path = key_path + '/' + key
+        else:
+            sub_key_path = key
+        if isinstance(val, str) and val.startswith('ci-b64:'):
+            md_copy['base64-encoded-keys'].append(sub_key_path)
+            md_copy[key] = val.replace('ci-b64:', '')
+        if isinstance(val, dict):
+            return_val = process_base64_metadata(val, sub_key_path)
+            md_copy['base64-encoded-keys'].extend(
+                return_val.pop('base64-encoded-keys'))
+            md_copy[key] = return_val
+    return md_copy
+
+
 @six.add_metaclass(abc.ABCMeta)
 class DataSource(object):
 
     dsmode = DSMODE_NETWORK
     default_locale = 'en_US.UTF-8'
 
+    # Datasource name needs to be set by subclasses to determine which
+    # cloud-config datasource key is loaded
+    dsname = '_undef'
+
+    # Cached cloud_name as determined by _get_cloud_name
+    _cloud_name = None
+
     def __init__(self, sys_cfg, distro, paths, ud_proc=None):
         self.sys_cfg = sys_cfg
         self.distro = distro
@@ -56,17 +91,8 @@ class DataSource(object):
         self.vendordata = None
         self.vendordata_raw = None
 
-        # find the datasource config name.
-        # remove 'DataSource' from classname on front, and remove 'Net' on end.
-        # Both Foo and FooNet sources expect config in cfg['sources']['Foo']
-        name = type_utils.obj_name(self)
-        if name.startswith(DS_PREFIX):
-            name = name[len(DS_PREFIX):]
-        if name.endswith('Net'):
-            name = name[0:-3]
-
-        self.ds_cfg = util.get_cfg_by_path(self.sys_cfg,
-                                           ("datasource", name), {})
+        self.ds_cfg = util.get_cfg_by_path(
+            self.sys_cfg, ("datasource", self.dsname), {})
         if not self.ds_cfg:
             self.ds_cfg = {}
 
@@ -78,6 +104,51 @@ class DataSource(object):
     def __str__(self):
         return type_utils.obj_name(self)
 
+    def _get_standardized_metadata(self):
+        """Return a dictionary of standardized metadata keys."""
+        return {'v1': {
+            'local-hostname': self.get_hostname(),
+            'instance-id': self.get_instance_id(),
+            'cloud-name': self.cloud_name,
+            'region': self.region,
+            'availability-zone': self.availability_zone}}
+
+    def get_data(self):
+        """Datasources implement _get_data to setup metadata and userdata_raw.
+
+        Minimally, the datasource should return a boolean True on success.
+        """
+        return_value = self._get_data()
+        json_file = os.path.join(self.paths.run_dir, INSTANCE_JSON_FILE)
+        if not return_value:
+            return return_value
+
+        instance_data = {
+            'ds': {
+                'meta-data': self.metadata,
+                'user-data': self.get_userdata_raw(),
+                'vendor-data': self.get_vendordata_raw()}}
+        instance_data.update(
+            self._get_standardized_metadata())
+        try:
+            # Process content base64encoding unserializable values
+            content = util.json_dumps(instance_data)
+            # Strip base64: prefix and return base64-encoded-keys
+            processed_data = process_base64_metadata(json.loads(content))
+        except TypeError as e:
+            LOG.warning('Error persisting instance-data.json: %s', str(e))
+            return return_value
+        except UnicodeDecodeError as e:
+            LOG.warning('Error persisting instance-data.json: %s', str(e))
+            return return_value
+        write_json(json_file, processed_data, mode=0o600)
+        return return_value
+
+    def _get_data(self):
+        raise NotImplementedError(
+            'Subclasses of DataSource must implement _get_data which'
+            ' sets self.metadata, vendordata_raw and userdata_raw.')
+
     def get_userdata(self, apply_filter=False):
         if self.userdata is None:
             self.userdata = self.ud_proc.process(self.get_userdata_raw())
@@ -91,6 +162,34 @@ class DataSource(object):
         return self.vendordata
 
     @property
+    def cloud_name(self):
+        """Return lowercase cloud name as determined by the datasource.
+
+        Datasource can determine or define its own cloud product name in
+        metadata.
+        """
+        if self._cloud_name:
+            return self._cloud_name
+        if self.metadata and self.metadata.get(METADATA_CLOUD_NAME_KEY):
+            cloud_name = self.metadata.get(METADATA_CLOUD_NAME_KEY)
+            if isinstance(cloud_name, six.string_types):
+                self._cloud_name = cloud_name.lower()
+            LOG.debug(
+                'Ignoring metadata provided key %s: non-string type %s',
+                METADATA_CLOUD_NAME_KEY, type(cloud_name))
+        else:
+            self._cloud_name = self._get_cloud_name().lower()
+        return self._cloud_name
+
+    def _get_cloud_name(self):
+        """Return the datasource name as it frequently matches cloud name.
+
+        Should be overridden in subclasses which can run on multiple
+        cloud names, such as DatasourceEc2.
+        """
+        return self.dsname
+
+    @property
     def launch_index(self):
         if not self.metadata:
             return None
@@ -161,8 +260,11 @@ class DataSource(object):
 
     @property
     def availability_zone(self):
-        return self.metadata.get('availability-zone',
-                                 self.metadata.get('availability_zone'))
+        top_level_az = self.metadata.get(
+            'availability-zone', self.metadata.get('availability_zone'))
+        if top_level_az:
+            return top_level_az
+        return self.metadata.get('placement', {}).get('availability-zone')
 
     @property
     def region(self):
@@ -417,4 +519,5 @@ def list_from_depends(depends, ds_list):
             ret_list.append(cls)
     return ret_list
 
+
 # vi: ts=4 expandtab