support base64 encoded data in the smart os datasource.

The big benefit of this is that now the user can put in arbitrary
data into the user-data or user-script keys and there is no concern
about the data being incorrectly read.

Previously, if data contained '\n.\n', there was no way to differenciate
that from a end of message in the serial communication format.

It would be recommended that anyone using user-data on smartos base64 encode
that data and specify a key of 'b64-user-data' with value 'true'.

4 files changed, 261 insertions, 50 deletions

diff --git a/cloudinit/sources/DataSourceSmartOS.py b/cloudinit/sources/DataSourceSmartOS.py
index 1ce20c10..d348d20b 100644
--- a/cloudinit/sources/DataSourceSmartOS.py
+++ b/cloudinit/sources/DataSourceSmartOS.py
@@ -27,6 +27,7 @@
 #
 
 
+import base64
 from cloudinit import log as logging
 from cloudinit import sources
 from cloudinit import util
@@ -34,7 +35,6 @@ import os
 import os.path
 import serial
 
-
 DEF_TTY_LOC = '/dev/ttyS1'
 DEF_TTY_TIMEOUT = 60
 LOG = logging.getLogger(__name__)
@@ -49,15 +49,24 @@ SMARTOS_ATTRIB_MAP = {
     'motd_sys_info': ('motd_sys_info', True),
 }
 
+# These are values which will never be base64 encoded.
+# They come from the cloud platform, not user
+SMARTOS_NO_BASE64 = ['root_authorized_keys', 'motd_sys_info',
+                     'iptables_disable']
+
 
 class DataSourceSmartOS(sources.DataSource):
     def __init__(self, sys_cfg, distro, paths):
         sources.DataSource.__init__(self, sys_cfg, distro, paths)
         self.seed_dir = os.path.join(paths.seed_dir, 'sdc')
         self.is_smartdc = None
-        self.seed = self.sys_cfg.get("serial_device", DEF_TTY_LOC)
-        self.seed_timeout = self.sys_cfg.get("serial_timeout",
-                                             DEF_TTY_TIMEOUT)
+
+        self.seed = self.ds_cfg.get("serial_device", DEF_TTY_LOC)
+        self.seed_timeout = self.ds_cfg.get("serial_timeout", DEF_TTY_TIMEOUT)
+        self.smartos_no_base64 = self.ds_cfg.get('no_base64_decode',
+                                                 SMARTOS_NO_BASE64)
+        self.b64_keys = self.ds_cfg.get('base64_keys', [])
+        self.b64_all = self.ds_cfg.get('base64_all', False)
 
     def __str__(self):
         root = sources.DataSource.__str__(self)
@@ -79,22 +88,30 @@ class DataSourceSmartOS(sources.DataSource):
 
         system_uuid, system_type = dmi_info
         if 'smartdc' not in system_type.lower():
-            LOG.debug("Host is not on SmartOS")
+            LOG.debug("Host is not on SmartOS. system_type=%s", system_type)
             return False
         self.is_smartdc = True
         md['instance-id'] = system_uuid
 
+        b64_keys = self.query('base64_keys', strip=True, b64=False)
+        if b64_keys is not None:
+            self.b64_keys = [k.strip() for k in str(b64_keys).split(',')]
+
+        b64_all = self.query('base64_all', strip=True, b64=False)
+        if b64_all is not None:
+            self.b64_all = util.is_true(b64_all)
+
         for ci_noun, attribute in SMARTOS_ATTRIB_MAP.iteritems():
             smartos_noun, strip = attribute
-            md[ci_noun] = query_data(smartos_noun, self.seed,
-                                     self.seed_timeout, strip=strip)
+            md[ci_noun] = self.query(smartos_noun, strip=strip)
 
         if not md['local-hostname']:
             md['local-hostname'] = system_uuid
 
+        ud = None
         if md['user-data']:
             ud = md['user-data']
-        else:
+        elif md['user-script']:
             ud = md['user-script']
 
         self.metadata = md
@@ -104,10 +121,21 @@ class DataSourceSmartOS(sources.DataSource):
     def get_instance_id(self):
         return self.metadata['instance-id']
 
+    def query(self, noun, strip=False, default=None, b64=None):
+        if b64 is None:
+            if noun in self.smartos_no_base64:
+                b64 = False
+            elif self.b64_all or noun in self.b64_keys:
+                b64 = True
+
+        return query_data(noun=noun, strip=strip, seed_device=self.seed,
+                          seed_timeout=self.seed_timeout, default=default,
+                          b64=b64)
+
 
 def get_serial(seed_device, seed_timeout):
     """This is replaced in unit testing, allowing us to replace
-        serial.Serial with a mocked class
+        serial.Serial with a mocked class.
 
         The timeout value of 60 seconds should never be hit. The value
         is taken from SmartOS own provisioning tools. Since we are reading
@@ -124,12 +152,18 @@ def get_serial(seed_device, seed_timeout):
     return ser
 
 
-def query_data(noun, seed_device, seed_timeout, strip=False):
+def query_data(noun, seed_device, seed_timeout, strip=False, default=None,
+               b64=None):
     """Makes a request to via the serial console via "GET <NOUN>"
 
         In the response, the first line is the status, while subsequent lines
         are is the value. A blank line with a "." is used to indicate end of
         response.
+
+        If the response is expected to be base64 encoded, then set b64encoded
+        to true. Unfortantely, there is no way to know if something is 100%
+        encoded, so this method relies on being told if the data is base64 or
+        not.
     """
 
     if not noun:
@@ -143,7 +177,7 @@ def query_data(noun, seed_device, seed_timeout, strip=False):
 
     if 'SUCCESS' not in status:
         ser.close()
-        return None
+        return default
 
     while not eom_found:
         m = ser.readline()
@@ -153,12 +187,27 @@ def query_data(noun, seed_device, seed_timeout, strip=False):
             response.append(m)
 
     ser.close()
-    if not strip:
-        return "".join(response)
+
+    if b64 is None:
+        b64 = query_data('b64-%s' % noun, seed_device=seed_device,
+                            seed_timeout=seed_timeout, b64=False,
+                            default=False, strip=True)
+        b64 = util.is_true(b64)
+
+    resp = None
+    if b64 or strip:
+        resp = "".join(response).rstrip()
     else:
-        return "".join(response).rstrip()
+        resp = "".join(response)
+
+    if b64:
+        try:
+            return base64.b64decode(resp)
+        except TypeError:
+            LOG.warn("Failed base64 decoding key '%s'", noun)
+            return resp
 
-    return None
+    return resp
 
 
 def dmi_data():
diff --git a/doc/examples/cloud-config-datasources.txt b/doc/examples/cloud-config-datasources.txt
index 6544448e..65a3cdf5 100644
--- a/doc/examples/cloud-config-datasources.txt
+++ b/doc/examples/cloud-config-datasources.txt
@@ -55,5 +55,13 @@ datasource:
     # Smart OS datasource works over a serial console interacting with
     # a server on the other end. By default, the second serial console is the
     # device. SmartOS also uses a serial timeout of 60 seconds.
-    serial device: /dev/ttyS1
-    serial timeout: 60
+    serial_device: /dev/ttyS1
+    serial_timeout: 60
+
+    # a list of keys that will not be base64 decoded even if base64_all
+    no_base64_decode: ['root_authorized_keys', 'motd_sys_info',
+                       'iptables_disable']
+    # a plaintext, comma delimited list of keys whose values are b64 encoded
+    base64_keys: []
+    # a boolean indicating that all keys not in 'no_base64_decode' are encoded
+    base64_all: False
diff --git a/doc/sources/smartos/README.rst b/doc/sources/smartos/README.rst
new file mode 100644
index 00000000..fd4e496d
--- /dev/null
+++ b/doc/sources/smartos/README.rst
@@ -0,0 +1,72 @@
+==================
+SmartOS Datasource
+==================
+
+This datasource finds metadata and user-data from the SmartOS virtualization
+platform (i.e. Joyent).
+
+SmartOS Platform
+----------------
+The SmartOS virtualization platform meta-data to the instance via the second
+serial console. On Linux, this is /dev/ttyS1. The data is a provided via a
+simple protocol, where something queries for the userdata, where the console
+responds with the status and if "SUCCESS" returns until a single ".\n".
+
+New versions of the SmartOS tooling will include support for base64 encoded data.
+
+Userdata
+--------
+
+In SmartOS parlance, user-data is a actually meta-data. This userdata can be
+provided a key-value pairs.
+
+Cloud-init supports reading the traditional meta-data fields supported by the
+SmartOS tools. These are:
+ * root_authorized_keys
+ * hostname
+ * enable_motd_sys_info
+ * iptables_disable
+
+Note: At this time iptables_disable and enable_motd_sys_info are read but
+    are not actioned.
+
+user-script
+-----------
+
+SmartOS traditionally supports sending over a user-script for execution at the
+rc.local level. Cloud-init supports running user-scripts as if they were
+cloud-init user-data. In this sense, anything with a shell interpreter
+directive will run
+
+user-data and user-script
+-------------------------
+
+In the event that a user defines the meta-data key of "user-data" it will
+always supercede any user-script data. This is for consistency.
+
+base64
+------
+
+The following are exempt from base64 encoding, owing to the fact that they
+are provided by SmartOS:
+ * root_authorized_keys
+ * enable_motd_sys_info
+ * iptables_disable
+
+This list can be changed through system config of variable 'no_base64_decode'.
+
+This means that user-script and user-data as well as other values can be
+base64 encoded. Since Cloud-init can only guess as to whether or not something
+is truly base64 encoded, the following meta-data keys are hints as to whether
+or not to base64 decode something:
+  * base64_all: Except for excluded keys, attempt to base64 decode
+        the values. If the value fails to decode properly, it will be
+        returned in its text
+  * base64_keys: A comma deliminated list of which keys are base64 encoded.
+  * b64-<key>:
+    for any key, if there exists an entry in the metadata for 'b64-<key>'
+    Then 'b64-<key>' is expected to be a plaintext boolean indicating whether
+    or not its value is encoded.
+  * no_base64_decode: This is a configuration setting
+        (i.e. /etc/cloud/cloud.cfg.d) that sets which values should not be
+        base64 decoded.
diff --git a/tests/unittests/test_datasource/test_smartos.py b/tests/unittests/test_datasource/test_smartos.py
index 6c12f1e2..f53715b0 100644
--- a/tests/unittests/test_datasource/test_smartos.py
+++ b/tests/unittests/test_datasource/test_smartos.py
@@ -22,25 +22,24 @@
 #   return responses.
 #
 
+import base64
 from cloudinit import helpers
 from cloudinit.sources import DataSourceSmartOS
 
 from mocker import MockerTestCase
 import uuid
 
-mock_returns = {
+MOCK_RETURNS = {
     'hostname': 'test-host',
     'root_authorized_keys': 'ssh-rsa AAAAB3Nz...aC1yc2E= keyname',
     'disable_iptables_flag': None,
     'enable_motd_sys_info': None,
-    'system_uuid': str(uuid.uuid4()),
-    'smartdc': 'smartdc',
-    'userdata': """
-#!/bin/sh
-/bin/true
-""",
+    'test-var1': 'some data',
+    'user-data': '\n'.join(['#!/bin/sh', '/bin/true', '']),
 }
 
+DMI_DATA_RETURN = (str(uuid.uuid4()), 'smartdc')
+
 
 class MockSerial(object):
     """Fake a serial terminal for testing the code that
@@ -48,12 +47,13 @@ class MockSerial(object):
 
     port = None
 
-    def __init__(self):
+    def __init__(self, mockdata):
         self.last = None
         self.last = None
         self.new = True
         self.count = 0
         self.mocked_out = []
+        self.mockdata = mockdata
 
     def open(self):
         return True
@@ -71,12 +71,12 @@ class MockSerial(object):
     def readline(self):
         if self.new:
             self.new = False
-            if self.last in mock_returns:
+            if self.last in self.mockdata:
                 return 'SUCCESS\n'
             else:
                 return 'NOTFOUND %s\n' % self.last
 
-        if self.last in mock_returns:
+        if self.last in self.mockdata:
             if not self.mocked_out:
                 self.mocked_out = [x for x in self._format_out()]
                 print self.mocked_out
@@ -86,15 +86,16 @@ class MockSerial(object):
                 return self.mocked_out[self.count - 1]
 
     def _format_out(self):
-        if self.last in mock_returns:
+        if self.last in self.mockdata:
+            _mret = self.mockdata[self.last]
             try:
-                for l in mock_returns[self.last].splitlines():
-                    yield "%s\n" % l
+                for l in _mret.splitlines():
+                    yield "%s\n" % l.rstrip()
             except:
-                yield "%s\n" % mock_returns[self.last]
+                yield "%s\n" % _mret.rstrip()
 
-            yield '\n'
             yield '.'
+            yield '\n'
 
 
 class TestSmartOSDataSource(MockerTestCase):
@@ -116,23 +117,36 @@ class TestSmartOSDataSource(MockerTestCase):
         ret = apply_patches(patches)
         self.unapply += ret
 
-    def _get_ds(self):
+    def _get_ds(self, sys_cfg=None, ds_cfg=None, mockdata=None, dmi_data=None):
+        mod = DataSourceSmartOS
+
+        if mockdata is None:
+            mockdata = MOCK_RETURNS
+
+        if dmi_data is None:
+            dmi_data = DMI_DATA_RETURN
 
         def _get_serial(*_):
-            return MockSerial()
+            return MockSerial(mockdata)
 
         def _dmi_data():
-            return mock_returns['system_uuid'], 'smartdc'
+            return dmi_data
+
+        if sys_cfg is None:
+            sys_cfg = {}
+
+        if ds_cfg is not None:
+            sys_cfg['datasource'] = sys_cfg.get('datasource', {})
+            sys_cfg['datasource']['SmartOS'] = ds_cfg
 
-        data = {'sys_cfg': {}}
-        mod = DataSourceSmartOS
         self.apply_patches([(mod, 'get_serial', _get_serial)])
         self.apply_patches([(mod, 'dmi_data', _dmi_data)])
-        dsrc = mod.DataSourceSmartOS(
-            data.get('sys_cfg', {}), distro=None, paths=self.paths)
+        dsrc = mod.DataSourceSmartOS(sys_cfg, distro=None,
+                                     paths=self.paths)
         return dsrc
 
     def test_seed(self):
+        # default seed should be /dev/ttyS1
         dsrc = self._get_ds()
         ret = dsrc.get_data()
         self.assertTrue(ret)
@@ -144,39 +158,107 @@ class TestSmartOSDataSource(MockerTestCase):
         self.assertTrue(ret)
         self.assertTrue(dsrc.is_smartdc)
 
+    def test_no_base64(self):
+        ds_cfg = {'no_base64_decode': ['test_var1'], 'all_base': True}
+        dsrc = self._get_ds(ds_cfg=ds_cfg)
+        ret = dsrc.get_data()
+        self.assertTrue(ret)
+
     def test_uuid(self):
-        dsrc = self._get_ds()
+        dsrc = self._get_ds(mockdata=MOCK_RETURNS)
         ret = dsrc.get_data()
         self.assertTrue(ret)
-        self.assertEquals(mock_returns['system_uuid'],
-                          dsrc.metadata['instance-id'])
+        self.assertEquals(DMI_DATA_RETURN[0], dsrc.metadata['instance-id'])
 
     def test_root_keys(self):
-        dsrc = self._get_ds()
+        dsrc = self._get_ds(mockdata=MOCK_RETURNS)
         ret = dsrc.get_data()
         self.assertTrue(ret)
-        self.assertEquals(mock_returns['root_authorized_keys'],
+        self.assertEquals(MOCK_RETURNS['root_authorized_keys'],
                           dsrc.metadata['public-keys'])
 
+    def test_hostname_b64(self):
+        dsrc = self._get_ds(mockdata=MOCK_RETURNS)
+        ret = dsrc.get_data()
+        self.assertTrue(ret)
+        self.assertEquals(MOCK_RETURNS['hostname'],
+                          dsrc.metadata['local-hostname'])
+
     def test_hostname(self):
-        dsrc = self._get_ds()
+        dsrc = self._get_ds(mockdata=MOCK_RETURNS)
         ret = dsrc.get_data()
         self.assertTrue(ret)
-        self.assertEquals(mock_returns['hostname'],
+        self.assertEquals(MOCK_RETURNS['hostname'],
                           dsrc.metadata['local-hostname'])
 
+    def test_base64_all(self):
+        # metadata provided base64_all of true
+        my_returns = MOCK_RETURNS.copy()
+        my_returns['base64_all'] = "true"
+        for k in ('hostname', 'user-data'):
+            my_returns[k] = base64.b64encode(my_returns[k])
+
+        dsrc = self._get_ds(mockdata=my_returns)
+        ret = dsrc.get_data()
+        self.assertTrue(ret)
+        self.assertEquals(MOCK_RETURNS['hostname'],
+                          dsrc.metadata['local-hostname'])
+        self.assertEquals(MOCK_RETURNS['user-data'],
+                          dsrc.userdata_raw)
+        self.assertEquals(MOCK_RETURNS['root_authorized_keys'],
+                          dsrc.metadata['public-keys'])
+        self.assertEquals(MOCK_RETURNS['disable_iptables_flag'],
+                          dsrc.metadata['iptables_disable'])
+        self.assertEquals(MOCK_RETURNS['enable_motd_sys_info'],
+                          dsrc.metadata['motd_sys_info'])
+
+    def test_b64_userdata(self):
+        my_returns = MOCK_RETURNS.copy()
+        my_returns['b64-user-data'] = "true"
+        my_returns['b64-hostname'] = "true"
+        for k in ('hostname', 'user-data'):
+            my_returns[k] = base64.b64encode(my_returns[k])
+
+        dsrc = self._get_ds(mockdata=my_returns)
+        ret = dsrc.get_data()
+        self.assertTrue(ret)
+        self.assertEquals(MOCK_RETURNS['hostname'],
+                          dsrc.metadata['local-hostname'])
+        self.assertEquals(MOCK_RETURNS['user-data'], dsrc.userdata_raw)
+        self.assertEquals(MOCK_RETURNS['root_authorized_keys'],
+                          dsrc.metadata['public-keys'])
+
+    def test_b64_keys(self):
+        my_returns = MOCK_RETURNS.copy()
+        my_returns['base64_keys'] = 'hostname,ignored'
+        for k in ('hostname',):
+            my_returns[k] = base64.b64encode(my_returns[k])
+
+        dsrc = self._get_ds(mockdata=my_returns)
+        ret = dsrc.get_data()
+        self.assertTrue(ret)
+        self.assertEquals(MOCK_RETURNS['hostname'],
+                          dsrc.metadata['local-hostname'])
+        self.assertEquals(MOCK_RETURNS['user-data'], dsrc.userdata_raw)
+
+    def test_userdata(self):
+        dsrc = self._get_ds(mockdata=MOCK_RETURNS)
+        ret = dsrc.get_data()
+        self.assertTrue(ret)
+        self.assertEquals(MOCK_RETURNS['user-data'], dsrc.userdata_raw)
+
     def test_disable_iptables_flag(self):
-        dsrc = self._get_ds()
+        dsrc = self._get_ds(mockdata=MOCK_RETURNS)
         ret = dsrc.get_data()
         self.assertTrue(ret)
-        self.assertEquals(mock_returns['disable_iptables_flag'],
+        self.assertEquals(MOCK_RETURNS['disable_iptables_flag'],
                           dsrc.metadata['iptables_disable'])
 
     def test_motd_sys_info(self):
-        dsrc = self._get_ds()
+        dsrc = self._get_ds(mockdata=MOCK_RETURNS)
         ret = dsrc.get_data()
         self.assertTrue(ret)
-        self.assertEquals(mock_returns['enable_motd_sys_info'],
+        self.assertEquals(MOCK_RETURNS['enable_motd_sys_info'],
                           dsrc.metadata['motd_sys_info'])