changes to behavior on specifying keys.

The most likely end user operation (or at least a valid one) for base64
encoding would be to encode the user-data, but leave all other values
as plaintext.

In order to facilitate that, the user can simply add:
 b64-user-data=true
to indicate that user-data is base64 encoded.

Other changes here are to change the cloud-config and metadata keynames
that are used.
  base64_all = boolean(True)
  base64_keys = [list, of, keys]

Fixed up tests to accomodate.

4 files changed, 149 insertions, 124 deletions

diff --git a/cloudinit/sources/DataSourceSmartOS.py b/cloudinit/sources/DataSourceSmartOS.py
index 45f03a7e..d348d20b 100644
--- a/cloudinit/sources/DataSourceSmartOS.py
+++ b/cloudinit/sources/DataSourceSmartOS.py
@@ -35,7 +35,6 @@ import os
 import os.path
 import serial
 
-DS_NAME = 'SmartOS'
 DEF_TTY_LOC = '/dev/ttyS1'
 DEF_TTY_TIMEOUT = 60
 LOG = logging.getLogger(__name__)
@@ -51,6 +50,7 @@ SMARTOS_ATTRIB_MAP = {
 }
 
 # These are values which will never be base64 encoded.
+# They come from the cloud platform, not user
 SMARTOS_NO_BASE64 = ['root_authorized_keys', 'motd_sys_info',
                      'iptables_disable']
 
@@ -60,17 +60,13 @@ class DataSourceSmartOS(sources.DataSource):
         sources.DataSource.__init__(self, sys_cfg, distro, paths)
         self.seed_dir = os.path.join(paths.seed_dir, 'sdc')
         self.is_smartdc = None
-        self.base_64_encoded = []
-        self.seed = self.sys_cfg.get("serial_device", DEF_TTY_LOC)
-        self.seed_timeout = self.sys_cfg.get("serial_timeout",
-                                             DEF_TTY_TIMEOUT)
-        self.all_base64 = False
-        if 'decode_base64' in self.ds_cfg:
-            self.all_base64 =  self.ds_cfg['decode_base64']
-
-        self.smartos_no_base64 = SMARTOS_NO_BASE64
-        if 'no_base64_decode' in self.ds_cfg:
-            self.smartos_no_base64 = self.ds_cfg['no_base64_decode']
+
+        self.seed = self.ds_cfg.get("serial_device", DEF_TTY_LOC)
+        self.seed_timeout = self.ds_cfg.get("serial_timeout", DEF_TTY_TIMEOUT)
+        self.smartos_no_base64 = self.ds_cfg.get('no_base64_decode',
+                                                 SMARTOS_NO_BASE64)
+        self.b64_keys = self.ds_cfg.get('base64_keys', [])
+        self.b64_all = self.ds_cfg.get('base64_all', False)
 
     def __str__(self):
         root = sources.DataSource.__str__(self)
@@ -92,38 +88,22 @@ class DataSourceSmartOS(sources.DataSource):
 
         system_uuid, system_type = dmi_info
         if 'smartdc' not in system_type.lower():
-            LOG.debug("Host is not on SmartOS")
+            LOG.debug("Host is not on SmartOS. system_type=%s", system_type)
             return False
         self.is_smartdc = True
         md['instance-id'] = system_uuid
 
-        self.base_64_encoded = query_data('base_64_enocded',
-                                          self.seed,
-                                          self.seed_timeout,
-                                          strip=True)
-        if self.base_64_encoded:
-            self.base_64_encoded = str(self.base_64_encoded).split(',')
-        else:
-            self.base_64_encoded = []
+        b64_keys = self.query('base64_keys', strip=True, b64=False)
+        if b64_keys is not None:
+            self.b64_keys = [k.strip() for k in str(b64_keys).split(',')]
 
-        if not self.all_base64:
-            self.all_base64 = util.is_true(query_data('meta_encoded_base64',
-                                                      self.seed,
-                                                      self.seed_timeout,
-                                                      strip=True))
+        b64_all = self.query('base64_all', strip=True, b64=False)
+        if b64_all is not None:
+            self.b64_all = util.is_true(b64_all)
 
         for ci_noun, attribute in SMARTOS_ATTRIB_MAP.iteritems():
             smartos_noun, strip = attribute
-
-            b64encoded = False
-            if self.all_base64 and \
-               (smartos_noun not in self.smartos_no_base64 and \
-                ci_noun not in self.smartos_no_base64):
-                b64encoded = True
-
-            md[ci_noun] = query_data(smartos_noun, self.seed,
-                                     self.seed_timeout, strip=strip,
-                                     b64encoded=b64encoded)
+            md[ci_noun] = self.query(smartos_noun, strip=strip)
 
         if not md['local-hostname']:
             md['local-hostname'] = system_uuid
@@ -141,20 +121,16 @@ class DataSourceSmartOS(sources.DataSource):
     def get_instance_id(self):
         return self.metadata['instance-id']
 
-    def not_b64_var(self, var):
-        """Return true if value is read as b64."""
-        if var in self.smartos_no_base64 or \
-           not self.all_base64:
-            return True
-        return False
+    def query(self, noun, strip=False, default=None, b64=None):
+        if b64 is None:
+            if noun in self.smartos_no_base64:
+                b64 = False
+            elif self.b64_all or noun in self.b64_keys:
+                b64 = True
 
-    def is_b64_var(self, var):
-        """Return true if value is read as b64."""
-        if self.all_base64 or (
-           var not in self.smartos_no_base64 and
-           var in self.base_64_encoded):
-            return True
-        return False
+        return query_data(noun=noun, strip=strip, seed_device=self.seed,
+                          seed_timeout=self.seed_timeout, default=default,
+                          b64=b64)
 
 
 def get_serial(seed_device, seed_timeout):
@@ -176,7 +152,8 @@ def get_serial(seed_device, seed_timeout):
     return ser
 
 
-def query_data(noun, seed_device, seed_timeout, strip=False, b64encoded=False):
+def query_data(noun, seed_device, seed_timeout, strip=False, default=None,
+               b64=None):
     """Makes a request to via the serial console via "GET <NOUN>"
 
         In the response, the first line is the status, while subsequent lines
@@ -200,7 +177,7 @@ def query_data(noun, seed_device, seed_timeout, strip=False, b64encoded=False):
 
     if 'SUCCESS' not in status:
         ser.close()
-        return None
+        return default
 
     while not eom_found:
         m = ser.readline()
@@ -211,18 +188,23 @@ def query_data(noun, seed_device, seed_timeout, strip=False, b64encoded=False):
 
     ser.close()
 
+    if b64 is None:
+        b64 = query_data('b64-%s' % noun, seed_device=seed_device,
+                            seed_timeout=seed_timeout, b64=False,
+                            default=False, strip=True)
+        b64 = util.is_true(b64)
+
     resp = None
-    if not strip:
-        resp = "".join(response)
-    elif b64encoded:
+    if b64 or strip:
         resp = "".join(response).rstrip()
     else:
-        resp = "".join(response).rstrip()
+        resp = "".join(response)
 
-    if b64encoded:
+    if b64:
         try:
             return base64.b64decode(resp)
         except TypeError:
+            LOG.warn("Failed base64 decoding key '%s'", noun)
             return resp
 
     return resp
diff --git a/doc/examples/cloud-config-datasources.txt b/doc/examples/cloud-config-datasources.txt
index 6ec0d57e..65a3cdf5 100644
--- a/doc/examples/cloud-config-datasources.txt
+++ b/doc/examples/cloud-config-datasources.txt
@@ -56,4 +56,12 @@ datasource:
     # a server on the other end. By default, the second serial console is the
     # device. SmartOS also uses a serial timeout of 60 seconds.
     serial_device: /dev/ttyS1
-    serial timeout: 60
+    serial_timeout: 60
+
+    # a list of keys that will not be base64 decoded even if base64_all
+    no_base64_decode: ['root_authorized_keys', 'motd_sys_info',
+                       'iptables_disable']
+    # a plaintext, comma delimited list of keys whose values are b64 encoded
+    base64_keys: []
+    # a boolean indicating that all keys not in 'no_base64_decode' are encoded
+    base64_all: False
diff --git a/doc/sources/smartos/README.rst b/doc/sources/smartos/README.rst
index 8f72fa0f..fd4e496d 100644
--- a/doc/sources/smartos/README.rst
+++ b/doc/sources/smartos/README.rst
@@ -53,14 +53,20 @@ are provided by SmartOS:
  * enable_motd_sys_info
  * iptables_disable
 
+This list can be changed through system config of variable 'no_base64_decode'.
+
 This means that user-script and user-data as well as other values can be
 base64 encoded. Since Cloud-init can only guess as to whether or not something
 is truly base64 encoded, the following meta-data keys are hints as to whether
 or not to base64 decode something:
-  * decode_base64: Except for excluded keys, attempt to base64 decode
+  * base64_all: Except for excluded keys, attempt to base64 decode
         the values. If the value fails to decode properly, it will be
         returned in its text
-  * base_64_encoded: A comma deliminated list of which values are base64
-        encoded.
-  * no_base64_decode: This is a configuration setting (i.e. /etc/cloud/cloud.cfg.d)
-        that sets which values should not be base64 decoded.
+  * base64_keys: A comma deliminated list of which keys are base64 encoded.
+  * b64-<key>:
+    for any key, if there exists an entry in the metadata for 'b64-<key>'
+    Then 'b64-<key>' is expected to be a plaintext boolean indicating whether
+    or not its value is encoded.
+  * no_base64_decode: This is a configuration setting
+        (i.e. /etc/cloud/cloud.cfg.d) that sets which values should not be
+        base64 decoded.
diff --git a/tests/unittests/test_datasource/test_smartos.py b/tests/unittests/test_datasource/test_smartos.py
index b9b3a479..f53715b0 100644
--- a/tests/unittests/test_datasource/test_smartos.py
+++ b/tests/unittests/test_datasource/test_smartos.py
@@ -29,20 +29,17 @@ from cloudinit.sources import DataSourceSmartOS
 from mocker import MockerTestCase
 import uuid
 
-mock_returns = {
+MOCK_RETURNS = {
     'hostname': 'test-host',
     'root_authorized_keys': 'ssh-rsa AAAAB3Nz...aC1yc2E= keyname',
     'disable_iptables_flag': None,
     'enable_motd_sys_info': None,
-    'system_uuid': str(uuid.uuid4()),
-    'smartdc': 'smartdc',
     'test-var1': 'some data',
-    'user-data': """
-#!/bin/sh
-/bin/true
-""",
+    'user-data': '\n'.join(['#!/bin/sh', '/bin/true', '']),
 }
 
+DMI_DATA_RETURN = (str(uuid.uuid4()), 'smartdc')
+
 
 class MockSerial(object):
     """Fake a serial terminal for testing the code that
@@ -50,14 +47,13 @@ class MockSerial(object):
 
     port = None
 
-    def __init__(self, b64encode=False):
+    def __init__(self, mockdata):
         self.last = None
         self.last = None
         self.new = True
         self.count = 0
         self.mocked_out = []
-        self.b64encode = b64encode
-        self.b64excluded = DataSourceSmartOS.SMARTOS_NO_BASE64
+        self.mockdata = mockdata
 
     def open(self):
         return True
@@ -75,12 +71,12 @@ class MockSerial(object):
     def readline(self):
         if self.new:
             self.new = False
-            if self.last in mock_returns:
+            if self.last in self.mockdata:
                 return 'SUCCESS\n'
             else:
                 return 'NOTFOUND %s\n' % self.last
 
-        if self.last in mock_returns:
+        if self.last in self.mockdata:
             if not self.mocked_out:
                 self.mocked_out = [x for x in self._format_out()]
                 print self.mocked_out
@@ -90,21 +86,16 @@ class MockSerial(object):
                 return self.mocked_out[self.count - 1]
 
     def _format_out(self):
-        if self.last in mock_returns:
-            _mret = mock_returns[self.last]
-            if self.b64encode and \
-               self.last not in self.b64excluded:
-                yield base64.b64encode(_mret)
-
-            else:
-                try:
-                    for l in _mret.splitlines():
-                        yield "%s\n" % l.rstrip()
-                except:
-                    yield "%s\n" % _mret.rstrip()
+        if self.last in self.mockdata:
+            _mret = self.mockdata[self.last]
+            try:
+                for l in _mret.splitlines():
+                    yield "%s\n" % l.rstrip()
+            except:
+                yield "%s\n" % _mret.rstrip()
 
-            yield '\n'
             yield '.'
+            yield '\n'
 
 
 class TestSmartOSDataSource(MockerTestCase):
@@ -126,26 +117,36 @@ class TestSmartOSDataSource(MockerTestCase):
         ret = apply_patches(patches)
         self.unapply += ret
 
-    def _get_ds(self, b64encode=False, sys_cfg=None):
+    def _get_ds(self, sys_cfg=None, ds_cfg=None, mockdata=None, dmi_data=None):
         mod = DataSourceSmartOS
 
+        if mockdata is None:
+            mockdata = MOCK_RETURNS
+
+        if dmi_data is None:
+            dmi_data = DMI_DATA_RETURN
+
         def _get_serial(*_):
-            return MockSerial(b64encode=b64encode)
+            return MockSerial(mockdata)
 
         def _dmi_data():
-            return mock_returns['system_uuid'], 'smartdc'
+            return dmi_data
 
-        if not sys_cfg:
+        if sys_cfg is None:
             sys_cfg = {}
 
-        data = {'sys_cfg': sys_cfg}
+        if ds_cfg is not None:
+            sys_cfg['datasource'] = sys_cfg.get('datasource', {})
+            sys_cfg['datasource']['SmartOS'] = ds_cfg
+
         self.apply_patches([(mod, 'get_serial', _get_serial)])
         self.apply_patches([(mod, 'dmi_data', _dmi_data)])
-        dsrc = mod.DataSourceSmartOS(
-            data.get('sys_cfg', {}), distro=None, paths=self.paths)
+        dsrc = mod.DataSourceSmartOS(sys_cfg, distro=None,
+                                     paths=self.paths)
         return dsrc
 
     def test_seed(self):
+        # default seed should be /dev/ttyS1
         dsrc = self._get_ds()
         ret = dsrc.get_data()
         self.assertTrue(ret)
@@ -158,78 +159,106 @@ class TestSmartOSDataSource(MockerTestCase):
         self.assertTrue(dsrc.is_smartdc)
 
     def test_no_base64(self):
-        sys_cfg = {'no_base64_decode': ['test_var1'], 'all_base': True}
-        dsrc = self._get_ds(sys_cfg=sys_cfg)
+        ds_cfg = {'no_base64_decode': ['test_var1'], 'all_base': True}
+        dsrc = self._get_ds(ds_cfg=ds_cfg)
         ret = dsrc.get_data()
         self.assertTrue(ret)
-        self.assertTrue(dsrc.not_b64_var('test-var'))
 
     def test_uuid(self):
-        dsrc = self._get_ds()
+        dsrc = self._get_ds(mockdata=MOCK_RETURNS)
         ret = dsrc.get_data()
         self.assertTrue(ret)
-        self.assertEquals(mock_returns['system_uuid'],
-                          dsrc.metadata['instance-id'])
+        self.assertEquals(DMI_DATA_RETURN[0], dsrc.metadata['instance-id'])
 
     def test_root_keys(self):
-        dsrc = self._get_ds()
+        dsrc = self._get_ds(mockdata=MOCK_RETURNS)
         ret = dsrc.get_data()
         self.assertTrue(ret)
-        self.assertEquals(mock_returns['root_authorized_keys'],
+        self.assertEquals(MOCK_RETURNS['root_authorized_keys'],
                           dsrc.metadata['public-keys'])
 
     def test_hostname_b64(self):
-        dsrc = self._get_ds(b64encode=True)
+        dsrc = self._get_ds(mockdata=MOCK_RETURNS)
         ret = dsrc.get_data()
         self.assertTrue(ret)
-        self.assertEquals(base64.b64encode(mock_returns['hostname']),
+        self.assertEquals(MOCK_RETURNS['hostname'],
                           dsrc.metadata['local-hostname'])
 
     def test_hostname(self):
-        dsrc = self._get_ds()
+        dsrc = self._get_ds(mockdata=MOCK_RETURNS)
         ret = dsrc.get_data()
         self.assertTrue(ret)
-        self.assertEquals(mock_returns['hostname'],
+        self.assertEquals(MOCK_RETURNS['hostname'],
                           dsrc.metadata['local-hostname'])
 
-    def test_base64(self):
-        """This tests to make sure that SmartOS system key/value pairs
-            are not interpetted as being base64 encoded, while making
-            sure that the others are when 'decode_base64' is set"""
-        dsrc = self._get_ds(sys_cfg={'decode_base64': True},
-                            b64encode=True)
+    def test_base64_all(self):
+        # metadata provided base64_all of true
+        my_returns = MOCK_RETURNS.copy()
+        my_returns['base64_all'] = "true"
+        for k in ('hostname', 'user-data'):
+            my_returns[k] = base64.b64encode(my_returns[k])
+
+        dsrc = self._get_ds(mockdata=my_returns)
         ret = dsrc.get_data()
         self.assertTrue(ret)
-        self.assertEquals(mock_returns['hostname'],
+        self.assertEquals(MOCK_RETURNS['hostname'],
                           dsrc.metadata['local-hostname'])
-        self.assertEquals("%s" % mock_returns['user-data'],
+        self.assertEquals(MOCK_RETURNS['user-data'],
                           dsrc.userdata_raw)
-        self.assertEquals(mock_returns['root_authorized_keys'],
+        self.assertEquals(MOCK_RETURNS['root_authorized_keys'],
                           dsrc.metadata['public-keys'])
-        self.assertEquals(mock_returns['disable_iptables_flag'],
+        self.assertEquals(MOCK_RETURNS['disable_iptables_flag'],
                           dsrc.metadata['iptables_disable'])
-        self.assertEquals(mock_returns['enable_motd_sys_info'],
+        self.assertEquals(MOCK_RETURNS['enable_motd_sys_info'],
                           dsrc.metadata['motd_sys_info'])
 
+    def test_b64_userdata(self):
+        my_returns = MOCK_RETURNS.copy()
+        my_returns['b64-user-data'] = "true"
+        my_returns['b64-hostname'] = "true"
+        for k in ('hostname', 'user-data'):
+            my_returns[k] = base64.b64encode(my_returns[k])
+
+        dsrc = self._get_ds(mockdata=my_returns)
+        ret = dsrc.get_data()
+        self.assertTrue(ret)
+        self.assertEquals(MOCK_RETURNS['hostname'],
+                          dsrc.metadata['local-hostname'])
+        self.assertEquals(MOCK_RETURNS['user-data'], dsrc.userdata_raw)
+        self.assertEquals(MOCK_RETURNS['root_authorized_keys'],
+                          dsrc.metadata['public-keys'])
+
+    def test_b64_keys(self):
+        my_returns = MOCK_RETURNS.copy()
+        my_returns['base64_keys'] = 'hostname,ignored'
+        for k in ('hostname',):
+            my_returns[k] = base64.b64encode(my_returns[k])
+
+        dsrc = self._get_ds(mockdata=my_returns)
+        ret = dsrc.get_data()
+        self.assertTrue(ret)
+        self.assertEquals(MOCK_RETURNS['hostname'],
+                          dsrc.metadata['local-hostname'])
+        self.assertEquals(MOCK_RETURNS['user-data'], dsrc.userdata_raw)
+
     def test_userdata(self):
-        dsrc = self._get_ds()
+        dsrc = self._get_ds(mockdata=MOCK_RETURNS)
         ret = dsrc.get_data()
         self.assertTrue(ret)
-        self.assertEquals("%s\n" % mock_returns['user-data'],
-                          dsrc.userdata_raw)
+        self.assertEquals(MOCK_RETURNS['user-data'], dsrc.userdata_raw)
 
     def test_disable_iptables_flag(self):
-        dsrc = self._get_ds()
+        dsrc = self._get_ds(mockdata=MOCK_RETURNS)
         ret = dsrc.get_data()
         self.assertTrue(ret)
-        self.assertEquals(mock_returns['disable_iptables_flag'],
+        self.assertEquals(MOCK_RETURNS['disable_iptables_flag'],
                           dsrc.metadata['iptables_disable'])
 
     def test_motd_sys_info(self):
-        dsrc = self._get_ds()
+        dsrc = self._get_ds(mockdata=MOCK_RETURNS)
         ret = dsrc.get_data()
         self.assertTrue(ret)
-        self.assertEquals(mock_returns['enable_motd_sys_info'],
+        self.assertEquals(MOCK_RETURNS['enable_motd_sys_info'],
                           dsrc.metadata['motd_sys_info'])