From 8c4d88a630025b6fe9f90957343a94105768533f Mon Sep 17 00:00:00 2001
From: Ben Howard <ben.howard@canonical.com>
Date: Tue, 30 Jul 2013 17:00:33 -0600
Subject: Added base64 support to SmartOS datasource. Added documentation on
SmartOS datasource.
---
cloudinit/sources/DataSourceSmartOS.py | 60 +++++++++++++++++++---
doc/sources/smartos/README.rst | 66 ++++++++++++++++++++++++
tests/unittests/test_datasource/test_smartos.py | 67 +++++++++++++++++++++----
3 files changed, 176 insertions(+), 17 deletions(-)
create mode 100644 doc/sources/smartos/README.rst
diff --git a/cloudinit/sources/DataSourceSmartOS.py b/cloudinit/sources/DataSourceSmartOS.py
index 1ce20c10..e0bb871c 100644
--- a/cloudinit/sources/DataSourceSmartOS.py
+++ b/cloudinit/sources/DataSourceSmartOS.py
@@ -27,6 +27,7 @@
#
+import base64
from cloudinit import log as logging
from cloudinit import sources
from cloudinit import util
@@ -49,6 +50,10 @@ SMARTOS_ATTRIB_MAP = {
'motd_sys_info': ('motd_sys_info', True),
}
+# These are values which will never be base64 encoded.
+SMARTOS_NO_BASE64 = ['root_authorized_keys', 'motd_sys_info',
+ 'iptables_disable']
+
class DataSourceSmartOS(sources.DataSource):
def __init__(self, sys_cfg, distro, paths):
@@ -56,6 +61,10 @@ class DataSourceSmartOS(sources.DataSource):
self.seed_dir = os.path.join(paths.seed_dir, 'sdc')
self.is_smartdc = None
self.seed = self.sys_cfg.get("serial_device", DEF_TTY_LOC)
+ self.all_base64 = self.sys_cfg.get("decode_base64", False)
+ self.base_64_encoded = []
+ self.smartos_no_base64 = self.sys_cfg.get("no_base64_decode",
+ SMARTOS_NO_BASE64)
self.seed_timeout = self.sys_cfg.get("serial_timeout",
DEF_TTY_TIMEOUT)
@@ -84,17 +93,41 @@ class DataSourceSmartOS(sources.DataSource):
self.is_smartdc = True
md['instance-id'] = system_uuid
+ self.base_64_encoded = query_data('base_64_enocded',
+ self.seed,
+ self.seed_timeout,
+ strip=True)
+ if self.base_64_encoded:
+ self.base_64_encoded = str(self.base_64_encoded).split(',')
+ else:
+ self.base_64_encoded = []
+
+ if not self.all_base64:
+ self.all_base64 = util.is_true(query_data('meta_encoded_base64',
+ self.seed,
+ self.seed_timeout,
+ strip=True))
+
for ci_noun, attribute in SMARTOS_ATTRIB_MAP.iteritems():
smartos_noun, strip = attribute
+
+ b64encoded = False
+ if self.all_base64 and \
+ (smartos_noun not in self.smartos_no_base64 and \
+ ci_noun not in self.smartos_no_base64):
+ b64encoded = True
+
md[ci_noun] = query_data(smartos_noun, self.seed,
- self.seed_timeout, strip=strip)
+ self.seed_timeout, strip=strip,
+ b64encoded=b64encoded)
if not md['local-hostname']:
md['local-hostname'] = system_uuid
+ ud = None
if md['user-data']:
ud = md['user-data']
- else:
+ elif md['user-script']:
ud = md['user-script']
self.metadata = md
@@ -124,12 +157,17 @@ def get_serial(seed_device, seed_timeout):
return ser
-def query_data(noun, seed_device, seed_timeout, strip=False):
+def query_data(noun, seed_device, seed_timeout, strip=False, b64encoded=False):
"""Makes a request to via the serial console via "GET <NOUN>"
In the response, the first line is the status, while subsequent lines
are is the value. A blank line with a "." is used to indicate end of
response.
+
+ If the response is expected to be base64 encoded, then set b64encoded
+ to true. Unfortantely, there is no way to know if something is 100%
+ encoded, so this method relies on being told if the data is base64 or
+ not.
"""
if not noun:
@@ -153,12 +191,22 @@ def query_data(noun, seed_device, seed_timeout, strip=False):
response.append(m)
ser.close()
+
+ resp = None
if not strip:
- return "".join(response)
+ resp = "".join(response)
+ elif b64encoded:
+ resp = "".join(response).rstrip()
else:
- return "".join(response).rstrip()
+ resp = "".join(response).rstrip()
+
+ if b64encoded:
+ try:
+ return base64.b64decode(resp)
+ except TypeError:
+ return resp
- return None
+ return resp
def dmi_data():
diff --git a/doc/sources/smartos/README.rst b/doc/sources/smartos/README.rst
new file mode 100644
index 00000000..96310857
--- /dev/null
+++ b/doc/sources/smartos/README.rst
@@ -0,0 +1,66 @@
+==================
+SmartOS Datasource
+==================
+
+This datasource finds metadata and user-data from the SmartOS virtualization
+platform (i.e. Joyent).
+
+SmartOS Platform
+----------------
+The SmartOS virtualization platform meta-data to the instance via the second
+serial console. On Linux, this is /dev/ttyS1. The data is a provided via a
+simple protocol, where something queries for the userdata, where the console
+responds with the status and if "SUCCESS" returns until a single ".\n".
+
+The format is lossy. As such, new versions of the SmartOS tooling will include
+support for base64 encoded data.
+
+Userdata
+--------
+
+In SmartOS parlance, user-data is a actually meta-data. This userdata can be
+provided a key-value pairs.
+
+Cloud-init supports reading the traditional meta-data fields supported by the
+SmartOS tools. These are:
+ * root_authorized_keys
+ * hostname
+ * enable_motd_sys_info
+ * iptables_disable
+
+Note: At this time iptables_disable and enable_motd_sys_info are read but
+ are not actioned.
+
+user-script
+-----------
+
+SmartOS traditionally supports sending over a user-script for execution at the
+rc.local level. Cloud-init supports running user-scripts as if they were
+cloud-init user-data. In this sense, anything with a shell interpetter
+directive will run
+
+user-data and user-script
+-------------------------
+
+In the event that a user defines the meta-data key of "user-data" it will
+always supercede any user-script data. This is for consistency.
+
+base64
+------
+
+In order to provide a lossy format, all data except for:
+ * root_authorized_keys
+ * enable_motd_sys_info
+ * iptables_disable
+
+This means that user-script and user-data as well as other values can be
+base64 encoded to provide a lossy format. Since Cloud-init can only guess
+as to whether or not something is truly base64 encoded, the following
+meta-data keys are hints as to whether or not to base64 decode something:
+ * decode_base64: Except for excluded keys, attempt to base64 decode
+ the values. If the value fails to decode properly, it will be
+ returned in its text
+ * base_64_encoded: A comma deliminated list of which values are base64
+ encoded.
+ * no_base64_decode: This is a configuration setting (i.e. /etc/cloud/cloud.cfg.d)
+ that sets which values should not be base64 decoded.
diff --git a/tests/unittests/test_datasource/test_smartos.py b/tests/unittests/test_datasource/test_smartos.py
index 6c12f1e2..ae621433 100644
--- a/tests/unittests/test_datasource/test_smartos.py
+++ b/tests/unittests/test_datasource/test_smartos.py
@@ -22,6 +22,7 @@
# return responses.
#
+import base64
from cloudinit import helpers
from cloudinit.sources import DataSourceSmartOS
@@ -35,7 +36,7 @@ mock_returns = {
'enable_motd_sys_info': None,
'system_uuid': str(uuid.uuid4()),
'smartdc': 'smartdc',
- 'userdata': """
+ 'user-data': """
#!/bin/sh
/bin/true
""",
@@ -48,12 +49,14 @@ class MockSerial(object):
port = None
- def __init__(self):
+ def __init__(self, b64encode=False):
self.last = None
self.last = None
self.new = True
self.count = 0
self.mocked_out = []
+ self.b64encode = b64encode
+ self.b64excluded = DataSourceSmartOS.SMARTOS_NO_BASE64
def open(self):
return True
@@ -87,11 +90,17 @@ class MockSerial(object):
def _format_out(self):
if self.last in mock_returns:
- try:
- for l in mock_returns[self.last].splitlines():
- yield "%s\n" % l
- except:
- yield "%s\n" % mock_returns[self.last]
+ _mret = mock_returns[self.last]
+ if self.b64encode and \
+ self.last not in self.b64excluded:
+ yield base64.b64encode(_mret)
+
+ else:
+ try:
+ for l in _mret.splitlines():
+ yield "%s\n" % l.rstrip()
+ except:
+ yield "%s\n" % _mret.rstrip()
yield '\n'
yield '.'
@@ -116,16 +125,19 @@ class TestSmartOSDataSource(MockerTestCase):
ret = apply_patches(patches)
self.unapply += ret
- def _get_ds(self):
+ def _get_ds(self, b64encode=False, sys_cfg=None):
+ mod = DataSourceSmartOS
def _get_serial(*_):
- return MockSerial()
+ return MockSerial(b64encode=b64encode)
def _dmi_data():
return mock_returns['system_uuid'], 'smartdc'
- data = {'sys_cfg': {}}
- mod = DataSourceSmartOS
+ if not sys_cfg:
+ sys_cfg = {}
+
+ data = {'sys_cfg': sys_cfg}
self.apply_patches([(mod, 'get_serial', _get_serial)])
self.apply_patches([(mod, 'dmi_data', _dmi_data)])
dsrc = mod.DataSourceSmartOS(
@@ -158,6 +170,13 @@ class TestSmartOSDataSource(MockerTestCase):
self.assertEquals(mock_returns['root_authorized_keys'],
dsrc.metadata['public-keys'])
+ def test_hostname_b64(self):
+ dsrc = self._get_ds(b64encode=True)
+ ret = dsrc.get_data()
+ self.assertTrue(ret)
+ self.assertEquals(base64.b64encode(mock_returns['hostname']),
+ dsrc.metadata['local-hostname'])
+
def test_hostname(self):
dsrc = self._get_ds()
ret = dsrc.get_data()
@@ -165,6 +184,32 @@ class TestSmartOSDataSource(MockerTestCase):
self.assertEquals(mock_returns['hostname'],
dsrc.metadata['local-hostname'])
+ def test_base64(self):
+ """This tests to make sure that SmartOS system key/value pairs
+ are not interpetted as being base64 encoded, while making
+ sure that the others are when 'decode_base64' is set"""
+ dsrc = self._get_ds(sys_cfg={'decode_base64': True},
+ b64encode=True)
+ ret = dsrc.get_data()
+ self.assertTrue(ret)
+ self.assertEquals(mock_returns['hostname'],
+ dsrc.metadata['local-hostname'])
+ self.assertEquals("%s" % mock_returns['user-data'],
+ dsrc.userdata_raw)
+ self.assertEquals(mock_returns['root_authorized_keys'],
+ dsrc.metadata['public-keys'])
+ self.assertEquals(mock_returns['disable_iptables_flag'],
+ dsrc.metadata['iptables_disable'])
+ self.assertEquals(mock_returns['enable_motd_sys_info'],
+ dsrc.metadata['motd_sys_info'])
+
+ def test_userdata(self):
+ dsrc = self._get_ds()
+ ret = dsrc.get_data()
+ self.assertTrue(ret)
+ self.assertEquals("%s\n" % mock_returns['user-data'],
+ dsrc.userdata_raw)
+
def test_disable_iptables_flag(self):
dsrc = self._get_ds()
ret = dsrc.get_data()
--
cgit v1.2.3
From e683ab2baa3e67614edcd409122bd1aec99737e0 Mon Sep 17 00:00:00 2001
From: Ben Howard <ben.howard@canonical.com>
Date: Tue, 20 Aug 2013 09:56:25 -0600
Subject: Fixed no_base64_decode settings
---
cloudinit/sources/DataSourceSmartOS.py | 26 ++++++++++++++++++++-----
doc/examples/cloud-config-datasources.txt | 2 +-
doc/sources/smartos/README.rst | 12 ++++++------
tests/unittests/test_datasource/test_smartos.py | 8 ++++++++
4 files changed, 36 insertions(+), 12 deletions(-)
diff --git a/cloudinit/sources/DataSourceSmartOS.py b/cloudinit/sources/DataSourceSmartOS.py
index e0bb871c..1cf9e4f0 100644
--- a/cloudinit/sources/DataSourceSmartOS.py
+++ b/cloudinit/sources/DataSourceSmartOS.py
@@ -35,7 +35,7 @@ import os
import os.path
import serial
-
+DS_NAME = 'SmartOS'
DEF_TTY_LOC = '/dev/ttyS1'
DEF_TTY_TIMEOUT = 60
LOG = logging.getLogger(__name__)
@@ -60,13 +60,14 @@ class DataSourceSmartOS(sources.DataSource):
sources.DataSource.__init__(self, sys_cfg, distro, paths)
self.seed_dir = os.path.join(paths.seed_dir, 'sdc')
self.is_smartdc = None
+ self.base_64_encoded = []
self.seed = self.sys_cfg.get("serial_device", DEF_TTY_LOC)
self.all_base64 = self.sys_cfg.get("decode_base64", False)
- self.base_64_encoded = []
- self.smartos_no_base64 = self.sys_cfg.get("no_base64_decode",
- SMARTOS_NO_BASE64)
self.seed_timeout = self.sys_cfg.get("serial_timeout",
DEF_TTY_TIMEOUT)
+ self.smartos_no_base64 = SMARTOS_NO_BASE64
+ if 'no_base64_decode' in self.ds_cfg:
+ self.smartos_no_base64 = self.ds_cfg['no_base64_decode']
def __str__(self):
root = sources.DataSource.__str__(self)
@@ -137,10 +138,25 @@ class DataSourceSmartOS(sources.DataSource):
def get_instance_id(self):
return self.metadata['instance-id']
+ def not_b64_var(self, var):
+ """Return true if value is read as b64."""
+ if var in self.smartos_no_base64 or \
+ not self.all_base64:
+ return True
+ return False
+
+ def is_b64_var(self, var):
+ """Return true if value is read as b64."""
+ if self.all_base64 or (
+ var not in self.smartos_no_base64 and
+ var in self.base_64_encoded):
+ return True
+ return False
+
def get_serial(seed_device, seed_timeout):
"""This is replaced in unit testing, allowing us to replace
- serial.Serial with a mocked class
+ serial.Serial with a mocked class.
The timeout value of 60 seconds should never be hit. The value
is taken from SmartOS own provisioning tools. Since we are reading
diff --git a/doc/examples/cloud-config-datasources.txt b/doc/examples/cloud-config-datasources.txt
index 6544448e..6ec0d57e 100644
--- a/doc/examples/cloud-config-datasources.txt
+++ b/doc/examples/cloud-config-datasources.txt
@@ -55,5 +55,5 @@ datasource:
# Smart OS datasource works over a serial console interacting with
# a server on the other end. By default, the second serial console is the
# device. SmartOS also uses a serial timeout of 60 seconds.
- serial device: /dev/ttyS1
+ serial_device: /dev/ttyS1
serial timeout: 60
diff --git a/doc/sources/smartos/README.rst b/doc/sources/smartos/README.rst
index 96310857..ba90e7af 100644
--- a/doc/sources/smartos/README.rst
+++ b/doc/sources/smartos/README.rst
@@ -12,8 +12,7 @@ serial console. On Linux, this is /dev/ttyS1. The data is a provided via a
simple protocol, where something queries for the userdata, where the console
responds with the status and if "SUCCESS" returns until a single ".\n".
-The format is lossy. As such, new versions of the SmartOS tooling will include
-support for base64 encoded data.
+New versions of the SmartOS tooling will include support for base64 encoded data.
Userdata
--------
@@ -48,15 +47,16 @@ always supercede any user-script data. This is for consistency.
base64
------
-In order to provide a lossy format, all data except for:
+The following are excempt from base64 encoding, owing to the fact that they
+are provided by SmartOS:
* root_authorized_keys
* enable_motd_sys_info
* iptables_disable
This means that user-script and user-data as well as other values can be
-base64 encoded to provide a lossy format. Since Cloud-init can only guess
-as to whether or not something is truly base64 encoded, the following
-meta-data keys are hints as to whether or not to base64 decode something:
+base64 encoded. Since Cloud-init can only guess as to whether or not something
+is truly base64 encoded, the following meta-data keys are hints as to whether
+or not to base64 decode something:
* decode_base64: Except for excluded keys, attempt to base64 decode
the values. If the value fails to decode properly, it will be
returned in its text
diff --git a/tests/unittests/test_datasource/test_smartos.py b/tests/unittests/test_datasource/test_smartos.py
index ae621433..b9b3a479 100644
--- a/tests/unittests/test_datasource/test_smartos.py
+++ b/tests/unittests/test_datasource/test_smartos.py
@@ -36,6 +36,7 @@ mock_returns = {
'enable_motd_sys_info': None,
'system_uuid': str(uuid.uuid4()),
'smartdc': 'smartdc',
+ 'test-var1': 'some data',
'user-data': """
#!/bin/sh
/bin/true
@@ -156,6 +157,13 @@ class TestSmartOSDataSource(MockerTestCase):
self.assertTrue(ret)
self.assertTrue(dsrc.is_smartdc)
+ def test_no_base64(self):
+ sys_cfg = {'no_base64_decode': ['test_var1'], 'all_base': True}
+ dsrc = self._get_ds(sys_cfg=sys_cfg)
+ ret = dsrc.get_data()
+ self.assertTrue(ret)
+ self.assertTrue(dsrc.not_b64_var('test-var'))
+
def test_uuid(self):
dsrc = self._get_ds()
ret = dsrc.get_data()
--
cgit v1.2.3
From 0a667454289a788a6f406e66d78a34c7ec914daa Mon Sep 17 00:00:00 2001
From: Ben Howard <ben.howard@canonical.com>
Date: Fri, 23 Aug 2013 09:10:30 -0600
Subject: Fixed some typos. Change decode_base64 from sys_cfg to ds_cfg
---
cloudinit/sources/DataSourceSmartOS.py | 5 ++++-
doc/sources/smartos/README.rst | 6 +++---
2 files changed, 7 insertions(+), 4 deletions(-)
diff --git a/cloudinit/sources/DataSourceSmartOS.py b/cloudinit/sources/DataSourceSmartOS.py
index 1cf9e4f0..45f03a7e 100644
--- a/cloudinit/sources/DataSourceSmartOS.py
+++ b/cloudinit/sources/DataSourceSmartOS.py
@@ -62,9 +62,12 @@ class DataSourceSmartOS(sources.DataSource):
self.is_smartdc = None
self.base_64_encoded = []
self.seed = self.sys_cfg.get("serial_device", DEF_TTY_LOC)
- self.all_base64 = self.sys_cfg.get("decode_base64", False)
self.seed_timeout = self.sys_cfg.get("serial_timeout",
DEF_TTY_TIMEOUT)
+ self.all_base64 = False
+ if 'decode_base64' in self.ds_cfg:
+ self.all_base64 = self.ds_cfg['decode_base64']
+
self.smartos_no_base64 = SMARTOS_NO_BASE64
if 'no_base64_decode' in self.ds_cfg:
self.smartos_no_base64 = self.ds_cfg['no_base64_decode']
diff --git a/doc/sources/smartos/README.rst b/doc/sources/smartos/README.rst
index ba90e7af..8f72fa0f 100644
--- a/doc/sources/smartos/README.rst
+++ b/doc/sources/smartos/README.rst
@@ -35,7 +35,7 @@ user-script
SmartOS traditionally supports sending over a user-script for execution at the
rc.local level. Cloud-init supports running user-scripts as if they were
-cloud-init user-data. In this sense, anything with a shell interpetter
+cloud-init user-data. In this sense, anything with a shell interpreter
directive will run
user-data and user-script
@@ -47,7 +47,7 @@ always supercede any user-script data. This is for consistency.
base64
------
-The following are excempt from base64 encoding, owing to the fact that they
+The following are exempt from base64 encoding, owing to the fact that they
are provided by SmartOS:
* root_authorized_keys
* enable_motd_sys_info
@@ -63,4 +63,4 @@ or not to base64 decode something:
* base_64_encoded: A comma deliminated list of which values are base64
encoded.
* no_base64_decode: This is a configuration setting (i.e. /etc/cloud/cloud.cfg.d)
- that sets which values should not be base64 decoded.
+ that sets which values should not be base64 decoded.
--
cgit v1.2.3
From 10c8ec1e5c1b16572a38afd08ee794d28c450054 Mon Sep 17 00:00:00 2001
From: Scott Moser <smoser@ubuntu.com>
Date: Sat, 24 Aug 2013 00:07:35 -0400
Subject: changes to behavior on specifying keys.
The most likely end user operation (or at least a valid one) for base64
encoding would be to encode the user-data, but leave all other values
as plaintext.
In order to facilitate that, the user can simply add:
b64-user-data=true
to indicate that user-data is base64 encoded.
Other changes here are to change the cloud-config and metadata keynames
that are used.
base64_all = boolean(True)
base64_keys = [list, of, keys]
Fixed up tests to accomodate.
---
cloudinit/sources/DataSourceSmartOS.py | 94 ++++++---------
doc/examples/cloud-config-datasources.txt | 10 +-
doc/sources/smartos/README.rst | 16 ++-
tests/unittests/test_datasource/test_smartos.py | 153 ++++++++++++++----------
4 files changed, 149 insertions(+), 124 deletions(-)
diff --git a/cloudinit/sources/DataSourceSmartOS.py b/cloudinit/sources/DataSourceSmartOS.py
index 45f03a7e..d348d20b 100644
--- a/cloudinit/sources/DataSourceSmartOS.py
+++ b/cloudinit/sources/DataSourceSmartOS.py
@@ -35,7 +35,6 @@ import os
import os.path
import serial
-DS_NAME = 'SmartOS'
DEF_TTY_LOC = '/dev/ttyS1'
DEF_TTY_TIMEOUT = 60
LOG = logging.getLogger(__name__)
@@ -51,6 +50,7 @@ SMARTOS_ATTRIB_MAP = {
}
# These are values which will never be base64 encoded.
+# They come from the cloud platform, not user
SMARTOS_NO_BASE64 = ['root_authorized_keys', 'motd_sys_info',
'iptables_disable']
@@ -60,17 +60,13 @@ class DataSourceSmartOS(sources.DataSource):
sources.DataSource.__init__(self, sys_cfg, distro, paths)
self.seed_dir = os.path.join(paths.seed_dir, 'sdc')
self.is_smartdc = None
- self.base_64_encoded = []
- self.seed = self.sys_cfg.get("serial_device", DEF_TTY_LOC)
- self.seed_timeout = self.sys_cfg.get("serial_timeout",
- DEF_TTY_TIMEOUT)
- self.all_base64 = False
- if 'decode_base64' in self.ds_cfg:
- self.all_base64 = self.ds_cfg['decode_base64']
-
- self.smartos_no_base64 = SMARTOS_NO_BASE64
- if 'no_base64_decode' in self.ds_cfg:
- self.smartos_no_base64 = self.ds_cfg['no_base64_decode']
+
+ self.seed = self.ds_cfg.get("serial_device", DEF_TTY_LOC)
+ self.seed_timeout = self.ds_cfg.get("serial_timeout", DEF_TTY_TIMEOUT)
+ self.smartos_no_base64 = self.ds_cfg.get('no_base64_decode',
+ SMARTOS_NO_BASE64)
+ self.b64_keys = self.ds_cfg.get('base64_keys', [])
+ self.b64_all = self.ds_cfg.get('base64_all', False)
def __str__(self):
root = sources.DataSource.__str__(self)
@@ -92,38 +88,22 @@ class DataSourceSmartOS(sources.DataSource):
system_uuid, system_type = dmi_info
if 'smartdc' not in system_type.lower():
- LOG.debug("Host is not on SmartOS")
+ LOG.debug("Host is not on SmartOS. system_type=%s", system_type)
return False
self.is_smartdc = True
md['instance-id'] = system_uuid
- self.base_64_encoded = query_data('base_64_enocded',
- self.seed,
- self.seed_timeout,
- strip=True)
- if self.base_64_encoded:
- self.base_64_encoded = str(self.base_64_encoded).split(',')
- else:
- self.base_64_encoded = []
+ b64_keys = self.query('base64_keys', strip=True, b64=False)
+ if b64_keys is not None:
+ self.b64_keys = [k.strip() for k in str(b64_keys).split(',')]
- if not self.all_base64:
- self.all_base64 = util.is_true(query_data('meta_encoded_base64',
- self.seed,
- self.seed_timeout,
- strip=True))
+ b64_all = self.query('base64_all', strip=True, b64=False)
+ if b64_all is not None:
+ self.b64_all = util.is_true(b64_all)
for ci_noun, attribute in SMARTOS_ATTRIB_MAP.iteritems():
smartos_noun, strip = attribute
-
- b64encoded = False
- if self.all_base64 and \
- (smartos_noun not in self.smartos_no_base64 and \
- ci_noun not in self.smartos_no_base64):
- b64encoded = True
-
- md[ci_noun] = query_data(smartos_noun, self.seed,
- self.seed_timeout, strip=strip,
- b64encoded=b64encoded)
+ md[ci_noun] = self.query(smartos_noun, strip=strip)
if not md['local-hostname']:
md['local-hostname'] = system_uuid
@@ -141,20 +121,16 @@ class DataSourceSmartOS(sources.DataSource):
def get_instance_id(self):
return self.metadata['instance-id']
- def not_b64_var(self, var):
- """Return true if value is read as b64."""
- if var in self.smartos_no_base64 or \
- not self.all_base64:
- return True
- return False
+ def query(self, noun, strip=False, default=None, b64=None):
+ if b64 is None:
+ if noun in self.smartos_no_base64:
+ b64 = False
+ elif self.b64_all or noun in self.b64_keys:
+ b64 = True
- def is_b64_var(self, var):
- """Return true if value is read as b64."""
- if self.all_base64 or (
- var not in self.smartos_no_base64 and
- var in self.base_64_encoded):
- return True
- return False
+ return query_data(noun=noun, strip=strip, seed_device=self.seed,
+ seed_timeout=self.seed_timeout, default=default,
+ b64=b64)
def get_serial(seed_device, seed_timeout):
@@ -176,7 +152,8 @@ def get_serial(seed_device, seed_timeout):
return ser
-def query_data(noun, seed_device, seed_timeout, strip=False, b64encoded=False):
+def query_data(noun, seed_device, seed_timeout, strip=False, default=None,
+ b64=None):
"""Makes a request to via the serial console via "GET <NOUN>"
In the response, the first line is the status, while subsequent lines
@@ -200,7 +177,7 @@ def query_data(noun, seed_device, seed_timeout, strip=False, b64encoded=False):
if 'SUCCESS' not in status:
ser.close()
- return None
+ return default
while not eom_found:
m = ser.readline()
@@ -211,18 +188,23 @@ def query_data(noun, seed_device, seed_timeout, strip=False, b64encoded=False):
ser.close()
+ if b64 is None:
+ b64 = query_data('b64-%s' % noun, seed_device=seed_device,
+ seed_timeout=seed_timeout, b64=False,
+ default=False, strip=True)
+ b64 = util.is_true(b64)
+
resp = None
- if not strip:
- resp = "".join(response)
- elif b64encoded:
+ if b64 or strip:
resp = "".join(response).rstrip()
else:
- resp = "".join(response).rstrip()
+ resp = "".join(response)
- if b64encoded:
+ if b64:
try:
return base64.b64decode(resp)
except TypeError:
+ LOG.warn("Failed base64 decoding key '%s'", noun)
return resp
return resp
diff --git a/doc/examples/cloud-config-datasources.txt b/doc/examples/cloud-config-datasources.txt
index 6ec0d57e..65a3cdf5 100644
--- a/doc/examples/cloud-config-datasources.txt
+++ b/doc/examples/cloud-config-datasources.txt
@@ -56,4 +56,12 @@ datasource:
# a server on the other end. By default, the second serial console is the
# device. SmartOS also uses a serial timeout of 60 seconds.
serial_device: /dev/ttyS1
- serial timeout: 60
+ serial_timeout: 60
+
+ # a list of keys that will not be base64 decoded even if base64_all
+ no_base64_decode: ['root_authorized_keys', 'motd_sys_info',
+ 'iptables_disable']
+ # a plaintext, comma delimited list of keys whose values are b64 encoded
+ base64_keys: []
+ # a boolean indicating that all keys not in 'no_base64_decode' are encoded
+ base64_all: False
diff --git a/doc/sources/smartos/README.rst b/doc/sources/smartos/README.rst
index 8f72fa0f..fd4e496d 100644
--- a/doc/sources/smartos/README.rst
+++ b/doc/sources/smartos/README.rst
@@ -53,14 +53,20 @@ are provided by SmartOS:
* enable_motd_sys_info
* iptables_disable
+This list can be changed through system config of variable 'no_base64_decode'.
+
This means that user-script and user-data as well as other values can be
base64 encoded. Since Cloud-init can only guess as to whether or not something
is truly base64 encoded, the following meta-data keys are hints as to whether
or not to base64 decode something:
- * decode_base64: Except for excluded keys, attempt to base64 decode
+ * base64_all: Except for excluded keys, attempt to base64 decode
the values. If the value fails to decode properly, it will be
returned in its text
- * base_64_encoded: A comma deliminated list of which values are base64
- encoded.
- * no_base64_decode: This is a configuration setting (i.e. /etc/cloud/cloud.cfg.d)
- that sets which values should not be base64 decoded.
+ * base64_keys: A comma deliminated list of which keys are base64 encoded.
+ * b64-<key>:
+ for any key, if there exists an entry in the metadata for 'b64-<key>'
+ Then 'b64-<key>' is expected to be a plaintext boolean indicating whether
+ or not its value is encoded.
+ * no_base64_decode: This is a configuration setting
+ (i.e. /etc/cloud/cloud.cfg.d) that sets which values should not be
+ base64 decoded.
diff --git a/tests/unittests/test_datasource/test_smartos.py b/tests/unittests/test_datasource/test_smartos.py
index b9b3a479..f53715b0 100644
--- a/tests/unittests/test_datasource/test_smartos.py
+++ b/tests/unittests/test_datasource/test_smartos.py
@@ -29,20 +29,17 @@ from cloudinit.sources import DataSourceSmartOS
from mocker import MockerTestCase
import uuid
-mock_returns = {
+MOCK_RETURNS = {
'hostname': 'test-host',
'root_authorized_keys': 'ssh-rsa AAAAB3Nz...aC1yc2E= keyname',
'disable_iptables_flag': None,
'enable_motd_sys_info': None,
- 'system_uuid': str(uuid.uuid4()),
- 'smartdc': 'smartdc',
'test-var1': 'some data',
- 'user-data': """
-#!/bin/sh
-/bin/true
-""",
+ 'user-data': '\n'.join(['#!/bin/sh', '/bin/true', '']),
}
+DMI_DATA_RETURN = (str(uuid.uuid4()), 'smartdc')
+
class MockSerial(object):
"""Fake a serial terminal for testing the code that
@@ -50,14 +47,13 @@ class MockSerial(object):
port = None
- def __init__(self, b64encode=False):
+ def __init__(self, mockdata):
self.last = None
self.last = None
self.new = True
self.count = 0
self.mocked_out = []
- self.b64encode = b64encode
- self.b64excluded = DataSourceSmartOS.SMARTOS_NO_BASE64
+ self.mockdata = mockdata
def open(self):
return True
@@ -75,12 +71,12 @@ class MockSerial(object):
def readline(self):
if self.new:
self.new = False
- if self.last in mock_returns:
+ if self.last in self.mockdata:
return 'SUCCESS\n'
else:
return 'NOTFOUND %s\n' % self.last
- if self.last in mock_returns:
+ if self.last in self.mockdata:
if not self.mocked_out:
self.mocked_out = [x for x in self._format_out()]
print self.mocked_out
@@ -90,21 +86,16 @@ class MockSerial(object):
return self.mocked_out[self.count - 1]
def _format_out(self):
- if self.last in mock_returns:
- _mret = mock_returns[self.last]
- if self.b64encode and \
- self.last not in self.b64excluded:
- yield base64.b64encode(_mret)
-
- else:
- try:
- for l in _mret.splitlines():
- yield "%s\n" % l.rstrip()
- except:
- yield "%s\n" % _mret.rstrip()
+ if self.last in self.mockdata:
+ _mret = self.mockdata[self.last]
+ try:
+ for l in _mret.splitlines():
+ yield "%s\n" % l.rstrip()
+ except:
+ yield "%s\n" % _mret.rstrip()
- yield '\n'
yield '.'
+ yield '\n'
class TestSmartOSDataSource(MockerTestCase):
@@ -126,26 +117,36 @@ class TestSmartOSDataSource(MockerTestCase):
ret = apply_patches(patches)
self.unapply += ret
- def _get_ds(self, b64encode=False, sys_cfg=None):
+ def _get_ds(self, sys_cfg=None, ds_cfg=None, mockdata=None, dmi_data=None):
mod = DataSourceSmartOS
+ if mockdata is None:
+ mockdata = MOCK_RETURNS
+
+ if dmi_data is None:
+ dmi_data = DMI_DATA_RETURN
+
def _get_serial(*_):
- return MockSerial(b64encode=b64encode)
+ return MockSerial(mockdata)
def _dmi_data():
- return mock_returns['system_uuid'], 'smartdc'
+ return dmi_data
- if not sys_cfg:
+ if sys_cfg is None:
sys_cfg = {}
- data = {'sys_cfg': sys_cfg}
+ if ds_cfg is not None:
+ sys_cfg['datasource'] = sys_cfg.get('datasource', {})
+ sys_cfg['datasource']['SmartOS'] = ds_cfg
+
self.apply_patches([(mod, 'get_serial', _get_serial)])
self.apply_patches([(mod, 'dmi_data', _dmi_data)])
- dsrc = mod.DataSourceSmartOS(
- data.get('sys_cfg', {}), distro=None, paths=self.paths)
+ dsrc = mod.DataSourceSmartOS(sys_cfg, distro=None,
+ paths=self.paths)
return dsrc
def test_seed(self):
+ # default seed should be /dev/ttyS1
dsrc = self._get_ds()
ret = dsrc.get_data()
self.assertTrue(ret)
@@ -158,78 +159,106 @@ class TestSmartOSDataSource(MockerTestCase):
self.assertTrue(dsrc.is_smartdc)
def test_no_base64(self):
- sys_cfg = {'no_base64_decode': ['test_var1'], 'all_base': True}
- dsrc = self._get_ds(sys_cfg=sys_cfg)
+ ds_cfg = {'no_base64_decode': ['test_var1'], 'all_base': True}
+ dsrc = self._get_ds(ds_cfg=ds_cfg)
ret = dsrc.get_data()
self.assertTrue(ret)
- self.assertTrue(dsrc.not_b64_var('test-var'))
def test_uuid(self):
- dsrc = self._get_ds()
+ dsrc = self._get_ds(mockdata=MOCK_RETURNS)
ret = dsrc.get_data()
self.assertTrue(ret)
- self.assertEquals(mock_returns['system_uuid'],
- dsrc.metadata['instance-id'])
+ self.assertEquals(DMI_DATA_RETURN[0], dsrc.metadata['instance-id'])
def test_root_keys(self):
- dsrc = self._get_ds()
+ dsrc = self._get_ds(mockdata=MOCK_RETURNS)
ret = dsrc.get_data()
self.assertTrue(ret)
- self.assertEquals(mock_returns['root_authorized_keys'],
+ self.assertEquals(MOCK_RETURNS['root_authorized_keys'],
dsrc.metadata['public-keys'])
def test_hostname_b64(self):
- dsrc = self._get_ds(b64encode=True)
+ dsrc = self._get_ds(mockdata=MOCK_RETURNS)
ret = dsrc.get_data()
self.assertTrue(ret)
- self.assertEquals(base64.b64encode(mock_returns['hostname']),
+ self.assertEquals(MOCK_RETURNS['hostname'],
dsrc.metadata['local-hostname'])
def test_hostname(self):
- dsrc = self._get_ds()
+ dsrc = self._get_ds(mockdata=MOCK_RETURNS)
ret = dsrc.get_data()
self.assertTrue(ret)
- self.assertEquals(mock_returns['hostname'],
+ self.assertEquals(MOCK_RETURNS['hostname'],
dsrc.metadata['local-hostname'])
- def test_base64(self):
- """This tests to make sure that SmartOS system key/value pairs
- are not interpetted as being base64 encoded, while making
- sure that the others are when 'decode_base64' is set"""
- dsrc = self._get_ds(sys_cfg={'decode_base64': True},
- b64encode=True)
+ def test_base64_all(self):
+ # metadata provided base64_all of true
+ my_returns = MOCK_RETURNS.copy()
+ my_returns['base64_all'] = "true"
+ for k in ('hostname', 'user-data'):
+ my_returns[k] = base64.b64encode(my_returns[k])
+
+ dsrc = self._get_ds(mockdata=my_returns)
ret = dsrc.get_data()
self.assertTrue(ret)
- self.assertEquals(mock_returns['hostname'],
+ self.assertEquals(MOCK_RETURNS['hostname'],
dsrc.metadata['local-hostname'])
- self.assertEquals("%s" % mock_returns['user-data'],
+ self.assertEquals(MOCK_RETURNS['user-data'],
dsrc.userdata_raw)
- self.assertEquals(mock_returns['root_authorized_keys'],
+ self.assertEquals(MOCK_RETURNS['root_authorized_keys'],
dsrc.metadata['public-keys'])
- self.assertEquals(mock_returns['disable_iptables_flag'],
+ self.assertEquals(MOCK_RETURNS['disable_iptables_flag'],
dsrc.metadata['iptables_disable'])
- self.assertEquals(mock_returns['enable_motd_sys_info'],
+ self.assertEquals(MOCK_RETURNS['enable_motd_sys_info'],
dsrc.metadata['motd_sys_info'])
+ def test_b64_userdata(self):
+ my_returns = MOCK_RETURNS.copy()
+ my_returns['b64-user-data'] = "true"
+ my_returns['b64-hostname'] = "true"
+ for k in ('hostname', 'user-data'):
+ my_returns[k] = base64.b64encode(my_returns[k])
+
+ dsrc = self._get_ds(mockdata=my_returns)
+ ret = dsrc.get_data()
+ self.assertTrue(ret)
+ self.assertEquals(MOCK_RETURNS['hostname'],
+ dsrc.metadata['local-hostname'])
+ self.assertEquals(MOCK_RETURNS['user-data'], dsrc.userdata_raw)
+ self.assertEquals(MOCK_RETURNS['root_authorized_keys'],
+ dsrc.metadata['public-keys'])
+
+ def test_b64_keys(self):
+ my_returns = MOCK_RETURNS.copy()
+ my_returns['base64_keys'] = 'hostname,ignored'
+ for k in ('hostname',):
+ my_returns[k] = base64.b64encode(my_returns[k])
+
+ dsrc = self._get_ds(mockdata=my_returns)
+ ret = dsrc.get_data()
+ self.assertTrue(ret)
+ self.assertEquals(MOCK_RETURNS['hostname'],
+ dsrc.metadata['local-hostname'])
+ self.assertEquals(MOCK_RETURNS['user-data'], dsrc.userdata_raw)
+
def test_userdata(self):
- dsrc = self._get_ds()
+ dsrc = self._get_ds(mockdata=MOCK_RETURNS)
ret = dsrc.get_data()
self.assertTrue(ret)
- self.assertEquals("%s\n" % mock_returns['user-data'],
- dsrc.userdata_raw)
+ self.assertEquals(MOCK_RETURNS['user-data'], dsrc.userdata_raw)
def test_disable_iptables_flag(self):
- dsrc = self._get_ds()
+ dsrc = self._get_ds(mockdata=MOCK_RETURNS)
ret = dsrc.get_data()
self.assertTrue(ret)
- self.assertEquals(mock_returns['disable_iptables_flag'],
+ self.assertEquals(MOCK_RETURNS['disable_iptables_flag'],
dsrc.metadata['iptables_disable'])
def test_motd_sys_info(self):
- dsrc = self._get_ds()
+ dsrc = self._get_ds(mockdata=MOCK_RETURNS)
ret = dsrc.get_data()
self.assertTrue(ret)
- self.assertEquals(mock_returns['enable_motd_sys_info'],
+ self.assertEquals(MOCK_RETURNS['enable_motd_sys_info'],
dsrc.metadata['motd_sys_info'])
--
cgit v1.2.3