diff options
| -rw-r--r-- | cloudinit/sources/DataSourceSmartOS.py | 79 | ||||
| -rw-r--r-- | doc/examples/cloud-config-datasources.txt | 12 | ||||
| -rw-r--r-- | doc/sources/smartos/README.rst | 72 | ||||
| -rw-r--r-- | tests/unittests/test_datasource/test_smartos.py | 148 |
4 files changed, 261 insertions, 50 deletions
diff --git a/cloudinit/sources/DataSourceSmartOS.py b/cloudinit/sources/DataSourceSmartOS.py index 1ce20c10..d348d20b 100644 --- a/cloudinit/sources/DataSourceSmartOS.py +++ b/cloudinit/sources/DataSourceSmartOS.py @@ -27,6 +27,7 @@ # +import base64 from cloudinit import log as logging from cloudinit import sources from cloudinit import util @@ -34,7 +35,6 @@ import os import os.path import serial - DEF_TTY_LOC = '/dev/ttyS1' DEF_TTY_TIMEOUT = 60 LOG = logging.getLogger(__name__) @@ -49,15 +49,24 @@ SMARTOS_ATTRIB_MAP = { 'motd_sys_info': ('motd_sys_info', True), } +# These are values which will never be base64 encoded. +# They come from the cloud platform, not user +SMARTOS_NO_BASE64 = ['root_authorized_keys', 'motd_sys_info', + 'iptables_disable'] + class DataSourceSmartOS(sources.DataSource): def __init__(self, sys_cfg, distro, paths): sources.DataSource.__init__(self, sys_cfg, distro, paths) self.seed_dir = os.path.join(paths.seed_dir, 'sdc') self.is_smartdc = None - self.seed = self.sys_cfg.get("serial_device", DEF_TTY_LOC) - self.seed_timeout = self.sys_cfg.get("serial_timeout", - DEF_TTY_TIMEOUT) + + self.seed = self.ds_cfg.get("serial_device", DEF_TTY_LOC) + self.seed_timeout = self.ds_cfg.get("serial_timeout", DEF_TTY_TIMEOUT) + self.smartos_no_base64 = self.ds_cfg.get('no_base64_decode', + SMARTOS_NO_BASE64) + self.b64_keys = self.ds_cfg.get('base64_keys', []) + self.b64_all = self.ds_cfg.get('base64_all', False) def __str__(self): root = sources.DataSource.__str__(self) @@ -79,22 +88,30 @@ class DataSourceSmartOS(sources.DataSource): system_uuid, system_type = dmi_info if 'smartdc' not in system_type.lower(): - LOG.debug("Host is not on SmartOS") + LOG.debug("Host is not on SmartOS. system_type=%s", system_type) return False self.is_smartdc = True md['instance-id'] = system_uuid + b64_keys = self.query('base64_keys', strip=True, b64=False) + if b64_keys is not None: + self.b64_keys = [k.strip() for k in str(b64_keys).split(',')] + + b64_all = self.query('base64_all', strip=True, b64=False) + if b64_all is not None: + self.b64_all = util.is_true(b64_all) + for ci_noun, attribute in SMARTOS_ATTRIB_MAP.iteritems(): smartos_noun, strip = attribute - md[ci_noun] = query_data(smartos_noun, self.seed, - self.seed_timeout, strip=strip) + md[ci_noun] = self.query(smartos_noun, strip=strip) if not md['local-hostname']: md['local-hostname'] = system_uuid + ud = None if md['user-data']: ud = md['user-data'] - else: + elif md['user-script']: ud = md['user-script'] self.metadata = md @@ -104,10 +121,21 @@ class DataSourceSmartOS(sources.DataSource): def get_instance_id(self): return self.metadata['instance-id'] + def query(self, noun, strip=False, default=None, b64=None): + if b64 is None: + if noun in self.smartos_no_base64: + b64 = False + elif self.b64_all or noun in self.b64_keys: + b64 = True + + return query_data(noun=noun, strip=strip, seed_device=self.seed, + seed_timeout=self.seed_timeout, default=default, + b64=b64) + def get_serial(seed_device, seed_timeout): """This is replaced in unit testing, allowing us to replace - serial.Serial with a mocked class + serial.Serial with a mocked class. The timeout value of 60 seconds should never be hit. The value is taken from SmartOS own provisioning tools. Since we are reading @@ -124,12 +152,18 @@ def get_serial(seed_device, seed_timeout): return ser -def query_data(noun, seed_device, seed_timeout, strip=False): +def query_data(noun, seed_device, seed_timeout, strip=False, default=None, + b64=None): """Makes a request to via the serial console via "GET <NOUN>" In the response, the first line is the status, while subsequent lines are is the value. A blank line with a "." is used to indicate end of response. + + If the response is expected to be base64 encoded, then set b64encoded + to true. Unfortantely, there is no way to know if something is 100% + encoded, so this method relies on being told if the data is base64 or + not. """ if not noun: @@ -143,7 +177,7 @@ def query_data(noun, seed_device, seed_timeout, strip=False): if 'SUCCESS' not in status: ser.close() - return None + return default while not eom_found: m = ser.readline() @@ -153,12 +187,27 @@ def query_data(noun, seed_device, seed_timeout, strip=False): response.append(m) ser.close() - if not strip: - return "".join(response) + + if b64 is None: + b64 = query_data('b64-%s' % noun, seed_device=seed_device, + seed_timeout=seed_timeout, b64=False, + default=False, strip=True) + b64 = util.is_true(b64) + + resp = None + if b64 or strip: + resp = "".join(response).rstrip() else: - return "".join(response).rstrip() + resp = "".join(response) + + if b64: + try: + return base64.b64decode(resp) + except TypeError: + LOG.warn("Failed base64 decoding key '%s'", noun) + return resp - return None + return resp def dmi_data(): diff --git a/doc/examples/cloud-config-datasources.txt b/doc/examples/cloud-config-datasources.txt index 6544448e..65a3cdf5 100644 --- a/doc/examples/cloud-config-datasources.txt +++ b/doc/examples/cloud-config-datasources.txt @@ -55,5 +55,13 @@ datasource: # Smart OS datasource works over a serial console interacting with # a server on the other end. By default, the second serial console is the # device. SmartOS also uses a serial timeout of 60 seconds. - serial device: /dev/ttyS1 - serial timeout: 60 + serial_device: /dev/ttyS1 + serial_timeout: 60 + + # a list of keys that will not be base64 decoded even if base64_all + no_base64_decode: ['root_authorized_keys', 'motd_sys_info', + 'iptables_disable'] + # a plaintext, comma delimited list of keys whose values are b64 encoded + base64_keys: [] + # a boolean indicating that all keys not in 'no_base64_decode' are encoded + base64_all: False diff --git a/doc/sources/smartos/README.rst b/doc/sources/smartos/README.rst new file mode 100644 index 00000000..fd4e496d --- /dev/null +++ b/doc/sources/smartos/README.rst @@ -0,0 +1,72 @@ +================== +SmartOS Datasource +================== + +This datasource finds metadata and user-data from the SmartOS virtualization +platform (i.e. Joyent). + +SmartOS Platform +---------------- +The SmartOS virtualization platform meta-data to the instance via the second +serial console. On Linux, this is /dev/ttyS1. The data is a provided via a +simple protocol, where something queries for the userdata, where the console +responds with the status and if "SUCCESS" returns until a single ".\n". + +New versions of the SmartOS tooling will include support for base64 encoded data. + +Userdata +-------- + +In SmartOS parlance, user-data is a actually meta-data. This userdata can be +provided a key-value pairs. + +Cloud-init supports reading the traditional meta-data fields supported by the +SmartOS tools. These are: + * root_authorized_keys + * hostname + * enable_motd_sys_info + * iptables_disable + +Note: At this time iptables_disable and enable_motd_sys_info are read but + are not actioned. + +user-script +----------- + +SmartOS traditionally supports sending over a user-script for execution at the +rc.local level. Cloud-init supports running user-scripts as if they were +cloud-init user-data. In this sense, anything with a shell interpreter +directive will run + +user-data and user-script +------------------------- + +In the event that a user defines the meta-data key of "user-data" it will +always supercede any user-script data. This is for consistency. + +base64 +------ + +The following are exempt from base64 encoding, owing to the fact that they +are provided by SmartOS: + * root_authorized_keys + * enable_motd_sys_info + * iptables_disable + +This list can be changed through system config of variable 'no_base64_decode'. + +This means that user-script and user-data as well as other values can be +base64 encoded. Since Cloud-init can only guess as to whether or not something +is truly base64 encoded, the following meta-data keys are hints as to whether +or not to base64 decode something: + * base64_all: Except for excluded keys, attempt to base64 decode + the values. If the value fails to decode properly, it will be + returned in its text + * base64_keys: A comma deliminated list of which keys are base64 encoded. + * b64-<key>: + for any key, if there exists an entry in the metadata for 'b64-<key>' + Then 'b64-<key>' is expected to be a plaintext boolean indicating whether + or not its value is encoded. + * no_base64_decode: This is a configuration setting + (i.e. /etc/cloud/cloud.cfg.d) that sets which values should not be + base64 decoded. diff --git a/tests/unittests/test_datasource/test_smartos.py b/tests/unittests/test_datasource/test_smartos.py index 6c12f1e2..f53715b0 100644 --- a/tests/unittests/test_datasource/test_smartos.py +++ b/tests/unittests/test_datasource/test_smartos.py @@ -22,25 +22,24 @@ # return responses. # +import base64 from cloudinit import helpers from cloudinit.sources import DataSourceSmartOS from mocker import MockerTestCase import uuid -mock_returns = { +MOCK_RETURNS = { 'hostname': 'test-host', 'root_authorized_keys': 'ssh-rsa AAAAB3Nz...aC1yc2E= keyname', 'disable_iptables_flag': None, 'enable_motd_sys_info': None, - 'system_uuid': str(uuid.uuid4()), - 'smartdc': 'smartdc', - 'userdata': """ -#!/bin/sh -/bin/true -""", + 'test-var1': 'some data', + 'user-data': '\n'.join(['#!/bin/sh', '/bin/true', '']), } +DMI_DATA_RETURN = (str(uuid.uuid4()), 'smartdc') + class MockSerial(object): """Fake a serial terminal for testing the code that @@ -48,12 +47,13 @@ class MockSerial(object): port = None - def __init__(self): + def __init__(self, mockdata): self.last = None self.last = None self.new = True self.count = 0 self.mocked_out = [] + self.mockdata = mockdata def open(self): return True @@ -71,12 +71,12 @@ class MockSerial(object): def readline(self): if self.new: self.new = False - if self.last in mock_returns: + if self.last in self.mockdata: return 'SUCCESS\n' else: return 'NOTFOUND %s\n' % self.last - if self.last in mock_returns: + if self.last in self.mockdata: if not self.mocked_out: self.mocked_out = [x for x in self._format_out()] print self.mocked_out @@ -86,15 +86,16 @@ class MockSerial(object): return self.mocked_out[self.count - 1] def _format_out(self): - if self.last in mock_returns: + if self.last in self.mockdata: + _mret = self.mockdata[self.last] try: - for l in mock_returns[self.last].splitlines(): - yield "%s\n" % l + for l in _mret.splitlines(): + yield "%s\n" % l.rstrip() except: - yield "%s\n" % mock_returns[self.last] + yield "%s\n" % _mret.rstrip() - yield '\n' yield '.' + yield '\n' class TestSmartOSDataSource(MockerTestCase): @@ -116,23 +117,36 @@ class TestSmartOSDataSource(MockerTestCase): ret = apply_patches(patches) self.unapply += ret - def _get_ds(self): + def _get_ds(self, sys_cfg=None, ds_cfg=None, mockdata=None, dmi_data=None): + mod = DataSourceSmartOS + + if mockdata is None: + mockdata = MOCK_RETURNS + + if dmi_data is None: + dmi_data = DMI_DATA_RETURN def _get_serial(*_): - return MockSerial() + return MockSerial(mockdata) def _dmi_data(): - return mock_returns['system_uuid'], 'smartdc' + return dmi_data + + if sys_cfg is None: + sys_cfg = {} + + if ds_cfg is not None: + sys_cfg['datasource'] = sys_cfg.get('datasource', {}) + sys_cfg['datasource']['SmartOS'] = ds_cfg - data = {'sys_cfg': {}} - mod = DataSourceSmartOS self.apply_patches([(mod, 'get_serial', _get_serial)]) self.apply_patches([(mod, 'dmi_data', _dmi_data)]) - dsrc = mod.DataSourceSmartOS( - data.get('sys_cfg', {}), distro=None, paths=self.paths) + dsrc = mod.DataSourceSmartOS(sys_cfg, distro=None, + paths=self.paths) return dsrc def test_seed(self): + # default seed should be /dev/ttyS1 dsrc = self._get_ds() ret = dsrc.get_data() self.assertTrue(ret) @@ -144,39 +158,107 @@ class TestSmartOSDataSource(MockerTestCase): self.assertTrue(ret) self.assertTrue(dsrc.is_smartdc) + def test_no_base64(self): + ds_cfg = {'no_base64_decode': ['test_var1'], 'all_base': True} + dsrc = self._get_ds(ds_cfg=ds_cfg) + ret = dsrc.get_data() + self.assertTrue(ret) + def test_uuid(self): - dsrc = self._get_ds() + dsrc = self._get_ds(mockdata=MOCK_RETURNS) ret = dsrc.get_data() self.assertTrue(ret) - self.assertEquals(mock_returns['system_uuid'], - dsrc.metadata['instance-id']) + self.assertEquals(DMI_DATA_RETURN[0], dsrc.metadata['instance-id']) def test_root_keys(self): - dsrc = self._get_ds() + dsrc = self._get_ds(mockdata=MOCK_RETURNS) ret = dsrc.get_data() self.assertTrue(ret) - self.assertEquals(mock_returns['root_authorized_keys'], + self.assertEquals(MOCK_RETURNS['root_authorized_keys'], dsrc.metadata['public-keys']) + def test_hostname_b64(self): + dsrc = self._get_ds(mockdata=MOCK_RETURNS) + ret = dsrc.get_data() + self.assertTrue(ret) + self.assertEquals(MOCK_RETURNS['hostname'], + dsrc.metadata['local-hostname']) + def test_hostname(self): - dsrc = self._get_ds() + dsrc = self._get_ds(mockdata=MOCK_RETURNS) ret = dsrc.get_data() self.assertTrue(ret) - self.assertEquals(mock_returns['hostname'], + self.assertEquals(MOCK_RETURNS['hostname'], dsrc.metadata['local-hostname']) + def test_base64_all(self): + # metadata provided base64_all of true + my_returns = MOCK_RETURNS.copy() + my_returns['base64_all'] = "true" + for k in ('hostname', 'user-data'): + my_returns[k] = base64.b64encode(my_returns[k]) + + dsrc = self._get_ds(mockdata=my_returns) + ret = dsrc.get_data() + self.assertTrue(ret) + self.assertEquals(MOCK_RETURNS['hostname'], + dsrc.metadata['local-hostname']) + self.assertEquals(MOCK_RETURNS['user-data'], + dsrc.userdata_raw) + self.assertEquals(MOCK_RETURNS['root_authorized_keys'], + dsrc.metadata['public-keys']) + self.assertEquals(MOCK_RETURNS['disable_iptables_flag'], + dsrc.metadata['iptables_disable']) + self.assertEquals(MOCK_RETURNS['enable_motd_sys_info'], + dsrc.metadata['motd_sys_info']) + + def test_b64_userdata(self): + my_returns = MOCK_RETURNS.copy() + my_returns['b64-user-data'] = "true" + my_returns['b64-hostname'] = "true" + for k in ('hostname', 'user-data'): + my_returns[k] = base64.b64encode(my_returns[k]) + + dsrc = self._get_ds(mockdata=my_returns) + ret = dsrc.get_data() + self.assertTrue(ret) + self.assertEquals(MOCK_RETURNS['hostname'], + dsrc.metadata['local-hostname']) + self.assertEquals(MOCK_RETURNS['user-data'], dsrc.userdata_raw) + self.assertEquals(MOCK_RETURNS['root_authorized_keys'], + dsrc.metadata['public-keys']) + + def test_b64_keys(self): + my_returns = MOCK_RETURNS.copy() + my_returns['base64_keys'] = 'hostname,ignored' + for k in ('hostname',): + my_returns[k] = base64.b64encode(my_returns[k]) + + dsrc = self._get_ds(mockdata=my_returns) + ret = dsrc.get_data() + self.assertTrue(ret) + self.assertEquals(MOCK_RETURNS['hostname'], + dsrc.metadata['local-hostname']) + self.assertEquals(MOCK_RETURNS['user-data'], dsrc.userdata_raw) + + def test_userdata(self): + dsrc = self._get_ds(mockdata=MOCK_RETURNS) + ret = dsrc.get_data() + self.assertTrue(ret) + self.assertEquals(MOCK_RETURNS['user-data'], dsrc.userdata_raw) + def test_disable_iptables_flag(self): - dsrc = self._get_ds() + dsrc = self._get_ds(mockdata=MOCK_RETURNS) ret = dsrc.get_data() self.assertTrue(ret) - self.assertEquals(mock_returns['disable_iptables_flag'], + self.assertEquals(MOCK_RETURNS['disable_iptables_flag'], dsrc.metadata['iptables_disable']) def test_motd_sys_info(self): - dsrc = self._get_ds() + dsrc = self._get_ds(mockdata=MOCK_RETURNS) ret = dsrc.get_data() self.assertTrue(ret) - self.assertEquals(mock_returns['enable_motd_sys_info'], + self.assertEquals(MOCK_RETURNS['enable_motd_sys_info'], dsrc.metadata['motd_sys_info']) |