From 7c63a4096d9b6c9dc10605c289ee048c7b0778c6 Mon Sep 17 00:00:00 2001 From: Daniel Watkins Date: Wed, 25 Mar 2015 15:54:07 +0000 Subject: Convert DataSourceSmartOS to use v2 metadata. --- cloudinit/sources/DataSourceSmartOS.py | 75 +++++++++++++++++++++++++--------- 1 file changed, 55 insertions(+), 20 deletions(-) (limited to 'cloudinit/sources/DataSourceSmartOS.py') diff --git a/cloudinit/sources/DataSourceSmartOS.py b/cloudinit/sources/DataSourceSmartOS.py index 896fde3f..694a011a 100644 --- a/cloudinit/sources/DataSourceSmartOS.py +++ b/cloudinit/sources/DataSourceSmartOS.py @@ -29,9 +29,10 @@ # http://us-east.manta.joyent.com/jmc/public/mdata/datadict.html # Comments with "@datadictionary" are snippets of the definition -import base64 import binascii import os +import random +import re import serial from cloudinit import log as logging @@ -301,6 +302,53 @@ def get_serial(seed_device, seed_timeout): return ser +class JoyentMetadataFetchException(Exception): + pass + + +class JoyentMetadataClient(object): + + def __init__(self, serial): + self.serial = serial + + def _checksum(self, body): + return '{0:08x}'.format( + binascii.crc32(body.encode('utf-8')) & 0xffffffff) + + def _get_value_from_frame(self, expected_request_id, frame): + regex = ( + r'V2 (?P\d+) (?P[0-9a-f]+)' + r' (?P(?P[0-9a-f]+) (?PSUCCESS|NOTFOUND)' + r'( (?P.+))?)') + frame_data = re.match(regex, frame).groupdict() + if int(frame_data['length']) != len(frame_data['body']): + raise JoyentMetadataFetchException( + 'Incorrect frame length given ({0} != {1}).'.format( + frame_data['length'], len(frame_data['body']))) + expected_checksum = self._checksum(frame_data['body']) + if frame_data['checksum'] != expected_checksum: + raise JoyentMetadataFetchException( + 'Invalid checksum (expected: {0}; got {1}).'.format( + expected_checksum, frame_data['checksum'])) + if frame_data['request_id'] != expected_request_id: + raise JoyentMetadataFetchException( + 'Request ID mismatch (expected: {0}; got {1}).'.format( + expected_request_id, frame_data['request_id'])) + if not frame_data.get('payload', None): + return None + return util.b64d(frame_data['payload']) + + def get_metadata(self, metadata_key): + request_id = '{0:08x}'.format(random.randint(0, 0xffffffff)) + message_body = '{0} GET {1}'.format(request_id, + util.b64e(metadata_key)) + msg = 'V2 {0} {1} {2}\n'.format( + len(message_body), self._checksum(message_body), message_body) + self.serial.write(msg.encode('ascii')) + response = self.serial.readline().decode('ascii') + return self._get_value_from_frame(request_id, response) + + def query_data(noun, seed_device, seed_timeout, strip=False, default=None, b64=None): """Makes a request to via the serial console via "GET " @@ -314,34 +362,21 @@ def query_data(noun, seed_device, seed_timeout, strip=False, default=None, encoded, so this method relies on being told if the data is base64 or not. """ - if not noun: return False ser = get_serial(seed_device, seed_timeout) - request_line = "GET %s\n" % noun.rstrip() - ser.write(request_line.encode('ascii')) - status = str(ser.readline()).rstrip() - response = [] - eom_found = False - - if 'SUCCESS' not in status: - ser.close() - return default - - while not eom_found: - m = ser.readline().decode('ascii') - if m.rstrip() == ".": - eom_found = True - else: - response.append(m) + client = JoyentMetadataClient(ser) + response = client.get_metadata(noun) ser.close() + if response is None: + return default if b64 is None: b64 = query_data('b64-%s' % noun, seed_device=seed_device, - seed_timeout=seed_timeout, b64=False, - default=False, strip=True) + seed_timeout=seed_timeout, b64=False, + default=False, strip=True) b64 = util.is_true(b64) resp = None -- cgit v1.2.3 From 1828ac3fa151ec7ff761b34305ed5fb85a9020d1 Mon Sep 17 00:00:00 2001 From: Daniel Watkins Date: Wed, 25 Mar 2015 15:54:14 +0000 Subject: Add logging to JoyentMetadataClient. --- cloudinit/sources/DataSourceSmartOS.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) (limited to 'cloudinit/sources/DataSourceSmartOS.py') diff --git a/cloudinit/sources/DataSourceSmartOS.py b/cloudinit/sources/DataSourceSmartOS.py index 694a011a..61dd044f 100644 --- a/cloudinit/sources/DataSourceSmartOS.py +++ b/cloudinit/sources/DataSourceSmartOS.py @@ -335,17 +335,23 @@ class JoyentMetadataClient(object): 'Request ID mismatch (expected: {0}; got {1}).'.format( expected_request_id, frame_data['request_id'])) if not frame_data.get('payload', None): + LOG.info('No value found.') return None - return util.b64d(frame_data['payload']) + value = util.b64d(frame_data['payload']) + LOG.info('Value "%s" found.', value) + return value def get_metadata(self, metadata_key): + LOG.info('Fetching metadata key "%s"...', metadata_key) request_id = '{0:08x}'.format(random.randint(0, 0xffffffff)) message_body = '{0} GET {1}'.format(request_id, util.b64e(metadata_key)) msg = 'V2 {0} {1} {2}\n'.format( len(message_body), self._checksum(message_body), message_body) + LOG.debug('Writing "%s" to serial port.', msg) self.serial.write(msg.encode('ascii')) response = self.serial.readline().decode('ascii') + LOG.debug('Read "%s" from serial port.', response) return self._get_value_from_frame(request_id, response) -- cgit v1.2.3 From d52feae7ad38670964edebb0eea5db2c8c80f760 Mon Sep 17 00:00:00 2001 From: Daniel Watkins Date: Wed, 25 Mar 2015 15:54:19 +0000 Subject: Ensure that the serial console is always closed. --- cloudinit/sources/DataSourceSmartOS.py | 9 +++++---- tests/unittests/test_datasource/test_smartos.py | 12 ++++++++++++ 2 files changed, 17 insertions(+), 4 deletions(-) (limited to 'cloudinit/sources/DataSourceSmartOS.py') diff --git a/cloudinit/sources/DataSourceSmartOS.py b/cloudinit/sources/DataSourceSmartOS.py index 61dd044f..237fc140 100644 --- a/cloudinit/sources/DataSourceSmartOS.py +++ b/cloudinit/sources/DataSourceSmartOS.py @@ -30,9 +30,11 @@ # Comments with "@datadictionary" are snippets of the definition import binascii +import contextlib import os import random import re + import serial from cloudinit import log as logging @@ -371,11 +373,10 @@ def query_data(noun, seed_device, seed_timeout, strip=False, default=None, if not noun: return False - ser = get_serial(seed_device, seed_timeout) + with contextlib.closing(get_serial(seed_device, seed_timeout)) as ser: + client = JoyentMetadataClient(ser) + response = client.get_metadata(noun) - client = JoyentMetadataClient(ser) - response = client.get_metadata(noun) - ser.close() if response is None: return default diff --git a/tests/unittests/test_datasource/test_smartos.py b/tests/unittests/test_datasource/test_smartos.py index 39991cc2..28b41eaf 100644 --- a/tests/unittests/test_datasource/test_smartos.py +++ b/tests/unittests/test_datasource/test_smartos.py @@ -409,6 +409,18 @@ class TestSmartOSDataSource(helpers.FilesystemMockingTestCase): self.assertEqual(dsrc.device_name_to_device('FOO'), mydscfg['disk_aliases']['FOO']) + @mock.patch('cloudinit.sources.DataSourceSmartOS.JoyentMetadataClient') + @mock.patch('cloudinit.sources.DataSourceSmartOS.get_serial') + def test_serial_console_closed_on_error(self, get_serial, metadata_client): + class OurException(Exception): + pass + metadata_client.side_effect = OurException + try: + DataSourceSmartOS.query_data('noun', 'device', 0) + except OurException: + pass + self.assertEqual(1, get_serial.return_value.close.call_count) + def apply_patches(patches): ret = [] -- cgit v1.2.3 From f4eb74ccc512d12afbb17dd9c678a5308ca64e9f Mon Sep 17 00:00:00 2001 From: Daniel Watkins Date: Wed, 25 Mar 2015 17:26:33 +0000 Subject: Switch logging from info to debug level. --- cloudinit/sources/DataSourceSmartOS.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'cloudinit/sources/DataSourceSmartOS.py') diff --git a/cloudinit/sources/DataSourceSmartOS.py b/cloudinit/sources/DataSourceSmartOS.py index 237fc140..d299cf26 100644 --- a/cloudinit/sources/DataSourceSmartOS.py +++ b/cloudinit/sources/DataSourceSmartOS.py @@ -337,14 +337,14 @@ class JoyentMetadataClient(object): 'Request ID mismatch (expected: {0}; got {1}).'.format( expected_request_id, frame_data['request_id'])) if not frame_data.get('payload', None): - LOG.info('No value found.') + LOG.debug('No value found.') return None value = util.b64d(frame_data['payload']) - LOG.info('Value "%s" found.', value) + LOG.debug('Value "%s" found.', value) return value def get_metadata(self, metadata_key): - LOG.info('Fetching metadata key "%s"...', metadata_key) + LOG.debug('Fetching metadata key "%s"...', metadata_key) request_id = '{0:08x}'.format(random.randint(0, 0xffffffff)) message_body = '{0} GET {1}'.format(request_id, util.b64e(metadata_key)) -- cgit v1.2.3 From 5ae131cad02f383c9f3109ad0f51d918787b0196 Mon Sep 17 00:00:00 2001 From: Daniel Watkins Date: Wed, 25 Mar 2015 17:27:22 +0000 Subject: Add link to Joyent metadata specification. --- cloudinit/sources/DataSourceSmartOS.py | 6 ++++++ 1 file changed, 6 insertions(+) (limited to 'cloudinit/sources/DataSourceSmartOS.py') diff --git a/cloudinit/sources/DataSourceSmartOS.py b/cloudinit/sources/DataSourceSmartOS.py index d299cf26..ec2d10ae 100644 --- a/cloudinit/sources/DataSourceSmartOS.py +++ b/cloudinit/sources/DataSourceSmartOS.py @@ -309,6 +309,12 @@ class JoyentMetadataFetchException(Exception): class JoyentMetadataClient(object): + """ + A client implementing v2 of the Joyent Metadata Protocol Specification. + + The full specification can be found at + http://eng.joyent.com/mdata/protocol.html + """ def __init__(self, serial): self.serial = serial -- cgit v1.2.3 From 5524fd6336a9162aef7687e84705114aa3eb47cd Mon Sep 17 00:00:00 2001 From: Daniel Watkins Date: Wed, 25 Mar 2015 17:59:42 +0000 Subject: Compile SmartOS line-parsing regex once. --- cloudinit/sources/DataSourceSmartOS.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) (limited to 'cloudinit/sources/DataSourceSmartOS.py') diff --git a/cloudinit/sources/DataSourceSmartOS.py b/cloudinit/sources/DataSourceSmartOS.py index ec2d10ae..c9b497df 100644 --- a/cloudinit/sources/DataSourceSmartOS.py +++ b/cloudinit/sources/DataSourceSmartOS.py @@ -315,6 +315,10 @@ class JoyentMetadataClient(object): The full specification can be found at http://eng.joyent.com/mdata/protocol.html """ + line_regex = re.compile( + r'V2 (?P\d+) (?P[0-9a-f]+)' + r' (?P(?P[0-9a-f]+) (?PSUCCESS|NOTFOUND)' + r'( (?P.+))?)') def __init__(self, serial): self.serial = serial @@ -324,11 +328,7 @@ class JoyentMetadataClient(object): binascii.crc32(body.encode('utf-8')) & 0xffffffff) def _get_value_from_frame(self, expected_request_id, frame): - regex = ( - r'V2 (?P\d+) (?P[0-9a-f]+)' - r' (?P(?P[0-9a-f]+) (?PSUCCESS|NOTFOUND)' - r'( (?P.+))?)') - frame_data = re.match(regex, frame).groupdict() + frame_data = self.line_regex.match(frame).groupdict() if int(frame_data['length']) != len(frame_data['body']): raise JoyentMetadataFetchException( 'Incorrect frame length given ({0} != {1}).'.format( -- cgit v1.2.3