From b9f26689e8b3bb7a3486771c6362107232a7dcf4 Mon Sep 17 00:00:00 2001 From: Daniel Watkins Date: Fri, 8 May 2015 13:16:42 +0100 Subject: Split WALinuxAgentShim code out to separate file. --- .../unittests/test_datasource/test_azure_helper.py | 377 +++++++++++++++++++++ 1 file changed, 377 insertions(+) create mode 100644 tests/unittests/test_datasource/test_azure_helper.py (limited to 'tests/unittests/test_datasource/test_azure_helper.py') diff --git a/tests/unittests/test_datasource/test_azure_helper.py b/tests/unittests/test_datasource/test_azure_helper.py new file mode 100644 index 00000000..47b77840 --- /dev/null +++ b/tests/unittests/test_datasource/test_azure_helper.py @@ -0,0 +1,377 @@ +import os +import struct +import unittest + +from cloudinit.sources.helpers import azure as azure_helper +from ..helpers import TestCase + +try: + from unittest import mock +except ImportError: + import mock + +try: + from contextlib import ExitStack +except ImportError: + from contextlib2 import ExitStack + + +GOAL_STATE_TEMPLATE = """\ + + + 2012-11-30 + {incarnation} + + Started + 300000 + + 16001 + + FALSE + + + {container_id} + + + {instance_id} + Started + + http://100.86.192.70:80/machine/46504ebc-f968-4f23-b9aa-cd2b3e4d470c/68ce47b32ea94952be7b20951c383628.utl%2Dtrusty%2D%2D292258?comp=config&type=hostingEnvironmentConfig&incarnation=1 + {shared_config_url} + http://100.86.192.70:80/machine/46504ebc-f968-4f23-b9aa-cd2b3e4d470c/68ce47b32ea94952be7b20951c383628.utl%2Dtrusty%2D%2D292258?comp=config&type=extensionsConfig&incarnation=1 + http://100.86.192.70:80/machine/46504ebc-f968-4f23-b9aa-cd2b3e4d470c/68ce47b32ea94952be7b20951c383628.utl%2Dtrusty%2D%2D292258?comp=config&type=fullConfig&incarnation=1 + {certificates_url} + 68ce47b32ea94952be7b20951c383628.0.68ce47b32ea94952be7b20951c383628.0.utl-trusty--292258.1.xml + + + + + +""" + + +class TestReadAzureSharedConfig(unittest.TestCase): + + def test_valid_content(self): + xml = """ + + + + + + + """ + ret = azure_helper.iid_from_shared_config_content(xml) + self.assertEqual("MY_INSTANCE_ID", ret) + + +class TestFindEndpoint(TestCase): + + def setUp(self): + super(TestFindEndpoint, self).setUp() + patches = ExitStack() + self.addCleanup(patches.close) + + self.load_file = patches.enter_context( + mock.patch.object(azure_helper.util, 'load_file')) + + def test_missing_file(self): + self.load_file.side_effect = IOError + self.assertRaises(IOError, + azure_helper.WALinuxAgentShim.find_endpoint) + + def test_missing_special_azure_line(self): + self.load_file.return_value = '' + self.assertRaises(Exception, + azure_helper.WALinuxAgentShim.find_endpoint) + + def _build_lease_content(self, ip_address, use_hex=True): + ip_address_repr = ':'.join( + [hex(int(part)).replace('0x', '') + for part in ip_address.split('.')]) + if not use_hex: + ip_address_repr = struct.pack( + '>L', int(ip_address_repr.replace(':', ''), 16)) + ip_address_repr = '"{0}"'.format(ip_address_repr.decode('utf-8')) + return '\n'.join([ + 'lease {', + ' interface "eth0";', + ' option unknown-245 {0};'.format(ip_address_repr), + '}']) + + def test_hex_string(self): + ip_address = '98.76.54.32' + file_content = self._build_lease_content(ip_address) + self.load_file.return_value = file_content + self.assertEqual(ip_address, + azure_helper.WALinuxAgentShim.find_endpoint()) + + def test_hex_string_with_single_character_part(self): + ip_address = '4.3.2.1' + file_content = self._build_lease_content(ip_address) + self.load_file.return_value = file_content + self.assertEqual(ip_address, + azure_helper.WALinuxAgentShim.find_endpoint()) + + def test_packed_string(self): + ip_address = '98.76.54.32' + file_content = self._build_lease_content(ip_address, use_hex=False) + self.load_file.return_value = file_content + self.assertEqual(ip_address, + azure_helper.WALinuxAgentShim.find_endpoint()) + + def test_latest_lease_used(self): + ip_addresses = ['4.3.2.1', '98.76.54.32'] + file_content = '\n'.join([self._build_lease_content(ip_address) + for ip_address in ip_addresses]) + self.load_file.return_value = file_content + self.assertEqual(ip_addresses[-1], + azure_helper.WALinuxAgentShim.find_endpoint()) + + +class TestGoalStateParsing(TestCase): + + default_parameters = { + 'incarnation': 1, + 'container_id': 'MyContainerId', + 'instance_id': 'MyInstanceId', + 'shared_config_url': 'MySharedConfigUrl', + 'certificates_url': 'MyCertificatesUrl', + } + + def _get_goal_state(self, http_client=None, **kwargs): + if http_client is None: + http_client = mock.MagicMock() + parameters = self.default_parameters.copy() + parameters.update(kwargs) + xml = GOAL_STATE_TEMPLATE.format(**parameters) + if parameters['certificates_url'] is None: + new_xml_lines = [] + for line in xml.splitlines(): + if 'Certificates' in line: + continue + new_xml_lines.append(line) + xml = '\n'.join(new_xml_lines) + return azure_helper.GoalState(xml, http_client) + + def test_incarnation_parsed_correctly(self): + incarnation = '123' + goal_state = self._get_goal_state(incarnation=incarnation) + self.assertEqual(incarnation, goal_state.incarnation) + + def test_container_id_parsed_correctly(self): + container_id = 'TestContainerId' + goal_state = self._get_goal_state(container_id=container_id) + self.assertEqual(container_id, goal_state.container_id) + + def test_instance_id_parsed_correctly(self): + instance_id = 'TestInstanceId' + goal_state = self._get_goal_state(instance_id=instance_id) + self.assertEqual(instance_id, goal_state.instance_id) + + def test_shared_config_xml_parsed_and_fetched_correctly(self): + http_client = mock.MagicMock() + shared_config_url = 'TestSharedConfigUrl' + goal_state = self._get_goal_state( + http_client=http_client, shared_config_url=shared_config_url) + shared_config_xml = goal_state.shared_config_xml + self.assertEqual(1, http_client.get.call_count) + self.assertEqual(shared_config_url, http_client.get.call_args[0][0]) + self.assertEqual(http_client.get.return_value.contents, + shared_config_xml) + + def test_certificates_xml_parsed_and_fetched_correctly(self): + http_client = mock.MagicMock() + certificates_url = 'TestSharedConfigUrl' + goal_state = self._get_goal_state( + http_client=http_client, certificates_url=certificates_url) + certificates_xml = goal_state.certificates_xml + self.assertEqual(1, http_client.get.call_count) + self.assertEqual(certificates_url, http_client.get.call_args[0][0]) + self.assertTrue(http_client.get.call_args[1].get('secure', False)) + self.assertEqual(http_client.get.return_value.contents, + certificates_xml) + + def test_missing_certificates_skips_http_get(self): + http_client = mock.MagicMock() + goal_state = self._get_goal_state( + http_client=http_client, certificates_url=None) + certificates_xml = goal_state.certificates_xml + self.assertEqual(0, http_client.get.call_count) + self.assertIsNone(certificates_xml) + + +class TestAzureEndpointHttpClient(TestCase): + + regular_headers = { + 'x-ms-agent-name': 'WALinuxAgent', + 'x-ms-version': '2012-11-30', + } + + def setUp(self): + super(TestAzureEndpointHttpClient, self).setUp() + patches = ExitStack() + self.addCleanup(patches.close) + + self.read_file_or_url = patches.enter_context( + mock.patch.object(azure_helper.util, 'read_file_or_url')) + + def test_non_secure_get(self): + client = azure_helper.AzureEndpointHttpClient(mock.MagicMock()) + url = 'MyTestUrl' + response = client.get(url, secure=False) + self.assertEqual(1, self.read_file_or_url.call_count) + self.assertEqual(self.read_file_or_url.return_value, response) + self.assertEqual(mock.call(url, headers=self.regular_headers), + self.read_file_or_url.call_args) + + def test_secure_get(self): + url = 'MyTestUrl' + certificate = mock.MagicMock() + expected_headers = self.regular_headers.copy() + expected_headers.update({ + "x-ms-cipher-name": "DES_EDE3_CBC", + "x-ms-guest-agent-public-x509-cert": certificate, + }) + client = azure_helper.AzureEndpointHttpClient(certificate) + response = client.get(url, secure=True) + self.assertEqual(1, self.read_file_or_url.call_count) + self.assertEqual(self.read_file_or_url.return_value, response) + self.assertEqual(mock.call(url, headers=expected_headers), + self.read_file_or_url.call_args) + + def test_post(self): + data = mock.MagicMock() + url = 'MyTestUrl' + client = azure_helper.AzureEndpointHttpClient(mock.MagicMock()) + response = client.post(url, data=data) + self.assertEqual(1, self.read_file_or_url.call_count) + self.assertEqual(self.read_file_or_url.return_value, response) + self.assertEqual( + mock.call(url, data=data, headers=self.regular_headers), + self.read_file_or_url.call_args) + + def test_post_with_extra_headers(self): + url = 'MyTestUrl' + client = azure_helper.AzureEndpointHttpClient(mock.MagicMock()) + extra_headers = {'test': 'header'} + client.post(url, extra_headers=extra_headers) + self.assertEqual(1, self.read_file_or_url.call_count) + expected_headers = self.regular_headers.copy() + expected_headers.update(extra_headers) + self.assertEqual( + mock.call(mock.ANY, data=mock.ANY, headers=expected_headers), + self.read_file_or_url.call_args) + + +class TestOpenSSLManager(TestCase): + + def setUp(self): + super(TestOpenSSLManager, self).setUp() + patches = ExitStack() + self.addCleanup(patches.close) + + self.subp = patches.enter_context( + mock.patch.object(azure_helper.util, 'subp')) + + @mock.patch.object(azure_helper, 'cd', mock.MagicMock()) + @mock.patch.object(azure_helper.tempfile, 'TemporaryDirectory') + def test_openssl_manager_creates_a_tmpdir(self, TemporaryDirectory): + manager = azure_helper.OpenSSLManager() + self.assertEqual(TemporaryDirectory.return_value, manager.tmpdir) + + @mock.patch('builtins.open') + def test_generate_certificate_uses_tmpdir(self, open): + subp_directory = {} + + def capture_directory(*args, **kwargs): + subp_directory['path'] = os.getcwd() + + self.subp.side_effect = capture_directory + manager = azure_helper.OpenSSLManager() + self.assertEqual(manager.tmpdir.name, subp_directory['path']) + + +class TestWALinuxAgentShim(TestCase): + + def setUp(self): + super(TestWALinuxAgentShim, self).setUp() + patches = ExitStack() + self.addCleanup(patches.close) + + self.AzureEndpointHttpClient = patches.enter_context( + mock.patch.object(azure_helper, 'AzureEndpointHttpClient')) + self.find_endpoint = patches.enter_context( + mock.patch.object( + azure_helper.WALinuxAgentShim, 'find_endpoint')) + self.GoalState = patches.enter_context( + mock.patch.object(azure_helper, 'GoalState')) + self.iid_from_shared_config_content = patches.enter_context( + mock.patch.object(azure_helper, 'iid_from_shared_config_content')) + self.OpenSSLManager = patches.enter_context( + mock.patch.object(azure_helper, 'OpenSSLManager')) + + def test_http_client_uses_certificate(self): + shim = azure_helper.WALinuxAgentShim() + self.assertEqual( + [mock.call(self.OpenSSLManager.return_value.certificate)], + self.AzureEndpointHttpClient.call_args_list) + self.assertEqual(self.AzureEndpointHttpClient.return_value, + shim.http_client) + + def test_correct_url_used_for_goalstate(self): + self.find_endpoint.return_value = 'test_endpoint' + shim = azure_helper.WALinuxAgentShim() + shim.register_with_azure_and_fetch_data() + get = self.AzureEndpointHttpClient.return_value.get + self.assertEqual( + [mock.call('http://test_endpoint/machine/?comp=goalstate')], + get.call_args_list) + self.assertEqual( + [mock.call(get.return_value.contents, shim.http_client)], + self.GoalState.call_args_list) + + def test_certificates_used_to_determine_public_keys(self): + shim = azure_helper.WALinuxAgentShim() + data = shim.register_with_azure_and_fetch_data() + self.assertEqual( + [mock.call(self.GoalState.return_value.certificates_xml)], + self.OpenSSLManager.return_value.parse_certificates.call_args_list) + self.assertEqual( + self.OpenSSLManager.return_value.parse_certificates.return_value, + data['public-keys']) + + def test_absent_certificates_produces_empty_public_keys(self): + self.GoalState.return_value.certificates_xml = None + shim = azure_helper.WALinuxAgentShim() + data = shim.register_with_azure_and_fetch_data() + self.assertEqual([], data['public-keys']) + + def test_instance_id_returned_in_data(self): + shim = azure_helper.WALinuxAgentShim() + data = shim.register_with_azure_and_fetch_data() + self.assertEqual( + [mock.call(self.GoalState.return_value.shared_config_xml)], + self.iid_from_shared_config_content.call_args_list) + self.assertEqual(self.iid_from_shared_config_content.return_value, + data['instance-id']) + + def test_correct_url_used_for_report_ready(self): + self.find_endpoint.return_value = 'test_endpoint' + shim = azure_helper.WALinuxAgentShim() + shim.register_with_azure_and_fetch_data() + expected_url = 'http://test_endpoint/machine?comp=health' + self.assertEqual( + [mock.call(expected_url, data=mock.ANY, extra_headers=mock.ANY)], + shim.http_client.post.call_args_list) + + def test_goal_state_values_used_for_report_ready(self): + self.GoalState.return_value.incarnation = 'TestIncarnation' + self.GoalState.return_value.container_id = 'TestContainerId' + self.GoalState.return_value.instance_id = 'TestInstanceId' + shim = azure_helper.WALinuxAgentShim() + shim.register_with_azure_and_fetch_data() + posted_document = shim.http_client.post.call_args[1]['data'] + self.assertIn('TestIncarnation', posted_document) + self.assertIn('TestContainerId', posted_document) + self.assertIn('TestInstanceId', posted_document) -- cgit v1.2.3 From 9c7643c4a0dee7843963709c361b755baf843a4b Mon Sep 17 00:00:00 2001 From: Daniel Watkins Date: Fri, 8 May 2015 13:16:44 +0100 Subject: Stop using Python 3 only tempfile.TemporaryDirectory (but lose free cleanup). --- cloudinit/sources/helpers/azure.py | 8 ++++---- tests/unittests/test_datasource/test_azure_helper.py | 17 +++++++++++------ 2 files changed, 15 insertions(+), 10 deletions(-) (limited to 'tests/unittests/test_datasource/test_azure_helper.py') diff --git a/cloudinit/sources/helpers/azure.py b/cloudinit/sources/helpers/azure.py index 60f116e0..cb13187f 100644 --- a/cloudinit/sources/helpers/azure.py +++ b/cloudinit/sources/helpers/azure.py @@ -104,7 +104,7 @@ class OpenSSLManager(object): } def __init__(self): - self.tmpdir = tempfile.TemporaryDirectory() + self.tmpdir = tempfile.mkdtemp() self.certificate = None self.generate_certificate() @@ -113,7 +113,7 @@ class OpenSSLManager(object): if self.certificate is not None: LOG.debug('Certificate already generated.') return - with cd(self.tmpdir.name): + with cd(self.tmpdir): util.subp([ 'openssl', 'req', '-x509', '-nodes', '-subj', '/CN=LinuxTransport', '-days', '32768', '-newkey', 'rsa:2048', @@ -139,7 +139,7 @@ class OpenSSLManager(object): b'', certificates_content.encode('utf-8'), ] - with cd(self.tmpdir.name): + with cd(self.tmpdir): with open('Certificates.p7m', 'wb') as f: f.write(b'\n'.join(lines)) out, _ = util.subp( @@ -159,7 +159,7 @@ class OpenSSLManager(object): current = [] keys = [] for certificate in certificates: - with cd(self.tmpdir.name): + with cd(self.tmpdir): public_key, _ = util.subp( 'openssl x509 -noout -pubkey |' 'ssh-keygen -i -m PKCS8 -f /dev/stdin', diff --git a/tests/unittests/test_datasource/test_azure_helper.py b/tests/unittests/test_datasource/test_azure_helper.py index 47b77840..398a9007 100644 --- a/tests/unittests/test_datasource/test_azure_helper.py +++ b/tests/unittests/test_datasource/test_azure_helper.py @@ -273,15 +273,20 @@ class TestOpenSSLManager(TestCase): self.subp = patches.enter_context( mock.patch.object(azure_helper.util, 'subp')) + try: + self.open = patches.enter_context( + mock.patch('__builtin__.open')) + except ImportError: + self.open = patches.enter_context( + mock.patch('builtins.open')) @mock.patch.object(azure_helper, 'cd', mock.MagicMock()) - @mock.patch.object(azure_helper.tempfile, 'TemporaryDirectory') - def test_openssl_manager_creates_a_tmpdir(self, TemporaryDirectory): + @mock.patch.object(azure_helper.tempfile, 'mkdtemp') + def test_openssl_manager_creates_a_tmpdir(self, mkdtemp): manager = azure_helper.OpenSSLManager() - self.assertEqual(TemporaryDirectory.return_value, manager.tmpdir) + self.assertEqual(mkdtemp.return_value, manager.tmpdir) - @mock.patch('builtins.open') - def test_generate_certificate_uses_tmpdir(self, open): + def test_generate_certificate_uses_tmpdir(self): subp_directory = {} def capture_directory(*args, **kwargs): @@ -289,7 +294,7 @@ class TestOpenSSLManager(TestCase): self.subp.side_effect = capture_directory manager = azure_helper.OpenSSLManager() - self.assertEqual(manager.tmpdir.name, subp_directory['path']) + self.assertEqual(manager.tmpdir, subp_directory['path']) class TestWALinuxAgentShim(TestCase): -- cgit v1.2.3 From 84868622c404cda5efd2a753e2de30c1afca49a2 Mon Sep 17 00:00:00 2001 From: Daniel Watkins Date: Fri, 8 May 2015 13:18:02 +0100 Subject: Move our walinuxagent implementation to a single function call. --- cloudinit/sources/DataSourceAzure.py | 8 ++-- cloudinit/sources/helpers/azure.py | 31 ++++++++---- tests/unittests/test_datasource/test_azure.py | 19 ++++++-- .../unittests/test_datasource/test_azure_helper.py | 56 ++++++++++++++++++++-- 4 files changed, 92 insertions(+), 22 deletions(-) (limited to 'tests/unittests/test_datasource/test_azure_helper.py') diff --git a/cloudinit/sources/DataSourceAzure.py b/cloudinit/sources/DataSourceAzure.py index 5e147950..4053cfa6 100644 --- a/cloudinit/sources/DataSourceAzure.py +++ b/cloudinit/sources/DataSourceAzure.py @@ -29,7 +29,7 @@ from cloudinit.settings import PER_ALWAYS from cloudinit import sources from cloudinit import util from cloudinit.sources.helpers.azure import ( - iid_from_shared_config_content, WALinuxAgentShim) + get_metadata_from_fabric, iid_from_shared_config_content) LOG = logging.getLogger(__name__) @@ -185,15 +185,13 @@ class DataSourceAzureNet(sources.DataSource): write_files(ddir, files, dirmode=0o700) try: - shim = WALinuxAgentShim() - data = shim.register_with_azure_and_fetch_data() + fabric_data = get_metadata_from_fabric() except Exception as exc: LOG.info("Error communicating with Azure fabric; assume we aren't" " on Azure.", exc_info=True) return False - self.metadata['instance-id'] = data['instance-id'] - self.metadata['public-keys'] = data['public-keys'] + self.metadata.update(fabric_data) found_ephemeral = find_ephemeral_disk() if found_ephemeral: diff --git a/cloudinit/sources/helpers/azure.py b/cloudinit/sources/helpers/azure.py index cb13187f..dfdfa7c2 100644 --- a/cloudinit/sources/helpers/azure.py +++ b/cloudinit/sources/helpers/azure.py @@ -108,6 +108,9 @@ class OpenSSLManager(object): self.certificate = None self.generate_certificate() + def clean_up(self): + util.del_dir(self.tmpdir) + def generate_certificate(self): LOG.debug('Generating certificate for communication with fabric...') if self.certificate is not None: @@ -205,11 +208,13 @@ class WALinuxAgentShim(object): def __init__(self): LOG.debug('WALinuxAgentShim instantiated...') self.endpoint = self.find_endpoint() - self.openssl_manager = OpenSSLManager() - self.http_client = AzureEndpointHttpClient( - self.openssl_manager.certificate) + self.openssl_manager = None self.values = {} + def clean_up(self): + if self.openssl_manager is not None: + self.openssl_manager.clean_up() + @staticmethod def find_endpoint(): LOG.debug('Finding Azure endpoint...') @@ -234,17 +239,19 @@ class WALinuxAgentShim(object): return endpoint_ip_address def register_with_azure_and_fetch_data(self): + self.openssl_manager = OpenSSLManager() + http_client = AzureEndpointHttpClient(self.openssl_manager.certificate) LOG.info('Registering with Azure...') for i in range(10): try: - response = self.http_client.get( + response = http_client.get( 'http://{}/machine/?comp=goalstate'.format(self.endpoint)) except Exception: time.sleep(i + 1) else: break LOG.debug('Successfully fetched GoalState XML.') - goal_state = GoalState(response.contents, self.http_client) + goal_state = GoalState(response.contents, http_client) public_keys = [] if goal_state.certificates_xml is not None: LOG.debug('Certificate XML found; parsing out public keys.') @@ -255,19 +262,27 @@ class WALinuxAgentShim(object): goal_state.shared_config_xml), 'public-keys': public_keys, } - self._report_ready(goal_state) + self._report_ready(goal_state, http_client) return data - def _report_ready(self, goal_state): + def _report_ready(self, goal_state, http_client): LOG.debug('Reporting ready to Azure fabric.') document = self.REPORT_READY_XML_TEMPLATE.format( incarnation=goal_state.incarnation, container_id=goal_state.container_id, instance_id=goal_state.instance_id, ) - self.http_client.post( + http_client.post( "http://{}/machine?comp=health".format(self.endpoint), data=document, extra_headers={'Content-Type': 'text/xml; charset=utf-8'}, ) LOG.info('Reported ready to Azure fabric.') + + +def get_metadata_from_fabric(): + shim = WALinuxAgentShim() + try: + return shim.register_with_azure_and_fetch_data() + finally: + shim.clean_up() diff --git a/tests/unittests/test_datasource/test_azure.py b/tests/unittests/test_datasource/test_azure.py index ee7109e1..983be4cd 100644 --- a/tests/unittests/test_datasource/test_azure.py +++ b/tests/unittests/test_datasource/test_azure.py @@ -122,11 +122,10 @@ class TestAzureDataSource(TestCase): mod = DataSourceAzure mod.BUILTIN_DS_CONFIG['data_dir'] = self.waagent_d - fake_shim = mock.MagicMock() - fake_shim().register_with_azure_and_fetch_data.return_value = { + self.get_metadata_from_fabric = mock.MagicMock(return_value={ 'instance-id': 'i-my-azure-id', 'public-keys': [], - } + }) self.apply_patches([ (mod, 'list_possible_azure_ds_devs', dsdevs), @@ -137,7 +136,7 @@ class TestAzureDataSource(TestCase): (mod, 'perform_hostname_bounce', mock.MagicMock()), (mod, 'get_hostname', mock.MagicMock()), (mod, 'set_hostname', mock.MagicMock()), - (mod, 'WALinuxAgentShim', fake_shim), + (mod, 'get_metadata_from_fabric', self.get_metadata_from_fabric), ]) dsrc = mod.DataSourceAzureNet( @@ -388,6 +387,18 @@ class TestAzureDataSource(TestCase): self.assertEqual(new_ovfenv, load_file(os.path.join(self.waagent_d, 'ovf-env.xml'))) + def test_exception_fetching_fabric_data_doesnt_propagate(self): + ds = self._get_ds({'ovfcontent': construct_valid_ovf_env()}) + self.get_metadata_from_fabric.side_effect = Exception + self.assertFalse(ds.get_data()) + + def test_fabric_data_included_in_metadata(self): + ds = self._get_ds({'ovfcontent': construct_valid_ovf_env()}) + self.get_metadata_from_fabric.return_value = {'test': 'value'} + ret = ds.get_data() + self.assertTrue(ret) + self.assertEqual('value', ds.metadata['test']) + class TestAzureBounce(TestCase): diff --git a/tests/unittests/test_datasource/test_azure_helper.py b/tests/unittests/test_datasource/test_azure_helper.py index 398a9007..5fac2ade 100644 --- a/tests/unittests/test_datasource/test_azure_helper.py +++ b/tests/unittests/test_datasource/test_azure_helper.py @@ -296,6 +296,14 @@ class TestOpenSSLManager(TestCase): manager = azure_helper.OpenSSLManager() self.assertEqual(manager.tmpdir, subp_directory['path']) + @mock.patch.object(azure_helper, 'cd', mock.MagicMock()) + @mock.patch.object(azure_helper.tempfile, 'mkdtemp', mock.MagicMock()) + @mock.patch.object(azure_helper.util, 'del_dir') + def test_clean_up(self, del_dir): + manager = azure_helper.OpenSSLManager() + manager.clean_up() + self.assertEqual([mock.call(manager.tmpdir)], del_dir.call_args_list) + class TestWALinuxAgentShim(TestCase): @@ -318,11 +326,10 @@ class TestWALinuxAgentShim(TestCase): def test_http_client_uses_certificate(self): shim = azure_helper.WALinuxAgentShim() + shim.register_with_azure_and_fetch_data() self.assertEqual( [mock.call(self.OpenSSLManager.return_value.certificate)], self.AzureEndpointHttpClient.call_args_list) - self.assertEqual(self.AzureEndpointHttpClient.return_value, - shim.http_client) def test_correct_url_used_for_goalstate(self): self.find_endpoint.return_value = 'test_endpoint' @@ -333,7 +340,8 @@ class TestWALinuxAgentShim(TestCase): [mock.call('http://test_endpoint/machine/?comp=goalstate')], get.call_args_list) self.assertEqual( - [mock.call(get.return_value.contents, shim.http_client)], + [mock.call(get.return_value.contents, + self.AzureEndpointHttpClient.return_value)], self.GoalState.call_args_list) def test_certificates_used_to_determine_public_keys(self): @@ -368,7 +376,7 @@ class TestWALinuxAgentShim(TestCase): expected_url = 'http://test_endpoint/machine?comp=health' self.assertEqual( [mock.call(expected_url, data=mock.ANY, extra_headers=mock.ANY)], - shim.http_client.post.call_args_list) + self.AzureEndpointHttpClient.return_value.post.call_args_list) def test_goal_state_values_used_for_report_ready(self): self.GoalState.return_value.incarnation = 'TestIncarnation' @@ -376,7 +384,45 @@ class TestWALinuxAgentShim(TestCase): self.GoalState.return_value.instance_id = 'TestInstanceId' shim = azure_helper.WALinuxAgentShim() shim.register_with_azure_and_fetch_data() - posted_document = shim.http_client.post.call_args[1]['data'] + posted_document = ( + self.AzureEndpointHttpClient.return_value.post.call_args[1]['data'] + ) self.assertIn('TestIncarnation', posted_document) self.assertIn('TestContainerId', posted_document) self.assertIn('TestInstanceId', posted_document) + + def test_clean_up_can_be_called_at_any_time(self): + shim = azure_helper.WALinuxAgentShim() + shim.clean_up() + + def test_clean_up_will_clean_up_openssl_manager_if_instantiated(self): + shim = azure_helper.WALinuxAgentShim() + shim.register_with_azure_and_fetch_data() + shim.clean_up() + self.assertEqual( + 1, self.OpenSSLManager.return_value.clean_up.call_count) + + +class TestGetMetadataFromFabric(TestCase): + + @mock.patch.object(azure_helper, 'WALinuxAgentShim') + def test_data_from_shim_returned(self, shim): + ret = azure_helper.get_metadata_from_fabric() + self.assertEqual( + shim.return_value.register_with_azure_and_fetch_data.return_value, + ret) + + @mock.patch.object(azure_helper, 'WALinuxAgentShim') + def test_success_calls_clean_up(self, shim): + azure_helper.get_metadata_from_fabric() + self.assertEqual(1, shim.return_value.clean_up.call_count) + + @mock.patch.object(azure_helper, 'WALinuxAgentShim') + def test_failure_in_registration_calls_clean_up(self, shim): + class SentinelException(Exception): + pass + shim.return_value.register_with_azure_and_fetch_data.side_effect = ( + SentinelException) + self.assertRaises(SentinelException, + azure_helper.get_metadata_from_fabric) + self.assertEqual(1, shim.return_value.clean_up.call_count) -- cgit v1.2.3 From 512eb552e0ca740e1d285dc1b66a56579bcf68ec Mon Sep 17 00:00:00 2001 From: Daniel Watkins Date: Fri, 8 May 2015 16:52:49 +0100 Subject: Fix retrying. --- cloudinit/sources/helpers/azure.py | 9 +++++++-- tests/unittests/test_datasource/test_azure_helper.py | 11 +++++++++++ 2 files changed, 18 insertions(+), 2 deletions(-) (limited to 'tests/unittests/test_datasource/test_azure_helper.py') diff --git a/cloudinit/sources/helpers/azure.py b/cloudinit/sources/helpers/azure.py index dfdfa7c2..2ce728f5 100644 --- a/cloudinit/sources/helpers/azure.py +++ b/cloudinit/sources/helpers/azure.py @@ -242,14 +242,19 @@ class WALinuxAgentShim(object): self.openssl_manager = OpenSSLManager() http_client = AzureEndpointHttpClient(self.openssl_manager.certificate) LOG.info('Registering with Azure...') - for i in range(10): + attempts = 0 + while True: try: response = http_client.get( 'http://{}/machine/?comp=goalstate'.format(self.endpoint)) except Exception: - time.sleep(i + 1) + if attempts < 10: + time.sleep(attempts + 1) + else: + raise else: break + attempts += 1 LOG.debug('Successfully fetched GoalState XML.') goal_state = GoalState(response.contents, http_client) public_keys = [] diff --git a/tests/unittests/test_datasource/test_azure_helper.py b/tests/unittests/test_datasource/test_azure_helper.py index 5fac2ade..23bc997c 100644 --- a/tests/unittests/test_datasource/test_azure_helper.py +++ b/tests/unittests/test_datasource/test_azure_helper.py @@ -323,6 +323,8 @@ class TestWALinuxAgentShim(TestCase): mock.patch.object(azure_helper, 'iid_from_shared_config_content')) self.OpenSSLManager = patches.enter_context( mock.patch.object(azure_helper, 'OpenSSLManager')) + patches.enter_context( + mock.patch.object(azure_helper.time, 'sleep', mock.MagicMock())) def test_http_client_uses_certificate(self): shim = azure_helper.WALinuxAgentShim() @@ -402,6 +404,15 @@ class TestWALinuxAgentShim(TestCase): self.assertEqual( 1, self.OpenSSLManager.return_value.clean_up.call_count) + def test_failure_to_fetch_goalstate_bubbles_up(self): + class SentinelException(Exception): + pass + self.AzureEndpointHttpClient.return_value.get.side_effect = ( + SentinelException) + shim = azure_helper.WALinuxAgentShim() + self.assertRaises(SentinelException, + shim.register_with_azure_and_fetch_data) + class TestGetMetadataFromFabric(TestCase): -- cgit v1.2.3 From 33db529855c0c0746091868c69f5d694bff7b9a8 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Fri, 15 May 2015 16:28:24 -0400 Subject: pep8 fixes --- cloudinit/distros/rhel.py | 4 ++-- tests/unittests/test_datasource/test_azure.py | 1 - tests/unittests/test_datasource/test_azure_helper.py | 15 ++++++++++----- .../unittests/test_handler/test_handler_apt_configure.py | 1 + 4 files changed, 13 insertions(+), 8 deletions(-) (limited to 'tests/unittests/test_datasource/test_azure_helper.py') diff --git a/cloudinit/distros/rhel.py b/cloudinit/distros/rhel.py index eec17c61..30c805a6 100644 --- a/cloudinit/distros/rhel.py +++ b/cloudinit/distros/rhel.py @@ -133,7 +133,7 @@ class Distro(distros.Distro): rhel_util.update_sysconfig_file(out_fn, locale_cfg) def _write_hostname(self, hostname, out_fn): - # systemd will never update previous-hostname for us, so + # systemd will never update previous-hostname for us, so # we need to do it ourselves if self.uses_systemd() and out_fn.endswith('/previous-hostname'): util.write_file(out_fn, hostname) @@ -161,7 +161,7 @@ class Distro(distros.Distro): def _read_hostname(self, filename, default=None): if self.uses_systemd() and filename.endswith('/previous-hostname'): - return util.load_file(filename).strip() + return util.load_file(filename).strip() elif self.uses_systemd(): (out, _err) = util.subp(['hostname']) if len(out): diff --git a/tests/unittests/test_datasource/test_azure.py b/tests/unittests/test_datasource/test_azure.py index c72dc801..4c4b8eec 100644 --- a/tests/unittests/test_datasource/test_azure.py +++ b/tests/unittests/test_datasource/test_azure.py @@ -587,4 +587,3 @@ class TestReadAzureOvf(TestCase): (_md, _ud, cfg) = DataSourceAzure.read_azure_ovf(content) for mypk in mypklist: self.assertIn(mypk, cfg['_pubkeys']) - diff --git a/tests/unittests/test_datasource/test_azure_helper.py b/tests/unittests/test_datasource/test_azure_helper.py index 23bc997c..a5228870 100644 --- a/tests/unittests/test_datasource/test_azure_helper.py +++ b/tests/unittests/test_datasource/test_azure_helper.py @@ -18,7 +18,8 @@ except ImportError: GOAL_STATE_TEMPLATE = """\ - + 2012-11-30 {incarnation} @@ -36,12 +37,16 @@ GOAL_STATE_TEMPLATE = """\ {instance_id} Started - http://100.86.192.70:80/machine/46504ebc-f968-4f23-b9aa-cd2b3e4d470c/68ce47b32ea94952be7b20951c383628.utl%2Dtrusty%2D%2D292258?comp=config&type=hostingEnvironmentConfig&incarnation=1 + + http://100.86.192.70:80/...hostingEnvironmentConfig... + {shared_config_url} - http://100.86.192.70:80/machine/46504ebc-f968-4f23-b9aa-cd2b3e4d470c/68ce47b32ea94952be7b20951c383628.utl%2Dtrusty%2D%2D292258?comp=config&type=extensionsConfig&incarnation=1 - http://100.86.192.70:80/machine/46504ebc-f968-4f23-b9aa-cd2b3e4d470c/68ce47b32ea94952be7b20951c383628.utl%2Dtrusty%2D%2D292258?comp=config&type=fullConfig&incarnation=1 + + http://100.86.192.70:80/...extensionsConfig... + + http://100.86.192.70:80/...fullConfig... {certificates_url} - 68ce47b32ea94952be7b20951c383628.0.68ce47b32ea94952be7b20951c383628.0.utl-trusty--292258.1.xml + 68ce47.0.68ce47.0.utl-trusty--292258.1.xml diff --git a/tests/unittests/test_handler/test_handler_apt_configure.py b/tests/unittests/test_handler/test_handler_apt_configure.py index 4a74ea47..1ed185ca 100644 --- a/tests/unittests/test_handler/test_handler_apt_configure.py +++ b/tests/unittests/test_handler/test_handler_apt_configure.py @@ -8,6 +8,7 @@ import re import shutil import tempfile + def load_tfile_or_url(*args, **kwargs): return(util.decode_binary(util.read_file_or_url(*args, **kwargs).contents)) -- cgit v1.2.3 From 41900b72f31a1bd0eebe2f58a8598bfab25f0003 Mon Sep 17 00:00:00 2001 From: Daniel Watkins Date: Fri, 9 Oct 2015 14:01:11 +0100 Subject: Handle escaped quotes in WALinuxAgentShim.find_endpoint. This fixes bug 1488891. --- cloudinit/sources/helpers/azure.py | 2 +- tests/unittests/test_datasource/test_azure_helper.py | 10 +++++++++- 2 files changed, 10 insertions(+), 2 deletions(-) (limited to 'tests/unittests/test_datasource/test_azure_helper.py') diff --git a/cloudinit/sources/helpers/azure.py b/cloudinit/sources/helpers/azure.py index 281d733e..33003da0 100644 --- a/cloudinit/sources/helpers/azure.py +++ b/cloudinit/sources/helpers/azure.py @@ -233,7 +233,7 @@ class WALinuxAgentShim(object): hex_string += hex_pair value = struct.pack('>L', int(hex_string.replace(':', ''), 16)) else: - value = value.encode('utf-8') + value = value.replace('\\', '').encode('utf-8') endpoint_ip_address = socket.inet_ntoa(value) LOG.debug('Azure endpoint found at %s', endpoint_ip_address) return endpoint_ip_address diff --git a/tests/unittests/test_datasource/test_azure_helper.py b/tests/unittests/test_datasource/test_azure_helper.py index a5228870..68af31cd 100644 --- a/tests/unittests/test_datasource/test_azure_helper.py +++ b/tests/unittests/test_datasource/test_azure_helper.py @@ -97,7 +97,8 @@ class TestFindEndpoint(TestCase): if not use_hex: ip_address_repr = struct.pack( '>L', int(ip_address_repr.replace(':', ''), 16)) - ip_address_repr = '"{0}"'.format(ip_address_repr.decode('utf-8')) + ip_address_repr = '"{0}"'.format( + ip_address_repr.decode('utf-8').replace('"', '\\"')) return '\n'.join([ 'lease {', ' interface "eth0";', @@ -125,6 +126,13 @@ class TestFindEndpoint(TestCase): self.assertEqual(ip_address, azure_helper.WALinuxAgentShim.find_endpoint()) + def test_packed_string_with_escaped_quote(self): + ip_address = '100.72.34.108' + file_content = self._build_lease_content(ip_address, use_hex=False) + self.load_file.return_value = file_content + self.assertEqual(ip_address, + azure_helper.WALinuxAgentShim.find_endpoint()) + def test_latest_lease_used(self): ip_addresses = ['4.3.2.1', '98.76.54.32'] file_content = '\n'.join([self._build_lease_content(ip_address) -- cgit v1.2.3 From 20dc4190e27c7778cfa6c2943961f2ad27e14b48 Mon Sep 17 00:00:00 2001 From: Daniel Watkins Date: Fri, 9 Oct 2015 14:01:11 +0100 Subject: Handle colons in packed strings in WALinuxAgentShim.find_endpoint. This fixes bug 1488896. --- cloudinit/sources/helpers/azure.py | 12 +++++++----- tests/unittests/test_datasource/test_azure_helper.py | 7 +++++++ 2 files changed, 14 insertions(+), 5 deletions(-) (limited to 'tests/unittests/test_datasource/test_azure_helper.py') diff --git a/cloudinit/sources/helpers/azure.py b/cloudinit/sources/helpers/azure.py index 33003da0..21b4cd21 100644 --- a/cloudinit/sources/helpers/azure.py +++ b/cloudinit/sources/helpers/azure.py @@ -225,16 +225,18 @@ class WALinuxAgentShim(object): value = line.strip(' ').split(' ', 2)[-1].strip(';\n"') if value is None: raise Exception('No endpoint found in DHCP config.') - if ':' in value: + unescaped_value = value.replace('\\', '') + if len(unescaped_value) > 4: hex_string = '' - for hex_pair in value.split(':'): + for hex_pair in unescaped_value.split(':'): if len(hex_pair) == 1: hex_pair = '0' + hex_pair hex_string += hex_pair - value = struct.pack('>L', int(hex_string.replace(':', ''), 16)) + packed_bytes = struct.pack( + '>L', int(hex_string.replace(':', ''), 16)) else: - value = value.replace('\\', '').encode('utf-8') - endpoint_ip_address = socket.inet_ntoa(value) + packed_bytes = unescaped_value.encode('utf-8') + endpoint_ip_address = socket.inet_ntoa(packed_bytes) LOG.debug('Azure endpoint found at %s', endpoint_ip_address) return endpoint_ip_address diff --git a/tests/unittests/test_datasource/test_azure_helper.py b/tests/unittests/test_datasource/test_azure_helper.py index 68af31cd..5f906837 100644 --- a/tests/unittests/test_datasource/test_azure_helper.py +++ b/tests/unittests/test_datasource/test_azure_helper.py @@ -133,6 +133,13 @@ class TestFindEndpoint(TestCase): self.assertEqual(ip_address, azure_helper.WALinuxAgentShim.find_endpoint()) + def test_packed_string_containing_a_colon(self): + ip_address = '100.72.58.108' + file_content = self._build_lease_content(ip_address, use_hex=False) + self.load_file.return_value = file_content + self.assertEqual(ip_address, + azure_helper.WALinuxAgentShim.find_endpoint()) + def test_latest_lease_used(self): ip_addresses = ['4.3.2.1', '98.76.54.32'] file_content = '\n'.join([self._build_lease_content(ip_address) -- cgit v1.2.3 From a9f5d20bc55abc5ff12a5ce1cec1125173f11c7b Mon Sep 17 00:00:00 2001 From: Daniel Watkins Date: Fri, 9 Oct 2015 14:01:11 +0100 Subject: Convert test helper to staticmethod. --- tests/unittests/test_datasource/test_azure_helper.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'tests/unittests/test_datasource/test_azure_helper.py') diff --git a/tests/unittests/test_datasource/test_azure_helper.py b/tests/unittests/test_datasource/test_azure_helper.py index 5f906837..1746f6b8 100644 --- a/tests/unittests/test_datasource/test_azure_helper.py +++ b/tests/unittests/test_datasource/test_azure_helper.py @@ -90,7 +90,8 @@ class TestFindEndpoint(TestCase): self.assertRaises(Exception, azure_helper.WALinuxAgentShim.find_endpoint) - def _build_lease_content(self, ip_address, use_hex=True): + @staticmethod + def _build_lease_content(ip_address, use_hex=True): ip_address_repr = ':'.join( [hex(int(part)).replace('0x', '') for part in ip_address.split('.')]) -- cgit v1.2.3 From 2f5a5ed9f9c5ba390a693bc441ab18cbcdff9623 Mon Sep 17 00:00:00 2001 From: Daniel Watkins Date: Fri, 9 Oct 2015 14:01:11 +0100 Subject: Refactor tests to test helper method directly, and remove need for test helper. --- .../unittests/test_datasource/test_azure_helper.py | 84 +++++++++++----------- 1 file changed, 42 insertions(+), 42 deletions(-) (limited to 'tests/unittests/test_datasource/test_azure_helper.py') diff --git a/tests/unittests/test_datasource/test_azure_helper.py b/tests/unittests/test_datasource/test_azure_helper.py index 1746f6b8..fed2a7d8 100644 --- a/tests/unittests/test_datasource/test_azure_helper.py +++ b/tests/unittests/test_datasource/test_azure_helper.py @@ -91,63 +91,63 @@ class TestFindEndpoint(TestCase): azure_helper.WALinuxAgentShim.find_endpoint) @staticmethod - def _build_lease_content(ip_address, use_hex=True): - ip_address_repr = ':'.join( - [hex(int(part)).replace('0x', '') - for part in ip_address.split('.')]) - if not use_hex: - ip_address_repr = struct.pack( - '>L', int(ip_address_repr.replace(':', ''), 16)) - ip_address_repr = '"{0}"'.format( - ip_address_repr.decode('utf-8').replace('"', '\\"')) + def _build_lease_content(encoded_address): return '\n'.join([ 'lease {', ' interface "eth0";', - ' option unknown-245 {0};'.format(ip_address_repr), + ' option unknown-245 {0};'.format(encoded_address), '}']) - def test_hex_string(self): - ip_address = '98.76.54.32' - file_content = self._build_lease_content(ip_address) + def test_latest_lease_used(self): + encoded_addresses = ['5:4:3:2', '4:3:2:1'] + file_content = '\n'.join([self._build_lease_content(encoded_address) + for encoded_address in encoded_addresses]) self.load_file.return_value = file_content - self.assertEqual(ip_address, + self.assertEqual(encoded_addresses[-1].replace(':', '.'), azure_helper.WALinuxAgentShim.find_endpoint()) + +class TestExtractIpAddressFromLeaseValue(TestCase): + + def test_hex_string(self): + ip_address, encoded_address = '98.76.54.32', '62:4c:36:20' + self.assertEqual( + ip_address, + azure_helper.WALinuxAgentShim.get_ip_from_lease_value( + encoded_address + )) + def test_hex_string_with_single_character_part(self): - ip_address = '4.3.2.1' - file_content = self._build_lease_content(ip_address) - self.load_file.return_value = file_content - self.assertEqual(ip_address, - azure_helper.WALinuxAgentShim.find_endpoint()) + ip_address, encoded_address = '4.3.2.1', '4:3:2:1' + self.assertEqual( + ip_address, + azure_helper.WALinuxAgentShim.get_ip_from_lease_value( + encoded_address + )) def test_packed_string(self): - ip_address = '98.76.54.32' - file_content = self._build_lease_content(ip_address, use_hex=False) - self.load_file.return_value = file_content - self.assertEqual(ip_address, - azure_helper.WALinuxAgentShim.find_endpoint()) + ip_address, encoded_address = '98.76.54.32', 'bL6 ' + self.assertEqual( + ip_address, + azure_helper.WALinuxAgentShim.get_ip_from_lease_value( + encoded_address + )) def test_packed_string_with_escaped_quote(self): - ip_address = '100.72.34.108' - file_content = self._build_lease_content(ip_address, use_hex=False) - self.load_file.return_value = file_content - self.assertEqual(ip_address, - azure_helper.WALinuxAgentShim.find_endpoint()) + ip_address, encoded_address = '100.72.34.108', 'dH\\"l' + self.assertEqual( + ip_address, + azure_helper.WALinuxAgentShim.get_ip_from_lease_value( + encoded_address + )) def test_packed_string_containing_a_colon(self): - ip_address = '100.72.58.108' - file_content = self._build_lease_content(ip_address, use_hex=False) - self.load_file.return_value = file_content - self.assertEqual(ip_address, - azure_helper.WALinuxAgentShim.find_endpoint()) - - def test_latest_lease_used(self): - ip_addresses = ['4.3.2.1', '98.76.54.32'] - file_content = '\n'.join([self._build_lease_content(ip_address) - for ip_address in ip_addresses]) - self.load_file.return_value = file_content - self.assertEqual(ip_addresses[-1], - azure_helper.WALinuxAgentShim.find_endpoint()) + ip_address, encoded_address = '100.72.58.108', 'dH:l' + self.assertEqual( + ip_address, + azure_helper.WALinuxAgentShim.get_ip_from_lease_value( + encoded_address + )) class TestGoalStateParsing(TestCase): -- cgit v1.2.3 From 34b208a05361ae6ab4a51a6a999c9ac4ab77f06a Mon Sep 17 00:00:00 2001 From: Daniel Watkins Date: Fri, 30 Oct 2015 16:26:31 +0000 Subject: Use DMI data to find Azure instance IDs. This replaces the use of SharedConfig.xml in both the walinuxagent case, and the case where we communicate with the Azure fabric ourselves. --- cloudinit/sources/DataSourceAzure.py | 38 +---------- cloudinit/sources/helpers/azure.py | 21 ------ tests/unittests/test_datasource/test_azure.py | 77 +++++----------------- .../unittests/test_datasource/test_azure_helper.py | 42 +----------- 4 files changed, 23 insertions(+), 155 deletions(-) (limited to 'tests/unittests/test_datasource/test_azure_helper.py') diff --git a/cloudinit/sources/DataSourceAzure.py b/cloudinit/sources/DataSourceAzure.py index c6228e6c..bd80a8a6 100644 --- a/cloudinit/sources/DataSourceAzure.py +++ b/cloudinit/sources/DataSourceAzure.py @@ -31,8 +31,7 @@ from cloudinit import log as logging from cloudinit.settings import PER_ALWAYS from cloudinit import sources from cloudinit import util -from cloudinit.sources.helpers.azure import ( - get_metadata_from_fabric, iid_from_shared_config_content) +from cloudinit.sources.helpers.azure import get_metadata_from_fabric LOG = logging.getLogger(__name__) @@ -41,7 +40,6 @@ DEFAULT_METADATA = {"instance-id": "iid-AZURE-NODE"} AGENT_START = ['service', 'walinuxagent', 'start'] BOUNCE_COMMAND = ['sh', '-xc', "i=$interface; x=0; ifdown $i || x=$?; ifup $i || x=$?; exit $x"] -DATA_DIR_CLEAN_LIST = ['SharedConfig.xml'] BUILTIN_DS_CONFIG = { 'agent_command': AGENT_START, @@ -144,8 +142,6 @@ class DataSourceAzureNet(sources.DataSource): self.ds_cfg['agent_command']) ddir = self.ds_cfg['data_dir'] - shcfgxml = os.path.join(ddir, "SharedConfig.xml") - wait_for = [shcfgxml] fp_files = [] key_value = None @@ -160,19 +156,11 @@ class DataSourceAzureNet(sources.DataSource): missing = util.log_time(logfunc=LOG.debug, msg="waiting for files", func=wait_for_files, - args=(wait_for + fp_files,)) + args=(fp_files,)) if len(missing): LOG.warn("Did not find files, but going on: %s", missing) metadata = {} - if shcfgxml in missing: - LOG.warn("SharedConfig.xml missing, using static instance-id") - else: - try: - metadata['instance-id'] = iid_from_shared_config(shcfgxml) - except ValueError as e: - LOG.warn("failed to get instance id in %s: %s", shcfgxml, e) - metadata['public-keys'] = key_value or pubkeys_from_crt_files(fp_files) return metadata @@ -229,21 +217,6 @@ class DataSourceAzureNet(sources.DataSource): user_ds_cfg = util.get_cfg_by_path(self.cfg, DS_CFG_PATH, {}) self.ds_cfg = util.mergemanydict([user_ds_cfg, self.ds_cfg]) - if found != ddir: - cached_ovfenv = util.load_file( - os.path.join(ddir, 'ovf-env.xml'), quiet=True, decode=False) - if cached_ovfenv != files['ovf-env.xml']: - # source was not walinux-agent's datadir, so we have to clean - # up so 'wait_for_files' doesn't return early due to stale data - cleaned = [] - for f in [os.path.join(ddir, f) for f in DATA_DIR_CLEAN_LIST]: - if os.path.exists(f): - util.del_file(f) - cleaned.append(f) - if cleaned: - LOG.info("removed stale file(s) in '%s': %s", - ddir, str(cleaned)) - # walinux agent writes files world readable, but expects # the directory to be protected. write_files(ddir, files, dirmode=0o700) @@ -259,6 +232,7 @@ class DataSourceAzureNet(sources.DataSource): " on Azure.", exc_info=True) return False + self.metadata['instance-id'] = util.read_dmi_data('system-uuid') self.metadata.update(fabric_data) found_ephemeral = find_fabric_formatted_ephemeral_disk() @@ -649,12 +623,6 @@ def load_azure_ds_dir(source_dir): return (md, ud, cfg, {'ovf-env.xml': contents}) -def iid_from_shared_config(path): - with open(path, "rb") as fp: - content = fp.read() - return iid_from_shared_config_content(content) - - class BrokenAzureDataSource(Exception): pass diff --git a/cloudinit/sources/helpers/azure.py b/cloudinit/sources/helpers/azure.py index 281d733e..d90c22fd 100644 --- a/cloudinit/sources/helpers/azure.py +++ b/cloudinit/sources/helpers/azure.py @@ -78,12 +78,6 @@ class GoalState(object): return self._text_from_xpath( './Container/RoleInstanceList/RoleInstance/InstanceId') - @property - def shared_config_xml(self): - url = self._text_from_xpath('./Container/RoleInstanceList/RoleInstance' - '/Configuration/SharedConfig') - return self.http_client.get(url).contents - @property def certificates_xml(self): if self._certificates_xml is None: @@ -172,19 +166,6 @@ class OpenSSLManager(object): return keys -def iid_from_shared_config_content(content): - """ - find INSTANCE_ID in: - - - - - """ - root = ElementTree.fromstring(content) - depnode = root.find('Deployment') - return depnode.get('name') - - class WALinuxAgentShim(object): REPORT_READY_XML_TEMPLATE = '\n'.join([ @@ -263,8 +244,6 @@ class WALinuxAgentShim(object): public_keys = self.openssl_manager.parse_certificates( goal_state.certificates_xml) data = { - 'instance-id': iid_from_shared_config_content( - goal_state.shared_config_xml), 'public-keys': public_keys, } self._report_ready(goal_state, http_client) diff --git a/tests/unittests/test_datasource/test_azure.py b/tests/unittests/test_datasource/test_azure.py index ec0435f5..3933794f 100644 --- a/tests/unittests/test_datasource/test_azure.py +++ b/tests/unittests/test_datasource/test_azure.py @@ -115,10 +115,6 @@ class TestAzureDataSource(TestCase): data['pubkey_files'] = flist return ["pubkey_from: %s" % f for f in flist] - def _iid_from_shared_config(path): - data['iid_from_shared_cfg'] = path - return 'i-my-azure-id' - if data.get('ovfcontent') is not None: populate_dir(os.path.join(self.paths.seed_dir, "azure"), {'ovf-env.xml': data['ovfcontent']}) @@ -127,20 +123,22 @@ class TestAzureDataSource(TestCase): mod.BUILTIN_DS_CONFIG['data_dir'] = self.waagent_d self.get_metadata_from_fabric = mock.MagicMock(return_value={ - 'instance-id': 'i-my-azure-id', 'public-keys': [], }) + self.instance_id = 'test-instance-id' + self.apply_patches([ (mod, 'list_possible_azure_ds_devs', dsdevs), (mod, 'invoke_agent', _invoke_agent), (mod, 'wait_for_files', _wait_for_files), (mod, 'pubkeys_from_crt_files', _pubkeys_from_crt_files), - (mod, 'iid_from_shared_config', _iid_from_shared_config), (mod, 'perform_hostname_bounce', mock.MagicMock()), (mod, 'get_hostname', mock.MagicMock()), (mod, 'set_hostname', mock.MagicMock()), (mod, 'get_metadata_from_fabric', self.get_metadata_from_fabric), + (mod.util, 'read_dmi_data', mock.MagicMock( + return_value=self.instance_id)), ]) dsrc = mod.DataSourceAzureNet( @@ -193,7 +191,6 @@ class TestAzureDataSource(TestCase): self.assertEqual(dsrc.metadata['local-hostname'], odata['HostName']) self.assertTrue(os.path.isfile( os.path.join(self.waagent_d, 'ovf-env.xml'))) - self.assertEqual(dsrc.metadata['instance-id'], 'i-my-azure-id') def test_waagent_d_has_0700_perms(self): # we expect /var/lib/waagent to be created 0700 @@ -345,7 +342,6 @@ class TestAzureDataSource(TestCase): for mypk in mypklist: self.assertIn(mypk['value'], dsrc.metadata['public-keys']) - def test_default_ephemeral(self): # make sure the ephemeral device works odata = {} @@ -434,54 +430,6 @@ class TestAzureDataSource(TestCase): dsrc = self._get_ds({'ovfcontent': xml}) dsrc.get_data() - def test_existing_ovf_same(self): - # waagent/SharedConfig left alone if found ovf-env.xml same as cached - odata = {'UserData': b64e("SOMEUSERDATA")} - data = {'ovfcontent': construct_valid_ovf_env(data=odata)} - - populate_dir(self.waagent_d, - {'ovf-env.xml': data['ovfcontent'], - 'otherfile': 'otherfile-content', - 'SharedConfig.xml': 'mysharedconfig'}) - - dsrc = self._get_ds(data) - ret = dsrc.get_data() - self.assertTrue(ret) - self.assertTrue(os.path.exists( - os.path.join(self.waagent_d, 'ovf-env.xml'))) - self.assertTrue(os.path.exists( - os.path.join(self.waagent_d, 'otherfile'))) - self.assertTrue(os.path.exists( - os.path.join(self.waagent_d, 'SharedConfig.xml'))) - - def test_existing_ovf_diff(self): - # waagent/SharedConfig must be removed if ovfenv is found elsewhere - - # 'get_data' should remove SharedConfig.xml in /var/lib/waagent - # if ovf-env.xml differs. - cached_ovfenv = construct_valid_ovf_env( - {'userdata': b64e("FOO_USERDATA")}) - new_ovfenv = construct_valid_ovf_env( - {'userdata': b64e("NEW_USERDATA")}) - - populate_dir(self.waagent_d, - {'ovf-env.xml': cached_ovfenv, - 'SharedConfig.xml': "mysharedconfigxml", - 'otherfile': 'otherfilecontent'}) - - dsrc = self._get_ds({'ovfcontent': new_ovfenv}) - ret = dsrc.get_data() - self.assertTrue(ret) - self.assertEqual(dsrc.userdata_raw, b"NEW_USERDATA") - self.assertTrue(os.path.exists( - os.path.join(self.waagent_d, 'otherfile'))) - self.assertFalse(os.path.exists( - os.path.join(self.waagent_d, 'SharedConfig.xml'))) - self.assertTrue(os.path.exists( - os.path.join(self.waagent_d, 'ovf-env.xml'))) - new_xml = load_file(os.path.join(self.waagent_d, 'ovf-env.xml')) - self.xml_equals(new_ovfenv, new_xml) - def test_exception_fetching_fabric_data_doesnt_propagate(self): ds = self._get_ds({'ovfcontent': construct_valid_ovf_env()}) ds.ds_cfg['agent_command'] = '__builtin__' @@ -496,6 +444,17 @@ class TestAzureDataSource(TestCase): self.assertTrue(ret) self.assertEqual('value', ds.metadata['test']) + def test_instance_id_from_dmidecode_used(self): + ds = self._get_ds({'ovfcontent': construct_valid_ovf_env()}) + ds.get_data() + self.assertEqual(self.instance_id, ds.metadata['instance-id']) + + def test_instance_id_from_dmidecode_used_for_builtin(self): + ds = self._get_ds({'ovfcontent': construct_valid_ovf_env()}) + ds.ds_cfg['agent_command'] = '__builtin__' + ds.get_data() + self.assertEqual(self.instance_id, ds.metadata['instance-id']) + class TestAzureBounce(TestCase): @@ -504,9 +463,6 @@ class TestAzureBounce(TestCase): mock.patch.object(DataSourceAzure, 'invoke_agent')) self.patches.enter_context( mock.patch.object(DataSourceAzure, 'wait_for_files')) - self.patches.enter_context( - mock.patch.object(DataSourceAzure, 'iid_from_shared_config', - mock.MagicMock(return_value='i-my-azure-id'))) self.patches.enter_context( mock.patch.object(DataSourceAzure, 'list_possible_azure_ds_devs', mock.MagicMock(return_value=[]))) @@ -521,6 +477,9 @@ class TestAzureBounce(TestCase): self.patches.enter_context( mock.patch.object(DataSourceAzure, 'get_metadata_from_fabric', mock.MagicMock(return_value={}))) + self.patches.enter_context( + mock.patch.object(DataSourceAzure.util, 'read_dmi_data', + mock.MagicMock(return_value='test-instance-id'))) def setUp(self): super(TestAzureBounce, self).setUp() diff --git a/tests/unittests/test_datasource/test_azure_helper.py b/tests/unittests/test_datasource/test_azure_helper.py index a5228870..0638c974 100644 --- a/tests/unittests/test_datasource/test_azure_helper.py +++ b/tests/unittests/test_datasource/test_azure_helper.py @@ -40,7 +40,7 @@ GOAL_STATE_TEMPLATE = """\ http://100.86.192.70:80/...hostingEnvironmentConfig... - {shared_config_url} + http://100.86.192.70:80/..SharedConfig.. http://100.86.192.70:80/...extensionsConfig... @@ -55,21 +55,6 @@ GOAL_STATE_TEMPLATE = """\ """ -class TestReadAzureSharedConfig(unittest.TestCase): - - def test_valid_content(self): - xml = """ - - - - - - - """ - ret = azure_helper.iid_from_shared_config_content(xml) - self.assertEqual("MY_INSTANCE_ID", ret) - - class TestFindEndpoint(TestCase): def setUp(self): @@ -140,7 +125,6 @@ class TestGoalStateParsing(TestCase): 'incarnation': 1, 'container_id': 'MyContainerId', 'instance_id': 'MyInstanceId', - 'shared_config_url': 'MySharedConfigUrl', 'certificates_url': 'MyCertificatesUrl', } @@ -174,20 +158,9 @@ class TestGoalStateParsing(TestCase): goal_state = self._get_goal_state(instance_id=instance_id) self.assertEqual(instance_id, goal_state.instance_id) - def test_shared_config_xml_parsed_and_fetched_correctly(self): - http_client = mock.MagicMock() - shared_config_url = 'TestSharedConfigUrl' - goal_state = self._get_goal_state( - http_client=http_client, shared_config_url=shared_config_url) - shared_config_xml = goal_state.shared_config_xml - self.assertEqual(1, http_client.get.call_count) - self.assertEqual(shared_config_url, http_client.get.call_args[0][0]) - self.assertEqual(http_client.get.return_value.contents, - shared_config_xml) - def test_certificates_xml_parsed_and_fetched_correctly(self): http_client = mock.MagicMock() - certificates_url = 'TestSharedConfigUrl' + certificates_url = 'TestCertificatesUrl' goal_state = self._get_goal_state( http_client=http_client, certificates_url=certificates_url) certificates_xml = goal_state.certificates_xml @@ -324,8 +297,6 @@ class TestWALinuxAgentShim(TestCase): azure_helper.WALinuxAgentShim, 'find_endpoint')) self.GoalState = patches.enter_context( mock.patch.object(azure_helper, 'GoalState')) - self.iid_from_shared_config_content = patches.enter_context( - mock.patch.object(azure_helper, 'iid_from_shared_config_content')) self.OpenSSLManager = patches.enter_context( mock.patch.object(azure_helper, 'OpenSSLManager')) patches.enter_context( @@ -367,15 +338,6 @@ class TestWALinuxAgentShim(TestCase): data = shim.register_with_azure_and_fetch_data() self.assertEqual([], data['public-keys']) - def test_instance_id_returned_in_data(self): - shim = azure_helper.WALinuxAgentShim() - data = shim.register_with_azure_and_fetch_data() - self.assertEqual( - [mock.call(self.GoalState.return_value.shared_config_xml)], - self.iid_from_shared_config_content.call_args_list) - self.assertEqual(self.iid_from_shared_config_content.return_value, - data['instance-id']) - def test_correct_url_used_for_report_ready(self): self.find_endpoint.return_value = 'test_endpoint' shim = azure_helper.WALinuxAgentShim() -- cgit v1.2.3 From c496b6a11d504ef62371cb5e03ac80b4ceb37540 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Thu, 3 Mar 2016 12:20:48 -0500 Subject: run pyflakes in more places, fix fallout this makes 'make' run pyflakes, so failures there will stop a build. also adds it to tox. --- Makefile | 6 ++++-- cloudinit/sources/DataSourceOVF.py | 3 ++- cloudinit/sources/helpers/vmware/imc/config_nic.py | 1 - cloudinit/util.py | 2 +- tests/unittests/test_datasource/test_azure_helper.py | 2 -- tests/unittests/test_datasource/test_smartos.py | 1 - tests/unittests/test_handler/test_handler_power_state.py | 2 +- tox.ini | 6 +++++- 8 files changed, 13 insertions(+), 10 deletions(-) (limited to 'tests/unittests/test_datasource/test_azure_helper.py') diff --git a/Makefile b/Makefile index bb0c5253..8987d51c 100644 --- a/Makefile +++ b/Makefile @@ -14,13 +14,15 @@ ifeq ($(distro),) distro = redhat endif -all: test check_version +all: check + +check: test check_version pyflakes pep8: @$(CWD)/tools/run-pep8 $(PY_FILES) pyflakes: - @$(CWD)/tools/tox-venv py34 pyflakes $(PY_FILES) + @pyflakes $(PY_FILES) pip-requirements: @echo "Installing cloud-init dependencies..." diff --git a/cloudinit/sources/DataSourceOVF.py b/cloudinit/sources/DataSourceOVF.py index 72ba5aba..d12601a4 100644 --- a/cloudinit/sources/DataSourceOVF.py +++ b/cloudinit/sources/DataSourceOVF.py @@ -90,7 +90,8 @@ class DataSourceOVF(sources.DataSource): nicConfigurator.configure() vmwarePlatformFound = True except Exception as inst: - LOG.debug("Error while parsing the Customization Config File") + LOG.debug("Error while parsing the Customization " + "Config File: %s", inst) finally: dirPath = os.path.dirname(vmwareImcConfigFilePath) shutil.rmtree(dirPath) diff --git a/cloudinit/sources/helpers/vmware/imc/config_nic.py b/cloudinit/sources/helpers/vmware/imc/config_nic.py index 172a1649..6d721134 100644 --- a/cloudinit/sources/helpers/vmware/imc/config_nic.py +++ b/cloudinit/sources/helpers/vmware/imc/config_nic.py @@ -19,7 +19,6 @@ import logging import os -import subprocess import re from cloudinit import util diff --git a/cloudinit/util.py b/cloudinit/util.py index 45d49e66..0a639bb9 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -2147,7 +2147,7 @@ def _read_dmi_syspath(key): LOG.debug("dmi data %s returned %s", dmi_key_path, key_data) return key_data.strip() - except Exception as e: + except Exception: logexc(LOG, "failed read of %s", dmi_key_path) return None diff --git a/tests/unittests/test_datasource/test_azure_helper.py b/tests/unittests/test_datasource/test_azure_helper.py index 8dbdfb0b..1134199b 100644 --- a/tests/unittests/test_datasource/test_azure_helper.py +++ b/tests/unittests/test_datasource/test_azure_helper.py @@ -1,6 +1,4 @@ import os -import struct -import unittest from cloudinit.sources.helpers import azure as azure_helper from ..helpers import TestCase diff --git a/tests/unittests/test_datasource/test_smartos.py b/tests/unittests/test_datasource/test_smartos.py index 1235436d..ccb9f080 100644 --- a/tests/unittests/test_datasource/test_smartos.py +++ b/tests/unittests/test_datasource/test_smartos.py @@ -31,7 +31,6 @@ import shutil import stat import tempfile import uuid -import unittest from binascii import crc32 import serial diff --git a/tests/unittests/test_handler/test_handler_power_state.py b/tests/unittests/test_handler/test_handler_power_state.py index 5687b10d..cd376e9c 100644 --- a/tests/unittests/test_handler/test_handler_power_state.py +++ b/tests/unittests/test_handler/test_handler_power_state.py @@ -107,7 +107,7 @@ def check_lps_ret(psc_return, mode=None): if 'shutdown' not in psc_return[0][0]: errs.append("string 'shutdown' not in cmd") - if 'condition' is None: + if condition is None: errs.append("condition was not returned") if mode is not None: diff --git a/tox.ini b/tox.ini index b72df0c9..fd65f6ef 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py27,py3 +envlist = py27,py3,pyflakes recreate = True [testenv] @@ -10,6 +10,10 @@ deps = -r{toxinidir}/test-requirements.txt [testenv:py3] basepython = python3 +[testenv:pyflakes] +basepython = python3 +commands = {envpython} -m pyflakes {posargs:cloudinit/ tests/ tools/} + # https://github.com/gabrielfalcao/HTTPretty/issues/223 setenv = LC_ALL = en_US.utf-8 -- cgit v1.2.3