From f0bc02d7e221c9aa5982b267739481420c761ead Mon Sep 17 00:00:00 2001 From: Chad Smith Date: Tue, 9 Oct 2018 21:46:35 +0000 Subject: instance-data: Add standard keys platform and subplatform. Refactor ec2. Add the following instance-data.json standardized keys: * v1._beta_keys: List any v1 keys in beta development, e.g. ['subplatform']. * v1.public_ssh_keys: List of any cloud-provided ssh keys for the instance. * v1.platform: String representing the cloud platform api supporting the datasource. For example: 'ec2' for aws, aliyun and brightbox cloud names. * v1.subplatform: String with more details about the source of the metadata consumed. For example, metadata uri, config drive device path or seed directory. To support the new platform and subplatform standardized instance-data, DataSource and its subclasses grew platform and subplatform attributes. The platform attribute defaults to the lowercase string datasource name at self.dsname. This method is overridden in NoCloud, Ec2 and ConfigDrive datasources. The subplatform attribute calls a _get_subplatform method which will return a string containing a simple slug for subplatform type such as metadata, seed-dir or config-drive followed by a detailed uri, device or directory path where the datasource consumed its configuration. As part of this work, DatasourceEC2 methods _get_data and _crawl_metadata have been refactored for a few reasons: - crawl_metadata is now a read-only operation, persisting no attributes on the datasource instance and returns a dictionary of consumed metadata. - crawl_metadata now closely represents the raw stucture of the ec2 metadata consumed, so that end-users can leverage public ec2 metadata documentation where possible. - crawl_metadata adds a '_metadata_api_version' key to the crawled ds.metadata to advertise what version of EC2's api was consumed by cloud-init. - _get_data now does all the processing of crawl_metadata and saves datasource instance attributes userdata_raw, metadata etc. Additional drive-bys: * unit test rework for test_altcloud and test_azure to simplify mocks and make use of existing util and test_helpers functions. --- tests/unittests/test_datasource/test_ec2.py | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) (limited to 'tests/unittests/test_datasource/test_ec2.py') diff --git a/tests/unittests/test_datasource/test_ec2.py b/tests/unittests/test_datasource/test_ec2.py index 497e7610..9f81255a 100644 --- a/tests/unittests/test_datasource/test_ec2.py +++ b/tests/unittests/test_datasource/test_ec2.py @@ -351,7 +351,9 @@ class TestEc2(test_helpers.HttprettyTestCase): m_get_interface_mac.return_value = mac1 nc = ds.network_config # Will re-crawl network metadata self.assertIsNotNone(nc) - self.assertIn('Re-crawl of metadata service', self.logs.getvalue()) + self.assertIn( + 'Refreshing stale metadata from prior to upgrade', + self.logs.getvalue()) expected = {'version': 1, 'config': [ {'mac_address': '06:17:04:d7:26:09', 'name': 'eth9', @@ -386,7 +388,7 @@ class TestEc2(test_helpers.HttprettyTestCase): register_mock_metaserver( '{0}/{1}/dynamic/'.format(ds.metadata_address, all_versions[-1]), DYNAMIC_METADATA) - ds._cloud_platform = ec2.Platforms.AWS + ds._cloud_name = ec2.CloudNames.AWS # Setup cached metadata on the Datasource ds.metadata = DEFAULT_METADATA self.assertEqual('my-identity-id', ds.get_instance_id()) @@ -401,6 +403,9 @@ class TestEc2(test_helpers.HttprettyTestCase): ret = ds.get_data() self.assertTrue(ret) self.assertEqual(0, m_dhcp.call_count) + self.assertEqual('aws', ds.cloud_name) + self.assertEqual('ec2', ds.platform_type) + self.assertEqual('metadata (%s)' % ds.metadata_address, ds.subplatform) def test_valid_platform_with_strict_false(self): """Valid platform data should return true with strict_id false.""" @@ -439,16 +444,17 @@ class TestEc2(test_helpers.HttprettyTestCase): sys_cfg={'datasource': {'Ec2': {'strict_id': False}}}, md=DEFAULT_METADATA) platform_attrs = [ - attr for attr in ec2.Platforms.__dict__.keys() + attr for attr in ec2.CloudNames.__dict__.keys() if not attr.startswith('__')] for attr_name in platform_attrs: - platform_name = getattr(ec2.Platforms, attr_name) - if platform_name != 'AWS': - ds._cloud_platform = platform_name + platform_name = getattr(ec2.CloudNames, attr_name) + if platform_name != 'aws': + ds._cloud_name = platform_name ret = ds.get_data() + self.assertEqual('ec2', ds.platform_type) self.assertFalse(ret) message = ( - "Local Ec2 mode only supported on ('AWS',)," + "Local Ec2 mode only supported on ('aws',)," ' not {0}'.format(platform_name)) self.assertIn(message, self.logs.getvalue()) -- cgit v1.2.3 From 093f968013f418d591d9e778a57b2f050ac30db3 Mon Sep 17 00:00:00 2001 From: Chad Smith Date: Thu, 8 Nov 2018 04:19:09 +0000 Subject: tests: ec2 mock missing httpretty user-data and instance-identity routes --- tests/unittests/test_datasource/test_ec2.py | 40 +++++++++++++++++------------ 1 file changed, 24 insertions(+), 16 deletions(-) (limited to 'tests/unittests/test_datasource/test_ec2.py') diff --git a/tests/unittests/test_datasource/test_ec2.py b/tests/unittests/test_datasource/test_ec2.py index 9f81255a..1a5956d9 100644 --- a/tests/unittests/test_datasource/test_ec2.py +++ b/tests/unittests/test_datasource/test_ec2.py @@ -211,9 +211,9 @@ class TestEc2(test_helpers.HttprettyTestCase): self.metadata_addr = self.datasource.metadata_urls[0] self.tmp = self.tmp_dir() - def data_url(self, version): + def data_url(self, version, data_item='meta-data'): """Return a metadata url based on the version provided.""" - return '/'.join([self.metadata_addr, version, 'meta-data', '']) + return '/'.join([self.metadata_addr, version, data_item]) def _patch_add_cleanup(self, mpath, *args, **kwargs): p = mock.patch(mpath, *args, **kwargs) @@ -238,10 +238,18 @@ class TestEc2(test_helpers.HttprettyTestCase): all_versions = ( [ds.min_metadata_version] + ds.extended_metadata_versions) for version in all_versions: - metadata_url = self.data_url(version) + metadata_url = self.data_url(version) + '/' if version == md_version: # Register all metadata for desired version - register_mock_metaserver(metadata_url, md) + register_mock_metaserver( + metadata_url, md.get('md', DEFAULT_METADATA)) + userdata_url = self.data_url( + version, data_item='user-data') + register_mock_metaserver(userdata_url, md.get('ud', '')) + identity_url = self.data_url( + version, data_item='dynamic/instance-identity') + register_mock_metaserver( + identity_url, md.get('id', DYNAMIC_METADATA)) else: instance_id_url = metadata_url + 'instance-id' if version == ds.min_metadata_version: @@ -261,7 +269,7 @@ class TestEc2(test_helpers.HttprettyTestCase): ds = self._setup_ds( platform_data=self.valid_platform_data, sys_cfg={'datasource': {'Ec2': {'strict_id': True}}}, - md=DEFAULT_METADATA) + md={'md': DEFAULT_METADATA}) find_fallback_path = ( 'cloudinit.sources.DataSourceEc2.net.find_fallback_nic') with mock.patch(find_fallback_path) as m_find_fallback: @@ -293,7 +301,7 @@ class TestEc2(test_helpers.HttprettyTestCase): ds = self._setup_ds( platform_data=self.valid_platform_data, sys_cfg={'datasource': {'Ec2': {'strict_id': True}}}, - md=DEFAULT_METADATA) + md={'md': DEFAULT_METADATA}) find_fallback_path = ( 'cloudinit.sources.DataSourceEc2.net.find_fallback_nic') with mock.patch(find_fallback_path) as m_find_fallback: @@ -322,7 +330,7 @@ class TestEc2(test_helpers.HttprettyTestCase): ds = self._setup_ds( platform_data=self.valid_platform_data, sys_cfg={'datasource': {'Ec2': {'strict_id': True}}}, - md=DEFAULT_METADATA) + md={'md': DEFAULT_METADATA}) ds._network_config = {'cached': 'data'} self.assertEqual({'cached': 'data'}, ds.network_config) @@ -338,7 +346,7 @@ class TestEc2(test_helpers.HttprettyTestCase): ds = self._setup_ds( platform_data=self.valid_platform_data, sys_cfg={'datasource': {'Ec2': {'strict_id': True}}}, - md=old_metadata) + md={'md': old_metadata}) self.assertTrue(ds.get_data()) # Provide new revision of metadata that contains network data register_mock_metaserver( @@ -372,7 +380,7 @@ class TestEc2(test_helpers.HttprettyTestCase): ds = self._setup_ds( platform_data=self.valid_platform_data, sys_cfg={'datasource': {'Ec2': {'strict_id': False}}}, - md=DEFAULT_METADATA) + md={'md': DEFAULT_METADATA}) # Mock 404s on all versions except latest all_versions = ( [ds.min_metadata_version] + ds.extended_metadata_versions) @@ -399,7 +407,7 @@ class TestEc2(test_helpers.HttprettyTestCase): ds = self._setup_ds( platform_data=self.valid_platform_data, sys_cfg={'datasource': {'Ec2': {'strict_id': True}}}, - md=DEFAULT_METADATA) + md={'md': DEFAULT_METADATA}) ret = ds.get_data() self.assertTrue(ret) self.assertEqual(0, m_dhcp.call_count) @@ -412,7 +420,7 @@ class TestEc2(test_helpers.HttprettyTestCase): ds = self._setup_ds( platform_data=self.valid_platform_data, sys_cfg={'datasource': {'Ec2': {'strict_id': False}}}, - md=DEFAULT_METADATA) + md={'md': DEFAULT_METADATA}) ret = ds.get_data() self.assertTrue(ret) @@ -422,7 +430,7 @@ class TestEc2(test_helpers.HttprettyTestCase): ds = self._setup_ds( platform_data={'uuid': uuid, 'uuid_source': 'dmi', 'serial': ''}, sys_cfg={'datasource': {'Ec2': {'strict_id': True}}}, - md=DEFAULT_METADATA) + md={'md': DEFAULT_METADATA}) ret = ds.get_data() self.assertFalse(ret) @@ -432,7 +440,7 @@ class TestEc2(test_helpers.HttprettyTestCase): ds = self._setup_ds( platform_data={'uuid': uuid, 'uuid_source': 'dmi', 'serial': ''}, sys_cfg={'datasource': {'Ec2': {'strict_id': False}}}, - md=DEFAULT_METADATA) + md={'md': DEFAULT_METADATA}) ret = ds.get_data() self.assertTrue(ret) @@ -442,7 +450,7 @@ class TestEc2(test_helpers.HttprettyTestCase): ds = self._setup_ds( platform_data=self.valid_platform_data, sys_cfg={'datasource': {'Ec2': {'strict_id': False}}}, - md=DEFAULT_METADATA) + md={'md': DEFAULT_METADATA}) platform_attrs = [ attr for attr in ec2.CloudNames.__dict__.keys() if not attr.startswith('__')] @@ -469,7 +477,7 @@ class TestEc2(test_helpers.HttprettyTestCase): ds = self._setup_ds( platform_data=self.valid_platform_data, sys_cfg={'datasource': {'Ec2': {'strict_id': False}}}, - md=DEFAULT_METADATA) + md={'md': DEFAULT_METADATA}) ret = ds.get_data() self.assertFalse(ret) self.assertIn( @@ -499,7 +507,7 @@ class TestEc2(test_helpers.HttprettyTestCase): ds = self._setup_ds( platform_data=self.valid_platform_data, sys_cfg={'datasource': {'Ec2': {'strict_id': False}}}, - md=DEFAULT_METADATA) + md={'md': DEFAULT_METADATA}) ret = ds.get_data() self.assertTrue(ret) -- cgit v1.2.3