diff options
| -rw-r--r-- | cloudinit/cmd/tests/test_main.py | 3 | ||||
| -rw-r--r-- | cloudinit/helpers.py | 7 | ||||
| -rw-r--r-- | cloudinit/settings.py | 1 | ||||
| -rw-r--r-- | cloudinit/sources/DataSourceOpenStack.py | 8 | ||||
| -rw-r--r-- | cloudinit/sources/__init__.py | 13 | ||||
| -rw-r--r-- | cloudinit/sources/helpers/openstack.py | 5 | ||||
| -rw-r--r-- | cloudinit/stages.py | 106 | ||||
| -rw-r--r-- | doc/rtd/topics/datasources/openstack.rst | 8 | ||||
| -rw-r--r-- | tests/unittests/test_data.py | 37 | ||||
| -rw-r--r-- | tests/unittests/test_datasource/test_openstack.py | 32 |
10 files changed, 171 insertions, 49 deletions
diff --git a/cloudinit/cmd/tests/test_main.py b/cloudinit/cmd/tests/test_main.py index 585b3b0e..78b27441 100644 --- a/cloudinit/cmd/tests/test_main.py +++ b/cloudinit/cmd/tests/test_main.py @@ -127,7 +127,8 @@ class TestMain(FilesystemMockingTestCase): 'syslog_fix_perms': [ 'syslog:adm', 'root:adm', 'root:wheel', 'root:root' ], - 'vendor_data': {'enabled': True, 'prefix': []}}) + 'vendor_data': {'enabled': True, 'prefix': []}, + 'vendor_data2': {'enabled': True, 'prefix': []}}) updated_cfg.pop('system_info') self.assertEqual(updated_cfg, cfg) diff --git a/cloudinit/helpers.py b/cloudinit/helpers.py index 9752ad28..fc5011ec 100644 --- a/cloudinit/helpers.py +++ b/cloudinit/helpers.py @@ -230,6 +230,10 @@ class ConfigMerger(object): cc_paths = ['cloud_config'] if self._include_vendor: + # the order is important here: we want vendor2 + # (dynamic vendor data from OpenStack) + # to override vendor (static data from OpenStack) + cc_paths.append('vendor2_cloud_config') cc_paths.append('vendor_cloud_config') for cc_p in cc_paths: @@ -337,9 +341,12 @@ class Paths(object): "obj_pkl": "obj.pkl", "cloud_config": "cloud-config.txt", "vendor_cloud_config": "vendor-cloud-config.txt", + "vendor2_cloud_config": "vendor2-cloud-config.txt", "data": "data", "vendordata_raw": "vendor-data.txt", + "vendordata2_raw": "vendor-data2.txt", "vendordata": "vendor-data.txt.i", + "vendordata2": "vendor-data2.txt.i", "instance_id": ".instance-id", "manual_clean_marker": "manual-clean", "warnings": "warnings", diff --git a/cloudinit/settings.py b/cloudinit/settings.py index ca4ffa8e..7516e17b 100644 --- a/cloudinit/settings.py +++ b/cloudinit/settings.py @@ -56,6 +56,7 @@ CFG_BUILTIN = { 'network': {'renderers': None}, }, 'vendor_data': {'enabled': True, 'prefix': []}, + 'vendor_data2': {'enabled': True, 'prefix': []}, } # Valid frequencies of handlers/modules diff --git a/cloudinit/sources/DataSourceOpenStack.py b/cloudinit/sources/DataSourceOpenStack.py index b3406c67..619a171e 100644 --- a/cloudinit/sources/DataSourceOpenStack.py +++ b/cloudinit/sources/DataSourceOpenStack.py @@ -167,6 +167,14 @@ class DataSourceOpenStack(openstack.SourceMixin, sources.DataSource): LOG.warning("Invalid content in vendor-data: %s", e) self.vendordata_raw = None + vd2 = results.get('vendordata2') + self.vendordata2_pure = vd2 + try: + self.vendordata2_raw = sources.convert_vendordata(vd2) + except ValueError as e: + LOG.warning("Invalid content in vendor-data2: %s", e) + self.vendordata2_raw = None + return True def _crawl_metadata(self): diff --git a/cloudinit/sources/__init__.py b/cloudinit/sources/__init__.py index 9dccc687..1ad1880d 100644 --- a/cloudinit/sources/__init__.py +++ b/cloudinit/sources/__init__.py @@ -187,7 +187,8 @@ class DataSource(metaclass=abc.ABCMeta): cached_attr_defaults = ( ('ec2_metadata', UNSET), ('network_json', UNSET), ('metadata', {}), ('userdata', None), ('userdata_raw', None), - ('vendordata', None), ('vendordata_raw', None)) + ('vendordata', None), ('vendordata_raw', None), + ('vendordata2', None), ('vendordata2_raw', None)) _dirty_cache = False @@ -203,7 +204,9 @@ class DataSource(metaclass=abc.ABCMeta): self.metadata = {} self.userdata_raw = None self.vendordata = None + self.vendordata2 = None self.vendordata_raw = None + self.vendordata2_raw = None self.ds_cfg = util.get_cfg_by_path( self.sys_cfg, ("datasource", self.dsname), {}) @@ -392,6 +395,11 @@ class DataSource(metaclass=abc.ABCMeta): self.vendordata = self.ud_proc.process(self.get_vendordata_raw()) return self.vendordata + def get_vendordata2(self): + if self.vendordata2 is None: + self.vendordata2 = self.ud_proc.process(self.get_vendordata2_raw()) + return self.vendordata2 + @property def fallback_interface(self): """Determine the network interface used during local network config.""" @@ -494,6 +502,9 @@ class DataSource(metaclass=abc.ABCMeta): def get_vendordata_raw(self): return self.vendordata_raw + def get_vendordata2_raw(self): + return self.vendordata2_raw + # the data sources' config_obj is a cloud-config formated # object that came to it from ways other than cloud-config # because cloud-config content would be handled elsewhere diff --git a/cloudinit/sources/helpers/openstack.py b/cloudinit/sources/helpers/openstack.py index 3e6365f1..4f566e64 100644 --- a/cloudinit/sources/helpers/openstack.py +++ b/cloudinit/sources/helpers/openstack.py @@ -247,6 +247,11 @@ class BaseReader(metaclass=abc.ABCMeta): False, load_json_anytype, ) + files['vendordata2'] = ( + self._path_join("openstack", version, 'vendor_data2.json'), + False, + load_json_anytype, + ) files['networkdata'] = ( self._path_join("openstack", version, 'network_data.json'), False, diff --git a/cloudinit/stages.py b/cloudinit/stages.py index 0cce6e80..3ef4491c 100644 --- a/cloudinit/stages.py +++ b/cloudinit/stages.py @@ -360,8 +360,18 @@ class Init(object): reporter=self.reporter) def update(self): - self._store_userdata() - self._store_vendordata() + self._store_rawdata(self.datasource.get_userdata_raw(), + 'userdata') + self._store_processeddata(self.datasource.get_userdata(), + 'userdata') + self._store_rawdata(self.datasource.get_vendordata_raw(), + 'vendordata') + self._store_processeddata(self.datasource.get_vendordata(), + 'vendordata') + self._store_rawdata(self.datasource.get_vendordata2_raw(), + 'vendordata2') + self._store_processeddata(self.datasource.get_vendordata2(), + 'vendordata2') def setup_datasource(self): with events.ReportEventStack("setup-datasource", @@ -381,28 +391,18 @@ class Init(object): is_new_instance=self.is_new_instance()) self._write_to_cache() - def _store_userdata(self): - raw_ud = self.datasource.get_userdata_raw() - if raw_ud is None: - raw_ud = b'' - util.write_file(self._get_ipath('userdata_raw'), raw_ud, 0o600) - # processed userdata is a Mime message, so write it as string. - processed_ud = self.datasource.get_userdata() - if processed_ud is None: - raw_ud = '' - util.write_file(self._get_ipath('userdata'), str(processed_ud), 0o600) - - def _store_vendordata(self): - raw_vd = self.datasource.get_vendordata_raw() - if raw_vd is None: - raw_vd = b'' - util.write_file(self._get_ipath('vendordata_raw'), raw_vd, 0o600) - # processed vendor data is a Mime message, so write it as string. - processed_vd = str(self.datasource.get_vendordata()) - if processed_vd is None: - processed_vd = '' - util.write_file(self._get_ipath('vendordata'), str(processed_vd), - 0o600) + def _store_rawdata(self, data, datasource): + # Raw data is bytes, not a string + if data is None: + data = b'' + util.write_file(self._get_ipath('%s_raw' % datasource), data, 0o600) + + def _store_processeddata(self, processed_data, datasource): + # processed is a Mime message, so write as string. + if processed_data is None: + processed_data = '' + util.write_file(self._get_ipath(datasource), + str(processed_data), 0o600) def _default_handlers(self, opts=None): if opts is None: @@ -434,6 +434,11 @@ class Init(object): opts={'script_path': 'vendor_scripts', 'cloud_config_path': 'vendor_cloud_config'}) + def _default_vendordata2_handlers(self): + return self._default_handlers( + opts={'script_path': 'vendor_scripts', + 'cloud_config_path': 'vendor2_cloud_config'}) + def _do_handlers(self, data_msg, c_handlers_list, frequency, excluded=None): """ @@ -555,7 +560,12 @@ class Init(object): with events.ReportEventStack("consume-vendor-data", "reading and applying vendor-data", parent=self.reporter): - self._consume_vendordata(frequency) + self._consume_vendordata("vendordata", frequency) + + with events.ReportEventStack("consume-vendor-data2", + "reading and applying vendor-data2", + parent=self.reporter): + self._consume_vendordata("vendordata2", frequency) # Perform post-consumption adjustments so that # modules that run during the init stage reflect @@ -568,46 +578,62 @@ class Init(object): # objects before the load of the userdata happened, # this is expected. - def _consume_vendordata(self, frequency=PER_INSTANCE): + def _consume_vendordata(self, vendor_source, frequency=PER_INSTANCE): """ Consume the vendordata and run the part handlers on it """ + # User-data should have been consumed first. # So we merge the other available cloud-configs (everything except # vendor provided), and check whether or not we should consume # vendor data at all. That gives user or system a chance to override. - if not self.datasource.get_vendordata_raw(): - LOG.debug("no vendordata from datasource") - return + if vendor_source == 'vendordata': + if not self.datasource.get_vendordata_raw(): + LOG.debug("no vendordata from datasource") + return + cfg_name = 'vendor_data' + elif vendor_source == 'vendordata2': + if not self.datasource.get_vendordata2_raw(): + LOG.debug("no vendordata2 from datasource") + return + cfg_name = 'vendor_data2' + else: + raise RuntimeError("vendor_source arg must be either 'vendordata'" + " or 'vendordata2'") _cc_merger = helpers.ConfigMerger(paths=self._paths, datasource=self.datasource, additional_fns=[], base_cfg=self.cfg, include_vendor=False) - vdcfg = _cc_merger.cfg.get('vendor_data', {}) + vdcfg = _cc_merger.cfg.get(cfg_name, {}) if not isinstance(vdcfg, dict): vdcfg = {'enabled': False} - LOG.warning("invalid 'vendor_data' setting. resetting to: %s", - vdcfg) + LOG.warning("invalid %s setting. resetting to: %s", + cfg_name, vdcfg) enabled = vdcfg.get('enabled') no_handlers = vdcfg.get('disabled_handlers', None) if not util.is_true(enabled): - LOG.debug("vendordata consumption is disabled.") + LOG.debug("%s consumption is disabled.", vendor_source) return - LOG.debug("vendor data will be consumed. disabled_handlers=%s", - no_handlers) + LOG.debug("%s will be consumed. disabled_handlers=%s", + vendor_source, no_handlers) - # Ensure vendordata source fetched before activation (just incase) - vendor_data_msg = self.datasource.get_vendordata() + # Ensure vendordata source fetched before activation (just in case.) - # This keeps track of all the active handlers, while excluding what the - # users doesn't want run, i.e. boot_hook, cloud_config, shell_script - c_handlers_list = self._default_vendordata_handlers() + # c_handlers_list keeps track of all the active handlers, while + # excluding what the users doesn't want run, i.e. boot_hook, + # cloud_config, shell_script + if vendor_source == 'vendordata': + vendor_data_msg = self.datasource.get_vendordata() + c_handlers_list = self._default_vendordata_handlers() + else: + vendor_data_msg = self.datasource.get_vendordata2() + c_handlers_list = self._default_vendordata2_handlers() # Run the handlers self._do_handlers(vendor_data_msg, c_handlers_list, frequency, diff --git a/doc/rtd/topics/datasources/openstack.rst b/doc/rtd/topics/datasources/openstack.rst index b23b4b7c..62d0fc03 100644 --- a/doc/rtd/topics/datasources/openstack.rst +++ b/doc/rtd/topics/datasources/openstack.rst @@ -82,4 +82,12 @@ For more general information about how cloud-init handles vendor data, including how it can be disabled by users on instances, see :doc:`/topics/vendordata`. +OpenStack can also be configured to provide 'dynamic vendordata' +which is provided by the DynamicJSON provider and appears under a +different metadata path, /vendor_data2.json. + +Cloud-init will look for a ``cloud-init`` at the vendor_data2 path; if found, +settings are applied after (and, hence, overriding) the settings from static +vendor data. Both sets of vendor data can be overridden by user data. + .. vi: textwidth=78 diff --git a/tests/unittests/test_data.py b/tests/unittests/test_data.py index fb2b55e8..8c968ae9 100644 --- a/tests/unittests/test_data.py +++ b/tests/unittests/test_data.py @@ -33,11 +33,12 @@ INSTANCE_ID = "i-testing" class FakeDataSource(sources.DataSource): - def __init__(self, userdata=None, vendordata=None): + def __init__(self, userdata=None, vendordata=None, vendordata2=None): sources.DataSource.__init__(self, {}, None, None) self.metadata = {'instance-id': INSTANCE_ID} self.userdata_raw = userdata self.vendordata_raw = vendordata + self.vendordata2_raw = vendordata2 def count_messages(root): @@ -105,13 +106,14 @@ class TestConsumeUserData(helpers.FilesystemMockingTestCase): self.assertEqual('qux', cc['baz']) self.assertEqual('qux2', cc['bar']) - def test_simple_jsonp_vendor_and_user(self): + def test_simple_jsonp_vendor_and_vendor2_and_user(self): # test that user-data wins over vendor user_blob = ''' #cloud-config-jsonp [ { "op": "add", "path": "/baz", "value": "qux" }, - { "op": "add", "path": "/bar", "value": "qux2" } + { "op": "add", "path": "/bar", "value": "qux2" }, + { "op": "add", "path": "/foobar", "value": "qux3" } ] ''' vendor_blob = ''' @@ -119,12 +121,23 @@ class TestConsumeUserData(helpers.FilesystemMockingTestCase): [ { "op": "add", "path": "/baz", "value": "quxA" }, { "op": "add", "path": "/bar", "value": "quxB" }, - { "op": "add", "path": "/foo", "value": "quxC" } + { "op": "add", "path": "/foo", "value": "quxC" }, + { "op": "add", "path": "/corge", "value": "quxEE" } +] +''' + vendor2_blob = ''' +#cloud-config-jsonp +[ + { "op": "add", "path": "/corge", "value": "quxD" }, + { "op": "add", "path": "/grault", "value": "quxFF" }, + { "op": "add", "path": "/foobar", "value": "quxGG" } ] ''' self.reRoot() initer = stages.Init() - initer.datasource = FakeDataSource(user_blob, vendordata=vendor_blob) + initer.datasource = FakeDataSource(user_blob, + vendordata=vendor_blob, + vendordata2=vendor2_blob) initer.read_cfg() initer.initialize() initer.fetch() @@ -138,9 +151,15 @@ class TestConsumeUserData(helpers.FilesystemMockingTestCase): (_which_ran, _failures) = mods.run_section('cloud_init_modules') cfg = mods.cfg self.assertIn('vendor_data', cfg) + self.assertIn('vendor_data2', cfg) + # Confirm that vendordata2 overrides vendordata, and that + # userdata overrides both self.assertEqual('qux', cfg['baz']) self.assertEqual('qux2', cfg['bar']) + self.assertEqual('qux3', cfg['foobar']) self.assertEqual('quxC', cfg['foo']) + self.assertEqual('quxD', cfg['corge']) + self.assertEqual('quxFF', cfg['grault']) def test_simple_jsonp_no_vendor_consumed(self): # make sure that vendor data is not consumed @@ -294,6 +313,10 @@ run: #!/bin/bash echo "test" ''' + vendor2_blob = ''' +#!/bin/bash +echo "dynamic test" +''' user_blob = ''' #cloud-config @@ -303,7 +326,9 @@ vendor_data: ''' new_root = self.reRoot() initer = stages.Init() - initer.datasource = FakeDataSource(user_blob, vendordata=vendor_blob) + initer.datasource = FakeDataSource(user_blob, + vendordata=vendor_blob, + vendordata2=vendor2_blob) initer.read_cfg() initer.initialize() initer.fetch() diff --git a/tests/unittests/test_datasource/test_openstack.py b/tests/unittests/test_datasource/test_openstack.py index 415755aa..478f3503 100644 --- a/tests/unittests/test_datasource/test_openstack.py +++ b/tests/unittests/test_datasource/test_openstack.py @@ -40,6 +40,9 @@ USER_DATA = b'#!/bin/sh\necho This is user data\n' VENDOR_DATA = { 'magic': '', } +VENDOR_DATA2 = { + 'static': {} +} OSTACK_META = { 'availability_zone': 'nova', 'files': [{'content_path': '/content/0000', 'path': '/etc/foo.cfg'}, @@ -60,6 +63,7 @@ OS_FILES = { {'links': [], 'networks': [], 'services': []}), 'openstack/latest/user_data': USER_DATA, 'openstack/latest/vendor_data.json': json.dumps(VENDOR_DATA), + 'openstack/latest/vendor_data2.json': json.dumps(VENDOR_DATA2), } EC2_FILES = { 'latest/user-data': USER_DATA, @@ -142,6 +146,7 @@ class TestOpenStackDataSource(test_helpers.HttprettyTestCase): _register_uris(self.VERSION, EC2_FILES, EC2_META, OS_FILES) f = _read_metadata_service() self.assertEqual(VENDOR_DATA, f.get('vendordata')) + self.assertEqual(VENDOR_DATA2, f.get('vendordata2')) self.assertEqual(CONTENT_0, f['files']['/etc/foo.cfg']) self.assertEqual(CONTENT_1, f['files']['/etc/bar/bar.cfg']) self.assertEqual(2, len(f['files'])) @@ -163,6 +168,7 @@ class TestOpenStackDataSource(test_helpers.HttprettyTestCase): _register_uris(self.VERSION, {}, {}, OS_FILES) f = _read_metadata_service() self.assertEqual(VENDOR_DATA, f.get('vendordata')) + self.assertEqual(VENDOR_DATA2, f.get('vendordata2')) self.assertEqual(CONTENT_0, f['files']['/etc/foo.cfg']) self.assertEqual(CONTENT_1, f['files']['/etc/bar/bar.cfg']) self.assertEqual(USER_DATA, f.get('userdata')) @@ -195,6 +201,7 @@ class TestOpenStackDataSource(test_helpers.HttprettyTestCase): _register_uris(self.VERSION, {}, {}, os_files) f = _read_metadata_service() self.assertEqual(VENDOR_DATA, f.get('vendordata')) + self.assertEqual(VENDOR_DATA2, f.get('vendordata2')) self.assertEqual(CONTENT_0, f['files']['/etc/foo.cfg']) self.assertEqual(CONTENT_1, f['files']['/etc/bar/bar.cfg']) self.assertFalse(f.get('userdata')) @@ -210,6 +217,17 @@ class TestOpenStackDataSource(test_helpers.HttprettyTestCase): self.assertEqual(CONTENT_1, f['files']['/etc/bar/bar.cfg']) self.assertFalse(f.get('vendordata')) + def test_vendordata2_empty(self): + os_files = copy.deepcopy(OS_FILES) + for k in list(os_files.keys()): + if k.endswith('vendor_data2.json'): + os_files.pop(k, None) + _register_uris(self.VERSION, {}, {}, os_files) + f = _read_metadata_service() + self.assertEqual(CONTENT_0, f['files']['/etc/foo.cfg']) + self.assertEqual(CONTENT_1, f['files']['/etc/bar/bar.cfg']) + self.assertFalse(f.get('vendordata2')) + def test_vendordata_invalid(self): os_files = copy.deepcopy(OS_FILES) for k in list(os_files.keys()): @@ -218,6 +236,14 @@ class TestOpenStackDataSource(test_helpers.HttprettyTestCase): _register_uris(self.VERSION, {}, {}, os_files) self.assertRaises(BrokenMetadata, _read_metadata_service) + def test_vendordata2_invalid(self): + os_files = copy.deepcopy(OS_FILES) + for k in list(os_files.keys()): + if k.endswith('vendor_data2.json'): + os_files[k] = '{' # some invalid json + _register_uris(self.VERSION, {}, {}, os_files) + self.assertRaises(BrokenMetadata, _read_metadata_service) + def test_metadata_invalid(self): os_files = copy.deepcopy(OS_FILES) for k in list(os_files.keys()): @@ -246,6 +272,7 @@ class TestOpenStackDataSource(test_helpers.HttprettyTestCase): self.assertEqual(USER_DATA, ds_os.userdata_raw) self.assertEqual(2, len(ds_os.files)) self.assertEqual(VENDOR_DATA, ds_os.vendordata_pure) + self.assertEqual(VENDOR_DATA2, ds_os.vendordata2_pure) self.assertIsNone(ds_os.vendordata_raw) m_dhcp.assert_not_called() @@ -278,6 +305,7 @@ class TestOpenStackDataSource(test_helpers.HttprettyTestCase): self.assertEqual(USER_DATA, ds_os_local.userdata_raw) self.assertEqual(2, len(ds_os_local.files)) self.assertEqual(VENDOR_DATA, ds_os_local.vendordata_pure) + self.assertEqual(VENDOR_DATA2, ds_os_local.vendordata2_pure) self.assertIsNone(ds_os_local.vendordata_raw) m_dhcp.assert_called_with('eth9', None) @@ -401,7 +429,7 @@ class TestOpenStackDataSource(test_helpers.HttprettyTestCase): self.assertIsNone(ds_os.vendordata_raw) self.assertEqual( ['dsmode', 'ec2-metadata', 'files', 'metadata', 'networkdata', - 'userdata', 'vendordata', 'version'], + 'userdata', 'vendordata', 'vendordata2', 'version'], sorted(crawled_data.keys())) self.assertEqual('local', crawled_data['dsmode']) self.assertEqual(EC2_META, crawled_data['ec2-metadata']) @@ -415,6 +443,7 @@ class TestOpenStackDataSource(test_helpers.HttprettyTestCase): crawled_data['networkdata']) self.assertEqual(USER_DATA, crawled_data['userdata']) self.assertEqual(VENDOR_DATA, crawled_data['vendordata']) + self.assertEqual(VENDOR_DATA2, crawled_data['vendordata2']) self.assertEqual(2, crawled_data['version']) @@ -681,6 +710,7 @@ class TestMetadataReader(test_helpers.HttprettyTestCase): 'version': 2, 'metadata': expected_md, 'vendordata': vendor_data, + 'vendordata2': vendor_data2, 'networkdata': network_data, 'ec2-metadata': mock_read_ec2.return_value, 'files': {}, |