openstack: read the dynamic metadata group vendor_data2.json (#777)

Add support for openstack's dynamic vendor data, which appears under openstack/latest/vendor_data2.json

This adds vendor_data2 to all pathways; it should be a no-op for non-OpenStack providers.

LP: #1841104

7 files changed, 101 insertions, 42 deletions

diff --git a/cloudinit/cmd/tests/test_main.py b/cloudinit/cmd/tests/test_main.py
index 585b3b0e..78b27441 100644
--- a/cloudinit/cmd/tests/test_main.py
+++ b/cloudinit/cmd/tests/test_main.py
@@ -127,7 +127,8 @@ class TestMain(FilesystemMockingTestCase):
                  'syslog_fix_perms': [
                      'syslog:adm', 'root:adm', 'root:wheel', 'root:root'
                  ],
-                 'vendor_data': {'enabled': True, 'prefix': []}})
+                 'vendor_data': {'enabled': True, 'prefix': []},
+                 'vendor_data2': {'enabled': True, 'prefix': []}})
             updated_cfg.pop('system_info')
 
             self.assertEqual(updated_cfg, cfg)
diff --git a/cloudinit/helpers.py b/cloudinit/helpers.py
index 9752ad28..fc5011ec 100644
--- a/cloudinit/helpers.py
+++ b/cloudinit/helpers.py
@@ -230,6 +230,10 @@ class ConfigMerger(object):
 
         cc_paths = ['cloud_config']
         if self._include_vendor:
+            # the order is important here: we want vendor2
+            #  (dynamic vendor data from OpenStack)
+            #  to override vendor (static data from OpenStack)
+            cc_paths.append('vendor2_cloud_config')
             cc_paths.append('vendor_cloud_config')
 
         for cc_p in cc_paths:
@@ -337,9 +341,12 @@ class Paths(object):
             "obj_pkl": "obj.pkl",
             "cloud_config": "cloud-config.txt",
             "vendor_cloud_config": "vendor-cloud-config.txt",
+            "vendor2_cloud_config": "vendor2-cloud-config.txt",
             "data": "data",
             "vendordata_raw": "vendor-data.txt",
+            "vendordata2_raw": "vendor-data2.txt",
             "vendordata": "vendor-data.txt.i",
+            "vendordata2": "vendor-data2.txt.i",
             "instance_id": ".instance-id",
             "manual_clean_marker": "manual-clean",
             "warnings": "warnings",
diff --git a/cloudinit/settings.py b/cloudinit/settings.py
index ca4ffa8e..7516e17b 100644
--- a/cloudinit/settings.py
+++ b/cloudinit/settings.py
@@ -56,6 +56,7 @@ CFG_BUILTIN = {
         'network': {'renderers': None},
     },
     'vendor_data': {'enabled': True, 'prefix': []},
+    'vendor_data2': {'enabled': True, 'prefix': []},
 }
 
 # Valid frequencies of handlers/modules
diff --git a/cloudinit/sources/DataSourceOpenStack.py b/cloudinit/sources/DataSourceOpenStack.py
index b3406c67..619a171e 100644
--- a/cloudinit/sources/DataSourceOpenStack.py
+++ b/cloudinit/sources/DataSourceOpenStack.py
@@ -167,6 +167,14 @@ class DataSourceOpenStack(openstack.SourceMixin, sources.DataSource):
             LOG.warning("Invalid content in vendor-data: %s", e)
             self.vendordata_raw = None
 
+        vd2 = results.get('vendordata2')
+        self.vendordata2_pure = vd2
+        try:
+            self.vendordata2_raw = sources.convert_vendordata(vd2)
+        except ValueError as e:
+            LOG.warning("Invalid content in vendor-data2: %s", e)
+            self.vendordata2_raw = None
+
         return True
 
     def _crawl_metadata(self):
diff --git a/cloudinit/sources/__init__.py b/cloudinit/sources/__init__.py
index 9dccc687..1ad1880d 100644
--- a/cloudinit/sources/__init__.py
+++ b/cloudinit/sources/__init__.py
@@ -187,7 +187,8 @@ class DataSource(metaclass=abc.ABCMeta):
     cached_attr_defaults = (
         ('ec2_metadata', UNSET), ('network_json', UNSET),
         ('metadata', {}), ('userdata', None), ('userdata_raw', None),
-        ('vendordata', None), ('vendordata_raw', None))
+        ('vendordata', None), ('vendordata_raw', None),
+        ('vendordata2', None), ('vendordata2_raw', None))
 
     _dirty_cache = False
 
@@ -203,7 +204,9 @@ class DataSource(metaclass=abc.ABCMeta):
         self.metadata = {}
         self.userdata_raw = None
         self.vendordata = None
+        self.vendordata2 = None
         self.vendordata_raw = None
+        self.vendordata2_raw = None
 
         self.ds_cfg = util.get_cfg_by_path(
             self.sys_cfg, ("datasource", self.dsname), {})
@@ -392,6 +395,11 @@ class DataSource(metaclass=abc.ABCMeta):
             self.vendordata = self.ud_proc.process(self.get_vendordata_raw())
         return self.vendordata
 
+    def get_vendordata2(self):
+        if self.vendordata2 is None:
+            self.vendordata2 = self.ud_proc.process(self.get_vendordata2_raw())
+        return self.vendordata2
+
     @property
     def fallback_interface(self):
         """Determine the network interface used during local network config."""
@@ -494,6 +502,9 @@ class DataSource(metaclass=abc.ABCMeta):
     def get_vendordata_raw(self):
         return self.vendordata_raw
 
+    def get_vendordata2_raw(self):
+        return self.vendordata2_raw
+
     # the data sources' config_obj is a cloud-config formated
     # object that came to it from ways other than cloud-config
     # because cloud-config content would be handled elsewhere
diff --git a/cloudinit/sources/helpers/openstack.py b/cloudinit/sources/helpers/openstack.py
index 3e6365f1..4f566e64 100644
--- a/cloudinit/sources/helpers/openstack.py
+++ b/cloudinit/sources/helpers/openstack.py
@@ -247,6 +247,11 @@ class BaseReader(metaclass=abc.ABCMeta):
                 False,
                 load_json_anytype,
             )
+            files['vendordata2'] = (
+                self._path_join("openstack", version, 'vendor_data2.json'),
+                False,
+                load_json_anytype,
+            )
             files['networkdata'] = (
                 self._path_join("openstack", version, 'network_data.json'),
                 False,
diff --git a/cloudinit/stages.py b/cloudinit/stages.py
index 0cce6e80..3ef4491c 100644
--- a/cloudinit/stages.py
+++ b/cloudinit/stages.py
@@ -360,8 +360,18 @@ class Init(object):
                            reporter=self.reporter)
 
     def update(self):
-        self._store_userdata()
-        self._store_vendordata()
+        self._store_rawdata(self.datasource.get_userdata_raw(),
+                            'userdata')
+        self._store_processeddata(self.datasource.get_userdata(),
+                                  'userdata')
+        self._store_rawdata(self.datasource.get_vendordata_raw(),
+                            'vendordata')
+        self._store_processeddata(self.datasource.get_vendordata(),
+                                  'vendordata')
+        self._store_rawdata(self.datasource.get_vendordata2_raw(),
+                            'vendordata2')
+        self._store_processeddata(self.datasource.get_vendordata2(),
+                                  'vendordata2')
 
     def setup_datasource(self):
         with events.ReportEventStack("setup-datasource",
@@ -381,28 +391,18 @@ class Init(object):
                                      is_new_instance=self.is_new_instance())
             self._write_to_cache()
 
-    def _store_userdata(self):
-        raw_ud = self.datasource.get_userdata_raw()
-        if raw_ud is None:
-            raw_ud = b''
-        util.write_file(self._get_ipath('userdata_raw'), raw_ud, 0o600)
-        # processed userdata is a Mime message, so write it as string.
-        processed_ud = self.datasource.get_userdata()
-        if processed_ud is None:
-            raw_ud = ''
-        util.write_file(self._get_ipath('userdata'), str(processed_ud), 0o600)
-
-    def _store_vendordata(self):
-        raw_vd = self.datasource.get_vendordata_raw()
-        if raw_vd is None:
-            raw_vd = b''
-        util.write_file(self._get_ipath('vendordata_raw'), raw_vd, 0o600)
-        # processed vendor data is a Mime message, so write it as string.
-        processed_vd = str(self.datasource.get_vendordata())
-        if processed_vd is None:
-            processed_vd = ''
-        util.write_file(self._get_ipath('vendordata'), str(processed_vd),
-                        0o600)
+    def _store_rawdata(self, data, datasource):
+        # Raw data is bytes, not a string
+        if data is None:
+            data = b''
+        util.write_file(self._get_ipath('%s_raw' % datasource), data, 0o600)
+
+    def _store_processeddata(self, processed_data, datasource):
+        # processed is a Mime message, so write as string.
+        if processed_data is None:
+            processed_data = ''
+        util.write_file(self._get_ipath(datasource),
+                        str(processed_data), 0o600)
 
     def _default_handlers(self, opts=None):
         if opts is None:
@@ -434,6 +434,11 @@ class Init(object):
             opts={'script_path': 'vendor_scripts',
                   'cloud_config_path': 'vendor_cloud_config'})
 
+    def _default_vendordata2_handlers(self):
+        return self._default_handlers(
+            opts={'script_path': 'vendor_scripts',
+                  'cloud_config_path': 'vendor2_cloud_config'})
+
     def _do_handlers(self, data_msg, c_handlers_list, frequency,
                      excluded=None):
         """
@@ -555,7 +560,12 @@ class Init(object):
         with events.ReportEventStack("consume-vendor-data",
                                      "reading and applying vendor-data",
                                      parent=self.reporter):
-            self._consume_vendordata(frequency)
+            self._consume_vendordata("vendordata", frequency)
+
+        with events.ReportEventStack("consume-vendor-data2",
+                                     "reading and applying vendor-data2",
+                                     parent=self.reporter):
+            self._consume_vendordata("vendordata2", frequency)
 
         # Perform post-consumption adjustments so that
         # modules that run during the init stage reflect
@@ -568,46 +578,62 @@ class Init(object):
         # objects before the load of the userdata happened,
         # this is expected.
 
-    def _consume_vendordata(self, frequency=PER_INSTANCE):
+    def _consume_vendordata(self, vendor_source, frequency=PER_INSTANCE):
         """
         Consume the vendordata and run the part handlers on it
         """
+
         # User-data should have been consumed first.
         # So we merge the other available cloud-configs (everything except
         # vendor provided), and check whether or not we should consume
         # vendor data at all. That gives user or system a chance to override.
-        if not self.datasource.get_vendordata_raw():
-            LOG.debug("no vendordata from datasource")
-            return
+        if vendor_source == 'vendordata':
+            if not self.datasource.get_vendordata_raw():
+                LOG.debug("no vendordata from datasource")
+                return
+            cfg_name = 'vendor_data'
+        elif vendor_source == 'vendordata2':
+            if not self.datasource.get_vendordata2_raw():
+                LOG.debug("no vendordata2 from datasource")
+                return
+            cfg_name = 'vendor_data2'
+        else:
+            raise RuntimeError("vendor_source arg must be either 'vendordata'"
+                               " or 'vendordata2'")
 
         _cc_merger = helpers.ConfigMerger(paths=self._paths,
                                           datasource=self.datasource,
                                           additional_fns=[],
                                           base_cfg=self.cfg,
                                           include_vendor=False)
-        vdcfg = _cc_merger.cfg.get('vendor_data', {})
+        vdcfg = _cc_merger.cfg.get(cfg_name, {})
 
         if not isinstance(vdcfg, dict):
             vdcfg = {'enabled': False}
-            LOG.warning("invalid 'vendor_data' setting. resetting to: %s",
-                        vdcfg)
+            LOG.warning("invalid %s setting. resetting to: %s",
+                        cfg_name, vdcfg)
 
         enabled = vdcfg.get('enabled')
         no_handlers = vdcfg.get('disabled_handlers', None)
 
         if not util.is_true(enabled):
-            LOG.debug("vendordata consumption is disabled.")
+            LOG.debug("%s consumption is disabled.", vendor_source)
             return
 
-        LOG.debug("vendor data will be consumed. disabled_handlers=%s",
-                  no_handlers)
+        LOG.debug("%s will be consumed. disabled_handlers=%s",
+                  vendor_source, no_handlers)
 
-        # Ensure vendordata source fetched before activation (just incase)
-        vendor_data_msg = self.datasource.get_vendordata()
+        # Ensure vendordata source fetched before activation (just in case.)
 
-        # This keeps track of all the active handlers, while excluding what the
-        # users doesn't want run, i.e. boot_hook, cloud_config, shell_script
-        c_handlers_list = self._default_vendordata_handlers()
+        # c_handlers_list keeps track of all the active handlers, while
+        # excluding what the users doesn't want run, i.e. boot_hook,
+        # cloud_config, shell_script
+        if vendor_source == 'vendordata':
+            vendor_data_msg = self.datasource.get_vendordata()
+            c_handlers_list = self._default_vendordata_handlers()
+        else:
+            vendor_data_msg = self.datasource.get_vendordata2()
+            c_handlers_list = self._default_vendordata2_handlers()
 
         # Run the handlers
         self._do_handlers(vendor_data_msg, c_handlers_list, frequency,