cloud-config modules: honor distros definitions in each module

Modules can optionally define a list of supported distros on which they can run
by declaring a distros attribute in the cc_*py module. This branch fixes
handling of cloudinit.stages.Modules.run_section. The behavior of run_section
is now the following:
 - always run a module if the module doesn't declare a distros attribute
 - always run a module if the module declares distros = [ALL_DISTROS]
 - skip a module if the distribution on which we run isn't in module.distros
 - force a run of a skipped module if unverified_modules configuration contains
   the module name

LP: #1715738
LP: #1715690

3 files changed, 26 insertions, 14 deletions

diff --git a/cloudinit/config/cc_runcmd.py b/cloudinit/config/cc_runcmd.py
index 7f995693..449872f0 100644
--- a/cloudinit/config/cc_runcmd.py
+++ b/cloudinit/config/cc_runcmd.py
@@ -10,6 +10,7 @@
 
 from cloudinit.config.schema import (
     get_schema_doc, validate_cloudconfig_schema)
+from cloudinit.distros import ALL_DISTROS
 from cloudinit.settings import PER_INSTANCE
 from cloudinit import util
 
@@ -23,7 +24,7 @@ from textwrap import dedent
 # configuration options before actually attempting to deploy with said
 # configuration.
 
-distros = ['all']
+distros = [ALL_DISTROS]
 
 schema = {
     'id': 'cc_runcmd',
diff --git a/cloudinit/distros/__init__.py b/cloudinit/distros/__init__.py
index b714b9ab..d5becd12 100755
--- a/cloudinit/distros/__init__.py
+++ b/cloudinit/distros/__init__.py
@@ -30,6 +30,10 @@ from cloudinit import util
 from cloudinit.distros.parsers import hosts
 
 
+# Used when a cloud-config module can be run on all cloud-init distibutions.
+# The value 'all' is surfaced in module documentation for distro support.
+ALL_DISTROS = 'all'
+
 OSFAMILIES = {
     'debian': ['debian', 'ubuntu'],
     'redhat': ['centos', 'fedora', 'rhel'],
diff --git a/cloudinit/stages.py b/cloudinit/stages.py
index a1c4a517..d0452688 100644
--- a/cloudinit/stages.py
+++ b/cloudinit/stages.py
@@ -821,28 +821,35 @@ class Modules(object):
         skipped = []
         forced = []
         overridden = self.cfg.get('unverified_modules', [])
+        active_mods = []
+        all_distros = set([distros.ALL_DISTROS])
         for (mod, name, _freq, _args) in mostly_mods:
-            worked_distros = set(mod.distros)
+            worked_distros = set(mod.distros)  # Minimally [] per fixup_modules
             worked_distros.update(
                 distros.Distro.expand_osfamily(mod.osfamilies))
 
-            # module does not declare 'distros' or lists this distro
-            if not worked_distros or d_name in worked_distros:
-                continue
-
-            if name in overridden:
-                forced.append(name)
-            else:
-                skipped.append(name)
+            # Skip only when the following conditions are all met:
+            #  - distros are defined in the module != ALL_DISTROS
+            #  - the current d_name isn't in distros
+            #  - and the module is unverified and not in the unverified_modules
+            #    override list
+            if worked_distros and worked_distros != all_distros:
+                if d_name not in worked_distros:
+                    if name not in overridden:
+                        skipped.append(name)
+                        continue
+                    forced.append(name)
+            active_mods.append([mod, name, _freq, _args])
 
         if skipped:
-            LOG.info("Skipping modules %s because they are not verified "
+            LOG.info("Skipping modules '%s' because they are not verified "
                      "on distro '%s'.  To run anyway, add them to "
-                     "'unverified_modules' in config.", skipped, d_name)
+                     "'unverified_modules' in config.",
+                     ','.join(skipped), d_name)
         if forced:
-            LOG.info("running unverified_modules: %s", forced)
+            LOG.info("running unverified_modules: '%s'", ', '.join(forced))
 
-        return self._run_modules(mostly_mods)
+        return self._run_modules(active_mods)
 
 
 def read_runtime_config():