cloud.cfg: move to a template. setup.py changes along the way.

Here we move the config/cloud.cfg to be rendered as a template.
That allows us to maintain deltas between distros in one place.
Currently we use 'variant' variable to make decisions.
A tools/render-cloudcfg is provided to render the file.

There were changes to setup.py, MANIFEST.in to allow us to put all
files into a virtual env installation and to render the cloud-config
file in 'install' or 'bdist' targets.

We have also included some config changes that were found in the
redhat distro spec.
 * include some config changes from the redhat distro spec.

The rendered cloud.cfg has some differences.
Ubuntu: white space and comment changes only.
Freebsd:
 - whitespace changes and comment changes
 - datasource_list definition moved to be closer to 'datasource'.
 - enable modules: migrator, write_files
 - move package-update-upgrade-install to final.

The initial work was done by Josh Harlow.

7 files changed, 266 insertions, 161 deletions

diff --git a/MANIFEST.in b/MANIFEST.in
index 94264640..1a4d7711 100644
--- a/MANIFEST.in
+++ b/MANIFEST.in
@@ -1,6 +1,15 @@
-include *.py MANIFEST.in ChangeLog
+include *.py MANIFEST.in LICENSE* ChangeLog
 global-include *.txt *.rst *.ini *.in *.conf *.cfg *.sh
+graft config
+graft doc
+graft packages
+graft systemd
+graft sysvinit
+graft templates
+graft tests
 graft tools
+graft udev
+graft upstart
 prune build
 prune dist
 prune .tox
diff --git a/Makefile b/Makefile
index 66d1dcad..a3bfaf79 100644
--- a/Makefile
+++ b/Makefile
@@ -69,6 +69,9 @@ check_version:
 	    "not equal to code version '$(CODE_VERSION)'"; exit 2; \
 	    else true; fi
 
+config/cloud.cfg:
+	$(PYVER) ./tools/render-cloudcfg config/cloud.cfg.tmpl config/cloud.cfg
+
 clean_pyc:
 	@find . -type f -name "*.pyc" -delete
 
diff --git a/cloudinit/util.py b/cloudinit/util.py
index 135e4608..b8c3e4ee 100644
--- a/cloudinit/util.py
+++ b/cloudinit/util.py
@@ -592,13 +592,40 @@ def get_cfg_option_int(yobj, key, default=0):
 
 
 def system_info():
-    return {
+    info = {
         'platform': platform.platform(),
         'release': platform.release(),
         'python': platform.python_version(),
         'uname': platform.uname(),
         'dist': platform.linux_distribution(),  # pylint: disable=W1505
     }
+    plat = info['platform'].lower()
+    # Try to get more info about what it actually is, in a format
+    # that we can easily use across linux and variants...
+    if plat.startswith('darwin'):
+        info['variant'] = 'darwin'
+    elif plat.endswith("bsd"):
+        info['variant'] = 'bsd'
+    elif plat.startswith('win'):
+        info['variant'] = 'windows'
+    elif 'linux' in plat:
+        # Try to get a single string out of these...
+        linux_dist, _version, _id = info['dist']
+        linux_dist = linux_dist.lower()
+        if linux_dist in ('ubuntu', 'linuxmint', 'mint'):
+            info['variant'] = 'ubuntu'
+        else:
+            for prefix, variant in [('redhat', 'rhel'),
+                                    ('centos', 'centos'),
+                                    ('fedora', 'fedora'),
+                                    ('debian', 'debian')]:
+                if linux_dist.startswith(prefix):
+                    info['variant'] = variant
+        if 'variant' not in info:
+            info['variant'] = 'linux'
+    if 'variant' not in info:
+        info['variant'] = 'unknown'
+    return info
 
 
 def get_cfg_option_list(yobj, key, default=None):
diff --git a/config/cloud.cfg-freebsd b/config/cloud.cfg-freebsd
deleted file mode 100644
index d666c397..00000000
--- a/config/cloud.cfg-freebsd
+++ /dev/null
@@ -1,88 +0,0 @@
-# The top level settings are used as module
-# and system configuration.
-
-syslog_fix_perms: root:wheel
-
-# This should not be required, but leave it in place until the real cause of
-# not beeing able to find -any- datasources is resolved.
-datasource_list: ['ConfigDrive', 'Azure', 'OpenStack', 'Ec2']
-
-# A set of users which may be applied and/or used by various modules
-# when a 'default' entry is found it will reference the 'default_user'
-# from the distro configuration specified below
-users:
-   - default
-
-# If this is set, 'root' will not be able to ssh in and they 
-# will get a message to login instead as the above $user (ubuntu)
-disable_root: false
-
-# This will cause the set+update hostname module to not operate (if true)
-preserve_hostname: false
-
-# Example datasource config
-# datasource: 
-#    Ec2: 
-#      metadata_urls: [ 'blah.com' ]
-#      timeout: 5 # (defaults to 50 seconds)
-#      max_wait: 10 # (defaults to 120 seconds)
-
-# The modules that run in the 'init' stage
-cloud_init_modules:
-# - migrator
- - seed_random
- - bootcmd
-# - write-files
- - growpart
- - resizefs
- - set_hostname
- - update_hostname
-# - update_etc_hosts
-# - ca-certs
-# - rsyslog
- - users-groups
- - ssh
-
-# The modules that run in the 'config' stage
-cloud_config_modules:
-# - disk_setup
-# - mounts
- - ssh-import-id
- - locale
- - set-passwords
- - package-update-upgrade-install
-# - landscape
- - timezone
-# - puppet
-# - chef
-# - salt-minion
-# - mcollective
- - disable-ec2-metadata
- - runcmd
-# - byobu
-
-# The modules that run in the 'final' stage
-cloud_final_modules:
- - rightscale_userdata
- - scripts-vendor
- - scripts-per-once
- - scripts-per-boot
- - scripts-per-instance
- - scripts-user
- - ssh-authkey-fingerprints
- - keys-to-console
- - phone-home
- - final-message
- - power-state-change
-
-# System and/or distro specific settings
-# (not accessible to handlers/transforms)
-system_info:
-   distro: freebsd
-   default_user:
-     name: freebsd
-     lock_passwd: True
-     gecos: FreeBSD
-     groups: [wheel]
-     sudo: ["ALL=(ALL) NOPASSWD:ALL"]
-     shell: /bin/tcsh
diff --git a/config/cloud.cfg b/config/cloud.cfg.tmpl
index 1b93e7f9..5af2a88f 100644
--- a/config/cloud.cfg
+++ b/config/cloud.cfg.tmpl
@@ -1,22 +1,43 @@
+## template:jinja
 # The top level settings are used as module
 # and system configuration.
 
+{% if variant in ["bsd"] %}
+syslog_fix_perms: root:wheel
+{% endif %}
 # A set of users which may be applied and/or used by various modules
 # when a 'default' entry is found it will reference the 'default_user'
 # from the distro configuration specified below
 users:
    - default
 
-# If this is set, 'root' will not be able to ssh in and they 
-# will get a message to login instead as the above $user (ubuntu)
+# If this is set, 'root' will not be able to ssh in and they
+# will get a message to login instead as the default $user
+{% if variant in ["bsd"] %}
+disable_root: false
+{% else %}
 disable_root: true
+{% endif %}
 
+{% if variant in ["centos", "fedora", "rhel"] %}
+mount_default_fields: [~, ~, 'auto', 'defaults,nofail', '0', '2']
+resize_rootfs_tmp: /dev
+ssh_deletekeys:   0
+ssh_genkeytypes:  ~
+ssh_pwauth:   0
+
+{% endif %}
 # This will cause the set+update hostname module to not operate (if true)
 preserve_hostname: false
 
+{% if variant in ["bsd"] %}
+# This should not be required, but leave it in place until the real cause of
+# not beeing able to find -any- datasources is resolved.
+datasource_list: ['ConfigDrive', 'Azure', 'OpenStack', 'Ec2']
+{% endif %}
 # Example datasource config
-# datasource: 
-#    Ec2: 
+# datasource:
+#    Ec2:
 #      metadata_urls: [ 'blah.com' ]
 #      timeout: 5 # (defaults to 50 seconds)
 #      max_wait: 10 # (defaults to 120 seconds)
@@ -24,51 +45,75 @@ preserve_hostname: false
 # The modules that run in the 'init' stage
 cloud_init_modules:
  - migrator
+{% if variant in ["ubuntu", "unknown", "debian"] %}
  - ubuntu-init-switch
+{% endif %}
  - seed_random
  - bootcmd
  - write-files
  - growpart
  - resizefs
+{% if variant not in ["bsd"] %}
  - disk_setup
  - mounts
+{% endif %}
  - set_hostname
  - update_hostname
+{% if variant not in ["bsd"] %}
  - update_etc_hosts
  - ca-certs
  - rsyslog
+{% endif %}
  - users-groups
  - ssh
 
 # The modules that run in the 'config' stage
 cloud_config_modules:
+{% if variant in ["ubuntu", "unknown", "debian"] %}
 # Emit the cloud config ready event
 # this can be used by upstart jobs for 'start on cloud-config'.
  - emit_upstart
  - snap_config
+{% endif %}
  - ssh-import-id
  - locale
  - set-passwords
+{% if variant in ["rhel", "fedora"] %}
+ - spacewalk
+ - yum-add-repo
+{% endif %}
+{% if variant in ["ubuntu", "unknown", "debian"] %}
  - grub-dpkg
  - apt-pipelining
  - apt-configure
+{% endif %}
+{% if variant not in ["bsd"] %}
  - ntp
+{% endif %}
  - timezone
  - disable-ec2-metadata
  - runcmd
+{% if variant in ["ubuntu", "unknown", "debian"] %}
  - byobu
+{% endif %}
 
 # The modules that run in the 'final' stage
 cloud_final_modules:
+{% if variant in ["ubuntu", "unknown", "debian"] %}
  - snappy
+{% endif %}
  - package-update-upgrade-install
+{% if variant in ["ubuntu", "unknown", "debian"] %}
  - fan
  - landscape
  - lxd
+{% endif %}
+{% if variant not in ["bsd"] %}
  - puppet
  - chef
  - salt-minion
  - mcollective
+{% endif %}
  - rightscale_userdata
  - scripts-vendor
  - scripts-per-once
@@ -85,7 +130,15 @@ cloud_final_modules:
 # (not accessible to handlers/transforms)
 system_info:
    # This will affect which distro class gets used
+{% if variant in ["centos", "debian", "fedora", "rhel", "ubuntu"] %}
+   distro: {{ variant }}
+{% elif variant in ["bsd"] %}
+   distro: freebsd
+{% else %}
+   # Unknown/fallback distro.
    distro: ubuntu
+{% endif %}
+{% if variant in ["ubuntu", "unknown", "debian"] %}
    # Default user name + that default users groups (if added/used)
    default_user:
      name: ubuntu
@@ -115,3 +168,27 @@ system_info:
          primary: http://ports.ubuntu.com/ubuntu-ports
          security: http://ports.ubuntu.com/ubuntu-ports
    ssh_svcname: ssh
+{% elif variant in ["centos", "rhel", "fedora"] %}
+   # Default user name + that default users groups (if added/used)
+   default_user:
+     name: {{ variant }}
+     lock_passwd: True
+     gecos: {{ variant }} Cloud User
+     groups: [wheel, adm, systemd-journal]
+     sudo: ["ALL=(ALL) NOPASSWD:ALL"]
+     shell: /bin/bash
+   # Other config here will be given to the distro class and/or path classes
+   paths:
+      cloud_dir: /var/lib/cloud/
+      templates_dir: /etc/cloud/templates/
+   ssh_svcname: sshd
+{% elif variant in ["bsd"] %}
+   # Default user name + that default users groups (if added/used)
+   default_user:
+     name: freebsd
+     lock_passwd: True
+     gecos: FreeBSD
+     groups: [wheel]
+     sudo: ["ALL=(ALL) NOPASSWD:ALL"]
+     shell: /bin/tcsh
+{% endif %}
diff --git a/setup.py b/setup.py
index 4616599b..d5223285 100755
--- a/setup.py
+++ b/setup.py
@@ -10,8 +10,11 @@
 
 from glob import glob
 
+import atexit
 import os
+import shutil
 import sys
+import tempfile
 
 import setuptools
 from setuptools.command.install import install
@@ -53,47 +56,15 @@ def pkg_config_read(library, var):
     cmd = ['pkg-config', '--variable=%s' % var, library]
     try:
         (path, err) = tiny_p(cmd)
+        path = path.strip()
     except Exception:
-        return fallbacks[library][var]
-    return str(path).strip()
+        path = fallbacks[library][var]
+    if path.startswith("/"):
+        path = path[1:]
 
+    return path
 
-INITSYS_FILES = {
-    'sysvinit': [f for f in glob('sysvinit/redhat/*') if is_f(f)],
-    'sysvinit_freebsd': [f for f in glob('sysvinit/freebsd/*') if is_f(f)],
-    'sysvinit_deb': [f for f in glob('sysvinit/debian/*') if is_f(f)],
-    'sysvinit_openrc': [f for f in glob('sysvinit/gentoo/*') if is_f(f)],
-    'systemd': [f for f in (glob('systemd/*.service') +
-                            glob('systemd/*.target')) if is_f(f)],
-    'systemd.generators': [f for f in glob('systemd/*-generator') if is_f(f)],
-    'upstart': [f for f in glob('upstart/*') if is_f(f)],
-}
-INITSYS_ROOTS = {
-    'sysvinit': '/etc/rc.d/init.d',
-    'sysvinit_freebsd': '/usr/local/etc/rc.d',
-    'sysvinit_deb': '/etc/init.d',
-    'sysvinit_openrc': '/etc/init.d',
-    'systemd': pkg_config_read('systemd', 'systemdsystemunitdir'),
-    'systemd.generators': pkg_config_read('systemd',
-                                          'systemdsystemgeneratordir'),
-    'upstart': '/etc/init/',
-}
-INITSYS_TYPES = sorted([f.partition(".")[0] for f in INITSYS_ROOTS.keys()])
-
-# Install everything in the right location and take care of Linux (default) and
-# FreeBSD systems.
-USR = "/usr"
-ETC = "/etc"
-USR_LIB_EXEC = "/usr/lib"
-LIB = "/lib"
-if os.uname()[0] == 'FreeBSD':
-    USR = "/usr/local"
-    USR_LIB_EXEC = "/usr/local/lib"
-elif os.path.isfile('/etc/redhat-release'):
-    USR_LIB_EXEC = "/usr/libexec"
 
-
-# Avoid having datafiles installed in a virtualenv...
 def in_virtualenv():
     try:
         if sys.real_prefix == sys.prefix:
@@ -116,6 +87,66 @@ def read_requires():
     return str(deps).splitlines()
 
 
+def render_cloud_cfg():
+    """render cloud.cfg into a tmpdir under same dir as setup.py
+
+    This is rendered to a temporary directory under the top level
+    directory with the name 'cloud.cfg'.  The reason for not just rendering
+    to config/cloud.cfg is for a.) don't want to write over contents
+    in that file if user had something there. b.) debuild will complain
+    that files are different outside of the debian directory."""
+
+    # older versions of tox use bdist (xenial), and then install from there.
+    # newer versions just use install.
+    if not (sys.argv[1] == 'install' or sys.argv[1].startswith('bdist*')):
+        return 'config/cloud.cfg.tmpl'
+    topdir = os.path.dirname(sys.argv[0])
+    tmpd = tempfile.mkdtemp(dir=topdir)
+    atexit.register(shutil.rmtree, tmpd)
+    fpath = os.path.join(tmpd, 'cloud.cfg')
+    tiny_p([sys.executable, './tools/render-cloudcfg',
+            'config/cloud.cfg.tmpl', fpath])
+    # relpath is relative to setup.py
+    relpath = os.path.join(os.path.basename(tmpd), 'cloud.cfg')
+    return relpath
+
+
+INITSYS_FILES = {
+    'sysvinit': [f for f in glob('sysvinit/redhat/*') if is_f(f)],
+    'sysvinit_freebsd': [f for f in glob('sysvinit/freebsd/*') if is_f(f)],
+    'sysvinit_deb': [f for f in glob('sysvinit/debian/*') if is_f(f)],
+    'sysvinit_openrc': [f for f in glob('sysvinit/gentoo/*') if is_f(f)],
+    'systemd': [f for f in (glob('systemd/*.service') +
+                            glob('systemd/*.target')) if is_f(f)],
+    'systemd.generators': [f for f in glob('systemd/*-generator') if is_f(f)],
+    'upstart': [f for f in glob('upstart/*') if is_f(f)],
+}
+INITSYS_ROOTS = {
+    'sysvinit': 'etc/rc.d/init.d',
+    'sysvinit_freebsd': 'usr/local/etc/rc.d',
+    'sysvinit_deb': 'etc/init.d',
+    'sysvinit_openrc': 'etc/init.d',
+    'systemd': pkg_config_read('systemd', 'systemdsystemunitdir'),
+    'systemd.generators': pkg_config_read('systemd',
+                                          'systemdsystemgeneratordir'),
+    'upstart': 'etc/init/',
+}
+INITSYS_TYPES = sorted([f.partition(".")[0] for f in INITSYS_ROOTS.keys()])
+
+
+# Install everything in the right location and take care of Linux (default) and
+# FreeBSD systems.
+USR = "usr"
+ETC = "etc"
+USR_LIB_EXEC = "usr/lib"
+LIB = "lib"
+if os.uname()[0] == 'FreeBSD':
+    USR = "usr/local"
+    USR_LIB_EXEC = "usr/local/lib"
+elif os.path.isfile('/etc/redhat-release'):
+    USR_LIB_EXEC = "usr/libexec"
+
+
 # TODO: Is there a better way to do this??
 class InitsysInstallData(install):
     init_system = None
@@ -155,36 +186,39 @@ class InitsysInstallData(install):
         self.distribution.reinitialize_command('install_data', True)
 
 
-if in_virtualenv():
-    data_files = []
-    cmdclass = {}
-else:
-    data_files = [
-        (ETC + '/cloud', glob('config/*.cfg')),
-        (ETC + '/cloud/cloud.cfg.d', glob('config/cloud.cfg.d/*')),
-        (ETC + '/cloud/templates', glob('templates/*')),
-        (USR_LIB_EXEC + '/cloud-init', ['tools/ds-identify',
-                                        'tools/uncloud-init',
-                                        'tools/write-ssh-key-fingerprints']),
-        (USR + '/share/doc/cloud-init', [f for f in glob('doc/*') if is_f(f)]),
-        (USR + '/share/doc/cloud-init/examples',
-            [f for f in glob('doc/examples/*') if is_f(f)]),
-        (USR + '/share/doc/cloud-init/examples/seed',
-            [f for f in glob('doc/examples/seed/*') if is_f(f)]),
-    ]
-    if os.uname()[0] != 'FreeBSD':
-        data_files.extend([
-            (ETC + '/NetworkManager/dispatcher.d/',
-             ['tools/hook-network-manager']),
-            (ETC + '/dhcp/dhclient-exit-hooks.d/', ['tools/hook-dhclient']),
-            (LIB + '/udev/rules.d', [f for f in glob('udev/*.rules')])
-        ])
-    # Use a subclass for install that handles
-    # adding on the right init system configuration files
-    cmdclass = {
-        'install': InitsysInstallData,
-    }
-
+if not in_virtualenv():
+    USR = "/" + USR
+    ETC = "/" + ETC
+    USR_LIB_EXEC = "/" + USR_LIB_EXEC
+    LIB = "/" + LIB
+    for k in INITSYS_ROOTS.keys():
+        INITSYS_ROOTS[k] = "/" + INITSYS_ROOTS[k]
+
+data_files = [
+    (ETC + '/cloud', [render_cloud_cfg()]),
+    (ETC + '/cloud/cloud.cfg.d', glob('config/cloud.cfg.d/*')),
+    (ETC + '/cloud/templates', glob('templates/*')),
+    (USR_LIB_EXEC + '/cloud-init', ['tools/ds-identify',
+                                    'tools/uncloud-init',
+                                    'tools/write-ssh-key-fingerprints']),
+    (USR + '/share/doc/cloud-init', [f for f in glob('doc/*') if is_f(f)]),
+    (USR + '/share/doc/cloud-init/examples',
+        [f for f in glob('doc/examples/*') if is_f(f)]),
+    (USR + '/share/doc/cloud-init/examples/seed',
+        [f for f in glob('doc/examples/seed/*') if is_f(f)]),
+]
+if os.uname()[0] != 'FreeBSD':
+    data_files.extend([
+        (ETC + '/NetworkManager/dispatcher.d/',
+         ['tools/hook-network-manager']),
+        (ETC + '/dhcp/dhclient-exit-hooks.d/', ['tools/hook-dhclient']),
+        (LIB + '/udev/rules.d', [f for f in glob('udev/*.rules')])
+    ])
+# Use a subclass for install that handles
+# adding on the right init system configuration files
+cmdclass = {
+    'install': InitsysInstallData,
+}
 
 requirements = read_requires()
 if sys.version_info < (3,):
diff --git a/tools/render-cloudcfg b/tools/render-cloudcfg
new file mode 100755
index 00000000..e624541a
--- /dev/null
+++ b/tools/render-cloudcfg
@@ -0,0 +1,43 @@
+#!/usr/bin/env python3
+
+import argparse
+import os
+import sys
+
+if "avoid-pep8-E402-import-not-top-of-file":
+    _tdir = os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))
+    sys.path.insert(0, _tdir)
+    from cloudinit import templater
+    from cloudinit import util
+    from cloudinit.atomic_helper import write_file
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    variants = ["bsd", "centos", "fedora", "rhel", "ubuntu", "unknown"]
+    platform = util.system_info()
+    parser.add_argument(
+        "--variant", default=platform['variant'], action="store",
+        help="define the variant.", choices=variants)
+    parser.add_argument(
+        "template", nargs="?", action="store",
+        default='./config/cloud.cfg.tmpl',
+        help="Path to the cloud.cfg template")
+    parser.add_argument(
+        "output", nargs="?", action="store", default="-",
+        help="Output file.  Use '-' to write to stdout")
+
+    args = parser.parse_args()
+
+    with open(args.template, 'r') as fh:
+        contents = fh.read()
+    tpl_params = {'variant': args.variant}
+    contents = (templater.render_string(contents, tpl_params)).rstrip() + "\n"
+    util.load_yaml(contents)
+    if args.output == "-":
+        sys.stdout.write(contents)
+    else:
+        write_file(args.output, contents, omode="w")
+
+if __name__ == '__main__':
+    main()