cc_puppet: support AIO installations and more (#960)

- update the puppet module to support AIO installations by setting
  `install_type` to `aio`
- make the install collection configurable through the `collection`
  parameter; by default the rolling `puppet` collection will be used,
  which installs the latest version)
- when `install_type` is `aio`, puppetlabs repos will be purged after
  installation; set `cleanup` to `False` to prevent this
- AIO installations are performed by downloading and executing a shell
  script; the URL for this script can be overridden using the
  `aio_install_url` parameter
- make it possible to run puppet agent after installation/configuration
  via the `exec` key
- by default, puppet agent will run with the `--test` argument; this can
  be overridden via the `exec_args` key

1 files changed, 134 insertions, 25 deletions

diff --git a/cloudinit/config/cc_puppet.py b/cloudinit/config/cc_puppet.py
index bc981cf4..a0779eb0 100644
--- a/cloudinit/config/cc_puppet.py
+++ b/cloudinit/config/cc_puppet.py
@@ -29,22 +29,41 @@ The keys are ``package_name``, ``conf_file``, ``ssl_dir`` and
 ones that work with puppet 3.x and with distributions that ship modified
 puppet 4.x that uses the old paths.
 
+Agent packages from the puppetlabs repositories can be installed by setting
+``install_type`` to ``aio``. Based on this setting, the default config/SSL/CSR
+paths will be adjusted accordingly. To maintain backwards compatibility this
+setting defaults to ``packages`` which will install puppet from the distro
+packages.
+
+If installing ``aio`` packages, ``collection`` can also be set to one of
+``puppet`` (rolling release), ``puppet6``, ``puppet7`` (or their nightly
+counterparts) in order to install specific release streams. By default, the
+puppetlabs repository will be purged after installation finishes; set
+``cleanup`` to ``false`` to prevent this. AIO packages are installed through a
+shell script which is downloaded on the machine and then executed; the path to
+this script can be overridden using the ``aio_install_url`` key.
+
 Puppet configuration can be specified under the ``conf`` key. The
 configuration is specified as a dictionary containing high-level ``<section>``
 keys and lists of ``<key>=<value>`` pairs within each section. Each section
 name and ``<key>=<value>`` pair is written directly to ``puppet.conf``. As
-such,  section names should be one of: ``main``, ``master``, ``agent`` or
+such,  section names should be one of: ``main``, ``server``, ``agent`` or
 ``user`` and keys should be valid puppet configuration options. The
 ``certname`` key supports string substitutions for ``%i`` and ``%f``,
 corresponding to the instance id and fqdn of the machine respectively.
 If ``ca_cert`` is present, it will not be written to ``puppet.conf``, but
-instead will be used as the puppermaster certificate. It should be specified
+instead will be used as the puppetserver certificate. It should be specified
 in pem format as a multi-line string (using the ``|`` yaml notation).
 
-Additionally it's possible to create a csr_attributes.yaml for
-CSR attributes and certificate extension requests.
+Additionally it's possible to create a ``csr_attributes.yaml`` file for CSR
+attributes and certificate extension requests.
 See https://puppet.com/docs/puppet/latest/config_file_csr_attributes.html
 
+The puppet service will be automatically enabled after installation. A manual
+run can also be triggered by setting ``exec`` to ``true``, and additional
+arguments can be passed to ``puppet agent`` via the ``exec_args`` key (by
+default the agent will execute with the ``--test`` flag).
+
 **Internal name:** ``cc_puppet``
 
 **Module frequency:** per instance
@@ -56,13 +75,19 @@ See https://puppet.com/docs/puppet/latest/config_file_csr_attributes.html
     puppet:
         install: <true/false>
         version: <version>
+        collection: <aio collection>
+        install_type: <packages/aio>
+        aio_install_url: 'https://git.io/JBhoQ'
+        cleanup: <true/false>
         conf_file: '/etc/puppet/puppet.conf'
         ssl_dir: '/var/lib/puppet/ssl'
         csr_attributes_path: '/etc/puppet/csr_attributes.yaml'
         package_name: 'puppet'
+        exec: <true/false>
+        exec_args: ['--test']
         conf:
             agent:
-                server: "puppetmaster.example.org"
+                server: "puppetserver.example.org"
                 certname: "%i.%f"
                 ca_cert: |
                     -------BEGIN CERTIFICATE-------
@@ -84,12 +109,12 @@ from io import StringIO
 
 from cloudinit import helpers
 from cloudinit import subp
+from cloudinit import temp_utils
 from cloudinit import util
+from cloudinit import url_helper
 
-PUPPET_CONF_PATH = '/etc/puppet/puppet.conf'
-PUPPET_SSL_DIR = '/var/lib/puppet/ssl'
-PUPPET_CSR_ATTRIBUTES_PATH = '/etc/puppet/csr_attributes.yaml'
-PUPPET_PACKAGE_NAME = 'puppet'
+AIO_INSTALL_URL = 'https://raw.githubusercontent.com/puppetlabs/install-puppet/main/install.sh'  # noqa: E501
+PUPPET_AGENT_DEFAULT_ARGS = ['--test']
 
 
 class PuppetConstants(object):
@@ -119,6 +144,43 @@ def _autostart_puppet(log):
                      " puppet services on this system"))
 
 
+def get_config_value(puppet_bin, setting):
+    """Get the config value for a given setting using `puppet config print`
+    :param puppet_bin: path to puppet binary
+    :param setting: setting to query
+    """
+    out, _ = subp.subp([puppet_bin, 'config', 'print', setting])
+    return out.rstrip()
+
+
+def install_puppet_aio(url=AIO_INSTALL_URL, version=None,
+                       collection=None, cleanup=True):
+    """Install puppet-agent from the puppetlabs repositories using the one-shot
+    shell script
+
+    :param url: URL from where to download the install script
+    :param version: version to install, blank defaults to latest
+    :param collection: collection to install, blank defaults to latest
+    :param cleanup: whether to purge the puppetlabs repo after installation
+    """
+    args = []
+    if version is not None:
+        args = ['-v', version]
+    if collection is not None:
+        args += ['-c', collection]
+
+    # Purge puppetlabs repos after installation
+    if cleanup:
+        args += ['--cleanup']
+    content = url_helper.readurl(url=url, retries=5).contents
+
+    # Use tmpdir over tmpfile to avoid 'text file busy' on execute
+    with temp_utils.tempdir(needs_exe=True) as tmpd:
+        tmpf = os.path.join(tmpd, 'puppet-install')
+        util.write_file(tmpf, content, mode=0o700)
+        return subp.subp([tmpf] + args, capture=False)
+
+
 def handle(name, cfg, cloud, log, _args):
     # If there isn't a puppet key in the configuration don't do anything
     if 'puppet' not in cfg:
@@ -130,23 +192,50 @@ def handle(name, cfg, cloud, log, _args):
     # Start by installing the puppet package if necessary...
     install = util.get_cfg_option_bool(puppet_cfg, 'install', True)
     version = util.get_cfg_option_str(puppet_cfg, 'version', None)
-    package_name = util.get_cfg_option_str(
-        puppet_cfg, 'package_name', PUPPET_PACKAGE_NAME)
-    conf_file = util.get_cfg_option_str(
-        puppet_cfg, 'conf_file', PUPPET_CONF_PATH)
-    ssl_dir = util.get_cfg_option_str(puppet_cfg, 'ssl_dir', PUPPET_SSL_DIR)
-    csr_attributes_path = util.get_cfg_option_str(
-        puppet_cfg, 'csr_attributes_path', PUPPET_CSR_ATTRIBUTES_PATH)
+    collection = util.get_cfg_option_str(puppet_cfg, 'collection', None)
+    install_type = util.get_cfg_option_str(
+        puppet_cfg, 'install_type', 'packages')
+    cleanup = util.get_cfg_option_bool(puppet_cfg, 'cleanup', True)
+    run = util.get_cfg_option_bool(puppet_cfg, 'exec', default=False)
+    aio_install_url = util.get_cfg_option_str(
+        puppet_cfg, 'aio_install_url', default=AIO_INSTALL_URL)
 
-    p_constants = PuppetConstants(conf_file, ssl_dir, csr_attributes_path, log)
+    # AIO and distro packages use different paths
+    if install_type == 'aio':
+        puppet_user = 'root'
+        puppet_bin = '/opt/puppetlabs/bin/puppet'
+        puppet_package = 'puppet-agent'
+    else:  # default to 'packages'
+        puppet_user = 'puppet'
+        puppet_bin = 'puppet'
+        puppet_package = 'puppet'
+
+    package_name = util.get_cfg_option_str(
+        puppet_cfg, 'package_name', puppet_package)
     if not install and version:
-        log.warning(("Puppet install set false but version supplied,"
+        log.warning(("Puppet install set to false but version supplied,"
                      " doing nothing."))
     elif install:
-        log.debug(("Attempting to install puppet %s,"),
-                  version if version else 'latest')
+        log.debug(("Attempting to install puppet %s from %s"),
+                  version if version else 'latest', install_type)
 
-        cloud.distro.install_packages((package_name, version))
+        if install_type == "packages":
+            cloud.distro.install_packages((package_name, version))
+        elif install_type == "aio":
+            install_puppet_aio(aio_install_url, version, collection, cleanup)
+        else:
+            log.warning("Unknown puppet install type '%s'", install_type)
+            run = False
+
+    conf_file = util.get_cfg_option_str(
+        puppet_cfg, 'conf_file', get_config_value(puppet_bin, 'config'))
+    ssl_dir = util.get_cfg_option_str(
+        puppet_cfg, 'ssl_dir', get_config_value(puppet_bin, 'ssldir'))
+    csr_attributes_path = util.get_cfg_option_str(
+        puppet_cfg, 'csr_attributes_path',
+        get_config_value(puppet_bin, 'csr_attributes'))
+
+    p_constants = PuppetConstants(conf_file, ssl_dir, csr_attributes_path, log)
 
     # ... and then update the puppet configuration
     if 'conf' in puppet_cfg:
@@ -165,17 +254,18 @@ def handle(name, cfg, cloud, log, _args):
             source=p_constants.conf_path)
         for (cfg_name, cfg) in puppet_cfg['conf'].items():
             # Cert configuration is a special case
-            # Dump the puppet master ca certificate in the correct place
+            # Dump the puppetserver ca certificate in the correct place
             if cfg_name == 'ca_cert':
                 # Puppet ssl sub-directory isn't created yet
                 # Create it with the proper permissions and ownership
                 util.ensure_dir(p_constants.ssl_dir, 0o771)
-                util.chownbyname(p_constants.ssl_dir, 'puppet', 'root')
+                util.chownbyname(p_constants.ssl_dir, puppet_user, 'root')
                 util.ensure_dir(p_constants.ssl_cert_dir)
 
-                util.chownbyname(p_constants.ssl_cert_dir, 'puppet', 'root')
+                util.chownbyname(p_constants.ssl_cert_dir, puppet_user, 'root')
                 util.write_file(p_constants.ssl_cert_path, cfg)
-                util.chownbyname(p_constants.ssl_cert_path, 'puppet', 'root')
+                util.chownbyname(p_constants.ssl_cert_path,
+                                 puppet_user, 'root')
             else:
                 # Iterate through the config items, we'll use ConfigParser.set
                 # to overwrite or create new items as needed
@@ -203,6 +293,25 @@ def handle(name, cfg, cloud, log, _args):
     # Set it up so it autostarts
     _autostart_puppet(log)
 
+    # Run the agent if needed
+    if run:
+        log.debug('Running puppet-agent')
+        cmd = [puppet_bin, 'agent']
+        if 'exec_args' in puppet_cfg:
+            cmd_args = puppet_cfg['exec_args']
+            if isinstance(cmd_args, (list, tuple)):
+                cmd.extend(cmd_args)
+            elif isinstance(cmd_args, str):
+                cmd.extend(cmd_args.split())
+            else:
+                log.warning("Unknown type %s provided for puppet"
+                            " 'exec_args' expected list, tuple,"
+                            " or string", type(cmd_args))
+                cmd.extend(PUPPET_AGENT_DEFAULT_ARGS)
+        else:
+            cmd.extend(PUPPET_AGENT_DEFAULT_ARGS)
+        subp.subp(cmd, capture=False)
+
     # Start puppetd
     subp.subp(['service', 'puppet', 'start'], capture=False)