From 3cb8ecc229999dbe524ff2ba4c4bd693e3c66058 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Fri, 10 Oct 2014 17:27:56 -0700 Subject: Increase the robustness of the chef module Add the following adjustments to the chef template and module - Make it so that the chef directories can be provided (defaults to the existing directories) - Make the params much more configurable, and if a parameter is provided in the chef configuration it will override existing template parameters. - Make the template skip lines if the values are None in the configuration so that template lines can be removed if/when this is desirable. - Allow the firstboot json path to be configurable (defaults to the existing location). - Adds a basic set of tests to ensure that good things are happening. --- cloudinit/config/cc_chef.py | 97 ++++++++++++---- cloudinit/util.py | 135 +++++----------------- templates/chef_client.rb.tmpl | 52 +++++++-- tests/unittests/test_handler/test_handler_chef.py | 84 ++++++++++++++ 4 files changed, 230 insertions(+), 138 deletions(-) create mode 100644 tests/unittests/test_handler/test_handler_chef.py diff --git a/cloudinit/config/cc_chef.py b/cloudinit/config/cc_chef.py index 806deed9..691a51bc 100644 --- a/cloudinit/config/cc_chef.py +++ b/cloudinit/config/cc_chef.py @@ -18,6 +18,8 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . +from datetime import datetime + import json import os @@ -38,6 +40,61 @@ CHEF_DIRS = [ OMNIBUS_URL = "https://www.opscode.com/chef/install.sh" +CHEF_RB_TPL_DEFAULTS = { + # These are ruby symbols... + 'ssl_verify_mode': ':verify_none', + 'log_level': ':info', + # These are not symbols... + 'log_location': '/var/log/chef/client.log', + 'validation_key': "/etc/chef/validation.pem", + 'client_key': "/etc/chef/client.pem", + 'json_attribs': "/etc/chef/firstboot.json", + 'file_cache_path': "/var/cache/chef", + 'file_backup_path': "/var/backups/chef", + 'pid_file': "/var/run/chef/client.pid", + 'show_time': True, +} +CHEF_RB_TPL_BOOL_KEYS = frozenset(['show_time']) +CHEF_RB_TPL_KEYS = list(CHEF_RB_TPL_DEFAULTS.keys()) +CHEF_RB_TPL_KEYS.extend(CHEF_RB_TPL_BOOL_KEYS) +CHEF_RB_TPL_KEYS.extend([ + 'server_url', + 'node_name', + 'environment', + 'validation_name', +]) +CHEF_RB_TPL_KEYS = frozenset(CHEF_RB_TPL_KEYS) +CHEF_RB_PATH = '/etc/chef/client.rb' +CHEF_FB_PATH = '/etc/chef/firstboot.json' + + +def get_template_params(iid, chef_cfg, log): + params = CHEF_RB_TPL_DEFAULTS.copy() + params.update({ + 'server_url': chef_cfg['server_url'], + 'node_name': util.get_cfg_option_str(chef_cfg, 'node_name', iid), + 'environment': util.get_cfg_option_str(chef_cfg, 'environment', + '_default'), + 'validation_name': chef_cfg['validation_name'], + }) + # Allow users to overwrite any of the keys they want (if they so choose), + # when a value is None, then the value will be set to None and no boolean + # or string version will be populated... + for (k, v) in chef_cfg.items(): + if k not in CHEF_RB_TPL_KEYS: + log.debug("Skipping unknown chef template key '%s'", k) + continue + if v is None: + params[k] = None + else: + # This will make the value a boolean or string... + if k in CHEF_RB_TPL_BOOL_KEYS: + params[k] = util.get_cfg_option_bool(chef_cfg, k) + else: + params[k] = util.get_cfg_option_str(chef_cfg, k) + params['generated_on'] = datetime.now().isoformat() + return params + def handle(name, cfg, cloud, log, _args): @@ -49,7 +106,7 @@ def handle(name, cfg, cloud, log, _args): chef_cfg = cfg['chef'] # Ensure the chef directories we use exist - for d in CHEF_DIRS: + for d in chef_cfg.get('directories', CHEF_DIRS): util.ensure_dir(d) # Set the validation key based on the presence of either 'validation_key' @@ -64,26 +121,26 @@ def handle(name, cfg, cloud, log, _args): template_fn = cloud.get_template_filename('chef_client.rb') if template_fn: iid = str(cloud.datasource.get_instance_id()) - params = { - 'server_url': chef_cfg['server_url'], - 'node_name': util.get_cfg_option_str(chef_cfg, 'node_name', iid), - 'environment': util.get_cfg_option_str(chef_cfg, 'environment', - '_default'), - 'validation_name': chef_cfg['validation_name'] - } - templater.render_to_file(template_fn, '/etc/chef/client.rb', params) + params = get_template_params(iid, chef_cfg, log) + templater.render_to_file(template_fn, CHEF_RB_PATH, params) + else: + log.warn("No template found, not rendering to %s", + CHEF_RB_PATH) + + # Set the firstboot json + fb_filename = util.get_cfg_option_str(chef_cfg, 'firstboot_path', + default=CHEF_FB_PATH) + if not fb_filename: + log.info("First boot path empty, not writing first boot json file") else: - log.warn("No template found, not rendering to /etc/chef/client.rb") - - # set the firstboot json - initial_json = {} - if 'run_list' in chef_cfg: - initial_json['run_list'] = chef_cfg['run_list'] - if 'initial_attributes' in chef_cfg: - initial_attributes = chef_cfg['initial_attributes'] - for k in list(initial_attributes.keys()): - initial_json[k] = initial_attributes[k] - util.write_file('/etc/chef/firstboot.json', json.dumps(initial_json)) + initial_json = {} + if 'run_list' in chef_cfg: + initial_json['run_list'] = chef_cfg['run_list'] + if 'initial_attributes' in chef_cfg: + initial_attributes = chef_cfg['initial_attributes'] + for k in list(initial_attributes.keys()): + initial_json[k] = initial_attributes[k] + util.write_file(fb_filename, json.dumps(initial_json)) # If chef is not installed, we install chef based on 'install_type' if (not os.path.isfile('/usr/bin/chef-client') or diff --git a/cloudinit/util.py b/cloudinit/util.py index f236d0bf..bdb0f268 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -191,11 +191,11 @@ def ExtendedTemporaryFile(**kwargs): return fh -def fork_cb(child_cb, *args, **kwargs): +def fork_cb(child_cb, *args): fid = os.fork() if fid == 0: try: - child_cb(*args, **kwargs) + child_cb(*args) os._exit(0) except: logexc(LOG, "Failed forking and calling callback %s", @@ -1297,7 +1297,7 @@ def unmounter(umount): yield umount finally: if umount: - umount_cmd = ["umount", umount] + umount_cmd = ["umount", '-l', umount] subp(umount_cmd) @@ -1346,70 +1346,37 @@ def mount_cb(device, callback, data=None, rw=False, mtype=None, sync=True): Mount the device, call method 'callback' passing the directory in which it was mounted, then unmount. Return whatever 'callback' returned. If data != None, also pass data to callback. - - mtype is a filesystem type. it may be a list, string (a single fsname) - or a list of fsnames. """ - - if isinstance(mtype, str): - mtypes = [mtype] - elif isinstance(mtype, (list, tuple)): - mtypes = list(mtype) - elif mtype is None: - mtypes = None - - # clean up 'mtype' input a bit based on platform. - platsys = platform.system().lower() - if platsys == "linux": - if mtypes is None: - mtypes = ["auto"] - elif platsys.endswith("bsd"): - if mtypes is None: - mtypes = ['ufs', 'cd9660', 'vfat'] - for index, mtype in enumerate(mtypes): - if mtype == "iso9660": - mtypes[index] = "cd9660" - else: - # we cannot do a smart "auto", so just call 'mount' once with no -t - mtypes = [''] - mounted = mounts() with tempdir() as tmpd: umount = False if device in mounted: mountpoint = mounted[device]['mountpoint'] else: - for mtype in mtypes: - mountpoint = None - try: - mountcmd = ['mount'] - mountopts = [] - if rw: - mountopts.append('rw') - else: - mountopts.append('ro') - if sync: - # This seems like the safe approach to do - # (ie where this is on by default) - mountopts.append("sync") - if mountopts: - mountcmd.extend(["-o", ",".join(mountopts)]) - if mtype: - mountcmd.extend(['-t', mtype]) - mountcmd.append(device) - mountcmd.append(tmpd) - subp(mountcmd) - umount = tmpd # This forces it to be unmounted (when set) - mountpoint = tmpd - break - except (IOError, OSError) as exc: - LOG.debug("Failed mount of '%s' as '%s': %s", - device, mtype, exc) - pass - if not mountpoint: - raise MountFailedError("Failed mounting %s to %s due to: %s" % + try: + mountcmd = ['mount'] + mountopts = [] + if rw: + mountopts.append('rw') + else: + mountopts.append('ro') + if sync: + # This seems like the safe approach to do + # (ie where this is on by default) + mountopts.append("sync") + if mountopts: + mountcmd.extend(["-o", ",".join(mountopts)]) + if mtype: + mountcmd.extend(['-t', mtype]) + mountcmd.append(device) + mountcmd.append(tmpd) + subp(mountcmd) + umount = tmpd # This forces it to be unmounted (when set) + mountpoint = tmpd + except (IOError, OSError) as exc: + raise MountFailedError(("Failed mounting %s " + "to %s due to: %s") % (device, tmpd, exc)) - # Be nice and ensure it ends with a slash if not mountpoint.endswith("/"): mountpoint += "/" @@ -1957,53 +1924,3 @@ def pathprefix2dict(base, required=None, optional=None, delim=os.path.sep): raise ValueError("Missing required files: %s", ','.join(missing)) return ret - - -def read_meminfo(meminfo="/proc/meminfo", raw=False): - # read a /proc/meminfo style file and return - # a dict with 'total', 'free', and 'available' - mpliers = {'kB': 2**10, 'mB': 2 ** 20, 'B': 1, 'gB': 2 ** 30} - kmap = {'MemTotal:': 'total', 'MemFree:': 'free', - 'MemAvailable:': 'available'} - ret = {} - for line in load_file(meminfo).splitlines(): - try: - key, value, unit = line.split() - except ValueError: - key, value = line.split() - unit = 'B' - if raw: - ret[key] = int(value) * mpliers[unit] - elif key in kmap: - ret[kmap[key]] = int(value) * mpliers[unit] - - return ret - - -def human2bytes(size): - """Convert human string or integer to size in bytes - 10M => 10485760 - .5G => 536870912 - """ - size_in = size - if size.endswith("B"): - size = size[:-1] - - mpliers = {'B': 1, 'K': 2 ** 10, 'M': 2 ** 20, 'G': 2 ** 30, 'T': 2 ** 40} - - num = size - mplier = 'B' - for m in mpliers: - if size.endswith(m): - mplier = m - num = size[0:-len(m)] - - try: - num = float(num) - except ValueError: - raise ValueError("'%s' is not valid input." % size_in) - - if num < 0: - raise ValueError("'%s': cannot be negative" % size_in) - - return int(num * mpliers[mplier]) diff --git a/templates/chef_client.rb.tmpl b/templates/chef_client.rb.tmpl index 538850ca..7b9e6298 100644 --- a/templates/chef_client.rb.tmpl +++ b/templates/chef_client.rb.tmpl @@ -9,17 +9,51 @@ you need to add the following to config: validation_name: XYZ server_url: XYZ -#} -log_level :info -log_location "/var/log/chef/client.log" -ssl_verify_mode :verify_none + +{# +The reason these are not in quotes is because they are ruby +symbols that will be placed inside here, and not actual strings... +#} +# This is a generated file, created on {{generated_on}}. +{% if log_level %} +log_level {{log_level}} +{% endif %} +{% if ssl_verify_mode %} +ssl_verify_mode {{ssl_verify_mode}} +{% endif %} +{% if log_location %} +log_location "{{log_location}}" +{% endif %} +{% if validation_name %} validation_client_name "{{validation_name}}" -validation_key "/etc/chef/validation.pem" -client_key "/etc/chef/client.pem" +{% endif %} +{% if validation_key %} +validation_key "{{validation_key}}" +{% endif %} +{% if client_key %} +client_key "{{client_key}}" +{% endif %} +{% if server_url %} chef_server_url "{{server_url}}" +{% endif %} +{% if environment %} environment "{{environment}}" +{% endif %} +{% if node_name %} node_name "{{node_name}}" -json_attribs "/etc/chef/firstboot.json" -file_cache_path "/var/cache/chef" -file_backup_path "/var/backups/chef" -pid_file "/var/run/chef/client.pid" +{% endif %} +{% if json_attribs %} +json_attribs "{{json_attribs}}" +{% endif %} +{% if file_cache_path %} +file_cache_path "{{file_cache_path}}" +{% endif %} +{% if file_backup_path %} +file_backup_path "{{file_backup_path}}" +{% endif %} +{% if pid_file %} +pid_file "{{pid_file}}" +{% endif %} +{% if show_time %} Chef::Log::Formatter.show_time = true +{% endif %} diff --git a/tests/unittests/test_handler/test_handler_chef.py b/tests/unittests/test_handler/test_handler_chef.py new file mode 100644 index 00000000..5562d18a --- /dev/null +++ b/tests/unittests/test_handler/test_handler_chef.py @@ -0,0 +1,84 @@ +import os +import json + +from cloudinit.config import cc_chef + +from cloudinit import cloud +from cloudinit import distros +from cloudinit import helpers +from cloudinit import util +from cloudinit.sources import DataSourceNone + +from .. import helpers as t_help + +import logging + +LOG = logging.getLogger(__name__) + + +class TestChef(t_help.FilesystemMockingTestCase): + def setUp(self): + super(TestChef, self).setUp() + self.tmp = self.makeDir(prefix="unittest_") + + def fetch_cloud(self, distro_kind): + cls = distros.fetch(distro_kind) + paths = helpers.Paths({}) + distro = cls(distro_kind, {}, paths) + ds = DataSourceNone.DataSourceNone({}, distro, paths, None) + return cloud.Cloud(ds, paths, {}, distro, None) + + def test_no_config(self): + self.patchUtils(self.tmp) + self.patchOS(self.tmp) + + cfg = {} + cc_chef.handle('chef', cfg, self.fetch_cloud('ubuntu'), LOG, []) + for d in cc_chef.CHEF_DIRS: + self.assertFalse(os.path.isdir(d)) + + def test_basic_config(self): + tpl_file = util.load_file('templates/chef_client.rb.tmpl') + self.patchUtils(self.tmp) + self.patchOS(self.tmp) + + util.write_file('/etc/cloud/templates/chef_client.rb.tmpl', tpl_file) + cfg = { + 'chef': { + 'server_url': 'localhost', + 'validation_name': 'bob', + }, + } + cc_chef.handle('chef', cfg, self.fetch_cloud('ubuntu'), LOG, []) + for d in cc_chef.CHEF_DIRS: + self.assertTrue(os.path.isdir(d)) + c = util.load_file(cc_chef.CHEF_RB_PATH) + for k, v in cfg['chef'].items(): + self.assertIn(v, c) + for k, v in cc_chef.CHEF_RB_TPL_DEFAULTS.items(): + if isinstance(v, basestring): + self.assertIn(v, c) + c = util.load_file(cc_chef.CHEF_FB_PATH) + self.assertEqual({}, json.loads(c)) + + def test_firstboot_json(self): + self.patchUtils(self.tmp) + self.patchOS(self.tmp) + + cfg = { + 'chef': { + 'server_url': 'localhost', + 'validation_name': 'bob', + 'run_list': ['a', 'b', 'c'], + 'initial_attributes': { + 'c': 'd', + } + }, + } + cc_chef.handle('chef', cfg, self.fetch_cloud('ubuntu'), LOG, []) + c = util.load_file(cc_chef.CHEF_FB_PATH) + self.assertEqual( + { + 'run_list': ['a', 'b', 'c'], + 'c': 'd', + }, json.loads(c)) -- cgit v1.2.3 From e8f9d27c6d43ef368a4047ae5818018a20e11f62 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Fri, 10 Oct 2014 17:34:32 -0700 Subject: Undo changes to the util file, not sure why that happened... --- cloudinit/util.py | 136 +++++++++++++++++++++++++++++++++++++++++++----------- 1 file changed, 110 insertions(+), 26 deletions(-) diff --git a/cloudinit/util.py b/cloudinit/util.py index bdb0f268..58f7455c 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -191,11 +191,11 @@ def ExtendedTemporaryFile(**kwargs): return fh -def fork_cb(child_cb, *args): +def fork_cb(child_cb, *args, **kwargs): fid = os.fork() if fid == 0: try: - child_cb(*args) + child_cb(*args, **kwargs) os._exit(0) except: logexc(LOG, "Failed forking and calling callback %s", @@ -1297,7 +1297,7 @@ def unmounter(umount): yield umount finally: if umount: - umount_cmd = ["umount", '-l', umount] + umount_cmd = ["umount", umount] subp(umount_cmd) @@ -1346,37 +1346,70 @@ def mount_cb(device, callback, data=None, rw=False, mtype=None, sync=True): Mount the device, call method 'callback' passing the directory in which it was mounted, then unmount. Return whatever 'callback' returned. If data != None, also pass data to callback. + + mtype is a filesystem type. it may be a list, string (a single fsname) + or a list of fsnames. """ + + if isinstance(mtype, str): + mtypes = [mtype] + elif isinstance(mtype, (list, tuple)): + mtypes = list(mtype) + elif mtype is None: + mtypes = None + + # clean up 'mtype' input a bit based on platform. + platsys = platform.system().lower() + if platsys == "linux": + if mtypes is None: + mtypes = ["auto"] + elif platsys.endswith("bsd"): + if mtypes is None: + mtypes = ['ufs', 'cd9660', 'vfat'] + for index, mtype in enumerate(mtypes): + if mtype == "iso9660": + mtypes[index] = "cd9660" + else: + # we cannot do a smart "auto", so just call 'mount' once with no -t + mtypes = [''] + mounted = mounts() with tempdir() as tmpd: umount = False if device in mounted: mountpoint = mounted[device]['mountpoint'] else: - try: - mountcmd = ['mount'] - mountopts = [] - if rw: - mountopts.append('rw') - else: - mountopts.append('ro') - if sync: - # This seems like the safe approach to do - # (ie where this is on by default) - mountopts.append("sync") - if mountopts: - mountcmd.extend(["-o", ",".join(mountopts)]) - if mtype: - mountcmd.extend(['-t', mtype]) - mountcmd.append(device) - mountcmd.append(tmpd) - subp(mountcmd) - umount = tmpd # This forces it to be unmounted (when set) - mountpoint = tmpd - except (IOError, OSError) as exc: - raise MountFailedError(("Failed mounting %s " - "to %s due to: %s") % + for mtype in mtypes: + mountpoint = None + try: + mountcmd = ['mount'] + mountopts = [] + if rw: + mountopts.append('rw') + else: + mountopts.append('ro') + if sync: + # This seems like the safe approach to do + # (ie where this is on by default) + mountopts.append("sync") + if mountopts: + mountcmd.extend(["-o", ",".join(mountopts)]) + if mtype: + mountcmd.extend(['-t', mtype]) + mountcmd.append(device) + mountcmd.append(tmpd) + subp(mountcmd) + umount = tmpd # This forces it to be unmounted (when set) + mountpoint = tmpd + break + except (IOError, OSError) as exc: + LOG.debug("Failed mount of '%s' as '%s': %s", + device, mtype, exc) + pass + if not mountpoint: + raise MountFailedError("Failed mounting %s to %s due to: %s" % (device, tmpd, exc)) + # Be nice and ensure it ends with a slash if not mountpoint.endswith("/"): mountpoint += "/" @@ -1924,3 +1957,54 @@ def pathprefix2dict(base, required=None, optional=None, delim=os.path.sep): raise ValueError("Missing required files: %s", ','.join(missing)) return ret + + +def read_meminfo(meminfo="/proc/meminfo", raw=False): + # read a /proc/meminfo style file and return + # a dict with 'total', 'free', and 'available' + mpliers = {'kB': 2**10, 'mB': 2 ** 20, 'B': 1, 'gB': 2 ** 30} + kmap = {'MemTotal:': 'total', 'MemFree:': 'free', + 'MemAvailable:': 'available'} + ret = {} + for line in load_file(meminfo).splitlines(): + try: + key, value, unit = line.split() + except ValueError: + key, value = line.split() + unit = 'B' + if raw: + ret[key] = int(value) * mpliers[unit] + elif key in kmap: + ret[kmap[key]] = int(value) * mpliers[unit] + + return ret + + +def human2bytes(size): + """Convert human string or integer to size in bytes + 10M => 10485760 + .5G => 536870912 + """ + size_in = size + if size.endswith("B"): + size = size[:-1] + + mpliers = {'B': 1, 'K': 2 ** 10, 'M': 2 ** 20, 'G': 2 ** 30, 'T': 2 ** 40} + + num = size + mplier = 'B' + for m in mpliers: + if size.endswith(m): + mplier = m + num = size[0:-len(m)] + + try: + num = float(num) + except ValueError: + raise ValueError("'%s' is not valid input." % size_in) + + if num < 0: + raise ValueError("'%s': cannot be negative" % size_in) + + return int(num * mpliers[mplier]) + -- cgit v1.2.3 From b8417c1af4a147240ec5919c2378fcc3e97078f7 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Fri, 10 Oct 2014 17:40:10 -0700 Subject: Fix newline added at end of file --- cloudinit/util.py | 1 - 1 file changed, 1 deletion(-) diff --git a/cloudinit/util.py b/cloudinit/util.py index 58f7455c..f236d0bf 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -2007,4 +2007,3 @@ def human2bytes(size): raise ValueError("'%s': cannot be negative" % size_in) return int(num * mpliers[mplier]) - -- cgit v1.2.3 From 28d09d73651772ed6c95e67f24e0a04488e79bd5 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Fri, 10 Oct 2014 17:44:57 -0700 Subject: Add a few template delete tests --- tests/unittests/test_handler/test_handler_chef.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/tests/unittests/test_handler/test_handler_chef.py b/tests/unittests/test_handler/test_handler_chef.py index 5562d18a..de7ff2da 100644 --- a/tests/unittests/test_handler/test_handler_chef.py +++ b/tests/unittests/test_handler/test_handler_chef.py @@ -82,3 +82,22 @@ class TestChef(t_help.FilesystemMockingTestCase): 'run_list': ['a', 'b', 'c'], 'c': 'd', }, json.loads(c)) + + def test_template_deletes(self): + tpl_file = util.load_file('templates/chef_client.rb.tmpl') + self.patchUtils(self.tmp) + self.patchOS(self.tmp) + + util.write_file('/etc/cloud/templates/chef_client.rb.tmpl', tpl_file) + cfg = { + 'chef': { + 'server_url': 'localhost', + 'validation_name': 'bob', + 'json_attribs': None, + 'show_time': None, + }, + } + cc_chef.handle('chef', cfg, self.fetch_cloud('ubuntu'), LOG, []) + c = util.load_file(cc_chef.CHEF_RB_PATH) + self.assertNotIn('json_attribs', c) + self.assertNotIn('Formatter.show_time', c) -- cgit v1.2.3 From e03d5277111fce36cf877d855b6a458e569ca5da Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Fri, 10 Oct 2014 19:09:27 -0700 Subject: Move the installation code to its own function --- cloudinit/config/cc_chef.py | 72 ++++++++++++++++++++++++--------------------- 1 file changed, 39 insertions(+), 33 deletions(-) diff --git a/cloudinit/config/cc_chef.py b/cloudinit/config/cc_chef.py index 691a51bc..971b4fce 100644 --- a/cloudinit/config/cc_chef.py +++ b/cloudinit/config/cc_chef.py @@ -142,36 +142,42 @@ def handle(name, cfg, cloud, log, _args): initial_json[k] = initial_attributes[k] util.write_file(fb_filename, json.dumps(initial_json)) + # Try to install chef, if its not already installed... + install_chef(cloud, chef_cfg, log) + + +def install_chef(cloud, chef_cfg, log): # If chef is not installed, we install chef based on 'install_type' - if (not os.path.isfile('/usr/bin/chef-client') or - util.get_cfg_option_bool(chef_cfg, - 'force_install', default=False)): - - install_type = util.get_cfg_option_str(chef_cfg, 'install_type', - 'packages') - if install_type == "gems": - # this will install and run the chef-client from gems - chef_version = util.get_cfg_option_str(chef_cfg, 'version', None) - ruby_version = util.get_cfg_option_str(chef_cfg, 'ruby_version', - RUBY_VERSION_DEFAULT) - install_chef_from_gems(cloud.distro, ruby_version, chef_version) - # and finally, run chef-client - log.debug('Running chef-client') - util.subp(['/usr/bin/chef-client', - '-d', '-i', '1800', '-s', '20'], capture=False) - elif install_type == 'packages': - # this will install and run the chef-client from packages - cloud.distro.install_packages(('chef',)) - elif install_type == 'omnibus': - url = util.get_cfg_option_str(chef_cfg, "omnibus_url", OMNIBUS_URL) - content = url_helper.readurl(url=url, retries=5) - with util.tempdir() as tmpd: - # use tmpd over tmpfile to avoid 'Text file busy' on execute - tmpf = "%s/chef-omnibus-install" % tmpd - util.write_file(tmpf, str(content), mode=0700) - util.subp([tmpf], capture=False) - else: - log.warn("Unknown chef install type %s", install_type) + if os.path.isfile('/usr/bin/chef-client'): + return + if not util.get_cfg_option_bool(chef_cfg, 'force_install', default=False): + return + install_type = util.get_cfg_option_str(chef_cfg, 'install_type', + 'packages') + if install_type == "gems": + # This will install and run the chef-client from gems + chef_version = util.get_cfg_option_str(chef_cfg, 'version', None) + ruby_version = util.get_cfg_option_str(chef_cfg, 'ruby_version', + RUBY_VERSION_DEFAULT) + install_chef_from_gems(cloud.distro, ruby_version, chef_version) + # And finally, run chef-client + log.debug('Running chef-client') + util.subp(['/usr/bin/chef-client', + '-d', '-i', '1800', '-s', '20'], capture=False) + elif install_type == 'packages': + # This will install and run the chef-client from packages + cloud.distro.install_packages(('chef',)) + elif install_type == 'omnibus': + # This will install as a omnibus unified package + url = util.get_cfg_option_str(chef_cfg, "omnibus_url", OMNIBUS_URL) + content = url_helper.readurl(url=url, retries=5) + with util.tempdir() as tmpd: + # Use tmpdir over tmpfile to avoid 'text file busy' on execute + tmpf = "%s/chef-omnibus-install" % tmpd + util.write_file(tmpf, str(content), mode=0700) + util.subp([tmpf], capture=False) + else: + log.warn("Unknown chef install type '%s'", install_type) def get_ruby_packages(version): @@ -190,9 +196,9 @@ def install_chef_from_gems(ruby_version, chef_version, distro): util.sym_link('/usr/bin/ruby%s' % ruby_version, '/usr/bin/ruby') if chef_version: util.subp(['/usr/bin/gem', 'install', 'chef', - '-v %s' % chef_version, '--no-ri', - '--no-rdoc', '--bindir', '/usr/bin', '-q'], capture=False) + '-v %s' % chef_version, '--no-ri', + '--no-rdoc', '--bindir', '/usr/bin', '-q'], capture=False) else: util.subp(['/usr/bin/gem', 'install', 'chef', - '--no-ri', '--no-rdoc', '--bindir', - '/usr/bin', '-q'], capture=False) + '--no-ri', '--no-rdoc', '--bindir', + '/usr/bin', '-q'], capture=False) -- cgit v1.2.3 From 9452f1247f69c2a8e8dedaac804f3c95bff0821b Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Sat, 11 Oct 2014 16:37:30 -0700 Subject: Some more reworkings - Make a helper function to tell if already installed. - Have the install routine not run chef after installed but have it instead return a result to tell the caller to run the chef program once completed. --- cloudinit/config/cc_chef.py | 32 +++++++++++++++++++++----------- 1 file changed, 21 insertions(+), 11 deletions(-) diff --git a/cloudinit/config/cc_chef.py b/cloudinit/config/cc_chef.py index 971b4fce..fb825404 100644 --- a/cloudinit/config/cc_chef.py +++ b/cloudinit/config/cc_chef.py @@ -19,7 +19,6 @@ # along with this program. If not, see . from datetime import datetime - import json import os @@ -66,6 +65,16 @@ CHEF_RB_TPL_KEYS.extend([ CHEF_RB_TPL_KEYS = frozenset(CHEF_RB_TPL_KEYS) CHEF_RB_PATH = '/etc/chef/client.rb' CHEF_FB_PATH = '/etc/chef/firstboot.json' +CHEF_EXEC_PATH = '/usr/bin/chef-client' +CHEF_EXEC_CMD = tuple([CHEF_EXEC_PATH, '-d', '-i', '1800', '-s', '20']) + + +def is_installed(): + if not os.path.isfile(CHEF_EXEC_PATH): + return False + if not os.access(CHEF_EXEC_PATH, os.X_OK): + return False + return True def get_template_params(iid, chef_cfg, log): @@ -106,7 +115,7 @@ def handle(name, cfg, cloud, log, _args): chef_cfg = cfg['chef'] # Ensure the chef directories we use exist - for d in chef_cfg.get('directories', CHEF_DIRS): + for d in list(chef_cfg.get('directories', CHEF_DIRS)): util.ensure_dir(d) # Set the validation key based on the presence of either 'validation_key' @@ -143,27 +152,27 @@ def handle(name, cfg, cloud, log, _args): util.write_file(fb_filename, json.dumps(initial_json)) # Try to install chef, if its not already installed... - install_chef(cloud, chef_cfg, log) + force_install = util.get_cfg_option_bool(chef_cfg, + 'force_install', default=False) + if not is_installed() or force_install: + run = install_chef(cloud, chef_cfg, log) + if run: + log.debug('Running chef-client') + util.subp(CHEF_EXEC_CMD, capture=False) def install_chef(cloud, chef_cfg, log): # If chef is not installed, we install chef based on 'install_type' - if os.path.isfile('/usr/bin/chef-client'): - return - if not util.get_cfg_option_bool(chef_cfg, 'force_install', default=False): - return install_type = util.get_cfg_option_str(chef_cfg, 'install_type', 'packages') + run_after = False if install_type == "gems": # This will install and run the chef-client from gems chef_version = util.get_cfg_option_str(chef_cfg, 'version', None) ruby_version = util.get_cfg_option_str(chef_cfg, 'ruby_version', RUBY_VERSION_DEFAULT) install_chef_from_gems(cloud.distro, ruby_version, chef_version) - # And finally, run chef-client - log.debug('Running chef-client') - util.subp(['/usr/bin/chef-client', - '-d', '-i', '1800', '-s', '20'], capture=False) + run_after = True elif install_type == 'packages': # This will install and run the chef-client from packages cloud.distro.install_packages(('chef',)) @@ -178,6 +187,7 @@ def install_chef(cloud, chef_cfg, log): util.subp([tmpf], capture=False) else: log.warn("Unknown chef install type '%s'", install_type) + return run_after def get_ruby_packages(version): -- cgit v1.2.3 From d87e89d9c674bac7e87d483037850a9ee4fc984a Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Sat, 11 Oct 2014 16:59:50 -0700 Subject: More adjustments - Use the generated_by() utility function to give the ruby template a better header comment - Set special parameters after selecting the basic chef parameters. --- cloudinit/config/cc_chef.py | 19 +++++++++++-------- templates/chef_client.rb.tmpl | 3 +-- tests/unittests/test_handler/test_handler_chef.py | 20 +++++++++++++++++++- 3 files changed, 31 insertions(+), 11 deletions(-) diff --git a/cloudinit/config/cc_chef.py b/cloudinit/config/cc_chef.py index fb825404..999b658d 100644 --- a/cloudinit/config/cc_chef.py +++ b/cloudinit/config/cc_chef.py @@ -79,13 +79,6 @@ def is_installed(): def get_template_params(iid, chef_cfg, log): params = CHEF_RB_TPL_DEFAULTS.copy() - params.update({ - 'server_url': chef_cfg['server_url'], - 'node_name': util.get_cfg_option_str(chef_cfg, 'node_name', iid), - 'environment': util.get_cfg_option_str(chef_cfg, 'environment', - '_default'), - 'validation_name': chef_cfg['validation_name'], - }) # Allow users to overwrite any of the keys they want (if they so choose), # when a value is None, then the value will be set to None and no boolean # or string version will be populated... @@ -101,7 +94,17 @@ def get_template_params(iid, chef_cfg, log): params[k] = util.get_cfg_option_bool(chef_cfg, k) else: params[k] = util.get_cfg_option_str(chef_cfg, k) - params['generated_on'] = datetime.now().isoformat() + # These ones are overwritten to be exact values... + params.update({ + 'generated_by': util.make_header(), + 'server_url': util.get_cfg_option_str(chef_cfg, 'server_url'), + 'node_name': util.get_cfg_option_str(chef_cfg, 'node_name', + default=iid), + 'environment': util.get_cfg_option_str(chef_cfg, 'environment', + default='_default'), + 'validation_name': util.get_cfg_option_str(chef_cfg, + 'validation_name'), + }) return params diff --git a/templates/chef_client.rb.tmpl b/templates/chef_client.rb.tmpl index 7b9e6298..c4069d22 100644 --- a/templates/chef_client.rb.tmpl +++ b/templates/chef_client.rb.tmpl @@ -9,12 +9,11 @@ you need to add the following to config: validation_name: XYZ server_url: XYZ -#} - +{{generated_by}} {# The reason these are not in quotes is because they are ruby symbols that will be placed inside here, and not actual strings... #} -# This is a generated file, created on {{generated_on}}. {% if log_level %} log_level {{log_level}} {% endif %} diff --git a/tests/unittests/test_handler/test_handler_chef.py b/tests/unittests/test_handler/test_handler_chef.py index de7ff2da..ef1aa208 100644 --- a/tests/unittests/test_handler/test_handler_chef.py +++ b/tests/unittests/test_handler/test_handler_chef.py @@ -1,5 +1,5 @@ -import os import json +import os from cloudinit.config import cc_chef @@ -38,6 +38,24 @@ class TestChef(t_help.FilesystemMockingTestCase): self.assertFalse(os.path.isdir(d)) def test_basic_config(self): + # This should create a file of the format... + """ + # Created by cloud-init v. 0.7.6 on Sat, 11 Oct 2014 23:57:21 +0000 + log_level :info + ssl_verify_mode :verify_none + log_location "/var/log/chef/client.log" + validation_client_name "bob" + validation_key "/etc/chef/validation.pem" + client_key "/etc/chef/client.pem" + chef_server_url "localhost" + environment "_default" + node_name "iid-datasource-none" + json_attribs "/etc/chef/firstboot.json" + file_cache_path "/var/cache/chef" + file_backup_path "/var/backups/chef" + pid_file "/var/run/chef/client.pid" + Chef::Log::Formatter.show_time = true + """ tpl_file = util.load_file('templates/chef_client.rb.tmpl') self.patchUtils(self.tmp) self.patchOS(self.tmp) -- cgit v1.2.3 From c5341fe07b767735e9ed74e45cee9629c6434892 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Sat, 11 Oct 2014 18:18:45 -0700 Subject: Allow for the running after install and run arguments to be configured Instead of only running the client when installed from gems, allow it to be ran from other install modes as well (if configured) and allow the arguments that are passed to the client when ran to be altered (if so desired). --- cloudinit/config/cc_chef.py | 32 +++++++++++++++++++++++++------- 1 file changed, 25 insertions(+), 7 deletions(-) diff --git a/cloudinit/config/cc_chef.py b/cloudinit/config/cc_chef.py index 999b658d..e503371d 100644 --- a/cloudinit/config/cc_chef.py +++ b/cloudinit/config/cc_chef.py @@ -18,7 +18,6 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -from datetime import datetime import json import os @@ -66,7 +65,7 @@ CHEF_RB_TPL_KEYS = frozenset(CHEF_RB_TPL_KEYS) CHEF_RB_PATH = '/etc/chef/client.rb' CHEF_FB_PATH = '/etc/chef/firstboot.json' CHEF_EXEC_PATH = '/usr/bin/chef-client' -CHEF_EXEC_CMD = tuple([CHEF_EXEC_PATH, '-d', '-i', '1800', '-s', '20']) +CHEF_EXEC_DEF_ARGS = tuple(['-d', '-i', '1800', '-s', '20']) def is_installed(): @@ -158,24 +157,42 @@ def handle(name, cfg, cloud, log, _args): force_install = util.get_cfg_option_bool(chef_cfg, 'force_install', default=False) if not is_installed() or force_install: - run = install_chef(cloud, chef_cfg, log) - if run: + run_after = install_chef(cloud, chef_cfg, log) + if run_after: log.debug('Running chef-client') - util.subp(CHEF_EXEC_CMD, capture=False) + cmd = [CHEF_EXEC_PATH] + if 'exec_arguments' in chef_cfg: + cmd_args = chef_cfg['exec_arguments'] + if isinstance(cmd_args, (list, tuple)): + cmd.extend(cmd_args) + elif isinstance(cmd_args, (str, basestring)): + cmd.append(cmd_args) + else: + log.warn("Unknown type %s provided for chef" + " 'exec_arguments' expected list, tuple," + " or string", type(cmd_args)) + cmd.extend(CHEF_EXEC_DEF_ARGS) + else: + cmd.extend(CHEF_EXEC_DEF_ARGS) + util.subp(cmd, capture=False) def install_chef(cloud, chef_cfg, log): # If chef is not installed, we install chef based on 'install_type' install_type = util.get_cfg_option_str(chef_cfg, 'install_type', 'packages') - run_after = False + run_after = util.get_cfg_option_bool(chef_cfg, 'exec_after_install', + default=False) if install_type == "gems": # This will install and run the chef-client from gems chef_version = util.get_cfg_option_str(chef_cfg, 'version', None) ruby_version = util.get_cfg_option_str(chef_cfg, 'ruby_version', RUBY_VERSION_DEFAULT) install_chef_from_gems(cloud.distro, ruby_version, chef_version) - run_after = True + # Retain backwards compat, but preferring True instead of False + # when not provided/overriden... + run_after = util.get_cfg_option_bool(chef_cfg, 'exec_after_install', + default=True) elif install_type == 'packages': # This will install and run the chef-client from packages cloud.distro.install_packages(('chef',)) @@ -190,6 +207,7 @@ def install_chef(cloud, chef_cfg, log): util.subp([tmpf], capture=False) else: log.warn("Unknown chef install type '%s'", install_type) + run_after = False return run_after -- cgit v1.2.3 From 4994f7cb475713be523f96e077a76f801e6d1db5 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Sat, 11 Oct 2014 18:23:20 -0700 Subject: Allow the omnibus url fetching retries to be configurable --- cloudinit/config/cc_chef.py | 6 +++++- cloudinit/util.py | 4 ++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/cloudinit/config/cc_chef.py b/cloudinit/config/cc_chef.py index e503371d..205f4b49 100644 --- a/cloudinit/config/cc_chef.py +++ b/cloudinit/config/cc_chef.py @@ -37,6 +37,7 @@ CHEF_DIRS = [ ] OMNIBUS_URL = "https://www.opscode.com/chef/install.sh" +OMNIBUS_URL_RETRIES = 5 CHEF_RB_TPL_DEFAULTS = { # These are ruby symbols... @@ -199,7 +200,10 @@ def install_chef(cloud, chef_cfg, log): elif install_type == 'omnibus': # This will install as a omnibus unified package url = util.get_cfg_option_str(chef_cfg, "omnibus_url", OMNIBUS_URL) - content = url_helper.readurl(url=url, retries=5) + retries = max(0, util.get_cfg_option_int(chef_cfg, + "omnibus_url_retries", + default=OMNIBUS_URL_RETRIES)) + content = url_helper.readurl(url=url, retries=retries) with util.tempdir() as tmpd: # Use tmpdir over tmpfile to avoid 'text file busy' on execute tmpf = "%s/chef-omnibus-install" % tmpd diff --git a/cloudinit/util.py b/cloudinit/util.py index f236d0bf..71221e09 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -399,6 +399,10 @@ def get_cfg_option_str(yobj, key, default=None): return val +def get_cfg_option_int(yobj, key, default=0): + return int(get_cfg_option_str(yobj, key, default=default)) + + def system_info(): return { 'platform': platform.platform(), -- cgit v1.2.3 From 1f969e30983f93eed2ece906a6ca9d16bc0aed86 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Sat, 11 Oct 2014 18:47:06 -0700 Subject: Use the util function to get the chef base directories --- cloudinit/config/cc_chef.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/cloudinit/config/cc_chef.py b/cloudinit/config/cc_chef.py index 205f4b49..fb8d7641 100644 --- a/cloudinit/config/cc_chef.py +++ b/cloudinit/config/cc_chef.py @@ -118,7 +118,10 @@ def handle(name, cfg, cloud, log, _args): chef_cfg = cfg['chef'] # Ensure the chef directories we use exist - for d in list(chef_cfg.get('directories', CHEF_DIRS)): + chef_dirs = util.get_cfg_option_list(chef_cfg, 'directories') + if not chef_dirs: + chef_dirs = list(CHEF_DIRS) + for d in chef_dirs: util.ensure_dir(d) # Set the validation key based on the presence of either 'validation_key' -- cgit v1.2.3 From 0a2e8b0130c2f83c0ea555d26473c92098c6adda Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Sat, 11 Oct 2014 18:51:33 -0700 Subject: Always ensure we create the /etc/chef dir --- cloudinit/config/cc_chef.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/cloudinit/config/cc_chef.py b/cloudinit/config/cc_chef.py index fb8d7641..ff08050d 100644 --- a/cloudinit/config/cc_chef.py +++ b/cloudinit/config/cc_chef.py @@ -18,6 +18,7 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . +import itertools import json import os @@ -35,6 +36,9 @@ CHEF_DIRS = [ '/var/backups/chef', '/var/run/chef', ] +REQUIRED_CHEF_DIRS = [ + '/etc/chef', +] OMNIBUS_URL = "https://www.opscode.com/chef/install.sh" OMNIBUS_URL_RETRIES = 5 @@ -121,7 +125,7 @@ def handle(name, cfg, cloud, log, _args): chef_dirs = util.get_cfg_option_list(chef_cfg, 'directories') if not chef_dirs: chef_dirs = list(CHEF_DIRS) - for d in chef_dirs: + for d in itertools.chain(chef_dirs, REQUIRED_CHEF_DIRS): util.ensure_dir(d) # Set the validation key based on the presence of either 'validation_key' -- cgit v1.2.3 From 736603ffdce0b836b43c77ae680ef2818521f30c Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Sat, 11 Oct 2014 18:54:14 -0700 Subject: Move the chef running to its own helper function --- cloudinit/config/cc_chef.py | 36 ++++++++++++++++++++---------------- 1 file changed, 20 insertions(+), 16 deletions(-) diff --git a/cloudinit/config/cc_chef.py b/cloudinit/config/cc_chef.py index ff08050d..f501f1f7 100644 --- a/cloudinit/config/cc_chef.py +++ b/cloudinit/config/cc_chef.py @@ -167,22 +167,26 @@ def handle(name, cfg, cloud, log, _args): if not is_installed() or force_install: run_after = install_chef(cloud, chef_cfg, log) if run_after: - log.debug('Running chef-client') - cmd = [CHEF_EXEC_PATH] - if 'exec_arguments' in chef_cfg: - cmd_args = chef_cfg['exec_arguments'] - if isinstance(cmd_args, (list, tuple)): - cmd.extend(cmd_args) - elif isinstance(cmd_args, (str, basestring)): - cmd.append(cmd_args) - else: - log.warn("Unknown type %s provided for chef" - " 'exec_arguments' expected list, tuple," - " or string", type(cmd_args)) - cmd.extend(CHEF_EXEC_DEF_ARGS) - else: - cmd.extend(CHEF_EXEC_DEF_ARGS) - util.subp(cmd, capture=False) + run_chef(chef_cfg, log) + + +def run_chef(chef_cfg, log): + log.debug('Running chef-client') + cmd = [CHEF_EXEC_PATH] + if 'exec_arguments' in chef_cfg: + cmd_args = chef_cfg['exec_arguments'] + if isinstance(cmd_args, (list, tuple)): + cmd.extend(cmd_args) + elif isinstance(cmd_args, (str, basestring)): + cmd.append(cmd_args) + else: + log.warn("Unknown type %s provided for chef" + " 'exec_arguments' expected list, tuple," + " or string", type(cmd_args)) + cmd.extend(CHEF_EXEC_DEF_ARGS) + else: + cmd.extend(CHEF_EXEC_DEF_ARGS) + util.subp(cmd, capture=False) def install_chef(cloud, chef_cfg, log): -- cgit v1.2.3 From fc380079288da045b1bfd61fb9ef2490a7cc1f82 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Sat, 11 Oct 2014 19:26:49 -0700 Subject: Ensure that any template paths have associated directories When the template provides a path, make sure that before the template is written that the path that is now in the template has the associated directory created (if not already created). --- cloudinit/config/cc_chef.py | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/cloudinit/config/cc_chef.py b/cloudinit/config/cc_chef.py index f501f1f7..28562caf 100644 --- a/cloudinit/config/cc_chef.py +++ b/cloudinit/config/cc_chef.py @@ -58,8 +58,18 @@ CHEF_RB_TPL_DEFAULTS = { 'show_time': True, } CHEF_RB_TPL_BOOL_KEYS = frozenset(['show_time']) +CHEF_RB_PATH_KEYS = frozenset([ + 'log_location', + 'validation_key', + 'client_key', + 'file_cache_path', + 'json_attribs', + 'file_cache_path', + 'pid_file', +]) CHEF_RB_TPL_KEYS = list(CHEF_RB_TPL_DEFAULTS.keys()) CHEF_RB_TPL_KEYS.extend(CHEF_RB_TPL_BOOL_KEYS) +CHEF_RB_TPL_KEYS.extend(CHEF_RB_PATH_KEYS) CHEF_RB_TPL_KEYS.extend([ 'server_url', 'node_name', @@ -109,7 +119,11 @@ def get_template_params(iid, chef_cfg, log): 'validation_name': util.get_cfg_option_str(chef_cfg, 'validation_name'), }) - return params + paths = set() + for (k, v) in params.items(): + if k in CHEF_RB_PATH_KEYS and v: + paths.add(os.path.dirname(v)) + return params, paths def handle(name, cfg, cloud, log, _args): @@ -140,7 +154,9 @@ def handle(name, cfg, cloud, log, _args): template_fn = cloud.get_template_filename('chef_client.rb') if template_fn: iid = str(cloud.datasource.get_instance_id()) - params = get_template_params(iid, chef_cfg, log) + params, paths = get_template_params(iid, chef_cfg, log) + for d in paths: + util.ensure_dir(d) templater.render_to_file(template_fn, CHEF_RB_PATH, params) else: log.warn("No template found, not rendering to %s", -- cgit v1.2.3 From 32bb2f37593482b179c8627bb254d50121a8be01 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Sat, 11 Oct 2014 19:30:12 -0700 Subject: Have the caller find the param paths instead of the param creator --- cloudinit/config/cc_chef.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/cloudinit/config/cc_chef.py b/cloudinit/config/cc_chef.py index 28562caf..66187659 100644 --- a/cloudinit/config/cc_chef.py +++ b/cloudinit/config/cc_chef.py @@ -119,11 +119,7 @@ def get_template_params(iid, chef_cfg, log): 'validation_name': util.get_cfg_option_str(chef_cfg, 'validation_name'), }) - paths = set() - for (k, v) in params.items(): - if k in CHEF_RB_PATH_KEYS and v: - paths.add(os.path.dirname(v)) - return params, paths + return params def handle(name, cfg, cloud, log, _args): @@ -154,9 +150,12 @@ def handle(name, cfg, cloud, log, _args): template_fn = cloud.get_template_filename('chef_client.rb') if template_fn: iid = str(cloud.datasource.get_instance_id()) - params, paths = get_template_params(iid, chef_cfg, log) - for d in paths: - util.ensure_dir(d) + params = get_template_params(iid, chef_cfg, log) + param_paths = set() + for (k, v) in params.items(): + if k in CHEF_RB_PATH_KEYS and v: + param_paths.add(os.path.dirname(v)) + util.ensure_dirs(param_paths) templater.render_to_file(template_fn, CHEF_RB_PATH, params) else: log.warn("No template found, not rendering to %s", -- cgit v1.2.3 From db6cee3fe1c6033af8ae66f76c7c81a5af2d9c5a Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Sat, 11 Oct 2014 19:39:36 -0700 Subject: Follow the same constant variable naming scheme for the path tpl keys --- cloudinit/config/cc_chef.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cloudinit/config/cc_chef.py b/cloudinit/config/cc_chef.py index 66187659..e9a37652 100644 --- a/cloudinit/config/cc_chef.py +++ b/cloudinit/config/cc_chef.py @@ -58,7 +58,7 @@ CHEF_RB_TPL_DEFAULTS = { 'show_time': True, } CHEF_RB_TPL_BOOL_KEYS = frozenset(['show_time']) -CHEF_RB_PATH_KEYS = frozenset([ +CHEF_RB_TPL_PATH_KEYS = frozenset([ 'log_location', 'validation_key', 'client_key', @@ -69,7 +69,7 @@ CHEF_RB_PATH_KEYS = frozenset([ ]) CHEF_RB_TPL_KEYS = list(CHEF_RB_TPL_DEFAULTS.keys()) CHEF_RB_TPL_KEYS.extend(CHEF_RB_TPL_BOOL_KEYS) -CHEF_RB_TPL_KEYS.extend(CHEF_RB_PATH_KEYS) +CHEF_RB_TPL_KEYS.extend(CHEF_RB_TPL_PATH_KEYS) CHEF_RB_TPL_KEYS.extend([ 'server_url', 'node_name', @@ -153,7 +153,7 @@ def handle(name, cfg, cloud, log, _args): params = get_template_params(iid, chef_cfg, log) param_paths = set() for (k, v) in params.items(): - if k in CHEF_RB_PATH_KEYS and v: + if k in CHEF_RB_TPL_PATH_KEYS and v: param_paths.add(os.path.dirname(v)) util.ensure_dirs(param_paths) templater.render_to_file(template_fn, CHEF_RB_PATH, params) -- cgit v1.2.3 From da89dca32a3dd394c6a19807856871a645f2acc4 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Sat, 11 Oct 2014 21:29:01 -0700 Subject: Allow running even if installed Standardize on using the chef_cfg key 'exec' which can be used when installing to tell the caller to run the chef client or can also be used if the client is already installed and its requested to be ran. To retain existing behavior 'exec' does not by default assume to be true, unless explicitly provided or a gems mode install is requested. --- cloudinit/config/cc_chef.py | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/cloudinit/config/cc_chef.py b/cloudinit/config/cc_chef.py index e9a37652..29238861 100644 --- a/cloudinit/config/cc_chef.py +++ b/cloudinit/config/cc_chef.py @@ -180,9 +180,13 @@ def handle(name, cfg, cloud, log, _args): force_install = util.get_cfg_option_bool(chef_cfg, 'force_install', default=False) if not is_installed() or force_install: - run_after = install_chef(cloud, chef_cfg, log) - if run_after: - run_chef(chef_cfg, log) + run = install_chef(cloud, chef_cfg, log) + elif is_installed(): + run = util.get_cfg_option_bool(chef_cfg, 'exec', default=False) + else: + run = False + if run: + run_chef(chef_cfg, log) def run_chef(chef_cfg, log): @@ -208,18 +212,16 @@ def install_chef(cloud, chef_cfg, log): # If chef is not installed, we install chef based on 'install_type' install_type = util.get_cfg_option_str(chef_cfg, 'install_type', 'packages') - run_after = util.get_cfg_option_bool(chef_cfg, 'exec_after_install', - default=False) + run = util.get_cfg_option_bool(chef_cfg, 'exec', default=False) if install_type == "gems": # This will install and run the chef-client from gems chef_version = util.get_cfg_option_str(chef_cfg, 'version', None) ruby_version = util.get_cfg_option_str(chef_cfg, 'ruby_version', RUBY_VERSION_DEFAULT) install_chef_from_gems(cloud.distro, ruby_version, chef_version) - # Retain backwards compat, but preferring True instead of False + # Retain backwards compat, by preferring True instead of False # when not provided/overriden... - run_after = util.get_cfg_option_bool(chef_cfg, 'exec_after_install', - default=True) + run = util.get_cfg_option_bool(chef_cfg, 'exec', default=True) elif install_type == 'packages': # This will install and run the chef-client from packages cloud.distro.install_packages(('chef',)) @@ -237,8 +239,8 @@ def install_chef(cloud, chef_cfg, log): util.subp([tmpf], capture=False) else: log.warn("Unknown chef install type '%s'", install_type) - run_after = False - return run_after + run = False + return run def get_ruby_packages(version): -- cgit v1.2.3 From f99d19b95009fea854adbb376518f92c195af917 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Sat, 11 Oct 2014 21:36:34 -0700 Subject: Add a comment explaining the param path logic --- cloudinit/config/cc_chef.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/cloudinit/config/cc_chef.py b/cloudinit/config/cc_chef.py index 29238861..687be69f 100644 --- a/cloudinit/config/cc_chef.py +++ b/cloudinit/config/cc_chef.py @@ -151,6 +151,9 @@ def handle(name, cfg, cloud, log, _args): if template_fn: iid = str(cloud.datasource.get_instance_id()) params = get_template_params(iid, chef_cfg, log) + # Do a best effort attempt to ensure that the template values that + # are associated with paths have there parent directory created + # before they are used by the chef-client itself. param_paths = set() for (k, v) in params.items(): if k in CHEF_RB_TPL_PATH_KEYS and v: -- cgit v1.2.3 From 088b3b7eaab6c36ef404978f10c514785651a8fd Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Sun, 12 Oct 2014 10:12:00 -0700 Subject: Retain the old behavior for mandatory keys The keys 'server_url' and 'validation_name' were previously mandatory, we should retain that behavior for now. --- cloudinit/config/cc_chef.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cloudinit/config/cc_chef.py b/cloudinit/config/cc_chef.py index 687be69f..1e44ec72 100644 --- a/cloudinit/config/cc_chef.py +++ b/cloudinit/config/cc_chef.py @@ -111,13 +111,13 @@ def get_template_params(iid, chef_cfg, log): # These ones are overwritten to be exact values... params.update({ 'generated_by': util.make_header(), - 'server_url': util.get_cfg_option_str(chef_cfg, 'server_url'), 'node_name': util.get_cfg_option_str(chef_cfg, 'node_name', default=iid), 'environment': util.get_cfg_option_str(chef_cfg, 'environment', default='_default'), - 'validation_name': util.get_cfg_option_str(chef_cfg, - 'validation_name'), + # These two are mandatory... + 'server_url': chef_cfg['server_url'], + 'validation_name': chef_cfg['validation_name'], }) return params -- cgit v1.2.3 From 5088b76198a8844dcf73ab7d2ef26bf9e5caf552 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Mon, 13 Oct 2014 18:29:23 -0700 Subject: Add a post-run method that can be used to delete validation.pem files For those who run chef in non-daemon mode, they would like to delete the validation.pem file if chef finishes as expected to remove that file from existing in an easy to read manner. --- cloudinit/config/cc_chef.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/cloudinit/config/cc_chef.py b/cloudinit/config/cc_chef.py index 1e44ec72..4350a353 100644 --- a/cloudinit/config/cc_chef.py +++ b/cloudinit/config/cc_chef.py @@ -78,6 +78,7 @@ CHEF_RB_TPL_KEYS.extend([ ]) CHEF_RB_TPL_KEYS = frozenset(CHEF_RB_TPL_KEYS) CHEF_RB_PATH = '/etc/chef/client.rb' +CHEF_VALIDATION_PEM_PATH = '/etc/chef/validation.pem' CHEF_FB_PATH = '/etc/chef/firstboot.json' CHEF_EXEC_PATH = '/usr/bin/chef-client' CHEF_EXEC_DEF_ARGS = tuple(['-d', '-i', '1800', '-s', '20']) @@ -91,6 +92,14 @@ def is_installed(): return True +def post_run_chef(chef_cfg, log): + delete_pem = util.get_cfg_option_bool(chef_cfg, + 'delete_validation_post_exec', + default=False) + if delete_pem and os.path.isfile(CHEF_VALIDATION_PEM_PATH): + os.unlink(CHEF_VALIDATION_PEM_PATH) + + def get_template_params(iid, chef_cfg, log): params = CHEF_RB_TPL_DEFAULTS.copy() # Allow users to overwrite any of the keys they want (if they so choose), @@ -143,7 +152,7 @@ def handle(name, cfg, cloud, log, _args): # takes precedence for key in ('validation_key', 'validation_cert'): if key in chef_cfg and chef_cfg[key]: - util.write_file('/etc/chef/validation.pem', chef_cfg[key]) + util.write_file(CHEF_VALIDATION_PEM_PATH, chef_cfg[key]) break # Create the chef config from template @@ -190,6 +199,7 @@ def handle(name, cfg, cloud, log, _args): run = False if run: run_chef(chef_cfg, log) + post_run_chef(chef_cfg, log) def run_chef(chef_cfg, log): -- cgit v1.2.3 From 9974012c8640b731a0e91826fc64e1f5ff5d1096 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Mon, 13 Oct 2014 18:51:19 -0700 Subject: Use the key contants in the default key => value set --- cloudinit/config/cc_chef.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/cloudinit/config/cc_chef.py b/cloudinit/config/cc_chef.py index 4350a353..f6f07bce 100644 --- a/cloudinit/config/cc_chef.py +++ b/cloudinit/config/cc_chef.py @@ -43,15 +43,17 @@ REQUIRED_CHEF_DIRS = [ OMNIBUS_URL = "https://www.opscode.com/chef/install.sh" OMNIBUS_URL_RETRIES = 5 +CHEF_VALIDATION_PEM_PATH = '/etc/chef/validation.pem' +CHEF_FB_PATH = '/etc/chef/firstboot.json' CHEF_RB_TPL_DEFAULTS = { # These are ruby symbols... 'ssl_verify_mode': ':verify_none', 'log_level': ':info', # These are not symbols... 'log_location': '/var/log/chef/client.log', - 'validation_key': "/etc/chef/validation.pem", + 'validation_key': CHEF_VALIDATION_PEM_PATH, 'client_key': "/etc/chef/client.pem", - 'json_attribs': "/etc/chef/firstboot.json", + 'json_attribs': CHEF_FB_PATH, 'file_cache_path': "/var/cache/chef", 'file_backup_path': "/var/backups/chef", 'pid_file': "/var/run/chef/client.pid", @@ -78,8 +80,6 @@ CHEF_RB_TPL_KEYS.extend([ ]) CHEF_RB_TPL_KEYS = frozenset(CHEF_RB_TPL_KEYS) CHEF_RB_PATH = '/etc/chef/client.rb' -CHEF_VALIDATION_PEM_PATH = '/etc/chef/validation.pem' -CHEF_FB_PATH = '/etc/chef/firstboot.json' CHEF_EXEC_PATH = '/usr/bin/chef-client' CHEF_EXEC_DEF_ARGS = tuple(['-d', '-i', '1800', '-s', '20']) -- cgit v1.2.3 From b04f582abe201e414905008d93b94fafb7ae707d Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Wed, 15 Oct 2014 11:40:37 -0700 Subject: Prefer immutable structures --- cloudinit/config/cc_chef.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/cloudinit/config/cc_chef.py b/cloudinit/config/cc_chef.py index f6f07bce..aa82cb0a 100644 --- a/cloudinit/config/cc_chef.py +++ b/cloudinit/config/cc_chef.py @@ -28,17 +28,17 @@ from cloudinit import util RUBY_VERSION_DEFAULT = "1.8" -CHEF_DIRS = [ +CHEF_DIRS = tuple([ '/etc/chef', '/var/log/chef', '/var/lib/chef', '/var/cache/chef', '/var/backups/chef', '/var/run/chef', -] -REQUIRED_CHEF_DIRS = [ +]) +REQUIRED_CHEF_DIRS = tuple([ '/etc/chef', -] +]) OMNIBUS_URL = "https://www.opscode.com/chef/install.sh" OMNIBUS_URL_RETRIES = 5 -- cgit v1.2.3 From 249c6c5980c36832ddd03ff6c3272f534538efc0 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Fri, 21 Nov 2014 17:12:56 -0800 Subject: Update chef module docstring to reflect the new style --- cloudinit/config/cc_chef.py | 50 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/cloudinit/config/cc_chef.py b/cloudinit/config/cc_chef.py index aa82cb0a..b30d7361 100644 --- a/cloudinit/config/cc_chef.py +++ b/cloudinit/config/cc_chef.py @@ -18,6 +18,56 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . +""" +**Summary:** module that configures, starts and installs chef. + +**Description:** This module enables chef to be installed (from packages or +from gems, or from omnibus). Before this occurs chef configurations are +written to disk (validation.pem, client.pem, firstboot.json, client.rb), +and needed chef folders/directories are created (/etc/chef and /var/log/chef +and so-on). Then once installing proceeds correctly if configured chef will +be started (in daemon mode or in non-daemon mode) and then once that has +finished (if ran in non-daemon mode this will be when chef finishes +converging, if ran in daemon mode then no further actions are possible since +chef will have forked into its own process) then a post run function can +run that can do finishing activities (such as removing the validation pem +file). + +It can be configured with the following option structure:: + + chef: + directories: (defaulting to /etc/chef, /var/log/chef, /var/lib/chef, + /var/cache/chef, /var/backups/chef, /var/run/chef) + validation_key or validation_cert: (optional string to be written to + /etc/chef/validation.pem) + firstboot_path: (path to write run_list and initial_attributes keys that + should also be present in this configuration, defaults + to /etc/chef/firstboot.json) + exec: boolean to run or not run chef (defaults to false, unless + a gem installed is requested + where this will then default + to true) + + chef.rb template keys (if falsey, then will be skipped and not + written to /etc/chef/client.rb) + + chef: + client_key: + environment: + file_backup_path: + file_cache_path: + json_attribs: + log_level: + log_location: + node_name: + pid_file: + server_url: + show_time: + ssl_verify_mode: + validation_key: + validation_name: +""" + import itertools import json import os -- cgit v1.2.3