From 696ea58ae484e9006477d5b8714a67861511cfcd Mon Sep 17 00:00:00 2001 From: John Bellone Date: Wed, 10 Sep 2014 03:20:28 -0400 Subject: cc_chef: Update omnibus URL to new Chef TLD. --- cloudinit/config/cc_chef.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cloudinit/config/cc_chef.py b/cloudinit/config/cc_chef.py index 806deed9..bc896253 100644 --- a/cloudinit/config/cc_chef.py +++ b/cloudinit/config/cc_chef.py @@ -36,7 +36,7 @@ CHEF_DIRS = [ '/var/run/chef', ] -OMNIBUS_URL = "https://www.opscode.com/chef/install.sh" +OMNIBUS_URL = "https://www.getchef.com/chef/install.sh" def handle(name, cfg, cloud, log, _args): -- cgit v1.2.3 From 3cb8ecc229999dbe524ff2ba4c4bd693e3c66058 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Fri, 10 Oct 2014 17:27:56 -0700 Subject: Increase the robustness of the chef module Add the following adjustments to the chef template and module - Make it so that the chef directories can be provided (defaults to the existing directories) - Make the params much more configurable, and if a parameter is provided in the chef configuration it will override existing template parameters. - Make the template skip lines if the values are None in the configuration so that template lines can be removed if/when this is desirable. - Allow the firstboot json path to be configurable (defaults to the existing location). - Adds a basic set of tests to ensure that good things are happening. --- cloudinit/config/cc_chef.py | 97 ++++++++++++---- cloudinit/util.py | 135 +++++----------------- templates/chef_client.rb.tmpl | 52 +++++++-- tests/unittests/test_handler/test_handler_chef.py | 84 ++++++++++++++ 4 files changed, 230 insertions(+), 138 deletions(-) create mode 100644 tests/unittests/test_handler/test_handler_chef.py diff --git a/cloudinit/config/cc_chef.py b/cloudinit/config/cc_chef.py index 806deed9..691a51bc 100644 --- a/cloudinit/config/cc_chef.py +++ b/cloudinit/config/cc_chef.py @@ -18,6 +18,8 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . +from datetime import datetime + import json import os @@ -38,6 +40,61 @@ CHEF_DIRS = [ OMNIBUS_URL = "https://www.opscode.com/chef/install.sh" +CHEF_RB_TPL_DEFAULTS = { + # These are ruby symbols... + 'ssl_verify_mode': ':verify_none', + 'log_level': ':info', + # These are not symbols... + 'log_location': '/var/log/chef/client.log', + 'validation_key': "/etc/chef/validation.pem", + 'client_key': "/etc/chef/client.pem", + 'json_attribs': "/etc/chef/firstboot.json", + 'file_cache_path': "/var/cache/chef", + 'file_backup_path': "/var/backups/chef", + 'pid_file': "/var/run/chef/client.pid", + 'show_time': True, +} +CHEF_RB_TPL_BOOL_KEYS = frozenset(['show_time']) +CHEF_RB_TPL_KEYS = list(CHEF_RB_TPL_DEFAULTS.keys()) +CHEF_RB_TPL_KEYS.extend(CHEF_RB_TPL_BOOL_KEYS) +CHEF_RB_TPL_KEYS.extend([ + 'server_url', + 'node_name', + 'environment', + 'validation_name', +]) +CHEF_RB_TPL_KEYS = frozenset(CHEF_RB_TPL_KEYS) +CHEF_RB_PATH = '/etc/chef/client.rb' +CHEF_FB_PATH = '/etc/chef/firstboot.json' + + +def get_template_params(iid, chef_cfg, log): + params = CHEF_RB_TPL_DEFAULTS.copy() + params.update({ + 'server_url': chef_cfg['server_url'], + 'node_name': util.get_cfg_option_str(chef_cfg, 'node_name', iid), + 'environment': util.get_cfg_option_str(chef_cfg, 'environment', + '_default'), + 'validation_name': chef_cfg['validation_name'], + }) + # Allow users to overwrite any of the keys they want (if they so choose), + # when a value is None, then the value will be set to None and no boolean + # or string version will be populated... + for (k, v) in chef_cfg.items(): + if k not in CHEF_RB_TPL_KEYS: + log.debug("Skipping unknown chef template key '%s'", k) + continue + if v is None: + params[k] = None + else: + # This will make the value a boolean or string... + if k in CHEF_RB_TPL_BOOL_KEYS: + params[k] = util.get_cfg_option_bool(chef_cfg, k) + else: + params[k] = util.get_cfg_option_str(chef_cfg, k) + params['generated_on'] = datetime.now().isoformat() + return params + def handle(name, cfg, cloud, log, _args): @@ -49,7 +106,7 @@ def handle(name, cfg, cloud, log, _args): chef_cfg = cfg['chef'] # Ensure the chef directories we use exist - for d in CHEF_DIRS: + for d in chef_cfg.get('directories', CHEF_DIRS): util.ensure_dir(d) # Set the validation key based on the presence of either 'validation_key' @@ -64,26 +121,26 @@ def handle(name, cfg, cloud, log, _args): template_fn = cloud.get_template_filename('chef_client.rb') if template_fn: iid = str(cloud.datasource.get_instance_id()) - params = { - 'server_url': chef_cfg['server_url'], - 'node_name': util.get_cfg_option_str(chef_cfg, 'node_name', iid), - 'environment': util.get_cfg_option_str(chef_cfg, 'environment', - '_default'), - 'validation_name': chef_cfg['validation_name'] - } - templater.render_to_file(template_fn, '/etc/chef/client.rb', params) + params = get_template_params(iid, chef_cfg, log) + templater.render_to_file(template_fn, CHEF_RB_PATH, params) + else: + log.warn("No template found, not rendering to %s", + CHEF_RB_PATH) + + # Set the firstboot json + fb_filename = util.get_cfg_option_str(chef_cfg, 'firstboot_path', + default=CHEF_FB_PATH) + if not fb_filename: + log.info("First boot path empty, not writing first boot json file") else: - log.warn("No template found, not rendering to /etc/chef/client.rb") - - # set the firstboot json - initial_json = {} - if 'run_list' in chef_cfg: - initial_json['run_list'] = chef_cfg['run_list'] - if 'initial_attributes' in chef_cfg: - initial_attributes = chef_cfg['initial_attributes'] - for k in list(initial_attributes.keys()): - initial_json[k] = initial_attributes[k] - util.write_file('/etc/chef/firstboot.json', json.dumps(initial_json)) + initial_json = {} + if 'run_list' in chef_cfg: + initial_json['run_list'] = chef_cfg['run_list'] + if 'initial_attributes' in chef_cfg: + initial_attributes = chef_cfg['initial_attributes'] + for k in list(initial_attributes.keys()): + initial_json[k] = initial_attributes[k] + util.write_file(fb_filename, json.dumps(initial_json)) # If chef is not installed, we install chef based on 'install_type' if (not os.path.isfile('/usr/bin/chef-client') or diff --git a/cloudinit/util.py b/cloudinit/util.py index f236d0bf..bdb0f268 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -191,11 +191,11 @@ def ExtendedTemporaryFile(**kwargs): return fh -def fork_cb(child_cb, *args, **kwargs): +def fork_cb(child_cb, *args): fid = os.fork() if fid == 0: try: - child_cb(*args, **kwargs) + child_cb(*args) os._exit(0) except: logexc(LOG, "Failed forking and calling callback %s", @@ -1297,7 +1297,7 @@ def unmounter(umount): yield umount finally: if umount: - umount_cmd = ["umount", umount] + umount_cmd = ["umount", '-l', umount] subp(umount_cmd) @@ -1346,70 +1346,37 @@ def mount_cb(device, callback, data=None, rw=False, mtype=None, sync=True): Mount the device, call method 'callback' passing the directory in which it was mounted, then unmount. Return whatever 'callback' returned. If data != None, also pass data to callback. - - mtype is a filesystem type. it may be a list, string (a single fsname) - or a list of fsnames. """ - - if isinstance(mtype, str): - mtypes = [mtype] - elif isinstance(mtype, (list, tuple)): - mtypes = list(mtype) - elif mtype is None: - mtypes = None - - # clean up 'mtype' input a bit based on platform. - platsys = platform.system().lower() - if platsys == "linux": - if mtypes is None: - mtypes = ["auto"] - elif platsys.endswith("bsd"): - if mtypes is None: - mtypes = ['ufs', 'cd9660', 'vfat'] - for index, mtype in enumerate(mtypes): - if mtype == "iso9660": - mtypes[index] = "cd9660" - else: - # we cannot do a smart "auto", so just call 'mount' once with no -t - mtypes = [''] - mounted = mounts() with tempdir() as tmpd: umount = False if device in mounted: mountpoint = mounted[device]['mountpoint'] else: - for mtype in mtypes: - mountpoint = None - try: - mountcmd = ['mount'] - mountopts = [] - if rw: - mountopts.append('rw') - else: - mountopts.append('ro') - if sync: - # This seems like the safe approach to do - # (ie where this is on by default) - mountopts.append("sync") - if mountopts: - mountcmd.extend(["-o", ",".join(mountopts)]) - if mtype: - mountcmd.extend(['-t', mtype]) - mountcmd.append(device) - mountcmd.append(tmpd) - subp(mountcmd) - umount = tmpd # This forces it to be unmounted (when set) - mountpoint = tmpd - break - except (IOError, OSError) as exc: - LOG.debug("Failed mount of '%s' as '%s': %s", - device, mtype, exc) - pass - if not mountpoint: - raise MountFailedError("Failed mounting %s to %s due to: %s" % + try: + mountcmd = ['mount'] + mountopts = [] + if rw: + mountopts.append('rw') + else: + mountopts.append('ro') + if sync: + # This seems like the safe approach to do + # (ie where this is on by default) + mountopts.append("sync") + if mountopts: + mountcmd.extend(["-o", ",".join(mountopts)]) + if mtype: + mountcmd.extend(['-t', mtype]) + mountcmd.append(device) + mountcmd.append(tmpd) + subp(mountcmd) + umount = tmpd # This forces it to be unmounted (when set) + mountpoint = tmpd + except (IOError, OSError) as exc: + raise MountFailedError(("Failed mounting %s " + "to %s due to: %s") % (device, tmpd, exc)) - # Be nice and ensure it ends with a slash if not mountpoint.endswith("/"): mountpoint += "/" @@ -1957,53 +1924,3 @@ def pathprefix2dict(base, required=None, optional=None, delim=os.path.sep): raise ValueError("Missing required files: %s", ','.join(missing)) return ret - - -def read_meminfo(meminfo="/proc/meminfo", raw=False): - # read a /proc/meminfo style file and return - # a dict with 'total', 'free', and 'available' - mpliers = {'kB': 2**10, 'mB': 2 ** 20, 'B': 1, 'gB': 2 ** 30} - kmap = {'MemTotal:': 'total', 'MemFree:': 'free', - 'MemAvailable:': 'available'} - ret = {} - for line in load_file(meminfo).splitlines(): - try: - key, value, unit = line.split() - except ValueError: - key, value = line.split() - unit = 'B' - if raw: - ret[key] = int(value) * mpliers[unit] - elif key in kmap: - ret[kmap[key]] = int(value) * mpliers[unit] - - return ret - - -def human2bytes(size): - """Convert human string or integer to size in bytes - 10M => 10485760 - .5G => 536870912 - """ - size_in = size - if size.endswith("B"): - size = size[:-1] - - mpliers = {'B': 1, 'K': 2 ** 10, 'M': 2 ** 20, 'G': 2 ** 30, 'T': 2 ** 40} - - num = size - mplier = 'B' - for m in mpliers: - if size.endswith(m): - mplier = m - num = size[0:-len(m)] - - try: - num = float(num) - except ValueError: - raise ValueError("'%s' is not valid input." % size_in) - - if num < 0: - raise ValueError("'%s': cannot be negative" % size_in) - - return int(num * mpliers[mplier]) diff --git a/templates/chef_client.rb.tmpl b/templates/chef_client.rb.tmpl index 538850ca..7b9e6298 100644 --- a/templates/chef_client.rb.tmpl +++ b/templates/chef_client.rb.tmpl @@ -9,17 +9,51 @@ you need to add the following to config: validation_name: XYZ server_url: XYZ -#} -log_level :info -log_location "/var/log/chef/client.log" -ssl_verify_mode :verify_none + +{# +The reason these are not in quotes is because they are ruby +symbols that will be placed inside here, and not actual strings... +#} +# This is a generated file, created on {{generated_on}}. +{% if log_level %} +log_level {{log_level}} +{% endif %} +{% if ssl_verify_mode %} +ssl_verify_mode {{ssl_verify_mode}} +{% endif %} +{% if log_location %} +log_location "{{log_location}}" +{% endif %} +{% if validation_name %} validation_client_name "{{validation_name}}" -validation_key "/etc/chef/validation.pem" -client_key "/etc/chef/client.pem" +{% endif %} +{% if validation_key %} +validation_key "{{validation_key}}" +{% endif %} +{% if client_key %} +client_key "{{client_key}}" +{% endif %} +{% if server_url %} chef_server_url "{{server_url}}" +{% endif %} +{% if environment %} environment "{{environment}}" +{% endif %} +{% if node_name %} node_name "{{node_name}}" -json_attribs "/etc/chef/firstboot.json" -file_cache_path "/var/cache/chef" -file_backup_path "/var/backups/chef" -pid_file "/var/run/chef/client.pid" +{% endif %} +{% if json_attribs %} +json_attribs "{{json_attribs}}" +{% endif %} +{% if file_cache_path %} +file_cache_path "{{file_cache_path}}" +{% endif %} +{% if file_backup_path %} +file_backup_path "{{file_backup_path}}" +{% endif %} +{% if pid_file %} +pid_file "{{pid_file}}" +{% endif %} +{% if show_time %} Chef::Log::Formatter.show_time = true +{% endif %} diff --git a/tests/unittests/test_handler/test_handler_chef.py b/tests/unittests/test_handler/test_handler_chef.py new file mode 100644 index 00000000..5562d18a --- /dev/null +++ b/tests/unittests/test_handler/test_handler_chef.py @@ -0,0 +1,84 @@ +import os +import json + +from cloudinit.config import cc_chef + +from cloudinit import cloud +from cloudinit import distros +from cloudinit import helpers +from cloudinit import util +from cloudinit.sources import DataSourceNone + +from .. import helpers as t_help + +import logging + +LOG = logging.getLogger(__name__) + + +class TestChef(t_help.FilesystemMockingTestCase): + def setUp(self): + super(TestChef, self).setUp() + self.tmp = self.makeDir(prefix="unittest_") + + def fetch_cloud(self, distro_kind): + cls = distros.fetch(distro_kind) + paths = helpers.Paths({}) + distro = cls(distro_kind, {}, paths) + ds = DataSourceNone.DataSourceNone({}, distro, paths, None) + return cloud.Cloud(ds, paths, {}, distro, None) + + def test_no_config(self): + self.patchUtils(self.tmp) + self.patchOS(self.tmp) + + cfg = {} + cc_chef.handle('chef', cfg, self.fetch_cloud('ubuntu'), LOG, []) + for d in cc_chef.CHEF_DIRS: + self.assertFalse(os.path.isdir(d)) + + def test_basic_config(self): + tpl_file = util.load_file('templates/chef_client.rb.tmpl') + self.patchUtils(self.tmp) + self.patchOS(self.tmp) + + util.write_file('/etc/cloud/templates/chef_client.rb.tmpl', tpl_file) + cfg = { + 'chef': { + 'server_url': 'localhost', + 'validation_name': 'bob', + }, + } + cc_chef.handle('chef', cfg, self.fetch_cloud('ubuntu'), LOG, []) + for d in cc_chef.CHEF_DIRS: + self.assertTrue(os.path.isdir(d)) + c = util.load_file(cc_chef.CHEF_RB_PATH) + for k, v in cfg['chef'].items(): + self.assertIn(v, c) + for k, v in cc_chef.CHEF_RB_TPL_DEFAULTS.items(): + if isinstance(v, basestring): + self.assertIn(v, c) + c = util.load_file(cc_chef.CHEF_FB_PATH) + self.assertEqual({}, json.loads(c)) + + def test_firstboot_json(self): + self.patchUtils(self.tmp) + self.patchOS(self.tmp) + + cfg = { + 'chef': { + 'server_url': 'localhost', + 'validation_name': 'bob', + 'run_list': ['a', 'b', 'c'], + 'initial_attributes': { + 'c': 'd', + } + }, + } + cc_chef.handle('chef', cfg, self.fetch_cloud('ubuntu'), LOG, []) + c = util.load_file(cc_chef.CHEF_FB_PATH) + self.assertEqual( + { + 'run_list': ['a', 'b', 'c'], + 'c': 'd', + }, json.loads(c)) -- cgit v1.2.3 From e8f9d27c6d43ef368a4047ae5818018a20e11f62 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Fri, 10 Oct 2014 17:34:32 -0700 Subject: Undo changes to the util file, not sure why that happened... --- cloudinit/util.py | 136 +++++++++++++++++++++++++++++++++++++++++++----------- 1 file changed, 110 insertions(+), 26 deletions(-) diff --git a/cloudinit/util.py b/cloudinit/util.py index bdb0f268..58f7455c 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -191,11 +191,11 @@ def ExtendedTemporaryFile(**kwargs): return fh -def fork_cb(child_cb, *args): +def fork_cb(child_cb, *args, **kwargs): fid = os.fork() if fid == 0: try: - child_cb(*args) + child_cb(*args, **kwargs) os._exit(0) except: logexc(LOG, "Failed forking and calling callback %s", @@ -1297,7 +1297,7 @@ def unmounter(umount): yield umount finally: if umount: - umount_cmd = ["umount", '-l', umount] + umount_cmd = ["umount", umount] subp(umount_cmd) @@ -1346,37 +1346,70 @@ def mount_cb(device, callback, data=None, rw=False, mtype=None, sync=True): Mount the device, call method 'callback' passing the directory in which it was mounted, then unmount. Return whatever 'callback' returned. If data != None, also pass data to callback. + + mtype is a filesystem type. it may be a list, string (a single fsname) + or a list of fsnames. """ + + if isinstance(mtype, str): + mtypes = [mtype] + elif isinstance(mtype, (list, tuple)): + mtypes = list(mtype) + elif mtype is None: + mtypes = None + + # clean up 'mtype' input a bit based on platform. + platsys = platform.system().lower() + if platsys == "linux": + if mtypes is None: + mtypes = ["auto"] + elif platsys.endswith("bsd"): + if mtypes is None: + mtypes = ['ufs', 'cd9660', 'vfat'] + for index, mtype in enumerate(mtypes): + if mtype == "iso9660": + mtypes[index] = "cd9660" + else: + # we cannot do a smart "auto", so just call 'mount' once with no -t + mtypes = [''] + mounted = mounts() with tempdir() as tmpd: umount = False if device in mounted: mountpoint = mounted[device]['mountpoint'] else: - try: - mountcmd = ['mount'] - mountopts = [] - if rw: - mountopts.append('rw') - else: - mountopts.append('ro') - if sync: - # This seems like the safe approach to do - # (ie where this is on by default) - mountopts.append("sync") - if mountopts: - mountcmd.extend(["-o", ",".join(mountopts)]) - if mtype: - mountcmd.extend(['-t', mtype]) - mountcmd.append(device) - mountcmd.append(tmpd) - subp(mountcmd) - umount = tmpd # This forces it to be unmounted (when set) - mountpoint = tmpd - except (IOError, OSError) as exc: - raise MountFailedError(("Failed mounting %s " - "to %s due to: %s") % + for mtype in mtypes: + mountpoint = None + try: + mountcmd = ['mount'] + mountopts = [] + if rw: + mountopts.append('rw') + else: + mountopts.append('ro') + if sync: + # This seems like the safe approach to do + # (ie where this is on by default) + mountopts.append("sync") + if mountopts: + mountcmd.extend(["-o", ",".join(mountopts)]) + if mtype: + mountcmd.extend(['-t', mtype]) + mountcmd.append(device) + mountcmd.append(tmpd) + subp(mountcmd) + umount = tmpd # This forces it to be unmounted (when set) + mountpoint = tmpd + break + except (IOError, OSError) as exc: + LOG.debug("Failed mount of '%s' as '%s': %s", + device, mtype, exc) + pass + if not mountpoint: + raise MountFailedError("Failed mounting %s to %s due to: %s" % (device, tmpd, exc)) + # Be nice and ensure it ends with a slash if not mountpoint.endswith("/"): mountpoint += "/" @@ -1924,3 +1957,54 @@ def pathprefix2dict(base, required=None, optional=None, delim=os.path.sep): raise ValueError("Missing required files: %s", ','.join(missing)) return ret + + +def read_meminfo(meminfo="/proc/meminfo", raw=False): + # read a /proc/meminfo style file and return + # a dict with 'total', 'free', and 'available' + mpliers = {'kB': 2**10, 'mB': 2 ** 20, 'B': 1, 'gB': 2 ** 30} + kmap = {'MemTotal:': 'total', 'MemFree:': 'free', + 'MemAvailable:': 'available'} + ret = {} + for line in load_file(meminfo).splitlines(): + try: + key, value, unit = line.split() + except ValueError: + key, value = line.split() + unit = 'B' + if raw: + ret[key] = int(value) * mpliers[unit] + elif key in kmap: + ret[kmap[key]] = int(value) * mpliers[unit] + + return ret + + +def human2bytes(size): + """Convert human string or integer to size in bytes + 10M => 10485760 + .5G => 536870912 + """ + size_in = size + if size.endswith("B"): + size = size[:-1] + + mpliers = {'B': 1, 'K': 2 ** 10, 'M': 2 ** 20, 'G': 2 ** 30, 'T': 2 ** 40} + + num = size + mplier = 'B' + for m in mpliers: + if size.endswith(m): + mplier = m + num = size[0:-len(m)] + + try: + num = float(num) + except ValueError: + raise ValueError("'%s' is not valid input." % size_in) + + if num < 0: + raise ValueError("'%s': cannot be negative" % size_in) + + return int(num * mpliers[mplier]) + -- cgit v1.2.3 From b8417c1af4a147240ec5919c2378fcc3e97078f7 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Fri, 10 Oct 2014 17:40:10 -0700 Subject: Fix newline added at end of file --- cloudinit/util.py | 1 - 1 file changed, 1 deletion(-) diff --git a/cloudinit/util.py b/cloudinit/util.py index 58f7455c..f236d0bf 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -2007,4 +2007,3 @@ def human2bytes(size): raise ValueError("'%s': cannot be negative" % size_in) return int(num * mpliers[mplier]) - -- cgit v1.2.3 From 28d09d73651772ed6c95e67f24e0a04488e79bd5 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Fri, 10 Oct 2014 17:44:57 -0700 Subject: Add a few template delete tests --- tests/unittests/test_handler/test_handler_chef.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/tests/unittests/test_handler/test_handler_chef.py b/tests/unittests/test_handler/test_handler_chef.py index 5562d18a..de7ff2da 100644 --- a/tests/unittests/test_handler/test_handler_chef.py +++ b/tests/unittests/test_handler/test_handler_chef.py @@ -82,3 +82,22 @@ class TestChef(t_help.FilesystemMockingTestCase): 'run_list': ['a', 'b', 'c'], 'c': 'd', }, json.loads(c)) + + def test_template_deletes(self): + tpl_file = util.load_file('templates/chef_client.rb.tmpl') + self.patchUtils(self.tmp) + self.patchOS(self.tmp) + + util.write_file('/etc/cloud/templates/chef_client.rb.tmpl', tpl_file) + cfg = { + 'chef': { + 'server_url': 'localhost', + 'validation_name': 'bob', + 'json_attribs': None, + 'show_time': None, + }, + } + cc_chef.handle('chef', cfg, self.fetch_cloud('ubuntu'), LOG, []) + c = util.load_file(cc_chef.CHEF_RB_PATH) + self.assertNotIn('json_attribs', c) + self.assertNotIn('Formatter.show_time', c) -- cgit v1.2.3 From 36bc6de57a2c9be1efe9ae7e9ebaf7fa9023d230 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Fri, 10 Oct 2014 18:54:28 -0700 Subject: Fix some of the erroring out test cases There are a couple new test cases that are now erroring out. - A usage of the non helper unit testcase base which is trigging and error on python 2.6 due to lack of method, fix this by using the base helper class. - A freebsd distro test check thats looking for /etc/resolv.conf and examining its contents, which won't exist due to our mocking routine that does not allow that file to be read. - A freebsd distro test where the distro class tries to call into ['ifconfig', '-a'] for a set of values, those values don't exist on the machine I am running on (and likely others machines) so we should mock the subp function out (that the distro class calls) and correctly return values that will work for the testcase. --- tests/unittests/test_datasource/test_openstack.py | 2 +- tests/unittests/test_distros/test_netconfig.py | 7 +++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/tests/unittests/test_datasource/test_openstack.py b/tests/unittests/test_datasource/test_openstack.py index 7b4e651a..8becbdd2 100644 --- a/tests/unittests/test_datasource/test_openstack.py +++ b/tests/unittests/test_datasource/test_openstack.py @@ -318,7 +318,7 @@ class TestOpenStackDataSource(test_helpers.HttprettyTestCase): self.assertIsNone(ds_os.version) -class TestVendorDataLoading(unittest.TestCase): +class TestVendorDataLoading(test_helpers.TestCase): def cvj(self, data): return openstack.convert_vendordata_json(data) diff --git a/tests/unittests/test_distros/test_netconfig.py b/tests/unittests/test_distros/test_netconfig.py index fbdb7b3f..35cc1f43 100644 --- a/tests/unittests/test_distros/test_netconfig.py +++ b/tests/unittests/test_distros/test_netconfig.py @@ -182,6 +182,12 @@ NETWORKING=yes spec=False, passthrough=False) load_mock = self.mocker.replace(util.load_file, spec=False, passthrough=False) + subp_mock = self.mocker.replace(util.subp, + spec=False, passthrough=False) + + subp_mock(['ifconfig', '-a']) + self.mocker.count(0, None) + self.mocker.result(('vtnet0', '')) exists_mock(mocker.ARGS) self.mocker.count(0, None) @@ -190,6 +196,7 @@ NETWORKING=yes write_bufs = {} read_bufs = { '/etc/rc.conf': '', + '/etc/resolv.conf': '', } def replace_write(filename, content, mode=0644, omode="wb"): -- cgit v1.2.3 From e03d5277111fce36cf877d855b6a458e569ca5da Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Fri, 10 Oct 2014 19:09:27 -0700 Subject: Move the installation code to its own function --- cloudinit/config/cc_chef.py | 72 ++++++++++++++++++++++++--------------------- 1 file changed, 39 insertions(+), 33 deletions(-) diff --git a/cloudinit/config/cc_chef.py b/cloudinit/config/cc_chef.py index 691a51bc..971b4fce 100644 --- a/cloudinit/config/cc_chef.py +++ b/cloudinit/config/cc_chef.py @@ -142,36 +142,42 @@ def handle(name, cfg, cloud, log, _args): initial_json[k] = initial_attributes[k] util.write_file(fb_filename, json.dumps(initial_json)) + # Try to install chef, if its not already installed... + install_chef(cloud, chef_cfg, log) + + +def install_chef(cloud, chef_cfg, log): # If chef is not installed, we install chef based on 'install_type' - if (not os.path.isfile('/usr/bin/chef-client') or - util.get_cfg_option_bool(chef_cfg, - 'force_install', default=False)): - - install_type = util.get_cfg_option_str(chef_cfg, 'install_type', - 'packages') - if install_type == "gems": - # this will install and run the chef-client from gems - chef_version = util.get_cfg_option_str(chef_cfg, 'version', None) - ruby_version = util.get_cfg_option_str(chef_cfg, 'ruby_version', - RUBY_VERSION_DEFAULT) - install_chef_from_gems(cloud.distro, ruby_version, chef_version) - # and finally, run chef-client - log.debug('Running chef-client') - util.subp(['/usr/bin/chef-client', - '-d', '-i', '1800', '-s', '20'], capture=False) - elif install_type == 'packages': - # this will install and run the chef-client from packages - cloud.distro.install_packages(('chef',)) - elif install_type == 'omnibus': - url = util.get_cfg_option_str(chef_cfg, "omnibus_url", OMNIBUS_URL) - content = url_helper.readurl(url=url, retries=5) - with util.tempdir() as tmpd: - # use tmpd over tmpfile to avoid 'Text file busy' on execute - tmpf = "%s/chef-omnibus-install" % tmpd - util.write_file(tmpf, str(content), mode=0700) - util.subp([tmpf], capture=False) - else: - log.warn("Unknown chef install type %s", install_type) + if os.path.isfile('/usr/bin/chef-client'): + return + if not util.get_cfg_option_bool(chef_cfg, 'force_install', default=False): + return + install_type = util.get_cfg_option_str(chef_cfg, 'install_type', + 'packages') + if install_type == "gems": + # This will install and run the chef-client from gems + chef_version = util.get_cfg_option_str(chef_cfg, 'version', None) + ruby_version = util.get_cfg_option_str(chef_cfg, 'ruby_version', + RUBY_VERSION_DEFAULT) + install_chef_from_gems(cloud.distro, ruby_version, chef_version) + # And finally, run chef-client + log.debug('Running chef-client') + util.subp(['/usr/bin/chef-client', + '-d', '-i', '1800', '-s', '20'], capture=False) + elif install_type == 'packages': + # This will install and run the chef-client from packages + cloud.distro.install_packages(('chef',)) + elif install_type == 'omnibus': + # This will install as a omnibus unified package + url = util.get_cfg_option_str(chef_cfg, "omnibus_url", OMNIBUS_URL) + content = url_helper.readurl(url=url, retries=5) + with util.tempdir() as tmpd: + # Use tmpdir over tmpfile to avoid 'text file busy' on execute + tmpf = "%s/chef-omnibus-install" % tmpd + util.write_file(tmpf, str(content), mode=0700) + util.subp([tmpf], capture=False) + else: + log.warn("Unknown chef install type '%s'", install_type) def get_ruby_packages(version): @@ -190,9 +196,9 @@ def install_chef_from_gems(ruby_version, chef_version, distro): util.sym_link('/usr/bin/ruby%s' % ruby_version, '/usr/bin/ruby') if chef_version: util.subp(['/usr/bin/gem', 'install', 'chef', - '-v %s' % chef_version, '--no-ri', - '--no-rdoc', '--bindir', '/usr/bin', '-q'], capture=False) + '-v %s' % chef_version, '--no-ri', + '--no-rdoc', '--bindir', '/usr/bin', '-q'], capture=False) else: util.subp(['/usr/bin/gem', 'install', 'chef', - '--no-ri', '--no-rdoc', '--bindir', - '/usr/bin', '-q'], capture=False) + '--no-ri', '--no-rdoc', '--bindir', + '/usr/bin', '-q'], capture=False) -- cgit v1.2.3 From 9452f1247f69c2a8e8dedaac804f3c95bff0821b Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Sat, 11 Oct 2014 16:37:30 -0700 Subject: Some more reworkings - Make a helper function to tell if already installed. - Have the install routine not run chef after installed but have it instead return a result to tell the caller to run the chef program once completed. --- cloudinit/config/cc_chef.py | 32 +++++++++++++++++++++----------- 1 file changed, 21 insertions(+), 11 deletions(-) diff --git a/cloudinit/config/cc_chef.py b/cloudinit/config/cc_chef.py index 971b4fce..fb825404 100644 --- a/cloudinit/config/cc_chef.py +++ b/cloudinit/config/cc_chef.py @@ -19,7 +19,6 @@ # along with this program. If not, see . from datetime import datetime - import json import os @@ -66,6 +65,16 @@ CHEF_RB_TPL_KEYS.extend([ CHEF_RB_TPL_KEYS = frozenset(CHEF_RB_TPL_KEYS) CHEF_RB_PATH = '/etc/chef/client.rb' CHEF_FB_PATH = '/etc/chef/firstboot.json' +CHEF_EXEC_PATH = '/usr/bin/chef-client' +CHEF_EXEC_CMD = tuple([CHEF_EXEC_PATH, '-d', '-i', '1800', '-s', '20']) + + +def is_installed(): + if not os.path.isfile(CHEF_EXEC_PATH): + return False + if not os.access(CHEF_EXEC_PATH, os.X_OK): + return False + return True def get_template_params(iid, chef_cfg, log): @@ -106,7 +115,7 @@ def handle(name, cfg, cloud, log, _args): chef_cfg = cfg['chef'] # Ensure the chef directories we use exist - for d in chef_cfg.get('directories', CHEF_DIRS): + for d in list(chef_cfg.get('directories', CHEF_DIRS)): util.ensure_dir(d) # Set the validation key based on the presence of either 'validation_key' @@ -143,27 +152,27 @@ def handle(name, cfg, cloud, log, _args): util.write_file(fb_filename, json.dumps(initial_json)) # Try to install chef, if its not already installed... - install_chef(cloud, chef_cfg, log) + force_install = util.get_cfg_option_bool(chef_cfg, + 'force_install', default=False) + if not is_installed() or force_install: + run = install_chef(cloud, chef_cfg, log) + if run: + log.debug('Running chef-client') + util.subp(CHEF_EXEC_CMD, capture=False) def install_chef(cloud, chef_cfg, log): # If chef is not installed, we install chef based on 'install_type' - if os.path.isfile('/usr/bin/chef-client'): - return - if not util.get_cfg_option_bool(chef_cfg, 'force_install', default=False): - return install_type = util.get_cfg_option_str(chef_cfg, 'install_type', 'packages') + run_after = False if install_type == "gems": # This will install and run the chef-client from gems chef_version = util.get_cfg_option_str(chef_cfg, 'version', None) ruby_version = util.get_cfg_option_str(chef_cfg, 'ruby_version', RUBY_VERSION_DEFAULT) install_chef_from_gems(cloud.distro, ruby_version, chef_version) - # And finally, run chef-client - log.debug('Running chef-client') - util.subp(['/usr/bin/chef-client', - '-d', '-i', '1800', '-s', '20'], capture=False) + run_after = True elif install_type == 'packages': # This will install and run the chef-client from packages cloud.distro.install_packages(('chef',)) @@ -178,6 +187,7 @@ def install_chef(cloud, chef_cfg, log): util.subp([tmpf], capture=False) else: log.warn("Unknown chef install type '%s'", install_type) + return run_after def get_ruby_packages(version): -- cgit v1.2.3 From d87e89d9c674bac7e87d483037850a9ee4fc984a Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Sat, 11 Oct 2014 16:59:50 -0700 Subject: More adjustments - Use the generated_by() utility function to give the ruby template a better header comment - Set special parameters after selecting the basic chef parameters. --- cloudinit/config/cc_chef.py | 19 +++++++++++-------- templates/chef_client.rb.tmpl | 3 +-- tests/unittests/test_handler/test_handler_chef.py | 20 +++++++++++++++++++- 3 files changed, 31 insertions(+), 11 deletions(-) diff --git a/cloudinit/config/cc_chef.py b/cloudinit/config/cc_chef.py index fb825404..999b658d 100644 --- a/cloudinit/config/cc_chef.py +++ b/cloudinit/config/cc_chef.py @@ -79,13 +79,6 @@ def is_installed(): def get_template_params(iid, chef_cfg, log): params = CHEF_RB_TPL_DEFAULTS.copy() - params.update({ - 'server_url': chef_cfg['server_url'], - 'node_name': util.get_cfg_option_str(chef_cfg, 'node_name', iid), - 'environment': util.get_cfg_option_str(chef_cfg, 'environment', - '_default'), - 'validation_name': chef_cfg['validation_name'], - }) # Allow users to overwrite any of the keys they want (if they so choose), # when a value is None, then the value will be set to None and no boolean # or string version will be populated... @@ -101,7 +94,17 @@ def get_template_params(iid, chef_cfg, log): params[k] = util.get_cfg_option_bool(chef_cfg, k) else: params[k] = util.get_cfg_option_str(chef_cfg, k) - params['generated_on'] = datetime.now().isoformat() + # These ones are overwritten to be exact values... + params.update({ + 'generated_by': util.make_header(), + 'server_url': util.get_cfg_option_str(chef_cfg, 'server_url'), + 'node_name': util.get_cfg_option_str(chef_cfg, 'node_name', + default=iid), + 'environment': util.get_cfg_option_str(chef_cfg, 'environment', + default='_default'), + 'validation_name': util.get_cfg_option_str(chef_cfg, + 'validation_name'), + }) return params diff --git a/templates/chef_client.rb.tmpl b/templates/chef_client.rb.tmpl index 7b9e6298..c4069d22 100644 --- a/templates/chef_client.rb.tmpl +++ b/templates/chef_client.rb.tmpl @@ -9,12 +9,11 @@ you need to add the following to config: validation_name: XYZ server_url: XYZ -#} - +{{generated_by}} {# The reason these are not in quotes is because they are ruby symbols that will be placed inside here, and not actual strings... #} -# This is a generated file, created on {{generated_on}}. {% if log_level %} log_level {{log_level}} {% endif %} diff --git a/tests/unittests/test_handler/test_handler_chef.py b/tests/unittests/test_handler/test_handler_chef.py index de7ff2da..ef1aa208 100644 --- a/tests/unittests/test_handler/test_handler_chef.py +++ b/tests/unittests/test_handler/test_handler_chef.py @@ -1,5 +1,5 @@ -import os import json +import os from cloudinit.config import cc_chef @@ -38,6 +38,24 @@ class TestChef(t_help.FilesystemMockingTestCase): self.assertFalse(os.path.isdir(d)) def test_basic_config(self): + # This should create a file of the format... + """ + # Created by cloud-init v. 0.7.6 on Sat, 11 Oct 2014 23:57:21 +0000 + log_level :info + ssl_verify_mode :verify_none + log_location "/var/log/chef/client.log" + validation_client_name "bob" + validation_key "/etc/chef/validation.pem" + client_key "/etc/chef/client.pem" + chef_server_url "localhost" + environment "_default" + node_name "iid-datasource-none" + json_attribs "/etc/chef/firstboot.json" + file_cache_path "/var/cache/chef" + file_backup_path "/var/backups/chef" + pid_file "/var/run/chef/client.pid" + Chef::Log::Formatter.show_time = true + """ tpl_file = util.load_file('templates/chef_client.rb.tmpl') self.patchUtils(self.tmp) self.patchOS(self.tmp) -- cgit v1.2.3 From c5341fe07b767735e9ed74e45cee9629c6434892 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Sat, 11 Oct 2014 18:18:45 -0700 Subject: Allow for the running after install and run arguments to be configured Instead of only running the client when installed from gems, allow it to be ran from other install modes as well (if configured) and allow the arguments that are passed to the client when ran to be altered (if so desired). --- cloudinit/config/cc_chef.py | 32 +++++++++++++++++++++++++------- 1 file changed, 25 insertions(+), 7 deletions(-) diff --git a/cloudinit/config/cc_chef.py b/cloudinit/config/cc_chef.py index 999b658d..e503371d 100644 --- a/cloudinit/config/cc_chef.py +++ b/cloudinit/config/cc_chef.py @@ -18,7 +18,6 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -from datetime import datetime import json import os @@ -66,7 +65,7 @@ CHEF_RB_TPL_KEYS = frozenset(CHEF_RB_TPL_KEYS) CHEF_RB_PATH = '/etc/chef/client.rb' CHEF_FB_PATH = '/etc/chef/firstboot.json' CHEF_EXEC_PATH = '/usr/bin/chef-client' -CHEF_EXEC_CMD = tuple([CHEF_EXEC_PATH, '-d', '-i', '1800', '-s', '20']) +CHEF_EXEC_DEF_ARGS = tuple(['-d', '-i', '1800', '-s', '20']) def is_installed(): @@ -158,24 +157,42 @@ def handle(name, cfg, cloud, log, _args): force_install = util.get_cfg_option_bool(chef_cfg, 'force_install', default=False) if not is_installed() or force_install: - run = install_chef(cloud, chef_cfg, log) - if run: + run_after = install_chef(cloud, chef_cfg, log) + if run_after: log.debug('Running chef-client') - util.subp(CHEF_EXEC_CMD, capture=False) + cmd = [CHEF_EXEC_PATH] + if 'exec_arguments' in chef_cfg: + cmd_args = chef_cfg['exec_arguments'] + if isinstance(cmd_args, (list, tuple)): + cmd.extend(cmd_args) + elif isinstance(cmd_args, (str, basestring)): + cmd.append(cmd_args) + else: + log.warn("Unknown type %s provided for chef" + " 'exec_arguments' expected list, tuple," + " or string", type(cmd_args)) + cmd.extend(CHEF_EXEC_DEF_ARGS) + else: + cmd.extend(CHEF_EXEC_DEF_ARGS) + util.subp(cmd, capture=False) def install_chef(cloud, chef_cfg, log): # If chef is not installed, we install chef based on 'install_type' install_type = util.get_cfg_option_str(chef_cfg, 'install_type', 'packages') - run_after = False + run_after = util.get_cfg_option_bool(chef_cfg, 'exec_after_install', + default=False) if install_type == "gems": # This will install and run the chef-client from gems chef_version = util.get_cfg_option_str(chef_cfg, 'version', None) ruby_version = util.get_cfg_option_str(chef_cfg, 'ruby_version', RUBY_VERSION_DEFAULT) install_chef_from_gems(cloud.distro, ruby_version, chef_version) - run_after = True + # Retain backwards compat, but preferring True instead of False + # when not provided/overriden... + run_after = util.get_cfg_option_bool(chef_cfg, 'exec_after_install', + default=True) elif install_type == 'packages': # This will install and run the chef-client from packages cloud.distro.install_packages(('chef',)) @@ -190,6 +207,7 @@ def install_chef(cloud, chef_cfg, log): util.subp([tmpf], capture=False) else: log.warn("Unknown chef install type '%s'", install_type) + run_after = False return run_after -- cgit v1.2.3 From 4994f7cb475713be523f96e077a76f801e6d1db5 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Sat, 11 Oct 2014 18:23:20 -0700 Subject: Allow the omnibus url fetching retries to be configurable --- cloudinit/config/cc_chef.py | 6 +++++- cloudinit/util.py | 4 ++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/cloudinit/config/cc_chef.py b/cloudinit/config/cc_chef.py index e503371d..205f4b49 100644 --- a/cloudinit/config/cc_chef.py +++ b/cloudinit/config/cc_chef.py @@ -37,6 +37,7 @@ CHEF_DIRS = [ ] OMNIBUS_URL = "https://www.opscode.com/chef/install.sh" +OMNIBUS_URL_RETRIES = 5 CHEF_RB_TPL_DEFAULTS = { # These are ruby symbols... @@ -199,7 +200,10 @@ def install_chef(cloud, chef_cfg, log): elif install_type == 'omnibus': # This will install as a omnibus unified package url = util.get_cfg_option_str(chef_cfg, "omnibus_url", OMNIBUS_URL) - content = url_helper.readurl(url=url, retries=5) + retries = max(0, util.get_cfg_option_int(chef_cfg, + "omnibus_url_retries", + default=OMNIBUS_URL_RETRIES)) + content = url_helper.readurl(url=url, retries=retries) with util.tempdir() as tmpd: # Use tmpdir over tmpfile to avoid 'text file busy' on execute tmpf = "%s/chef-omnibus-install" % tmpd diff --git a/cloudinit/util.py b/cloudinit/util.py index f236d0bf..71221e09 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -399,6 +399,10 @@ def get_cfg_option_str(yobj, key, default=None): return val +def get_cfg_option_int(yobj, key, default=0): + return int(get_cfg_option_str(yobj, key, default=default)) + + def system_info(): return { 'platform': platform.platform(), -- cgit v1.2.3 From 1f969e30983f93eed2ece906a6ca9d16bc0aed86 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Sat, 11 Oct 2014 18:47:06 -0700 Subject: Use the util function to get the chef base directories --- cloudinit/config/cc_chef.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/cloudinit/config/cc_chef.py b/cloudinit/config/cc_chef.py index 205f4b49..fb8d7641 100644 --- a/cloudinit/config/cc_chef.py +++ b/cloudinit/config/cc_chef.py @@ -118,7 +118,10 @@ def handle(name, cfg, cloud, log, _args): chef_cfg = cfg['chef'] # Ensure the chef directories we use exist - for d in list(chef_cfg.get('directories', CHEF_DIRS)): + chef_dirs = util.get_cfg_option_list(chef_cfg, 'directories') + if not chef_dirs: + chef_dirs = list(CHEF_DIRS) + for d in chef_dirs: util.ensure_dir(d) # Set the validation key based on the presence of either 'validation_key' -- cgit v1.2.3 From 0a2e8b0130c2f83c0ea555d26473c92098c6adda Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Sat, 11 Oct 2014 18:51:33 -0700 Subject: Always ensure we create the /etc/chef dir --- cloudinit/config/cc_chef.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/cloudinit/config/cc_chef.py b/cloudinit/config/cc_chef.py index fb8d7641..ff08050d 100644 --- a/cloudinit/config/cc_chef.py +++ b/cloudinit/config/cc_chef.py @@ -18,6 +18,7 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . +import itertools import json import os @@ -35,6 +36,9 @@ CHEF_DIRS = [ '/var/backups/chef', '/var/run/chef', ] +REQUIRED_CHEF_DIRS = [ + '/etc/chef', +] OMNIBUS_URL = "https://www.opscode.com/chef/install.sh" OMNIBUS_URL_RETRIES = 5 @@ -121,7 +125,7 @@ def handle(name, cfg, cloud, log, _args): chef_dirs = util.get_cfg_option_list(chef_cfg, 'directories') if not chef_dirs: chef_dirs = list(CHEF_DIRS) - for d in chef_dirs: + for d in itertools.chain(chef_dirs, REQUIRED_CHEF_DIRS): util.ensure_dir(d) # Set the validation key based on the presence of either 'validation_key' -- cgit v1.2.3 From 736603ffdce0b836b43c77ae680ef2818521f30c Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Sat, 11 Oct 2014 18:54:14 -0700 Subject: Move the chef running to its own helper function --- cloudinit/config/cc_chef.py | 36 ++++++++++++++++++++---------------- 1 file changed, 20 insertions(+), 16 deletions(-) diff --git a/cloudinit/config/cc_chef.py b/cloudinit/config/cc_chef.py index ff08050d..f501f1f7 100644 --- a/cloudinit/config/cc_chef.py +++ b/cloudinit/config/cc_chef.py @@ -167,22 +167,26 @@ def handle(name, cfg, cloud, log, _args): if not is_installed() or force_install: run_after = install_chef(cloud, chef_cfg, log) if run_after: - log.debug('Running chef-client') - cmd = [CHEF_EXEC_PATH] - if 'exec_arguments' in chef_cfg: - cmd_args = chef_cfg['exec_arguments'] - if isinstance(cmd_args, (list, tuple)): - cmd.extend(cmd_args) - elif isinstance(cmd_args, (str, basestring)): - cmd.append(cmd_args) - else: - log.warn("Unknown type %s provided for chef" - " 'exec_arguments' expected list, tuple," - " or string", type(cmd_args)) - cmd.extend(CHEF_EXEC_DEF_ARGS) - else: - cmd.extend(CHEF_EXEC_DEF_ARGS) - util.subp(cmd, capture=False) + run_chef(chef_cfg, log) + + +def run_chef(chef_cfg, log): + log.debug('Running chef-client') + cmd = [CHEF_EXEC_PATH] + if 'exec_arguments' in chef_cfg: + cmd_args = chef_cfg['exec_arguments'] + if isinstance(cmd_args, (list, tuple)): + cmd.extend(cmd_args) + elif isinstance(cmd_args, (str, basestring)): + cmd.append(cmd_args) + else: + log.warn("Unknown type %s provided for chef" + " 'exec_arguments' expected list, tuple," + " or string", type(cmd_args)) + cmd.extend(CHEF_EXEC_DEF_ARGS) + else: + cmd.extend(CHEF_EXEC_DEF_ARGS) + util.subp(cmd, capture=False) def install_chef(cloud, chef_cfg, log): -- cgit v1.2.3 From fc380079288da045b1bfd61fb9ef2490a7cc1f82 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Sat, 11 Oct 2014 19:26:49 -0700 Subject: Ensure that any template paths have associated directories When the template provides a path, make sure that before the template is written that the path that is now in the template has the associated directory created (if not already created). --- cloudinit/config/cc_chef.py | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/cloudinit/config/cc_chef.py b/cloudinit/config/cc_chef.py index f501f1f7..28562caf 100644 --- a/cloudinit/config/cc_chef.py +++ b/cloudinit/config/cc_chef.py @@ -58,8 +58,18 @@ CHEF_RB_TPL_DEFAULTS = { 'show_time': True, } CHEF_RB_TPL_BOOL_KEYS = frozenset(['show_time']) +CHEF_RB_PATH_KEYS = frozenset([ + 'log_location', + 'validation_key', + 'client_key', + 'file_cache_path', + 'json_attribs', + 'file_cache_path', + 'pid_file', +]) CHEF_RB_TPL_KEYS = list(CHEF_RB_TPL_DEFAULTS.keys()) CHEF_RB_TPL_KEYS.extend(CHEF_RB_TPL_BOOL_KEYS) +CHEF_RB_TPL_KEYS.extend(CHEF_RB_PATH_KEYS) CHEF_RB_TPL_KEYS.extend([ 'server_url', 'node_name', @@ -109,7 +119,11 @@ def get_template_params(iid, chef_cfg, log): 'validation_name': util.get_cfg_option_str(chef_cfg, 'validation_name'), }) - return params + paths = set() + for (k, v) in params.items(): + if k in CHEF_RB_PATH_KEYS and v: + paths.add(os.path.dirname(v)) + return params, paths def handle(name, cfg, cloud, log, _args): @@ -140,7 +154,9 @@ def handle(name, cfg, cloud, log, _args): template_fn = cloud.get_template_filename('chef_client.rb') if template_fn: iid = str(cloud.datasource.get_instance_id()) - params = get_template_params(iid, chef_cfg, log) + params, paths = get_template_params(iid, chef_cfg, log) + for d in paths: + util.ensure_dir(d) templater.render_to_file(template_fn, CHEF_RB_PATH, params) else: log.warn("No template found, not rendering to %s", -- cgit v1.2.3 From 32bb2f37593482b179c8627bb254d50121a8be01 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Sat, 11 Oct 2014 19:30:12 -0700 Subject: Have the caller find the param paths instead of the param creator --- cloudinit/config/cc_chef.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/cloudinit/config/cc_chef.py b/cloudinit/config/cc_chef.py index 28562caf..66187659 100644 --- a/cloudinit/config/cc_chef.py +++ b/cloudinit/config/cc_chef.py @@ -119,11 +119,7 @@ def get_template_params(iid, chef_cfg, log): 'validation_name': util.get_cfg_option_str(chef_cfg, 'validation_name'), }) - paths = set() - for (k, v) in params.items(): - if k in CHEF_RB_PATH_KEYS and v: - paths.add(os.path.dirname(v)) - return params, paths + return params def handle(name, cfg, cloud, log, _args): @@ -154,9 +150,12 @@ def handle(name, cfg, cloud, log, _args): template_fn = cloud.get_template_filename('chef_client.rb') if template_fn: iid = str(cloud.datasource.get_instance_id()) - params, paths = get_template_params(iid, chef_cfg, log) - for d in paths: - util.ensure_dir(d) + params = get_template_params(iid, chef_cfg, log) + param_paths = set() + for (k, v) in params.items(): + if k in CHEF_RB_PATH_KEYS and v: + param_paths.add(os.path.dirname(v)) + util.ensure_dirs(param_paths) templater.render_to_file(template_fn, CHEF_RB_PATH, params) else: log.warn("No template found, not rendering to %s", -- cgit v1.2.3 From db6cee3fe1c6033af8ae66f76c7c81a5af2d9c5a Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Sat, 11 Oct 2014 19:39:36 -0700 Subject: Follow the same constant variable naming scheme for the path tpl keys --- cloudinit/config/cc_chef.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cloudinit/config/cc_chef.py b/cloudinit/config/cc_chef.py index 66187659..e9a37652 100644 --- a/cloudinit/config/cc_chef.py +++ b/cloudinit/config/cc_chef.py @@ -58,7 +58,7 @@ CHEF_RB_TPL_DEFAULTS = { 'show_time': True, } CHEF_RB_TPL_BOOL_KEYS = frozenset(['show_time']) -CHEF_RB_PATH_KEYS = frozenset([ +CHEF_RB_TPL_PATH_KEYS = frozenset([ 'log_location', 'validation_key', 'client_key', @@ -69,7 +69,7 @@ CHEF_RB_PATH_KEYS = frozenset([ ]) CHEF_RB_TPL_KEYS = list(CHEF_RB_TPL_DEFAULTS.keys()) CHEF_RB_TPL_KEYS.extend(CHEF_RB_TPL_BOOL_KEYS) -CHEF_RB_TPL_KEYS.extend(CHEF_RB_PATH_KEYS) +CHEF_RB_TPL_KEYS.extend(CHEF_RB_TPL_PATH_KEYS) CHEF_RB_TPL_KEYS.extend([ 'server_url', 'node_name', @@ -153,7 +153,7 @@ def handle(name, cfg, cloud, log, _args): params = get_template_params(iid, chef_cfg, log) param_paths = set() for (k, v) in params.items(): - if k in CHEF_RB_PATH_KEYS and v: + if k in CHEF_RB_TPL_PATH_KEYS and v: param_paths.add(os.path.dirname(v)) util.ensure_dirs(param_paths) templater.render_to_file(template_fn, CHEF_RB_PATH, params) -- cgit v1.2.3 From da89dca32a3dd394c6a19807856871a645f2acc4 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Sat, 11 Oct 2014 21:29:01 -0700 Subject: Allow running even if installed Standardize on using the chef_cfg key 'exec' which can be used when installing to tell the caller to run the chef client or can also be used if the client is already installed and its requested to be ran. To retain existing behavior 'exec' does not by default assume to be true, unless explicitly provided or a gems mode install is requested. --- cloudinit/config/cc_chef.py | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/cloudinit/config/cc_chef.py b/cloudinit/config/cc_chef.py index e9a37652..29238861 100644 --- a/cloudinit/config/cc_chef.py +++ b/cloudinit/config/cc_chef.py @@ -180,9 +180,13 @@ def handle(name, cfg, cloud, log, _args): force_install = util.get_cfg_option_bool(chef_cfg, 'force_install', default=False) if not is_installed() or force_install: - run_after = install_chef(cloud, chef_cfg, log) - if run_after: - run_chef(chef_cfg, log) + run = install_chef(cloud, chef_cfg, log) + elif is_installed(): + run = util.get_cfg_option_bool(chef_cfg, 'exec', default=False) + else: + run = False + if run: + run_chef(chef_cfg, log) def run_chef(chef_cfg, log): @@ -208,18 +212,16 @@ def install_chef(cloud, chef_cfg, log): # If chef is not installed, we install chef based on 'install_type' install_type = util.get_cfg_option_str(chef_cfg, 'install_type', 'packages') - run_after = util.get_cfg_option_bool(chef_cfg, 'exec_after_install', - default=False) + run = util.get_cfg_option_bool(chef_cfg, 'exec', default=False) if install_type == "gems": # This will install and run the chef-client from gems chef_version = util.get_cfg_option_str(chef_cfg, 'version', None) ruby_version = util.get_cfg_option_str(chef_cfg, 'ruby_version', RUBY_VERSION_DEFAULT) install_chef_from_gems(cloud.distro, ruby_version, chef_version) - # Retain backwards compat, but preferring True instead of False + # Retain backwards compat, by preferring True instead of False # when not provided/overriden... - run_after = util.get_cfg_option_bool(chef_cfg, 'exec_after_install', - default=True) + run = util.get_cfg_option_bool(chef_cfg, 'exec', default=True) elif install_type == 'packages': # This will install and run the chef-client from packages cloud.distro.install_packages(('chef',)) @@ -237,8 +239,8 @@ def install_chef(cloud, chef_cfg, log): util.subp([tmpf], capture=False) else: log.warn("Unknown chef install type '%s'", install_type) - run_after = False - return run_after + run = False + return run def get_ruby_packages(version): -- cgit v1.2.3 From f99d19b95009fea854adbb376518f92c195af917 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Sat, 11 Oct 2014 21:36:34 -0700 Subject: Add a comment explaining the param path logic --- cloudinit/config/cc_chef.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/cloudinit/config/cc_chef.py b/cloudinit/config/cc_chef.py index 29238861..687be69f 100644 --- a/cloudinit/config/cc_chef.py +++ b/cloudinit/config/cc_chef.py @@ -151,6 +151,9 @@ def handle(name, cfg, cloud, log, _args): if template_fn: iid = str(cloud.datasource.get_instance_id()) params = get_template_params(iid, chef_cfg, log) + # Do a best effort attempt to ensure that the template values that + # are associated with paths have there parent directory created + # before they are used by the chef-client itself. param_paths = set() for (k, v) in params.items(): if k in CHEF_RB_TPL_PATH_KEYS and v: -- cgit v1.2.3 From 088b3b7eaab6c36ef404978f10c514785651a8fd Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Sun, 12 Oct 2014 10:12:00 -0700 Subject: Retain the old behavior for mandatory keys The keys 'server_url' and 'validation_name' were previously mandatory, we should retain that behavior for now. --- cloudinit/config/cc_chef.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cloudinit/config/cc_chef.py b/cloudinit/config/cc_chef.py index 687be69f..1e44ec72 100644 --- a/cloudinit/config/cc_chef.py +++ b/cloudinit/config/cc_chef.py @@ -111,13 +111,13 @@ def get_template_params(iid, chef_cfg, log): # These ones are overwritten to be exact values... params.update({ 'generated_by': util.make_header(), - 'server_url': util.get_cfg_option_str(chef_cfg, 'server_url'), 'node_name': util.get_cfg_option_str(chef_cfg, 'node_name', default=iid), 'environment': util.get_cfg_option_str(chef_cfg, 'environment', default='_default'), - 'validation_name': util.get_cfg_option_str(chef_cfg, - 'validation_name'), + # These two are mandatory... + 'server_url': chef_cfg['server_url'], + 'validation_name': chef_cfg['validation_name'], }) return params -- cgit v1.2.3 From 5088b76198a8844dcf73ab7d2ef26bf9e5caf552 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Mon, 13 Oct 2014 18:29:23 -0700 Subject: Add a post-run method that can be used to delete validation.pem files For those who run chef in non-daemon mode, they would like to delete the validation.pem file if chef finishes as expected to remove that file from existing in an easy to read manner. --- cloudinit/config/cc_chef.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/cloudinit/config/cc_chef.py b/cloudinit/config/cc_chef.py index 1e44ec72..4350a353 100644 --- a/cloudinit/config/cc_chef.py +++ b/cloudinit/config/cc_chef.py @@ -78,6 +78,7 @@ CHEF_RB_TPL_KEYS.extend([ ]) CHEF_RB_TPL_KEYS = frozenset(CHEF_RB_TPL_KEYS) CHEF_RB_PATH = '/etc/chef/client.rb' +CHEF_VALIDATION_PEM_PATH = '/etc/chef/validation.pem' CHEF_FB_PATH = '/etc/chef/firstboot.json' CHEF_EXEC_PATH = '/usr/bin/chef-client' CHEF_EXEC_DEF_ARGS = tuple(['-d', '-i', '1800', '-s', '20']) @@ -91,6 +92,14 @@ def is_installed(): return True +def post_run_chef(chef_cfg, log): + delete_pem = util.get_cfg_option_bool(chef_cfg, + 'delete_validation_post_exec', + default=False) + if delete_pem and os.path.isfile(CHEF_VALIDATION_PEM_PATH): + os.unlink(CHEF_VALIDATION_PEM_PATH) + + def get_template_params(iid, chef_cfg, log): params = CHEF_RB_TPL_DEFAULTS.copy() # Allow users to overwrite any of the keys they want (if they so choose), @@ -143,7 +152,7 @@ def handle(name, cfg, cloud, log, _args): # takes precedence for key in ('validation_key', 'validation_cert'): if key in chef_cfg and chef_cfg[key]: - util.write_file('/etc/chef/validation.pem', chef_cfg[key]) + util.write_file(CHEF_VALIDATION_PEM_PATH, chef_cfg[key]) break # Create the chef config from template @@ -190,6 +199,7 @@ def handle(name, cfg, cloud, log, _args): run = False if run: run_chef(chef_cfg, log) + post_run_chef(chef_cfg, log) def run_chef(chef_cfg, log): -- cgit v1.2.3 From 9974012c8640b731a0e91826fc64e1f5ff5d1096 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Mon, 13 Oct 2014 18:51:19 -0700 Subject: Use the key contants in the default key => value set --- cloudinit/config/cc_chef.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/cloudinit/config/cc_chef.py b/cloudinit/config/cc_chef.py index 4350a353..f6f07bce 100644 --- a/cloudinit/config/cc_chef.py +++ b/cloudinit/config/cc_chef.py @@ -43,15 +43,17 @@ REQUIRED_CHEF_DIRS = [ OMNIBUS_URL = "https://www.opscode.com/chef/install.sh" OMNIBUS_URL_RETRIES = 5 +CHEF_VALIDATION_PEM_PATH = '/etc/chef/validation.pem' +CHEF_FB_PATH = '/etc/chef/firstboot.json' CHEF_RB_TPL_DEFAULTS = { # These are ruby symbols... 'ssl_verify_mode': ':verify_none', 'log_level': ':info', # These are not symbols... 'log_location': '/var/log/chef/client.log', - 'validation_key': "/etc/chef/validation.pem", + 'validation_key': CHEF_VALIDATION_PEM_PATH, 'client_key': "/etc/chef/client.pem", - 'json_attribs': "/etc/chef/firstboot.json", + 'json_attribs': CHEF_FB_PATH, 'file_cache_path': "/var/cache/chef", 'file_backup_path': "/var/backups/chef", 'pid_file': "/var/run/chef/client.pid", @@ -78,8 +80,6 @@ CHEF_RB_TPL_KEYS.extend([ ]) CHEF_RB_TPL_KEYS = frozenset(CHEF_RB_TPL_KEYS) CHEF_RB_PATH = '/etc/chef/client.rb' -CHEF_VALIDATION_PEM_PATH = '/etc/chef/validation.pem' -CHEF_FB_PATH = '/etc/chef/firstboot.json' CHEF_EXEC_PATH = '/usr/bin/chef-client' CHEF_EXEC_DEF_ARGS = tuple(['-d', '-i', '1800', '-s', '20']) -- cgit v1.2.3 From b04f582abe201e414905008d93b94fafb7ae707d Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Wed, 15 Oct 2014 11:40:37 -0700 Subject: Prefer immutable structures --- cloudinit/config/cc_chef.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/cloudinit/config/cc_chef.py b/cloudinit/config/cc_chef.py index f6f07bce..aa82cb0a 100644 --- a/cloudinit/config/cc_chef.py +++ b/cloudinit/config/cc_chef.py @@ -28,17 +28,17 @@ from cloudinit import util RUBY_VERSION_DEFAULT = "1.8" -CHEF_DIRS = [ +CHEF_DIRS = tuple([ '/etc/chef', '/var/log/chef', '/var/lib/chef', '/var/cache/chef', '/var/backups/chef', '/var/run/chef', -] -REQUIRED_CHEF_DIRS = [ +]) +REQUIRED_CHEF_DIRS = tuple([ '/etc/chef', -] +]) OMNIBUS_URL = "https://www.opscode.com/chef/install.sh" OMNIBUS_URL_RETRIES = 5 -- cgit v1.2.3 From 03cec66c495e57fdf29e7bab4b3938e24a8dec18 Mon Sep 17 00:00:00 2001 From: Neal Shrader Date: Thu, 16 Oct 2014 11:30:08 -0400 Subject: Add DigitalOcean DataSource The DigitalOcean metadata service is an AWS-style service available over HTTP via the link local address 169.254.169.254. The specifics of the API are documented at: https://developers.digitalocean.com/metadata/ --- cloudinit/sources/DataSourceDigitalOcean.py | 102 ++++++++++++++++++++++++++++ 1 file changed, 102 insertions(+) create mode 100644 cloudinit/sources/DataSourceDigitalOcean.py diff --git a/cloudinit/sources/DataSourceDigitalOcean.py b/cloudinit/sources/DataSourceDigitalOcean.py new file mode 100644 index 00000000..c580e2d5 --- /dev/null +++ b/cloudinit/sources/DataSourceDigitalOcean.py @@ -0,0 +1,102 @@ +# vi: ts=4 expandtab +# +# Author: Neal Shrader +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +from cloudinit import log as logging +from cloudinit import util +from cloudinit import sources +from cloudinit import url_helper + +LOG = logging.getLogger(__name__) + +BUILTIN_DS_CONFIG = { + 'metadata_url': 'http://169.254.169.254/metadata/v1', + 'mirrors_url': 'http://mirrors.digitalocean.com/' +} + +class DataSourceDigitalOcean(sources.DataSource): + def __init__(self, sys_cfg, distro, paths): + sources.DataSource.__init__(self, sys_cfg, distro, paths) + self.metadata = dict() + self.ds_cfg = util.mergemanydict([ + util.get_cfg_by_path(sys_cfg, ["datasource", "DigitalOcean"], {}), + BUILTIN_DS_CONFIG]) + self.metadata_address = self.ds_cfg['metadata_url'] + self.retries = 3 + self.timeout = 1 + + def get_data(self): + url_map = [ + ('user-data', '/user-data'), + ('vendor-data', '/vendor-data'), + ('public-keys', '/public-keys'), + ('region', '/region'), + ('id', '/id'), + ('hostname', '/hostname'), + ] + + found = False + for (key, path) in url_map: + try: + resp = url_helper.readurl(url=self.metadata_address + path, + timeout=self.timeout, + retries=self.retries) + if resp.code == 200: + found = True + self.metadata[key] = resp.contents + else: + LOG.warn("Path: %s returned %s", path, resp.code) + return False + except url_helper.UrlError as e: + LOG.warn("Path: %s raised exception: %s", path, e) + return False + + return found + + def get_userdata_raw(self): + return self.metadata['user-data'] + + def get_vendordata_raw(self): + return self.metadata['vendor-data'] + + def get_public_ssh_keys(self): + return self.metadata['public-keys'].splitlines() + + @property + def availability_zone(self): + return self.metadata['region'] + + def get_instance_id(self): + return self.metadata['id'] + + def get_hostname(self, fqdn=False): + return self.metadata['hostname'] + + def get_package_mirror_info(self): + return self.ds_cfg['mirrors_url'] + + @property + def launch_index(self): + return None + +# Used to match classes to dependencies +datasources = [ + (DataSourceDigitalOcean, (sources.DEP_FILESYSTEM, sources.DEP_NETWORK)), + ] + + +# Return a list of data sources that match this set of dependencies +def get_datasource_list(depends): + return sources.list_from_depends(depends, datasources) -- cgit v1.2.3 From 4f24ed06e2ebfa59349fa8652b59f7743a78e79b Mon Sep 17 00:00:00 2001 From: Neal Shrader Date: Thu, 16 Oct 2014 17:53:59 -0400 Subject: Add tests for DigitalOcean datasource --- .../unittests/test_datasource/test_digitalocean.py | 95 ++++++++++++++++++++++ 1 file changed, 95 insertions(+) create mode 100644 tests/unittests/test_datasource/test_digitalocean.py diff --git a/tests/unittests/test_datasource/test_digitalocean.py b/tests/unittests/test_datasource/test_digitalocean.py new file mode 100644 index 00000000..9576e042 --- /dev/null +++ b/tests/unittests/test_datasource/test_digitalocean.py @@ -0,0 +1,95 @@ +# +# Copyright (C) 2014 Neal Shrader +# +# Author: Neal Shrader +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import httpretty +import re + +from urlparse import urlparse + +from cloudinit import settings +from cloudinit import helpers +from cloudinit.sources import DataSourceDigitalOcean + +from .. import helpers as test_helpers + +DO_META = { + 'user-data': '#!/bin/bash\necho "user-data"', + 'vendor-data': '#!/bin/bash\necho "vendor-data"', + 'public-keys': 'ssh-rsa AAAAB3NzaC1yc2EAAAA... neal@digitalocean.com', + 'region': 'nyc3', + 'id': '2000000', + 'hostname': 'cloudinit-test', +} + +MD_URL_RE = re.compile(r'http://169.254.169.254/metadata/v1/.*') + +def _request_callback(method, uri, headers): + url_path = urlparse(uri).path + if url_path.startswith('/metadata/v1/'): + path = url_path.split('/metadata/v1/')[1:][0] + else: + path = None + if path in DO_META: + return (200, headers, DO_META.get(path)) + else: + return (404, headers, '') + + +class TestDataSourceDigitalOcean(test_helpers.HttprettyTestCase): + + def setUp(self): + self.ds = DataSourceDigitalOcean.DataSourceDigitalOcean( + settings.CFG_BUILTIN, None, + helpers.Paths({})) + super(TestDataSourceDigitalOcean, self).setUp() + + @httpretty.activate + def test_connection(self): + httpretty.register_uri( + httpretty.GET, MD_URL_RE, + body=_request_callback) + + success = self.ds.get_data() + self.assertTrue(success) + + @httpretty.activate + def test_metadata(self): + httpretty.register_uri( + httpretty.GET, MD_URL_RE, + body=_request_callback) + self.ds.get_data() + + self.assertEqual(DO_META.get('user-data'), + self.ds.get_userdata_raw()) + + self.assertEqual(DO_META.get('vendor-data'), + self.ds.get_vendordata_raw()) + + self.assertEqual([DO_META.get('public-keys')], + self.ds.get_public_ssh_keys()) + + self.assertEqual(DO_META.get('region'), + self.ds.availability_zone) + + self.assertEqual(DO_META.get('id'), + self.ds.get_instance_id()) + + self.assertEqual(DO_META.get('hostname'), + self.ds.get_hostname()) + + self.assertEqual('http://mirrors.digitalocean.com/', + self.ds.get_package_mirror_info()) -- cgit v1.2.3 From 33b54a3ac2560b192f29ce1fbe797fdd3cb968aa Mon Sep 17 00:00:00 2001 From: Neal Shrader Date: Thu, 16 Oct 2014 18:20:19 -0400 Subject: Make metadata timeout/retries configurable Defaulting to only trying once. --- cloudinit/sources/DataSourceDigitalOcean.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/cloudinit/sources/DataSourceDigitalOcean.py b/cloudinit/sources/DataSourceDigitalOcean.py index c580e2d5..985f1663 100644 --- a/cloudinit/sources/DataSourceDigitalOcean.py +++ b/cloudinit/sources/DataSourceDigitalOcean.py @@ -25,6 +25,8 @@ BUILTIN_DS_CONFIG = { 'metadata_url': 'http://169.254.169.254/metadata/v1', 'mirrors_url': 'http://mirrors.digitalocean.com/' } +MD_RETRIES = 0 +MD_TIMEOUT = 1 class DataSourceDigitalOcean(sources.DataSource): def __init__(self, sys_cfg, distro, paths): @@ -34,8 +36,16 @@ class DataSourceDigitalOcean(sources.DataSource): util.get_cfg_by_path(sys_cfg, ["datasource", "DigitalOcean"], {}), BUILTIN_DS_CONFIG]) self.metadata_address = self.ds_cfg['metadata_url'] - self.retries = 3 - self.timeout = 1 + + if self.ds_cfg.get('retries'): + self.retries = self.ds_cfg['retries'] + else: + self.retries = MD_RETRIES + + if self.ds_cfg.get('timeout'): + self.timeout = self.ds_cfg['timeout'] + else: + self.timeout = MD_TIMEOUT def get_data(self): url_map = [ -- cgit v1.2.3 From 01e8df0557098093a0e3444f41ba3f1861ded316 Mon Sep 17 00:00:00 2001 From: Neal Shrader Date: Thu, 16 Oct 2014 19:19:29 -0400 Subject: Use existing metadata crawler to populate datasource --- cloudinit/sources/DataSourceDigitalOcean.py | 53 +++++++++------------- .../unittests/test_datasource/test_digitalocean.py | 9 ++++ 2 files changed, 30 insertions(+), 32 deletions(-) diff --git a/cloudinit/sources/DataSourceDigitalOcean.py b/cloudinit/sources/DataSourceDigitalOcean.py index 985f1663..b7afca93 100644 --- a/cloudinit/sources/DataSourceDigitalOcean.py +++ b/cloudinit/sources/DataSourceDigitalOcean.py @@ -18,11 +18,14 @@ from cloudinit import log as logging from cloudinit import util from cloudinit import sources from cloudinit import url_helper +from cloudinit import ec2_utils +import functools + LOG = logging.getLogger(__name__) BUILTIN_DS_CONFIG = { - 'metadata_url': 'http://169.254.169.254/metadata/v1', + 'metadata_url': 'http://169.254.169.254/metadata/v1/', 'mirrors_url': 'http://mirrors.digitalocean.com/' } MD_RETRIES = 0 @@ -37,9 +40,9 @@ class DataSourceDigitalOcean(sources.DataSource): BUILTIN_DS_CONFIG]) self.metadata_address = self.ds_cfg['metadata_url'] - if self.ds_cfg.get('retries'): + if self.ds_cfg.get('retries'): self.retries = self.ds_cfg['retries'] - else: + else: self.retries = MD_RETRIES if self.ds_cfg.get('timeout'): @@ -48,41 +51,27 @@ class DataSourceDigitalOcean(sources.DataSource): self.timeout = MD_TIMEOUT def get_data(self): - url_map = [ - ('user-data', '/user-data'), - ('vendor-data', '/vendor-data'), - ('public-keys', '/public-keys'), - ('region', '/region'), - ('id', '/id'), - ('hostname', '/hostname'), - ] - - found = False - for (key, path) in url_map: - try: - resp = url_helper.readurl(url=self.metadata_address + path, - timeout=self.timeout, - retries=self.retries) - if resp.code == 200: - found = True - self.metadata[key] = resp.contents - else: - LOG.warn("Path: %s returned %s", path, resp.code) - return False - except url_helper.UrlError as e: - LOG.warn("Path: %s raised exception: %s", path, e) - return False - - return found + caller = functools.partial(util.read_file_or_url, timeout=self.timeout, + retries=self.retries) + md = ec2_utils.MetadataMaterializer(str(caller(self.metadata_address)), + base_url=self.metadata_address, + caller=caller) + + self.metadata = md.materialize() + + if self.metadata.get('id'): + return True + else: + return False def get_userdata_raw(self): - return self.metadata['user-data'] + return "\n".join(self.metadata['user-data']) def get_vendordata_raw(self): - return self.metadata['vendor-data'] + return "\n".join(self.metadata['vendor-data']) def get_public_ssh_keys(self): - return self.metadata['public-keys'].splitlines() + return self.metadata['public-keys'].splitlines() @property def availability_zone(self): diff --git a/tests/unittests/test_datasource/test_digitalocean.py b/tests/unittests/test_datasource/test_digitalocean.py index 9576e042..559a4f9f 100644 --- a/tests/unittests/test_datasource/test_digitalocean.py +++ b/tests/unittests/test_datasource/test_digitalocean.py @@ -26,7 +26,16 @@ from cloudinit.sources import DataSourceDigitalOcean from .. import helpers as test_helpers +# Abbreviated for the test +DO_INDEX = """id + hostname + user-data + vendor-data + public-keys + region""" + DO_META = { + '': DO_INDEX, 'user-data': '#!/bin/bash\necho "user-data"', 'vendor-data': '#!/bin/bash\necho "vendor-data"', 'public-keys': 'ssh-rsa AAAAB3NzaC1yc2EAAAA... neal@digitalocean.com', -- cgit v1.2.3 From 8eedba0d3b9851bb0101407c5a070b7a975efa04 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Fri, 17 Oct 2014 12:32:41 -0700 Subject: Expose uses_systemd as a distro function Without this change the tests are currently failing on rhel7 since a location where a hostname file is written no longer exists at that location when systemd is active. To avoid this allow the test to inspect if the distro has systemd enabled and avoid testing the file when systemd is being used so the test passes. We likely need to figure out a better way to test features that no longer exist as files but exist as commands with systemd in general. --- cloudinit/distros/rhel.py | 12 ++++++------ tests/unittests/test_handler/test_handler_set_hostname.py | 9 +++++---- 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/cloudinit/distros/rhel.py b/cloudinit/distros/rhel.py index e8abf111..1a269e08 100644 --- a/cloudinit/distros/rhel.py +++ b/cloudinit/distros/rhel.py @@ -98,7 +98,7 @@ class Distro(distros.Distro): rhel_util.update_sysconfig_file(self.network_conf_fn, net_cfg) return dev_names - def _dist_uses_systemd(self): + def uses_systemd(self): # Fedora 18 and RHEL 7 were the first adopters in their series (dist, vers) = util.system_info()['dist'][:2] major = (int)(vers.split('.')[0]) @@ -106,7 +106,7 @@ class Distro(distros.Distro): or (dist.startswith('Fedora') and major >= 18)) def apply_locale(self, locale, out_fn=None): - if self._dist_uses_systemd(): + if self.uses_systemd(): if not out_fn: out_fn = self.systemd_locale_conf_fn out_fn = self.systemd_locale_conf_fn @@ -119,7 +119,7 @@ class Distro(distros.Distro): rhel_util.update_sysconfig_file(out_fn, locale_cfg) def _write_hostname(self, hostname, out_fn): - if self._dist_uses_systemd(): + if self.uses_systemd(): util.subp(['hostnamectl', 'set-hostname', str(hostname)]) else: host_cfg = { @@ -135,14 +135,14 @@ class Distro(distros.Distro): return hostname def _read_system_hostname(self): - if self._dist_uses_systemd(): + if self.uses_systemd(): host_fn = self.systemd_hostname_conf_fn else: host_fn = self.hostname_conf_fn return (host_fn, self._read_hostname(host_fn)) def _read_hostname(self, filename, default=None): - if self._dist_uses_systemd(): + if self.uses_systemd(): (out, _err) = util.subp(['hostname']) if len(out): return out @@ -163,7 +163,7 @@ class Distro(distros.Distro): def set_timezone(self, tz): tz_file = self._find_tz_file(tz) - if self._dist_uses_systemd(): + if self.uses_systemd(): # Currently, timedatectl complains if invoked during startup # so for compatibility, create the link manually. util.del_file(self.tz_local_fn) diff --git a/tests/unittests/test_handler/test_handler_set_hostname.py b/tests/unittests/test_handler/test_handler_set_hostname.py index 03004ab9..e1530e30 100644 --- a/tests/unittests/test_handler/test_handler_set_hostname.py +++ b/tests/unittests/test_handler/test_handler_set_hostname.py @@ -37,10 +37,11 @@ class TestHostname(t_help.FilesystemMockingTestCase): self.patchUtils(self.tmp) cc_set_hostname.handle('cc_set_hostname', cfg, cc, LOG, []) - contents = util.load_file("/etc/sysconfig/network") - n_cfg = ConfigObj(StringIO(contents)) - self.assertEquals({'HOSTNAME': 'blah.blah.blah.yahoo.com'}, - dict(n_cfg)) + if not distro.uses_systemd(): + contents = util.load_file("/etc/sysconfig/network") + n_cfg = ConfigObj(StringIO(contents)) + self.assertEquals({'HOSTNAME': 'blah.blah.blah.yahoo.com'}, + dict(n_cfg)) def test_write_hostname_debian(self): cfg = { -- cgit v1.2.3 From fdef6a9f84c5720bbd37f3eb98c9b7c58913bbfd Mon Sep 17 00:00:00 2001 From: Neal Shrader Date: Fri, 17 Oct 2014 16:26:46 -0400 Subject: Correct handling of single/multiple ssh keys --- cloudinit/sources/DataSourceDigitalOcean.py | 6 ++++- .../unittests/test_datasource/test_digitalocean.py | 30 +++++++++++++++++++--- 2 files changed, 31 insertions(+), 5 deletions(-) diff --git a/cloudinit/sources/DataSourceDigitalOcean.py b/cloudinit/sources/DataSourceDigitalOcean.py index b7afca93..c59232ca 100644 --- a/cloudinit/sources/DataSourceDigitalOcean.py +++ b/cloudinit/sources/DataSourceDigitalOcean.py @@ -19,6 +19,7 @@ from cloudinit import util from cloudinit import sources from cloudinit import url_helper from cloudinit import ec2_utils +from types import * import functools @@ -71,7 +72,10 @@ class DataSourceDigitalOcean(sources.DataSource): return "\n".join(self.metadata['vendor-data']) def get_public_ssh_keys(self): - return self.metadata['public-keys'].splitlines() + if type(self.metadata['public-keys']) is StringType: + return [self.metadata['public-keys']] + else: + return self.metadata['public-keys'] @property def availability_zone(self): diff --git a/tests/unittests/test_datasource/test_digitalocean.py b/tests/unittests/test_datasource/test_digitalocean.py index 559a4f9f..0997cf38 100644 --- a/tests/unittests/test_datasource/test_digitalocean.py +++ b/tests/unittests/test_datasource/test_digitalocean.py @@ -18,6 +18,7 @@ import httpretty import re +from types import * from urlparse import urlparse from cloudinit import settings @@ -34,11 +35,15 @@ DO_INDEX = """id public-keys region""" +DO_MULTIPLE_KEYS = """ssh-rsa AAAAB3NzaC1yc2EAAAA... neal@digitalocean.com + ssh-rsa AAAAB3NzaC1yc2EAAAA... neal2@digitalocean.com""" +DO_SINGLE_KEY = "ssh-rsa AAAAB3NzaC1yc2EAAAA... neal@digitalocean.com" + DO_META = { '': DO_INDEX, 'user-data': '#!/bin/bash\necho "user-data"', 'vendor-data': '#!/bin/bash\necho "vendor-data"', - 'public-keys': 'ssh-rsa AAAAB3NzaC1yc2EAAAA... neal@digitalocean.com', + 'public-keys': DO_SINGLE_KEY, 'region': 'nyc3', 'id': '2000000', 'hostname': 'cloudinit-test', @@ -88,9 +93,6 @@ class TestDataSourceDigitalOcean(test_helpers.HttprettyTestCase): self.assertEqual(DO_META.get('vendor-data'), self.ds.get_vendordata_raw()) - self.assertEqual([DO_META.get('public-keys')], - self.ds.get_public_ssh_keys()) - self.assertEqual(DO_META.get('region'), self.ds.availability_zone) @@ -102,3 +104,23 @@ class TestDataSourceDigitalOcean(test_helpers.HttprettyTestCase): self.assertEqual('http://mirrors.digitalocean.com/', self.ds.get_package_mirror_info()) + + # Single key + self.assertEqual([DO_META.get('public-keys')], + self.ds.get_public_ssh_keys()) + + self.assertIs(type(self.ds.get_public_ssh_keys()), ListType) + + @httpretty.activate + def test_multiple_ssh_keys(self): + DO_META['public_keys'] = DO_MULTIPLE_KEYS + httpretty.register_uri( + httpretty.GET, MD_URL_RE, + body=_request_callback) + self.ds.get_data() + + # Multiple keys + self.assertEqual(DO_META.get('public-keys').splitlines(), + self.ds.get_public_ssh_keys()) + + self.assertIs(type(self.ds.get_public_ssh_keys()), ListType) -- cgit v1.2.3 From 87a75c52902d7550acb812a626fddd72a6b2036f Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Fri, 17 Oct 2014 13:27:58 -0700 Subject: Include the systemd config files --- packages/redhat/cloud-init.spec.in | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/packages/redhat/cloud-init.spec.in b/packages/redhat/cloud-init.spec.in index 0e9862d8..75dd4d22 100644 --- a/packages/redhat/cloud-init.spec.in +++ b/packages/redhat/cloud-init.spec.in @@ -92,6 +92,11 @@ mkdir -p \$RPM_BUILD_ROOT/%{_sysconfdir}/rsyslog.d cp -p tools/21-cloudinit.conf \ \$RPM_BUILD_ROOT/%{_sysconfdir}/rsyslog.d/21-cloudinit.conf +#if $systemd +mkdir -p \$RPM_BUILD_ROOT/%{_unitdir} +cp -p systemd/* \$RPM_BUILD_ROOT/%{_unitdir} +#end if + %clean rm -rf \$RPM_BUILD_ROOT -- cgit v1.2.3 From 8479f44090df3ec53357d7faee0c0e4ccd5127c4 Mon Sep 17 00:00:00 2001 From: Neal Shrader Date: Fri, 17 Oct 2014 16:28:09 -0400 Subject: Add documentation for DigitalOcean datasource --- doc/sources/digitalocean/README.rst | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 doc/sources/digitalocean/README.rst diff --git a/doc/sources/digitalocean/README.rst b/doc/sources/digitalocean/README.rst new file mode 100644 index 00000000..1bb89fe1 --- /dev/null +++ b/doc/sources/digitalocean/README.rst @@ -0,0 +1,21 @@ + The `DigitalOcean`_ datasource consumes the content served from DigitalOcean's `metadata service`_. This +metadata service serves information about the running droplet via HTTP over the link local address +169.254.169.254. The metadata API endpoints are fully described at +`https://developers.digitalocean.com/metadata/ `_. + +Configuration +~~~~~~~~~~~~~ + +DigitalOcean's datasource can be configured as follows: + + datasource: + DigitalOcean: + retries: 3 + timeout: 2 + +- *retries*: Determines the number of times to attempt to connect to the metadata service +- *timeout*: Determines the timeout in seconds to wait for a response from the metadata service + +.. _DigitalOcean: http://digitalocean.com/ +.. _metadata service: https://developers.digitalocean.com/metadata/ +.. _Full documentation: https://developers.digitalocean.com/metadata/ -- cgit v1.2.3 From 216d3bf22414ca731a1eac2098e5883d2dab06b1 Mon Sep 17 00:00:00 2001 From: Neal Shrader Date: Fri, 17 Oct 2014 19:00:01 -0400 Subject: Explicitly import only types being compared --- cloudinit/sources/DataSourceDigitalOcean.py | 2 +- tests/unittests/test_datasource/test_digitalocean.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cloudinit/sources/DataSourceDigitalOcean.py b/cloudinit/sources/DataSourceDigitalOcean.py index c59232ca..b25dcb27 100644 --- a/cloudinit/sources/DataSourceDigitalOcean.py +++ b/cloudinit/sources/DataSourceDigitalOcean.py @@ -19,7 +19,7 @@ from cloudinit import util from cloudinit import sources from cloudinit import url_helper from cloudinit import ec2_utils -from types import * +from types import StringType import functools diff --git a/tests/unittests/test_datasource/test_digitalocean.py b/tests/unittests/test_datasource/test_digitalocean.py index 0997cf38..04bee340 100644 --- a/tests/unittests/test_datasource/test_digitalocean.py +++ b/tests/unittests/test_datasource/test_digitalocean.py @@ -18,7 +18,7 @@ import httpretty import re -from types import * +from types import ListType from urlparse import urlparse from cloudinit import settings -- cgit v1.2.3 From 2d9d3811b4b1d4ea078a0bba6cf5e067339c14f3 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Sat, 18 Oct 2014 09:27:47 -0700 Subject: Pretty up the debug module Previously the usage of the yaml_dumps module was causing any python unicode key and value to show up as: 'item': !!python/unicode "some string" This was not very pretty... Fix this by using safe_dumps (which is also a good thing to use and allow_unicode=True). Also create a tiny helper function in the cc_debug module that does not include the yaml start and end footers (since this module has its own footers and headers). Also includes a basic sanity test for this module. --- cloudinit/config/cc_debug.py | 18 +++-- cloudinit/util.py | 16 ++--- tests/unittests/test_handler/test_handler_debug.py | 78 ++++++++++++++++++++++ 3 files changed, 99 insertions(+), 13 deletions(-) create mode 100644 tests/unittests/test_handler/test_handler_debug.py diff --git a/cloudinit/config/cc_debug.py b/cloudinit/config/cc_debug.py index 7219b0f8..2705035b 100644 --- a/cloudinit/config/cc_debug.py +++ b/cloudinit/config/cc_debug.py @@ -14,11 +14,14 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -from cloudinit import type_utils -from cloudinit import util import copy from StringIO import StringIO +from cloudinit import type_utils +from cloudinit import util + +SKIP_KEYS = frozenset(['log_cfgs']) + def _make_header(text): header = StringIO() @@ -31,6 +34,11 @@ def _make_header(text): return header.getvalue() +def _dumps(obj): + text = util.yaml_dumps(obj, explicit_start=False, explicit_end=False) + return text.rstrip() + + def handle(name, cfg, cloud, log, args): verbose = util.get_cfg_by_path(cfg, ('debug', 'verbose'), default=True) if args: @@ -46,7 +54,7 @@ def handle(name, cfg, cloud, log, args): return # Clean out some keys that we just don't care about showing... dump_cfg = copy.deepcopy(cfg) - for k in ['log_cfgs']: + for k in SKIP_KEYS: dump_cfg.pop(k, None) all_keys = list(dump_cfg.keys()) for k in all_keys: @@ -55,10 +63,10 @@ def handle(name, cfg, cloud, log, args): # Now dump it... to_print = StringIO() to_print.write(_make_header("Config")) - to_print.write(util.yaml_dumps(dump_cfg)) + to_print.write(_dumps(dump_cfg)) to_print.write("\n") to_print.write(_make_header("MetaData")) - to_print.write(util.yaml_dumps(cloud.datasource.metadata)) + to_print.write(_dumps(cloud.datasource.metadata)) to_print.write("\n") to_print.write(_make_header("Misc")) to_print.write("Datasource: %s\n" % diff --git a/cloudinit/util.py b/cloudinit/util.py index f236d0bf..13084374 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -1270,14 +1270,14 @@ def read_write_cmdline_url(target_fn): logexc(LOG, "Failed writing url content to %s", target_fn) -def yaml_dumps(obj): - formatted = yaml.dump(obj, - line_break="\n", - indent=4, - explicit_start=True, - explicit_end=True, - default_flow_style=False) - return formatted +def yaml_dumps(obj, explicit_start=True, explicit_end=True): + return yaml.safe_dump(obj, + line_break="\n", + indent=4, + explicit_start=explicit_start, + explicit_end=explicit_end, + default_flow_style=False, + allow_unicode=True) def ensure_dir(path, mode=None): diff --git a/tests/unittests/test_handler/test_handler_debug.py b/tests/unittests/test_handler/test_handler_debug.py new file mode 100644 index 00000000..bd9e29d8 --- /dev/null +++ b/tests/unittests/test_handler/test_handler_debug.py @@ -0,0 +1,78 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2014 Yahoo! Inc. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +from cloudinit.config import cc_debug + +from cloudinit import cloud +from cloudinit import distros +from cloudinit import helpers +from cloudinit import util + +from cloudinit.sources import DataSourceNone + +from .. import helpers as t_help + +import logging + +LOG = logging.getLogger(__name__) + + +class TestDebug(t_help.FilesystemMockingTestCase): + def setUp(self): + super(TestDebug, self).setUp() + self.new_root = self.makeDir(prefix="unittest_") + + def _get_cloud(self, distro, metadata=None): + self.patchUtils(self.new_root) + paths = helpers.Paths({}) + cls = distros.fetch(distro) + d = cls(distro, {}, paths) + ds = DataSourceNone.DataSourceNone({}, d, paths) + if metadata: + ds.metadata.update(metadata) + return cloud.Cloud(ds, paths, {}, d, None) + + def test_debug_write(self): + cfg = { + 'abc': '123', + 'c': u'\u20a0', + 'debug': { + 'verbose': True, + # Does not actually write here due to mocking... + 'output': '/var/log/cloud-init-debug.log', + }, + } + cc = self._get_cloud('ubuntu') + cc_debug.handle('cc_debug', cfg, cc, LOG, []) + contents = util.load_file('/var/log/cloud-init-debug.log') + # Some basic sanity tests... + self.assertGreater(len(contents), 0) + for k in cfg.keys(): + self.assertIn(k, contents) + + def test_debug_no_write(self): + cfg = { + 'abc': '123', + 'debug': { + 'verbose': False, + # Does not actually write here due to mocking... + 'output': '/var/log/cloud-init-debug.log', + }, + } + cc = self._get_cloud('ubuntu') + cc_debug.handle('cc_debug', cfg, cc, LOG, []) + self.assertRaises(IOError, + util.load_file, '/var/log/cloud-init-debug.log') -- cgit v1.2.3 From cff60d55fcdffd29bd15ad199f3693aeeb2f814b Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Mon, 20 Oct 2014 14:27:00 -0400 Subject: open 0.7.7 --- ChangeLog | 2 ++ cloudinit/version.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/ChangeLog b/ChangeLog index b655946b..c33a45b9 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,5 @@ +0.7.7: + - open 0.7.7 0.7.6: - open 0.7.6 - Enable vendordata on CloudSigma datasource (LP: #1303986) diff --git a/cloudinit/version.py b/cloudinit/version.py index edb651a9..3d1d1d23 100644 --- a/cloudinit/version.py +++ b/cloudinit/version.py @@ -20,7 +20,7 @@ from distutils import version as vr def version(): - return vr.StrictVersion("0.7.6") + return vr.StrictVersion("0.7.7") def version_string(): -- cgit v1.2.3 From e88f6ed4c46fcb1069fe899606a8b6d95411c13f Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Tue, 21 Oct 2014 11:55:16 -0700 Subject: Handle strings/text type for 'ssh_authorized_keys' Instead of only expected a list, tuple, or set type allow for a string type to be passed in, and add log message that occurs if some other type is used that can not be correctly processed. --- cloudinit/distros/__init__.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/cloudinit/distros/__init__.py b/cloudinit/distros/__init__.py index 2599d9f2..d30098eb 100644 --- a/cloudinit/distros/__init__.py +++ b/cloudinit/distros/__init__.py @@ -387,8 +387,17 @@ class Distro(object): # Import SSH keys if 'ssh_authorized_keys' in kwargs: - keys = set(kwargs['ssh_authorized_keys']) or [] - ssh_util.setup_user_keys(keys, name, options=None) + # Try to handle this in a smart manner. + keys = kwargs['ssh_authorized_keys'] + if isinstance(keys, (basestring, str)): + keys = [keys] + if not isinstance(keys, (tuple, list, set)): + util.multi_log("Invalid type detected for" + " 'ssh_authorized_keys', expected list, string" + " or set.") + else: + keys = set(keys) or [] + ssh_util.setup_user_keys(keys, name, options=None) return True -- cgit v1.2.3 From 6fb6cfdea6ec31a69e749ddb638051c39256e7f3 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Tue, 21 Oct 2014 12:00:53 -0700 Subject: Also allow a dict to be used When a dict is passed in for 'ssh_authorized_keys' just extract the keys from the values of the dict (and discard the keys). --- cloudinit/distros/__init__.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/cloudinit/distros/__init__.py b/cloudinit/distros/__init__.py index d30098eb..762529a6 100644 --- a/cloudinit/distros/__init__.py +++ b/cloudinit/distros/__init__.py @@ -391,10 +391,12 @@ class Distro(object): keys = kwargs['ssh_authorized_keys'] if isinstance(keys, (basestring, str)): keys = [keys] + if isinstance(keys, dict): + keys = list(keys.values()) if not isinstance(keys, (tuple, list, set)): util.multi_log("Invalid type detected for" " 'ssh_authorized_keys', expected list, string" - " or set.") + " , dict, or set.") else: keys = set(keys) or [] ssh_util.setup_user_keys(keys, name, options=None) -- cgit v1.2.3 From a6336edea275c409791807d16c1575ebd6895c9c Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Tue, 21 Oct 2014 12:08:50 -0700 Subject: Fix the word spacing --- cloudinit/distros/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cloudinit/distros/__init__.py b/cloudinit/distros/__init__.py index 762529a6..7b05226a 100644 --- a/cloudinit/distros/__init__.py +++ b/cloudinit/distros/__init__.py @@ -396,7 +396,7 @@ class Distro(object): if not isinstance(keys, (tuple, list, set)): util.multi_log("Invalid type detected for" " 'ssh_authorized_keys', expected list, string" - " , dict, or set.") + ", dict, or set.") else: keys = set(keys) or [] ssh_util.setup_user_keys(keys, name, options=None) -- cgit v1.2.3 From 477a5418d55d45ddad55fcaa16ab3ac53652fdb9 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Tue, 21 Oct 2014 12:23:09 -0700 Subject: Use LOG.warn and handle the None case as well --- cloudinit/distros/__init__.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/cloudinit/distros/__init__.py b/cloudinit/distros/__init__.py index 7b05226a..83c2eebf 100644 --- a/cloudinit/distros/__init__.py +++ b/cloudinit/distros/__init__.py @@ -393,13 +393,14 @@ class Distro(object): keys = [keys] if isinstance(keys, dict): keys = list(keys.values()) - if not isinstance(keys, (tuple, list, set)): - util.multi_log("Invalid type detected for" - " 'ssh_authorized_keys', expected list, string" - ", dict, or set.") - else: - keys = set(keys) or [] - ssh_util.setup_user_keys(keys, name, options=None) + if keys is not None: + if not isinstance(keys, (tuple, list, set)): + LOG.warn("Invalid type '%s' detected for" + " 'ssh_authorized_keys', expected list," + " string, dict, or set.", type(keys)) + else: + keys = set(keys) or [] + ssh_util.setup_user_keys(keys, name, options=None) return True -- cgit v1.2.3 From ba09a65a307651144a72a0a0e554895ac0440c45 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Thu, 23 Oct 2014 17:47:19 -0700 Subject: Add the beginnings of module documentation --- cloudinit/config/cc_debug.py | 15 ++ cloudinit/config/cc_ubuntu_init_switch.py | 29 ++- doc/rtd/conf.py | 2 + doc/rtd/topics/modules.rst | 340 +++++++++++++++++++++++++++++- 4 files changed, 368 insertions(+), 18 deletions(-) diff --git a/cloudinit/config/cc_debug.py b/cloudinit/config/cc_debug.py index 7219b0f8..89627012 100644 --- a/cloudinit/config/cc_debug.py +++ b/cloudinit/config/cc_debug.py @@ -14,6 +14,21 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . +""" +**Summary:** helper to debug cloud-init *internal* datastructures. + +**Description:** This module will enable for outputting various internal +information that cloud-init sources provide to either a file or to the output +console/log location that this cloud-init has been configured with when +running. + +It can be configured with the following option structure:: + + debug: + verbose: (defaulting to true) + output: (location to write output, defaulting to console + log) +""" + from cloudinit import type_utils from cloudinit import util import copy diff --git a/cloudinit/config/cc_ubuntu_init_switch.py b/cloudinit/config/cc_ubuntu_init_switch.py index 6f994bff..b23f70c7 100644 --- a/cloudinit/config/cc_ubuntu_init_switch.py +++ b/cloudinit/config/cc_ubuntu_init_switch.py @@ -17,30 +17,27 @@ # along with this program. If not, see . """ -ubuntu_init_switch: reboot system into another init +**Summary:** reboot system into another init. -This provides a way for the user to boot with systemd even if the -image is set to boot with upstart. It should be run as one of the first -cloud_init_modules, and will switch the init system and then issue a reboot. -The next boot will come up in the target init system and no action will +**Description:** This module provides a way for the user to boot with systemd +even if the image is set to boot with upstart. It should be run as one of the +first cloud_init_modules, and will switch the init system and then issue a +reboot. The next boot will come up in the target init system and no action will be taken. This should be inert on non-ubuntu systems, and also exit quickly. -config is comes under the top level 'init_switch' dictionary. +It can be configured with the following option structure:: -#cloud-config -init_switch: - target: systemd - reboot: true + init_switch: + target: systemd (can be 'systemd' or 'upstart') + reboot: true (reboot if a change was made, or false to not reboot) -'target' can be 'systemd' or 'upstart'. Best effort is made, but its possible -this system will break, and probably won't interact well with any other -mechanism you've used to switch the init system. +.. note:: -'reboot': [default=true]. - true: reboot if a change was made. - false: do not reboot. + Best effort is made, but it's possible + this system will break, and probably won't interact well with any other + mechanism you've used to switch the init system. """ from cloudinit.settings import PER_INSTANCE diff --git a/doc/rtd/conf.py b/doc/rtd/conf.py index 52a8f92b..9be02766 100644 --- a/doc/rtd/conf.py +++ b/doc/rtd/conf.py @@ -27,6 +27,8 @@ project = 'Cloud-Init' # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom ones. extensions = [ 'sphinx.ext.intersphinx', + 'sphinx.ext.autodoc', + 'sphinx.ext.viewcode', ] intersphinx_mapping = { diff --git a/doc/rtd/topics/modules.rst b/doc/rtd/topics/modules.rst index d4dd55df..d34c2969 100644 --- a/doc/rtd/topics/modules.rst +++ b/doc/rtd/topics/modules.rst @@ -1,3 +1,339 @@ -========= +======= Modules -========= +======= + +Apt Configure +------------- + +*Internal name:* ``cc_apt_configure`` + +.. automodule:: cloudinit.config.cc_apt_configure + +Apt Pipelining +-------------- + +*Internal name:* ``cc_apt_pipelining`` + +.. automodule:: cloudinit.config.cc_apt_pipelining + +Bootcmd +------- + +*Internal name:* ``cc_bootcmd`` + +.. automodule:: cloudinit.config.cc_bootcmd + +Byobu +----- + +*Internal name:* ``cc_byobu`` + +.. automodule:: cloudinit.config.cc_byobu + +Ca Certs +-------- + +*Internal name:* ``cc_ca_certs`` + +.. automodule:: cloudinit.config.cc_ca_certs + +Chef +---- + +*Internal name:* ``cc_chef`` + +.. automodule:: cloudinit.config.cc_chef + +Debug +----- + +*Internal name:* ``cc_debug`` + +.. automodule:: cloudinit.config.cc_debug + +Disable Ec2 Metadata +-------------------- + +*Internal name:* ``cc_disable_ec2_metadata`` + +.. automodule:: cloudinit.config.cc_disable_ec2_metadata + +Disk Setup +---------- + +*Internal name:* ``cc_disk_setup`` + +.. automodule:: cloudinit.config.cc_disk_setup + +Emit Upstart +------------ + +*Internal name:* ``cc_emit_upstart`` + +.. automodule:: cloudinit.config.cc_emit_upstart + +Final Message +------------- + +*Internal name:* ``cc_final_message`` + +.. automodule:: cloudinit.config.cc_final_message + +Foo +--- + +*Internal name:* ``cc_foo`` + +.. automodule:: cloudinit.config.cc_foo + +Growpart +-------- + +*Internal name:* ``cc_growpart`` + +.. automodule:: cloudinit.config.cc_growpart + +Grub Dpkg +--------- + +*Internal name:* ``cc_grub_dpkg`` + +.. automodule:: cloudinit.config.cc_grub_dpkg + +Keys To Console +--------------- + +*Internal name:* ``cc_keys_to_console`` + +.. automodule:: cloudinit.config.cc_keys_to_console + +Landscape +--------- + +*Internal name:* ``cc_landscape`` + +.. automodule:: cloudinit.config.cc_landscape + +Locale +------ + +*Internal name:* ``cc_locale`` + +.. automodule:: cloudinit.config.cc_locale + +Mcollective +----------- + +*Internal name:* ``cc_mcollective`` + +.. automodule:: cloudinit.config.cc_mcollective + +Migrator +-------- + +*Internal name:* ``cc_migrator`` + +.. automodule:: cloudinit.config.cc_migrator + +Mounts +------ + +*Internal name:* ``cc_mounts`` + +.. automodule:: cloudinit.config.cc_mounts + +Package Update Upgrade Install +------------------------------ + +*Internal name:* ``cc_package_update_upgrade_install`` + +.. automodule:: cloudinit.config.cc_package_update_upgrade_install + +Phone Home +---------- + +*Internal name:* ``cc_phone_home`` + +.. automodule:: cloudinit.config.cc_phone_home + +Power State Change +------------------ + +*Internal name:* ``cc_power_state_change`` + +.. automodule:: cloudinit.config.cc_power_state_change + +Puppet +------ + +*Internal name:* ``cc_puppet`` + +.. automodule:: cloudinit.config.cc_puppet + +Resizefs +-------- + +*Internal name:* ``cc_resizefs`` + +.. automodule:: cloudinit.config.cc_resizefs + +Resolv Conf +----------- + +*Internal name:* ``cc_resolv_conf`` + +.. automodule:: cloudinit.config.cc_resolv_conf + +Rightscale Userdata +------------------- + +*Internal name:* ``cc_rightscale_userdata`` + +.. automodule:: cloudinit.config.cc_rightscale_userdata + +Rsyslog +------- + +*Internal name:* ``cc_rsyslog`` + +.. automodule:: cloudinit.config.cc_rsyslog + +Runcmd +------ + +*Internal name:* ``cc_runcmd`` + +.. automodule:: cloudinit.config.cc_runcmd + +Salt Minion +----------- + +*Internal name:* ``cc_salt_minion`` + +.. automodule:: cloudinit.config.cc_salt_minion + +Scripts Per Boot +---------------- + +*Internal name:* ``cc_scripts_per_boot`` + +.. automodule:: cloudinit.config.cc_scripts_per_boot + +Scripts Per Instance +-------------------- + +*Internal name:* ``cc_scripts_per_instance`` + +.. automodule:: cloudinit.config.cc_scripts_per_instance + +Scripts Per Once +---------------- + +*Internal name:* ``cc_scripts_per_once`` + +.. automodule:: cloudinit.config.cc_scripts_per_once + +Scripts User +------------ + +*Internal name:* ``cc_scripts_user`` + +.. automodule:: cloudinit.config.cc_scripts_user + +Scripts Vendor +-------------- + +*Internal name:* ``cc_scripts_vendor`` + +.. automodule:: cloudinit.config.cc_scripts_vendor + +Seed Random +----------- + +*Internal name:* ``cc_seed_random`` + +.. automodule:: cloudinit.config.cc_seed_random + +Set Hostname +------------ + +*Internal name:* ``cc_set_hostname`` + +.. automodule:: cloudinit.config.cc_set_hostname + +Set Passwords +------------- + +*Internal name:* ``cc_set_passwords`` + +.. automodule:: cloudinit.config.cc_set_passwords + +Ssh +--- + +*Internal name:* ``cc_ssh`` + +.. automodule:: cloudinit.config.cc_ssh + +Ssh Authkey Fingerprints +------------------------ + +*Internal name:* ``cc_ssh_authkey_fingerprints`` + +.. automodule:: cloudinit.config.cc_ssh_authkey_fingerprints + +Ssh Import Id +------------- + +*Internal name:* ``cc_ssh_import_id`` + +.. automodule:: cloudinit.config.cc_ssh_import_id + +Timezone +-------- + +*Internal name:* ``cc_timezone`` + +.. automodule:: cloudinit.config.cc_timezone + +Ubuntu Init Switch +------------------ + +*Internal name:* ``cc_ubuntu_init_switch`` + +.. automodule:: cloudinit.config.cc_ubuntu_init_switch + +Update Etc Hosts +---------------- + +*Internal name:* ``cc_update_etc_hosts`` + +.. automodule:: cloudinit.config.cc_update_etc_hosts + +Update Hostname +--------------- + +*Internal name:* ``cc_update_hostname`` + +.. automodule:: cloudinit.config.cc_update_hostname + +Users Groups +------------ + +*Internal name:* ``cc_users_groups`` + +.. automodule:: cloudinit.config.cc_users_groups + +Write Files +----------- + +*Internal name:* ``cc_write_files`` + +.. automodule:: cloudinit.config.cc_write_files + +Yum Add Repo +------------ + +*Internal name:* ``cc_yum_add_repo`` + +.. automodule:: cloudinit.config.cc_yum_add_repo -- cgit v1.2.3 From 17f180790f4a8b7f818b76abf34cc6b2ba61f0af Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Thu, 23 Oct 2014 17:51:47 -0700 Subject: Not that log_cfgs are not shown. --- cloudinit/config/cc_debug.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/cloudinit/config/cc_debug.py b/cloudinit/config/cc_debug.py index 89627012..39d85fdc 100644 --- a/cloudinit/config/cc_debug.py +++ b/cloudinit/config/cc_debug.py @@ -27,6 +27,10 @@ It can be configured with the following option structure:: debug: verbose: (defaulting to true) output: (location to write output, defaulting to console + log) + +.. note:: + + Log configurations are not output. """ from cloudinit import type_utils -- cgit v1.2.3 From 69f4a9c5d73be23d00be0fefb41788e6744112e7 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Thu, 23 Oct 2014 17:57:10 -0700 Subject: Add a link to the module code --- cloudinit/config/cc_debug.py | 2 ++ doc/rtd/topics/modules.rst | 1 + 2 files changed, 3 insertions(+) diff --git a/cloudinit/config/cc_debug.py b/cloudinit/config/cc_debug.py index 39d85fdc..a3af4500 100644 --- a/cloudinit/config/cc_debug.py +++ b/cloudinit/config/cc_debug.py @@ -51,6 +51,8 @@ def _make_header(text): def handle(name, cfg, cloud, log, args): + """Handler method activated by cloud-init.""" + verbose = util.get_cfg_by_path(cfg, ('debug', 'verbose'), default=True) if args: # if args are provided (from cmdline) then explicitly set verbose diff --git a/doc/rtd/topics/modules.rst b/doc/rtd/topics/modules.rst index d34c2969..c9a2e60c 100644 --- a/doc/rtd/topics/modules.rst +++ b/doc/rtd/topics/modules.rst @@ -50,6 +50,7 @@ Debug *Internal name:* ``cc_debug`` .. automodule:: cloudinit.config.cc_debug + :members: Disable Ec2 Metadata -------------------- -- cgit v1.2.3 From 10b4e49734526949da06426f9b102086aaaf7589 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Thu, 23 Oct 2014 17:59:02 -0700 Subject: Use bold for the internal name --- doc/rtd/topics/modules.rst | 96 +++++++++++++++++++++++----------------------- 1 file changed, 48 insertions(+), 48 deletions(-) diff --git a/doc/rtd/topics/modules.rst b/doc/rtd/topics/modules.rst index c9a2e60c..8cb786f4 100644 --- a/doc/rtd/topics/modules.rst +++ b/doc/rtd/topics/modules.rst @@ -5,49 +5,49 @@ Modules Apt Configure ------------- -*Internal name:* ``cc_apt_configure`` +**Internal name:** ``cc_apt_configure`` .. automodule:: cloudinit.config.cc_apt_configure Apt Pipelining -------------- -*Internal name:* ``cc_apt_pipelining`` +**Internal name:** ``cc_apt_pipelining`` .. automodule:: cloudinit.config.cc_apt_pipelining Bootcmd ------- -*Internal name:* ``cc_bootcmd`` +**Internal name:** ``cc_bootcmd`` .. automodule:: cloudinit.config.cc_bootcmd Byobu ----- -*Internal name:* ``cc_byobu`` +**Internal name:** ``cc_byobu`` .. automodule:: cloudinit.config.cc_byobu Ca Certs -------- -*Internal name:* ``cc_ca_certs`` +**Internal name:** ``cc_ca_certs`` .. automodule:: cloudinit.config.cc_ca_certs Chef ---- -*Internal name:* ``cc_chef`` +**Internal name:** ``cc_chef`` .. automodule:: cloudinit.config.cc_chef Debug ----- -*Internal name:* ``cc_debug`` +**Internal name:** ``cc_debug`` .. automodule:: cloudinit.config.cc_debug :members: @@ -55,286 +55,286 @@ Debug Disable Ec2 Metadata -------------------- -*Internal name:* ``cc_disable_ec2_metadata`` +**Internal name:** ``cc_disable_ec2_metadata`` .. automodule:: cloudinit.config.cc_disable_ec2_metadata Disk Setup ---------- -*Internal name:* ``cc_disk_setup`` +**Internal name:** ``cc_disk_setup`` .. automodule:: cloudinit.config.cc_disk_setup Emit Upstart ------------ -*Internal name:* ``cc_emit_upstart`` +**Internal name:** ``cc_emit_upstart`` .. automodule:: cloudinit.config.cc_emit_upstart Final Message ------------- -*Internal name:* ``cc_final_message`` +**Internal name:** ``cc_final_message`` .. automodule:: cloudinit.config.cc_final_message Foo --- -*Internal name:* ``cc_foo`` +**Internal name:** ``cc_foo`` .. automodule:: cloudinit.config.cc_foo Growpart -------- -*Internal name:* ``cc_growpart`` +**Internal name:** ``cc_growpart`` .. automodule:: cloudinit.config.cc_growpart Grub Dpkg --------- -*Internal name:* ``cc_grub_dpkg`` +**Internal name:** ``cc_grub_dpkg`` .. automodule:: cloudinit.config.cc_grub_dpkg Keys To Console --------------- -*Internal name:* ``cc_keys_to_console`` +**Internal name:** ``cc_keys_to_console`` .. automodule:: cloudinit.config.cc_keys_to_console Landscape --------- -*Internal name:* ``cc_landscape`` +**Internal name:** ``cc_landscape`` .. automodule:: cloudinit.config.cc_landscape Locale ------ -*Internal name:* ``cc_locale`` +**Internal name:** ``cc_locale`` .. automodule:: cloudinit.config.cc_locale Mcollective ----------- -*Internal name:* ``cc_mcollective`` +**Internal name:** ``cc_mcollective`` .. automodule:: cloudinit.config.cc_mcollective Migrator -------- -*Internal name:* ``cc_migrator`` +**Internal name:** ``cc_migrator`` .. automodule:: cloudinit.config.cc_migrator Mounts ------ -*Internal name:* ``cc_mounts`` +**Internal name:** ``cc_mounts`` .. automodule:: cloudinit.config.cc_mounts Package Update Upgrade Install ------------------------------ -*Internal name:* ``cc_package_update_upgrade_install`` +**Internal name:** ``cc_package_update_upgrade_install`` .. automodule:: cloudinit.config.cc_package_update_upgrade_install Phone Home ---------- -*Internal name:* ``cc_phone_home`` +**Internal name:** ``cc_phone_home`` .. automodule:: cloudinit.config.cc_phone_home Power State Change ------------------ -*Internal name:* ``cc_power_state_change`` +**Internal name:** ``cc_power_state_change`` .. automodule:: cloudinit.config.cc_power_state_change Puppet ------ -*Internal name:* ``cc_puppet`` +**Internal name:** ``cc_puppet`` .. automodule:: cloudinit.config.cc_puppet Resizefs -------- -*Internal name:* ``cc_resizefs`` +**Internal name:** ``cc_resizefs`` .. automodule:: cloudinit.config.cc_resizefs Resolv Conf ----------- -*Internal name:* ``cc_resolv_conf`` +**Internal name:** ``cc_resolv_conf`` .. automodule:: cloudinit.config.cc_resolv_conf Rightscale Userdata ------------------- -*Internal name:* ``cc_rightscale_userdata`` +**Internal name:** ``cc_rightscale_userdata`` .. automodule:: cloudinit.config.cc_rightscale_userdata Rsyslog ------- -*Internal name:* ``cc_rsyslog`` +**Internal name:** ``cc_rsyslog`` .. automodule:: cloudinit.config.cc_rsyslog Runcmd ------ -*Internal name:* ``cc_runcmd`` +**Internal name:** ``cc_runcmd`` .. automodule:: cloudinit.config.cc_runcmd Salt Minion ----------- -*Internal name:* ``cc_salt_minion`` +**Internal name:** ``cc_salt_minion`` .. automodule:: cloudinit.config.cc_salt_minion Scripts Per Boot ---------------- -*Internal name:* ``cc_scripts_per_boot`` +**Internal name:** ``cc_scripts_per_boot`` .. automodule:: cloudinit.config.cc_scripts_per_boot Scripts Per Instance -------------------- -*Internal name:* ``cc_scripts_per_instance`` +**Internal name:** ``cc_scripts_per_instance`` .. automodule:: cloudinit.config.cc_scripts_per_instance Scripts Per Once ---------------- -*Internal name:* ``cc_scripts_per_once`` +**Internal name:** ``cc_scripts_per_once`` .. automodule:: cloudinit.config.cc_scripts_per_once Scripts User ------------ -*Internal name:* ``cc_scripts_user`` +**Internal name:** ``cc_scripts_user`` .. automodule:: cloudinit.config.cc_scripts_user Scripts Vendor -------------- -*Internal name:* ``cc_scripts_vendor`` +**Internal name:** ``cc_scripts_vendor`` .. automodule:: cloudinit.config.cc_scripts_vendor Seed Random ----------- -*Internal name:* ``cc_seed_random`` +**Internal name:** ``cc_seed_random`` .. automodule:: cloudinit.config.cc_seed_random Set Hostname ------------ -*Internal name:* ``cc_set_hostname`` +**Internal name:** ``cc_set_hostname`` .. automodule:: cloudinit.config.cc_set_hostname Set Passwords ------------- -*Internal name:* ``cc_set_passwords`` +**Internal name:** ``cc_set_passwords`` .. automodule:: cloudinit.config.cc_set_passwords Ssh --- -*Internal name:* ``cc_ssh`` +**Internal name:** ``cc_ssh`` .. automodule:: cloudinit.config.cc_ssh Ssh Authkey Fingerprints ------------------------ -*Internal name:* ``cc_ssh_authkey_fingerprints`` +**Internal name:** ``cc_ssh_authkey_fingerprints`` .. automodule:: cloudinit.config.cc_ssh_authkey_fingerprints Ssh Import Id ------------- -*Internal name:* ``cc_ssh_import_id`` +**Internal name:** ``cc_ssh_import_id`` .. automodule:: cloudinit.config.cc_ssh_import_id Timezone -------- -*Internal name:* ``cc_timezone`` +**Internal name:** ``cc_timezone`` .. automodule:: cloudinit.config.cc_timezone Ubuntu Init Switch ------------------ -*Internal name:* ``cc_ubuntu_init_switch`` +**Internal name:** ``cc_ubuntu_init_switch`` .. automodule:: cloudinit.config.cc_ubuntu_init_switch Update Etc Hosts ---------------- -*Internal name:* ``cc_update_etc_hosts`` +**Internal name:** ``cc_update_etc_hosts`` .. automodule:: cloudinit.config.cc_update_etc_hosts Update Hostname --------------- -*Internal name:* ``cc_update_hostname`` +**Internal name:** ``cc_update_hostname`` .. automodule:: cloudinit.config.cc_update_hostname Users Groups ------------ -*Internal name:* ``cc_users_groups`` +**Internal name:** ``cc_users_groups`` .. automodule:: cloudinit.config.cc_users_groups Write Files ----------- -*Internal name:* ``cc_write_files`` +**Internal name:** ``cc_write_files`` .. automodule:: cloudinit.config.cc_write_files Yum Add Repo ------------ -*Internal name:* ``cc_yum_add_repo`` +**Internal name:** ``cc_yum_add_repo`` .. automodule:: cloudinit.config.cc_yum_add_repo -- cgit v1.2.3 From fe9ff4691afe50a4c9192b96a872fec319e2606b Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Thu, 23 Oct 2014 18:01:06 -0700 Subject: Show the handle method for the cc_ubuntu_init_switch module --- cloudinit/config/cc_ubuntu_init_switch.py | 1 + doc/rtd/topics/modules.rst | 1 + 2 files changed, 2 insertions(+) diff --git a/cloudinit/config/cc_ubuntu_init_switch.py b/cloudinit/config/cc_ubuntu_init_switch.py index b23f70c7..12b6d6d5 100644 --- a/cloudinit/config/cc_ubuntu_init_switch.py +++ b/cloudinit/config/cc_ubuntu_init_switch.py @@ -88,6 +88,7 @@ fi def handle(name, cfg, cloud, log, args): + """Handler method activated by cloud-init.""" if not isinstance(cloud.distro, ubuntu.Distro): log.debug("%s: distro is '%s', not ubuntu. returning", diff --git a/doc/rtd/topics/modules.rst b/doc/rtd/topics/modules.rst index 8cb786f4..b50acbfa 100644 --- a/doc/rtd/topics/modules.rst +++ b/doc/rtd/topics/modules.rst @@ -303,6 +303,7 @@ Ubuntu Init Switch **Internal name:** ``cc_ubuntu_init_switch`` .. automodule:: cloudinit.config.cc_ubuntu_init_switch + :members: Update Etc Hosts ---------------- -- cgit v1.2.3 From 24e6ef998899418d73a63a64141fbaa209320967 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Thu, 23 Oct 2014 18:02:37 -0700 Subject: Make the cloud_init_modules look like a heredoc/inline heredoc --- cloudinit/config/cc_ubuntu_init_switch.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cloudinit/config/cc_ubuntu_init_switch.py b/cloudinit/config/cc_ubuntu_init_switch.py index 12b6d6d5..7e88ed85 100644 --- a/cloudinit/config/cc_ubuntu_init_switch.py +++ b/cloudinit/config/cc_ubuntu_init_switch.py @@ -21,7 +21,7 @@ **Description:** This module provides a way for the user to boot with systemd even if the image is set to boot with upstart. It should be run as one of the -first cloud_init_modules, and will switch the init system and then issue a +first ``cloud_init_modules``, and will switch the init system and then issue a reboot. The next boot will come up in the target init system and no action will be taken. -- cgit v1.2.3 From d3efeef731470b1f840ab57c7fabc38954799a15 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Wed, 29 Oct 2014 15:39:04 -0400 Subject: fix bad logic resulting in failure to honor 'output' config. This busted logic causes 'output' to not be paid any attention to, and thus output is not written to /var/log/cloud-init-output.log. LP: #1387340 --- cloudinit/util.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cloudinit/util.py b/cloudinit/util.py index f236d0bf..4bb73c11 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -1146,7 +1146,7 @@ def chownbyname(fname, user=None, group=None): # this returns the specific 'mode' entry, cleanly formatted, with value def get_output_cfg(cfg, mode): ret = [None, None] - if cfg or 'output' not in cfg: + if not cfg or 'output' not in cfg: return ret outcfg = cfg['output'] -- cgit v1.2.3 From c634d32173fb09ff33896f3b03e495fada80dd33 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Wed, 29 Oct 2014 19:35:24 -0400 Subject: ChangeLog: update for last commit --- ChangeLog | 1 + 1 file changed, 1 insertion(+) diff --git a/ChangeLog b/ChangeLog index 63bb50ca..5e6f6521 100644 --- a/ChangeLog +++ b/ChangeLog @@ -2,6 +2,7 @@ - open 0.7.7 - Digital Ocean: add datasource for Digital Ocean. [Neal Shrader] - expose uses_systemd as a distro function (fix rhel7) + - fix broken 'output' config (LP: #1387340) 0.7.6: - open 0.7.6 - Enable vendordata on CloudSigma datasource (LP: #1303986) -- cgit v1.2.3 From ce80e672caa299afe786a852afa986abfa28e633 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Wed, 29 Oct 2014 17:12:31 -0700 Subject: Only use datafiles and initsys addon outside virtualenvs To make it so that cloud-init is installable in a virtualenv where it can be tested in an isolated scenario we need to avoid using and including datafiles (which won't be written into the virtualenv) and also avoid using our initsys helper class which also adds on its own files when we are being ran from a virtualenv. --- setup.py | 55 +++++++++++++++++++++++++++++++++++++------------------ 1 file changed, 37 insertions(+), 18 deletions(-) diff --git a/setup.py b/setup.py index bd41bc91..3e188089 100755 --- a/setup.py +++ b/setup.py @@ -23,6 +23,7 @@ from glob import glob import os +import sys import setuptools from setuptools.command.install import install @@ -86,6 +87,17 @@ if os.uname()[0] == 'FreeBSD': ETC = "/usr/local/etc" +# Avoid having datafiles installed in a virtualenv... +def in_virtualenv(): + try: + if sys.real_prefix == sys.prefix: + return False + else: + return True + except AttributeError: + return False + + def get_version(): cmd = ['tools/read-version'] (ver, _e) = tiny_p(cmd) @@ -135,6 +147,29 @@ class InitsysInstallData(install): self.distribution.reinitialize_command('install_data', True) +if in_virtualenv(): + data_files = [] + cmdclass = {} +else: + data_files = [ + (ETC + '/cloud', glob('config/*.cfg')), + (ETC + '/cloud/cloud.cfg.d', glob('config/cloud.cfg.d/*')), + (ETC + '/cloud/templates', glob('templates/*')), + (USR + '/lib/cloud-init', ['tools/uncloud-init', + 'tools/write-ssh-key-fingerprints']), + (USR + '/share/doc/cloud-init', [f for f in glob('doc/*') if is_f(f)]), + (USR + '/share/doc/cloud-init/examples', + [f for f in glob('doc/examples/*') if is_f(f)]), + (USR + '/share/doc/cloud-init/examples/seed', + [f for f in glob('doc/examples/seed/*') if is_f(f)]), + ] + # Use a subclass for install that handles + # adding on the right init system configuration files + cmdclass = { + 'install': InitsysInstallData, + } + + setuptools.setup(name='cloud-init', version=get_version(), description='EC2 initialisation magic', @@ -146,23 +181,7 @@ setuptools.setup(name='cloud-init', 'tools/cloud-init-per', ], license='GPLv3', - data_files=[(ETC + '/cloud', glob('config/*.cfg')), - (ETC + '/cloud/cloud.cfg.d', glob('config/cloud.cfg.d/*')), - (ETC + '/cloud/templates', glob('templates/*')), - (USR + '/lib/cloud-init', - ['tools/uncloud-init', - 'tools/write-ssh-key-fingerprints']), - (USR + '/share/doc/cloud-init', - [f for f in glob('doc/*') if is_f(f)]), - (USR + '/share/doc/cloud-init/examples', - [f for f in glob('doc/examples/*') if is_f(f)]), - (USR + '/share/doc/cloud-init/examples/seed', - [f for f in glob('doc/examples/seed/*') if is_f(f)]), - ], + data_files=data_files, install_requires=read_requires(), - cmdclass={ - # Use a subclass for install that handles - # adding on the right init system configuration files - 'install': InitsysInstallData, - }, + cmdclass=cmdclass, ) -- cgit v1.2.3 From 7a24608072fb47ee9e40d8127b1f0f552bae0818 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Thu, 30 Oct 2014 13:07:41 -0700 Subject: Fix the digital ocean test on py2.6 The digital ocean datasource test is using assertIs which is only created/existent on py2.7, so for the older py2.6 we need to add similar logic so that the test works correctly there. --- tests/unittests/helpers.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/unittests/helpers.py b/tests/unittests/helpers.py index 9700a4ca..52305397 100644 --- a/tests/unittests/helpers.py +++ b/tests/unittests/helpers.py @@ -35,6 +35,11 @@ else: if PY26: # For now add these on, taken from python 2.7 + slightly adjusted class TestCase(unittest.TestCase): + def assertIs(self, expr1, expr2, msg=None): + if expr1 is not expr2: + standardMsg = '%r is not %r' % (expr1, expr2) + self.fail(self._formatMessage(msg, standardMsg)) + def assertIn(self, member, container, msg=None): if member not in container: standardMsg = '%r not found in %r' % (member, container) -- cgit v1.2.3 From bd462bc68506d7da4d7e04b05e947e0cf3f8e19d Mon Sep 17 00:00:00 2001 From: Daniel Watkins Date: Wed, 12 Nov 2014 13:52:28 +0000 Subject: Add tests for current parse_ssh_config behaviour. This also adds mock as a test dependency, as we are looking to migrate away from mocker. --- test-requirements.txt | 1 + tests/unittests/test_sshutil.py | 58 ++++++++++++++++++++++++++++++++++++++++- 2 files changed, 58 insertions(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index 2edb8066..230f0404 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -1,4 +1,5 @@ httpretty>=0.7.1 +mock mocker nose pep8==1.5.7 diff --git a/tests/unittests/test_sshutil.py b/tests/unittests/test_sshutil.py index d8662cac..2a496418 100644 --- a/tests/unittests/test_sshutil.py +++ b/tests/unittests/test_sshutil.py @@ -1,6 +1,9 @@ -from cloudinit import ssh_util from unittest import TestCase +from mock import patch + +from cloudinit import ssh_util + VALID_CONTENT = { 'dsa': ( @@ -98,4 +101,57 @@ class TestAuthKeyLineParser(TestCase): self.assertFalse(key.valid()) +class TestParseSSHConfig(TestCase): + + def setUp(self): + self.load_file_patch = patch('cloudinit.ssh_util.util.load_file') + self.load_file = self.load_file_patch.start() + self.isfile_patch = patch('cloudinit.ssh_util.os.path.isfile') + self.isfile = self.isfile_patch.start() + self.isfile.return_value = True + + def tearDown(self): + self.load_file_patch.stop() + self.isfile_patch.stop() + + def test_not_a_file(self): + self.isfile.return_value = False + self.load_file.side_effect = IOError + ret = ssh_util.parse_ssh_config('not a real file') + self.assertEqual([], ret) + + def test_empty_file(self): + self.load_file.return_value = '' + ret = ssh_util.parse_ssh_config('some real file') + self.assertEqual([], ret) + + def test_comment_line(self): + comment_line = '# This is a comment' + self.load_file.return_value = comment_line + ret = ssh_util.parse_ssh_config('some real file') + self.assertEqual(1, len(ret)) + self.assertEqual(comment_line, ret[0].line) + + def test_blank_lines(self): + lines = ['', '\t', ' '] + self.load_file.return_value = '\n'.join(lines) + ret = ssh_util.parse_ssh_config('some real file') + self.assertEqual(len(lines), len(ret)) + for line in ret: + self.assertEqual('', line.line) + + def test_lower_case_config(self): + self.load_file.return_value = 'foo bar' + ret = ssh_util.parse_ssh_config('some real file') + self.assertEqual(1, len(ret)) + self.assertEqual('foo', ret[0].key) + self.assertEqual('bar', ret[0].value) + + def test_upper_case_config(self): + self.load_file.return_value = 'Foo Bar' + ret = ssh_util.parse_ssh_config('some real file') + self.assertEqual(1, len(ret)) + self.assertEqual('foo', ret[0].key) + self.assertEqual('Bar', ret[0].value) + # vi: ts=4 expandtab -- cgit v1.2.3 From 4ecca95e5973707f08fefc43448fec9e0f984966 Mon Sep 17 00:00:00 2001 From: Daniel Watkins Date: Wed, 12 Nov 2014 13:52:28 +0000 Subject: Handle = used as config delimiter in SSH config. --- cloudinit/ssh_util.py | 5 ++++- tests/unittests/test_sshutil.py | 14 ++++++++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/cloudinit/ssh_util.py b/cloudinit/ssh_util.py index 70a577bc..14d0cb0f 100644 --- a/cloudinit/ssh_util.py +++ b/cloudinit/ssh_util.py @@ -293,7 +293,10 @@ def parse_ssh_config(fname): if not line or line.startswith("#"): lines.append(SshdConfigLine(line)) continue - (key, val) = line.split(None, 1) + try: + key, val = line.split(None, 1) + except ValueError: + key, val = line.split('=', 1) lines.append(SshdConfigLine(line, key, val)) return lines diff --git a/tests/unittests/test_sshutil.py b/tests/unittests/test_sshutil.py index 2a496418..cd576e8f 100644 --- a/tests/unittests/test_sshutil.py +++ b/tests/unittests/test_sshutil.py @@ -154,4 +154,18 @@ class TestParseSSHConfig(TestCase): self.assertEqual('foo', ret[0].key) self.assertEqual('Bar', ret[0].value) + def test_lower_case_with_equals(self): + self.load_file.return_value = 'foo=bar' + ret = ssh_util.parse_ssh_config('some real file') + self.assertEqual(1, len(ret)) + self.assertEqual('foo', ret[0].key) + self.assertEqual('bar', ret[0].value) + + def test_upper_case_with_equals(self): + self.load_file.return_value = 'Foo=bar' + ret = ssh_util.parse_ssh_config('some real file') + self.assertEqual(1, len(ret)) + self.assertEqual('foo', ret[0].key) + self.assertEqual('bar', ret[0].value) + # vi: ts=4 expandtab -- cgit v1.2.3 From 5d168a9d11e905dd19e24d0596b7d1145e6b8aa1 Mon Sep 17 00:00:00 2001 From: Daniel Watkins Date: Thu, 13 Nov 2014 10:35:46 +0000 Subject: Use test_helpers.TestCase for test_sshutil tests. As requested by harlowja. --- tests/unittests/test_sshutil.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/tests/unittests/test_sshutil.py b/tests/unittests/test_sshutil.py index cd576e8f..3b317121 100644 --- a/tests/unittests/test_sshutil.py +++ b/tests/unittests/test_sshutil.py @@ -1,7 +1,6 @@ -from unittest import TestCase - from mock import patch +from . import helpers as test_helpers from cloudinit import ssh_util @@ -38,7 +37,7 @@ TEST_OPTIONS = ("no-port-forwarding,no-agent-forwarding,no-X11-forwarding," 'user \"root\".\';echo;sleep 10"') -class TestAuthKeyLineParser(TestCase): +class TestAuthKeyLineParser(test_helpers.TestCase): def test_simple_parse(self): # test key line with common 3 fields (keytype, base64, comment) parser = ssh_util.AuthKeyLineParser() @@ -101,7 +100,7 @@ class TestAuthKeyLineParser(TestCase): self.assertFalse(key.valid()) -class TestParseSSHConfig(TestCase): +class TestParseSSHConfig(test_helpers.TestCase): def setUp(self): self.load_file_patch = patch('cloudinit.ssh_util.util.load_file') -- cgit v1.2.3 From 3efc7142a6ca72bfb40e63c49ed64e2e04837c51 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Tue, 18 Nov 2014 09:40:57 -0600 Subject: retain trailing newline from template files when using jinja2 sources.list was where this showed itself, but all rendered files would have their newline stripped. LP: #1355343 --- ChangeLog | 2 ++ cloudinit/templater.py | 4 +++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/ChangeLog b/ChangeLog index 0cac0f04..1debc923 100644 --- a/ChangeLog +++ b/ChangeLog @@ -4,6 +4,8 @@ - expose uses_systemd as a distro function (fix rhel7) - fix broken 'output' config (LP: #1387340) - begin adding cloud config module docs to config modules (LP: #1383510) + - retain trailing eol from template files (sources.list) when + rendered with jinja (LP: #1355343) 0.7.6: - open 0.7.6 - Enable vendordata on CloudSigma datasource (LP: #1303986) diff --git a/cloudinit/templater.py b/cloudinit/templater.py index 02f6261d..4cd3f13d 100644 --- a/cloudinit/templater.py +++ b/cloudinit/templater.py @@ -89,9 +89,11 @@ def detect_template(text): return CTemplate(content, searchList=[params]).respond() def jinja_render(content, params): + # keep_trailing_newline is in jinja2 2.7+, not 2.6 + add = "\n" if content.endswith("\n") else "" return JTemplate(content, undefined=jinja2.StrictUndefined, - trim_blocks=True).render(**params) + trim_blocks=True).render(**params) + add if text.find("\n") != -1: ident, rest = text.split("\n", 1) -- cgit v1.2.3 From bfbe8099b98bb97cfd96385fe31c023548734cbf Mon Sep 17 00:00:00 2001 From: Shraddha Pandhe Date: Fri, 21 Nov 2014 18:32:30 +0000 Subject: Add IPv6 Support for Rhel. This patch does the following: 1. Adds support to process network config with IPv6 2. Adds support to display 'ifconfig -a' information for IPv6 3. Adds support to display routing information for IPv6 --- cloudinit/distros/net_util.py | 60 ++++++++++++++++++++++++++++--------------- cloudinit/distros/rhel.py | 11 ++++++++ cloudinit/netinfo.py | 60 ++++++++++++++++++++++++++++++++++++------- 3 files changed, 102 insertions(+), 29 deletions(-) diff --git a/cloudinit/distros/net_util.py b/cloudinit/distros/net_util.py index b9bcfd8b..f56f6ccd 100644 --- a/cloudinit/distros/net_util.py +++ b/cloudinit/distros/net_util.py @@ -113,6 +113,10 @@ def translate_network(settings): for info in ifaces: if 'iface' not in info: continue + use_ipv6 = False + # Check if current device has an ipv6 IP + if 'inet6' in info['iface']: + use_ipv6 = True iface_details = info['iface'].split(None) dev_name = None if len(iface_details) >= 1: @@ -122,6 +126,7 @@ def translate_network(settings): if not dev_name: continue iface_info = {} + iface_info['ipv6'] = {} if len(iface_details) >= 3: proto_type = iface_details[2].strip().lower() # Seems like this can be 'loopback' which we don't @@ -129,26 +134,39 @@ def translate_network(settings): if proto_type in ['dhcp', 'static']: iface_info['bootproto'] = proto_type # These can just be copied over - for k in ['netmask', 'address', 'gateway', 'broadcast']: - if k in info: - val = info[k].strip().lower() - if val: - iface_info[k] = val - # Name server info provided?? - if 'dns-nameservers' in info: - iface_info['dns-nameservers'] = info['dns-nameservers'].split() - # Name server search info provided?? - if 'dns-search' in info: - iface_info['dns-search'] = info['dns-search'].split() - # Is any mac address spoofing going on?? - if 'hwaddress' in info: - hw_info = info['hwaddress'].lower().strip() - hw_split = hw_info.split(None, 1) - if len(hw_split) == 2 and hw_split[0].startswith('ether'): - hw_addr = hw_split[1] - if hw_addr: - iface_info['hwaddress'] = hw_addr - real_ifaces[dev_name] = iface_info + if use_ipv6: + for k in ['address', 'gateway']: + if k in info: + val = info[k].strip().lower() + if val: + iface_info['ipv6'][k] = val + else: + for k in ['netmask', 'address', 'gateway', 'broadcast']: + if k in info: + val = info[k].strip().lower() + if val: + iface_info[k] = val + # Name server info provided?? + if 'dns-nameservers' in info: + iface_info['dns-nameservers'] = info['dns-nameservers'].split() + # Name server search info provided?? + if 'dns-search' in info: + iface_info['dns-search'] = info['dns-search'].split() + # Is any mac address spoofing going on?? + if 'hwaddress' in info: + hw_info = info['hwaddress'].lower().strip() + hw_split = hw_info.split(None, 1) + if len(hw_split) == 2 and hw_split[0].startswith('ether'): + hw_addr = hw_split[1] + if hw_addr: + iface_info['hwaddress'] = hw_addr + + # If ipv6 is enabled, device will have multiple IPs. + # Update the dictionary instead of overwriting it + if dev_name in real_ifaces: + real_ifaces[dev_name].update(iface_info) + else: + real_ifaces[dev_name] = iface_info # Check for those that should be started on boot via 'auto' for (cmd, args) in entries: if cmd == 'auto': @@ -160,4 +178,6 @@ def translate_network(settings): dev_name = args[0].strip().lower() if dev_name in real_ifaces: real_ifaces[dev_name]['auto'] = True + if cmd == 'iface' and 'inet6' in args: + real_ifaces[dev_name]['inet6'] = True return real_ifaces diff --git a/cloudinit/distros/rhel.py b/cloudinit/distros/rhel.py index 1a269e08..fa3ccb38 100644 --- a/cloudinit/distros/rhel.py +++ b/cloudinit/distros/rhel.py @@ -71,6 +71,7 @@ class Distro(distros.Distro): nameservers = [] searchservers = [] dev_names = entries.keys() + use_ipv6 = False for (dev, info) in entries.iteritems(): net_fn = self.network_script_tpl % (dev) net_cfg = { @@ -83,6 +84,13 @@ class Distro(distros.Distro): 'MACADDR': info.get('hwaddress'), 'ONBOOT': _make_sysconfig_bool(info.get('auto')), } + if info.get('inet6'): + use_ipv6 = True + net_cfg.update({ + 'IPV6INIT': _make_sysconfig_bool(True), + 'IPV6ADDR': info.get('ipv6').get('address'), + 'IPV6_DEFAULTGW': info.get('ipv6').get('gateway'), + }) rhel_util.update_sysconfig_file(net_fn, net_cfg) if 'dns-nameservers' in info: nameservers.extend(info['dns-nameservers']) @@ -95,6 +103,9 @@ class Distro(distros.Distro): net_cfg = { 'NETWORKING': _make_sysconfig_bool(True), } + # If IPv6 interface present, enable ipv6 networking + if use_ipv6: + net_cfg['NETWORKING_IPV6'] = _make_sysconfig_bool(True) rhel_util.update_sysconfig_file(self.network_conf_fn, net_cfg) return dev_names diff --git a/cloudinit/netinfo.py b/cloudinit/netinfo.py index 8d4df342..d891315b 100644 --- a/cloudinit/netinfo.py +++ b/cloudinit/netinfo.py @@ -72,6 +72,7 @@ def netdev_info(empty=""): "bcast:": "bcast", "broadcast": "bcast", "mask:": "mask", "netmask": "mask", "hwaddr": "hwaddr", "ether": "hwaddr", + "scope": "scope", } for origfield, field in ifconfigfields.items(): target = "%s%s" % (field, fieldpost) @@ -96,7 +97,12 @@ def netdev_info(empty=""): def route_info(): (route_out, _err) = util.subp(["netstat", "-rn"]) - routes = [] + (route_out6, _err6) = util.subp(["netstat", "-A inet6", "-n"]) + + routes = {} + routes['ipv4'] = [] + routes['ipv6'] = [] + entries = route_out.splitlines()[1:] for line in entries: if not line: @@ -132,7 +138,26 @@ def route_info(): 'iface': toks[7], } - routes.append(entry) + routes['ipv4'].append(entry) + + entries6 = route_out6.splitlines()[1:] + for line in entries6: + if not line: + continue + toks = line.split() + + if (len(toks) < 6 or toks[0] == "Kernel" or + toks[0] == "Proto" or toks[0] == "Active"): + continue + entry = { + 'proto': toks[0], + 'recv-q': toks[1], + 'send-q': toks[2], + 'local address': toks[3], + 'foreign address': toks[4], + 'state': toks[5], + } + routes['ipv6'].append(entry) return routes @@ -156,10 +181,12 @@ def netdev_pformat(): lines.append(util.center("Net device info failed", '!', 80)) netdev = None if netdev is not None: - fields = ['Device', 'Up', 'Address', 'Mask', 'Hw-Address'] + fields = ['Device', 'Up', 'Address', 'Mask', 'Scope', 'Hw-Address'] tbl = PrettyTable(fields) for (dev, d) in netdev.iteritems(): - tbl.add_row([dev, d["up"], d["addr"], d["mask"], d["hwaddr"]]) + tbl.add_row([dev, d["up"], d["addr"], d["mask"], ".", d["hwaddr"]]) + if d["addr6"]: + tbl.add_row([dev, d["up"], d["addr6"], ".", d["scope6"], d["hwaddr"]]) netdev_s = tbl.get_string() max_len = len(max(netdev_s.splitlines(), key=len)) header = util.center("Net device info", "+", max_len) @@ -176,15 +203,30 @@ def route_pformat(): util.logexc(LOG, "Route info failed: %s" % e) routes = None if routes is not None: - fields = ['Route', 'Destination', 'Gateway', + fields_v4 = ['Route', 'Destination', 'Gateway', 'Genmask', 'Interface', 'Flags'] - tbl = PrettyTable(fields) - for (n, r) in enumerate(routes): + + if routes.get('ipv6') is not None: + fields_v6 = ['Route', 'Proto', 'Recv-Q', 'Send-Q', 'Local Address', + 'Foreign Address', 'State'] + + tbl_v4 = PrettyTable(fields_v4) + for (n, r) in enumerate(routes.get('ipv4')): route_id = str(n) - tbl.add_row([route_id, r['destination'], + tbl_v4.add_row([route_id, r['destination'], r['gateway'], r['genmask'], r['iface'], r['flags']]) - route_s = tbl.get_string() + route_s = tbl_v4.get_string() + if fields_v6: + tbl_v6 = PrettyTable(fields_v6) + for (n, r) in enumerate(routes.get('ipv6')): + route_id = str(n) + tbl_v6.add_row([route_id, r['proto'], + r['recv-q'], r['send-q'], + r['local address'], r['foreign address'], + r['state']]) + route_s = route_s + tbl_v6.get_string() + max_len = len(max(route_s.splitlines(), key=len)) header = util.center("Route info", "+", max_len) lines.extend([header, route_s]) -- cgit v1.2.3 From 563467f2d62def448dbb8bacf33ae25782b2849e Mon Sep 17 00:00:00 2001 From: Shraddha Pandhe Date: Fri, 21 Nov 2014 22:17:26 +0000 Subject: Added unittests for IPv6 support for RHEL --- cloudinit/distros/rhel.py | 1 + tests/unittests/test_distros/test_netconfig.py | 93 ++++++++++++++++++++++++++ 2 files changed, 94 insertions(+) diff --git a/cloudinit/distros/rhel.py b/cloudinit/distros/rhel.py index fa3ccb38..13335df5 100644 --- a/cloudinit/distros/rhel.py +++ b/cloudinit/distros/rhel.py @@ -106,6 +106,7 @@ class Distro(distros.Distro): # If IPv6 interface present, enable ipv6 networking if use_ipv6: net_cfg['NETWORKING_IPV6'] = _make_sysconfig_bool(True) + net_cfg['IPV6_AUTOCONF'] = _make_sysconfig_bool(False) rhel_util.update_sysconfig_file(self.network_conf_fn, net_cfg) return dev_names diff --git a/tests/unittests/test_distros/test_netconfig.py b/tests/unittests/test_distros/test_netconfig.py index 35cc1f43..cb385da7 100644 --- a/tests/unittests/test_distros/test_netconfig.py +++ b/tests/unittests/test_distros/test_netconfig.py @@ -30,6 +30,24 @@ auto eth1 iface eth1 inet dhcp ''' +BASE_NET_CFG_IPV6 = ''' +auto lo +iface lo inet loopback + +auto eth0 +iface eth0 inet static + address 192.168.1.5 + netmask 255.255.255.0 + network 192.168.0.0 + broadcast 192.168.1.0 + gateway 192.168.1.254 + +iface eth0 inet6 static + address 2607:f0d0:1002:0011::2 + netmask 64 + gateway 2607:f0d0:1002:0011::1 +''' + class WriteBuffer(object): def __init__(self): @@ -174,6 +192,81 @@ NETWORKING=yes self.assertCfgEquals(expected_buf, str(write_buf)) self.assertEquals(write_buf.mode, 0644) + def test_write_ipv6_rhel(self): + rh_distro = self._get_distro('rhel') + write_mock = self.mocker.replace(util.write_file, + spec=False, passthrough=False) + load_mock = self.mocker.replace(util.load_file, + spec=False, passthrough=False) + exists_mock = self.mocker.replace(os.path.isfile, + spec=False, passthrough=False) + + write_bufs = {} + + def replace_write(filename, content, mode=0644, omode="wb"): + buf = WriteBuffer() + buf.mode = mode + buf.omode = omode + buf.write(content) + write_bufs[filename] = buf + + exists_mock(mocker.ARGS) + self.mocker.count(0, None) + self.mocker.result(False) + + load_mock(mocker.ARGS) + self.mocker.count(0, None) + self.mocker.result('') + + for _i in range(0, 2): + write_mock(mocker.ARGS) + self.mocker.call(replace_write) + + write_mock(mocker.ARGS) + self.mocker.call(replace_write) + + self.mocker.replay() + rh_distro.apply_network(BASE_NET_CFG_IPV6, False) + + self.assertEquals(len(write_bufs), 3) + self.assertIn('/etc/sysconfig/network-scripts/ifcfg-lo', write_bufs) + write_buf = write_bufs['/etc/sysconfig/network-scripts/ifcfg-lo'] + expected_buf = ''' +DEVICE="lo" +ONBOOT=yes +''' + self.assertCfgEquals(expected_buf, str(write_buf)) + self.assertEquals(write_buf.mode, 0644) + + self.assertIn('/etc/sysconfig/network-scripts/ifcfg-eth0', write_bufs) + write_buf = write_bufs['/etc/sysconfig/network-scripts/ifcfg-eth0'] + expected_buf = ''' +DEVICE="eth0" +BOOTPROTO="static" +NETMASK="255.255.255.0" +IPADDR="192.168.1.5" +ONBOOT=yes +GATEWAY="192.168.1.254" +BROADCAST="192.168.1.0" +IPV6INIT=yes +IPV6ADDR="2607:f0d0:1002:0011::2" +IPV6_DEFAULTGW="2607:f0d0:1002:0011::1" +''' + self.assertCfgEquals(expected_buf, str(write_buf)) + self.assertEquals(write_buf.mode, 0644) + + self.assertIn('/etc/sysconfig/network', write_bufs) + write_buf = write_bufs['/etc/sysconfig/network'] + expected_buf = ''' +# Created by cloud-init v. 0.7 +NETWORKING=yes +NETWORKING_IPV6=yes +IPV6_AUTOCONF=no +''' + self.assertCfgEquals(expected_buf, str(write_buf)) + self.assertEquals(write_buf.mode, 0644) + + def test_simple_write_freebsd(self): fbsd_distro = self._get_distro('freebsd') util_mock = self.mocker.replace(util.write_file, -- cgit v1.2.3 From 249c6c5980c36832ddd03ff6c3272f534538efc0 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Fri, 21 Nov 2014 17:12:56 -0800 Subject: Update chef module docstring to reflect the new style --- cloudinit/config/cc_chef.py | 50 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/cloudinit/config/cc_chef.py b/cloudinit/config/cc_chef.py index aa82cb0a..b30d7361 100644 --- a/cloudinit/config/cc_chef.py +++ b/cloudinit/config/cc_chef.py @@ -18,6 +18,56 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . +""" +**Summary:** module that configures, starts and installs chef. + +**Description:** This module enables chef to be installed (from packages or +from gems, or from omnibus). Before this occurs chef configurations are +written to disk (validation.pem, client.pem, firstboot.json, client.rb), +and needed chef folders/directories are created (/etc/chef and /var/log/chef +and so-on). Then once installing proceeds correctly if configured chef will +be started (in daemon mode or in non-daemon mode) and then once that has +finished (if ran in non-daemon mode this will be when chef finishes +converging, if ran in daemon mode then no further actions are possible since +chef will have forked into its own process) then a post run function can +run that can do finishing activities (such as removing the validation pem +file). + +It can be configured with the following option structure:: + + chef: + directories: (defaulting to /etc/chef, /var/log/chef, /var/lib/chef, + /var/cache/chef, /var/backups/chef, /var/run/chef) + validation_key or validation_cert: (optional string to be written to + /etc/chef/validation.pem) + firstboot_path: (path to write run_list and initial_attributes keys that + should also be present in this configuration, defaults + to /etc/chef/firstboot.json) + exec: boolean to run or not run chef (defaults to false, unless + a gem installed is requested + where this will then default + to true) + + chef.rb template keys (if falsey, then will be skipped and not + written to /etc/chef/client.rb) + + chef: + client_key: + environment: + file_backup_path: + file_cache_path: + json_attribs: + log_level: + log_location: + node_name: + pid_file: + server_url: + show_time: + ssl_verify_mode: + validation_key: + validation_name: +""" + import itertools import json import os -- cgit v1.2.3 From 94565bc8607f35557225589d9dde6d2954d49731 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Fri, 21 Nov 2014 17:44:32 -0800 Subject: Ensure the cc_chef doc links to the cc_chef code --- cloudinit/config/cc_chef.py | 1 + doc/rtd/topics/modules.rst | 1 + 2 files changed, 2 insertions(+) diff --git a/cloudinit/config/cc_chef.py b/cloudinit/config/cc_chef.py index b30d7361..d5611253 100644 --- a/cloudinit/config/cc_chef.py +++ b/cloudinit/config/cc_chef.py @@ -182,6 +182,7 @@ def get_template_params(iid, chef_cfg, log): def handle(name, cfg, cloud, log, _args): + """Handler method activated by cloud-init.""" # If there isn't a chef key in the configuration don't do anything if 'chef' not in cfg: diff --git a/doc/rtd/topics/modules.rst b/doc/rtd/topics/modules.rst index b50acbfa..4202338b 100644 --- a/doc/rtd/topics/modules.rst +++ b/doc/rtd/topics/modules.rst @@ -43,6 +43,7 @@ Chef **Internal name:** ``cc_chef`` .. automodule:: cloudinit.config.cc_chef + :members: Debug ----- -- cgit v1.2.3 From 4dfba8913a24da7254bf47017a264b28c7a49cd2 Mon Sep 17 00:00:00 2001 From: Shraddha Pandhe Date: Mon, 24 Nov 2014 19:36:18 +0000 Subject: IPv6 support for rhel distro - Saw an issue in my earlier commit with multiple NICs. This commit fixes that issue, along with the indentation issue --- cloudinit/distros/net_util.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/cloudinit/distros/net_util.py b/cloudinit/distros/net_util.py index f56f6ccd..f8c34846 100644 --- a/cloudinit/distros/net_util.py +++ b/cloudinit/distros/net_util.py @@ -169,15 +169,16 @@ def translate_network(settings): real_ifaces[dev_name] = iface_info # Check for those that should be started on boot via 'auto' for (cmd, args) in entries: + args = args.split(None) + if not args: + continue + dev_name = args[0].strip().lower() if cmd == 'auto': # Seems like auto can be like 'auto eth0 eth0:1' so just get the # first part out as the device name - args = args.split(None) - if not args: - continue - dev_name = args[0].strip().lower() if dev_name in real_ifaces: real_ifaces[dev_name]['auto'] = True if cmd == 'iface' and 'inet6' in args: - real_ifaces[dev_name]['inet6'] = True + real_ifaces[dev_name]['inet6'] = True return real_ifaces + -- cgit v1.2.3 From a2915dae58bb735a0033b3575db6540b8fee4a4e Mon Sep 17 00:00:00 2001 From: Shraddha Pandhe Date: Mon, 24 Nov 2014 19:54:22 +0000 Subject: Updated unittests + Scenario with multiple NICs --- tests/unittests/test_distros/test_netconfig.py | 32 ++++++++++++++++++++++++-- 1 file changed, 30 insertions(+), 2 deletions(-) diff --git a/tests/unittests/test_distros/test_netconfig.py b/tests/unittests/test_distros/test_netconfig.py index cb385da7..dbbf9617 100644 --- a/tests/unittests/test_distros/test_netconfig.py +++ b/tests/unittests/test_distros/test_netconfig.py @@ -46,6 +46,18 @@ iface eth0 inet6 static address 2607:f0d0:1002:0011::2 netmask 64 gateway 2607:f0d0:1002:0011::1 + +iface eth1 inet static + address 192.168.1.6 + netmask 255.255.255.0 + network 192.168.0.0 + broadcast 192.168.1.0 + gateway 192.168.1.254 + +iface eth1 inet6 static + address 2607:f0d0:1002:0011::3 + netmask 64 + gateway 2607:f0d0:1002:0011::1 ''' @@ -218,7 +230,7 @@ NETWORKING=yes self.mocker.count(0, None) self.mocker.result('') - for _i in range(0, 2): + for _i in range(0, 3): write_mock(mocker.ARGS) self.mocker.call(replace_write) @@ -228,7 +240,7 @@ NETWORKING=yes self.mocker.replay() rh_distro.apply_network(BASE_NET_CFG_IPV6, False) - self.assertEquals(len(write_bufs), 3) + self.assertEquals(len(write_bufs), 4) self.assertIn('/etc/sysconfig/network-scripts/ifcfg-lo', write_bufs) write_buf = write_bufs['/etc/sysconfig/network-scripts/ifcfg-lo'] expected_buf = ''' @@ -251,6 +263,22 @@ BROADCAST="192.168.1.0" IPV6INIT=yes IPV6ADDR="2607:f0d0:1002:0011::2" IPV6_DEFAULTGW="2607:f0d0:1002:0011::1" +''' + self.assertCfgEquals(expected_buf, str(write_buf)) + self.assertEquals(write_buf.mode, 0644) + self.assertIn('/etc/sysconfig/network-scripts/ifcfg-eth1', write_bufs) + write_buf = write_bufs['/etc/sysconfig/network-scripts/ifcfg-eth1'] + expected_buf = ''' +DEVICE="eth1" +BOOTPROTO="static" +NETMASK="255.255.255.0" +IPADDR="192.168.1.6" +ONBOOT=no +GATEWAY="192.168.1.254" +BROADCAST="192.168.1.0" +IPV6INIT=yes +IPV6ADDR="2607:f0d0:1002:0011::3" +IPV6_DEFAULTGW="2607:f0d0:1002:0011::1" ''' self.assertCfgEquals(expected_buf, str(write_buf)) self.assertEquals(write_buf.mode, 0644) -- cgit v1.2.3 From 1db41a6f5283d38dff6de0b0421d51eac869a39c Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Mon, 24 Nov 2014 16:41:21 -0800 Subject: Use assertNotEqual which exists on python2.6 Instead of using assertGreater which is new on python2.7 just use assertNotEqual which does exist on python2.6 to perform the same/similar operation. This makes the unittest not break on python2.6 --- tests/unittests/test_handler/test_handler_debug.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unittests/test_handler/test_handler_debug.py b/tests/unittests/test_handler/test_handler_debug.py index bd9e29d8..8891ca04 100644 --- a/tests/unittests/test_handler/test_handler_debug.py +++ b/tests/unittests/test_handler/test_handler_debug.py @@ -59,7 +59,7 @@ class TestDebug(t_help.FilesystemMockingTestCase): cc_debug.handle('cc_debug', cfg, cc, LOG, []) contents = util.load_file('/var/log/cloud-init-debug.log') # Some basic sanity tests... - self.assertGreater(len(contents), 0) + self.assertNotEqual(0, len(contents)) for k in cfg.keys(): self.assertIn(k, contents) -- cgit v1.2.3