diff options
| -rw-r--r-- | ChangeLog | 12 | ||||
| -rw-r--r-- | cloudinit/config/cc_chef.py | 302 | ||||
| -rw-r--r-- | cloudinit/config/cc_debug.py | 39 | ||||
| -rw-r--r-- | cloudinit/config/cc_ubuntu_init_switch.py | 30 | ||||
| -rw-r--r-- | cloudinit/ssh_util.py | 5 | ||||
| -rw-r--r-- | cloudinit/templater.py | 4 | ||||
| -rw-r--r-- | cloudinit/util.py | 22 | ||||
| -rw-r--r-- | doc/rtd/conf.py | 2 | ||||
| -rw-r--r-- | doc/rtd/topics/modules.rst | 343 | ||||
| -rwxr-xr-x | setup.py | 55 | ||||
| -rw-r--r-- | templates/chef_client.rb.tmpl | 51 | ||||
| -rw-r--r-- | test-requirements.txt | 1 | ||||
| -rw-r--r-- | tests/unittests/helpers.py | 5 | ||||
| -rw-r--r-- | tests/unittests/test_handler/test_handler_chef.py | 121 | ||||
| -rw-r--r-- | tests/unittests/test_handler/test_handler_debug.py | 78 | ||||
| -rw-r--r-- | tests/unittests/test_sshutil.py | 73 |
16 files changed, 1024 insertions, 119 deletions
@@ -2,6 +2,18 @@ - open 0.7.7 - Digital Ocean: add datasource for Digital Ocean. [Neal Shrader] - expose uses_systemd as a distro function (fix rhel7) + - fix broken 'output' config (LP: #1387340) + - begin adding cloud config module docs to config modules (LP: #1383510) + - retain trailing eol from template files (sources.list) when + rendered with jinja (LP: #1355343) + - Only use datafiles and initsys addon outside virtualenvs + - Fix the digital ocean test case on python 2.6 + - Increase the usefulness, robustness, configurability of the chef module + so that it is more useful, more documented and better for users + - Fix how '=' signs are not handled that well in ssh_utils (LP: #1391303) + - Be more tolerant of ssh keys passed into 'ssh_authorized_keys'; allowing + for list, tuple, set, dict, string types and warning on other unexpected + types 0.7.6: - open 0.7.6 - Enable vendordata on CloudSigma datasource (LP: #1303986) diff --git a/cloudinit/config/cc_chef.py b/cloudinit/config/cc_chef.py index 806deed9..d5611253 100644 --- a/cloudinit/config/cc_chef.py +++ b/cloudinit/config/cc_chef.py @@ -18,6 +18,57 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see <http://www.gnu.org/licenses/>. +""" +**Summary:** module that configures, starts and installs chef. + +**Description:** This module enables chef to be installed (from packages or +from gems, or from omnibus). Before this occurs chef configurations are +written to disk (validation.pem, client.pem, firstboot.json, client.rb), +and needed chef folders/directories are created (/etc/chef and /var/log/chef +and so-on). Then once installing proceeds correctly if configured chef will +be started (in daemon mode or in non-daemon mode) and then once that has +finished (if ran in non-daemon mode this will be when chef finishes +converging, if ran in daemon mode then no further actions are possible since +chef will have forked into its own process) then a post run function can +run that can do finishing activities (such as removing the validation pem +file). + +It can be configured with the following option structure:: + + chef: + directories: (defaulting to /etc/chef, /var/log/chef, /var/lib/chef, + /var/cache/chef, /var/backups/chef, /var/run/chef) + validation_key or validation_cert: (optional string to be written to + /etc/chef/validation.pem) + firstboot_path: (path to write run_list and initial_attributes keys that + should also be present in this configuration, defaults + to /etc/chef/firstboot.json) + exec: boolean to run or not run chef (defaults to false, unless + a gem installed is requested + where this will then default + to true) + + chef.rb template keys (if falsey, then will be skipped and not + written to /etc/chef/client.rb) + + chef: + client_key: + environment: + file_backup_path: + file_cache_path: + json_attribs: + log_level: + log_location: + node_name: + pid_file: + server_url: + show_time: + ssl_verify_mode: + validation_key: + validation_name: +""" + +import itertools import json import os @@ -27,19 +78,111 @@ from cloudinit import util RUBY_VERSION_DEFAULT = "1.8" -CHEF_DIRS = [ +CHEF_DIRS = tuple([ '/etc/chef', '/var/log/chef', '/var/lib/chef', '/var/cache/chef', '/var/backups/chef', '/var/run/chef', -] +]) +REQUIRED_CHEF_DIRS = tuple([ + '/etc/chef', +]) OMNIBUS_URL = "https://www.opscode.com/chef/install.sh" +OMNIBUS_URL_RETRIES = 5 + +CHEF_VALIDATION_PEM_PATH = '/etc/chef/validation.pem' +CHEF_FB_PATH = '/etc/chef/firstboot.json' +CHEF_RB_TPL_DEFAULTS = { + # These are ruby symbols... + 'ssl_verify_mode': ':verify_none', + 'log_level': ':info', + # These are not symbols... + 'log_location': '/var/log/chef/client.log', + 'validation_key': CHEF_VALIDATION_PEM_PATH, + 'client_key': "/etc/chef/client.pem", + 'json_attribs': CHEF_FB_PATH, + 'file_cache_path': "/var/cache/chef", + 'file_backup_path': "/var/backups/chef", + 'pid_file': "/var/run/chef/client.pid", + 'show_time': True, +} +CHEF_RB_TPL_BOOL_KEYS = frozenset(['show_time']) +CHEF_RB_TPL_PATH_KEYS = frozenset([ + 'log_location', + 'validation_key', + 'client_key', + 'file_cache_path', + 'json_attribs', + 'file_cache_path', + 'pid_file', +]) +CHEF_RB_TPL_KEYS = list(CHEF_RB_TPL_DEFAULTS.keys()) +CHEF_RB_TPL_KEYS.extend(CHEF_RB_TPL_BOOL_KEYS) +CHEF_RB_TPL_KEYS.extend(CHEF_RB_TPL_PATH_KEYS) +CHEF_RB_TPL_KEYS.extend([ + 'server_url', + 'node_name', + 'environment', + 'validation_name', +]) +CHEF_RB_TPL_KEYS = frozenset(CHEF_RB_TPL_KEYS) +CHEF_RB_PATH = '/etc/chef/client.rb' +CHEF_EXEC_PATH = '/usr/bin/chef-client' +CHEF_EXEC_DEF_ARGS = tuple(['-d', '-i', '1800', '-s', '20']) + + +def is_installed(): + if not os.path.isfile(CHEF_EXEC_PATH): + return False + if not os.access(CHEF_EXEC_PATH, os.X_OK): + return False + return True + + +def post_run_chef(chef_cfg, log): + delete_pem = util.get_cfg_option_bool(chef_cfg, + 'delete_validation_post_exec', + default=False) + if delete_pem and os.path.isfile(CHEF_VALIDATION_PEM_PATH): + os.unlink(CHEF_VALIDATION_PEM_PATH) + + +def get_template_params(iid, chef_cfg, log): + params = CHEF_RB_TPL_DEFAULTS.copy() + # Allow users to overwrite any of the keys they want (if they so choose), + # when a value is None, then the value will be set to None and no boolean + # or string version will be populated... + for (k, v) in chef_cfg.items(): + if k not in CHEF_RB_TPL_KEYS: + log.debug("Skipping unknown chef template key '%s'", k) + continue + if v is None: + params[k] = None + else: + # This will make the value a boolean or string... + if k in CHEF_RB_TPL_BOOL_KEYS: + params[k] = util.get_cfg_option_bool(chef_cfg, k) + else: + params[k] = util.get_cfg_option_str(chef_cfg, k) + # These ones are overwritten to be exact values... + params.update({ + 'generated_by': util.make_header(), + 'node_name': util.get_cfg_option_str(chef_cfg, 'node_name', + default=iid), + 'environment': util.get_cfg_option_str(chef_cfg, 'environment', + default='_default'), + # These two are mandatory... + 'server_url': chef_cfg['server_url'], + 'validation_name': chef_cfg['validation_name'], + }) + return params def handle(name, cfg, cloud, log, _args): + """Handler method activated by cloud-init.""" # If there isn't a chef key in the configuration don't do anything if 'chef' not in cfg: @@ -49,7 +192,10 @@ def handle(name, cfg, cloud, log, _args): chef_cfg = cfg['chef'] # Ensure the chef directories we use exist - for d in CHEF_DIRS: + chef_dirs = util.get_cfg_option_list(chef_cfg, 'directories') + if not chef_dirs: + chef_dirs = list(CHEF_DIRS) + for d in itertools.chain(chef_dirs, REQUIRED_CHEF_DIRS): util.ensure_dir(d) # Set the validation key based on the presence of either 'validation_key' @@ -57,64 +203,108 @@ def handle(name, cfg, cloud, log, _args): # takes precedence for key in ('validation_key', 'validation_cert'): if key in chef_cfg and chef_cfg[key]: - util.write_file('/etc/chef/validation.pem', chef_cfg[key]) + util.write_file(CHEF_VALIDATION_PEM_PATH, chef_cfg[key]) break # Create the chef config from template template_fn = cloud.get_template_filename('chef_client.rb') if template_fn: iid = str(cloud.datasource.get_instance_id()) - params = { - 'server_url': chef_cfg['server_url'], - 'node_name': util.get_cfg_option_str(chef_cfg, 'node_name', iid), - 'environment': util.get_cfg_option_str(chef_cfg, 'environment', - '_default'), - 'validation_name': chef_cfg['validation_name'] - } - templater.render_to_file(template_fn, '/etc/chef/client.rb', params) + params = get_template_params(iid, chef_cfg, log) + # Do a best effort attempt to ensure that the template values that + # are associated with paths have there parent directory created + # before they are used by the chef-client itself. + param_paths = set() + for (k, v) in params.items(): + if k in CHEF_RB_TPL_PATH_KEYS and v: + param_paths.add(os.path.dirname(v)) + util.ensure_dirs(param_paths) + templater.render_to_file(template_fn, CHEF_RB_PATH, params) else: - log.warn("No template found, not rendering to /etc/chef/client.rb") - - # set the firstboot json - initial_json = {} - if 'run_list' in chef_cfg: - initial_json['run_list'] = chef_cfg['run_list'] - if 'initial_attributes' in chef_cfg: - initial_attributes = chef_cfg['initial_attributes'] - for k in list(initial_attributes.keys()): - initial_json[k] = initial_attributes[k] - util.write_file('/etc/chef/firstboot.json', json.dumps(initial_json)) + log.warn("No template found, not rendering to %s", + CHEF_RB_PATH) - # If chef is not installed, we install chef based on 'install_type' - if (not os.path.isfile('/usr/bin/chef-client') or - util.get_cfg_option_bool(chef_cfg, - 'force_install', default=False)): - - install_type = util.get_cfg_option_str(chef_cfg, 'install_type', - 'packages') - if install_type == "gems": - # this will install and run the chef-client from gems - chef_version = util.get_cfg_option_str(chef_cfg, 'version', None) - ruby_version = util.get_cfg_option_str(chef_cfg, 'ruby_version', - RUBY_VERSION_DEFAULT) - install_chef_from_gems(cloud.distro, ruby_version, chef_version) - # and finally, run chef-client - log.debug('Running chef-client') - util.subp(['/usr/bin/chef-client', - '-d', '-i', '1800', '-s', '20'], capture=False) - elif install_type == 'packages': - # this will install and run the chef-client from packages - cloud.distro.install_packages(('chef',)) - elif install_type == 'omnibus': - url = util.get_cfg_option_str(chef_cfg, "omnibus_url", OMNIBUS_URL) - content = url_helper.readurl(url=url, retries=5) - with util.tempdir() as tmpd: - # use tmpd over tmpfile to avoid 'Text file busy' on execute - tmpf = "%s/chef-omnibus-install" % tmpd - util.write_file(tmpf, str(content), mode=0700) - util.subp([tmpf], capture=False) + # Set the firstboot json + fb_filename = util.get_cfg_option_str(chef_cfg, 'firstboot_path', + default=CHEF_FB_PATH) + if not fb_filename: + log.info("First boot path empty, not writing first boot json file") + else: + initial_json = {} + if 'run_list' in chef_cfg: + initial_json['run_list'] = chef_cfg['run_list'] + if 'initial_attributes' in chef_cfg: + initial_attributes = chef_cfg['initial_attributes'] + for k in list(initial_attributes.keys()): + initial_json[k] = initial_attributes[k] + util.write_file(fb_filename, json.dumps(initial_json)) + + # Try to install chef, if its not already installed... + force_install = util.get_cfg_option_bool(chef_cfg, + 'force_install', default=False) + if not is_installed() or force_install: + run = install_chef(cloud, chef_cfg, log) + elif is_installed(): + run = util.get_cfg_option_bool(chef_cfg, 'exec', default=False) + else: + run = False + if run: + run_chef(chef_cfg, log) + post_run_chef(chef_cfg, log) + + +def run_chef(chef_cfg, log): + log.debug('Running chef-client') + cmd = [CHEF_EXEC_PATH] + if 'exec_arguments' in chef_cfg: + cmd_args = chef_cfg['exec_arguments'] + if isinstance(cmd_args, (list, tuple)): + cmd.extend(cmd_args) + elif isinstance(cmd_args, (str, basestring)): + cmd.append(cmd_args) else: - log.warn("Unknown chef install type %s", install_type) + log.warn("Unknown type %s provided for chef" + " 'exec_arguments' expected list, tuple," + " or string", type(cmd_args)) + cmd.extend(CHEF_EXEC_DEF_ARGS) + else: + cmd.extend(CHEF_EXEC_DEF_ARGS) + util.subp(cmd, capture=False) + + +def install_chef(cloud, chef_cfg, log): + # If chef is not installed, we install chef based on 'install_type' + install_type = util.get_cfg_option_str(chef_cfg, 'install_type', + 'packages') + run = util.get_cfg_option_bool(chef_cfg, 'exec', default=False) + if install_type == "gems": + # This will install and run the chef-client from gems + chef_version = util.get_cfg_option_str(chef_cfg, 'version', None) + ruby_version = util.get_cfg_option_str(chef_cfg, 'ruby_version', + RUBY_VERSION_DEFAULT) + install_chef_from_gems(cloud.distro, ruby_version, chef_version) + # Retain backwards compat, by preferring True instead of False + # when not provided/overriden... + run = util.get_cfg_option_bool(chef_cfg, 'exec', default=True) + elif install_type == 'packages': + # This will install and run the chef-client from packages + cloud.distro.install_packages(('chef',)) + elif install_type == 'omnibus': + # This will install as a omnibus unified package + url = util.get_cfg_option_str(chef_cfg, "omnibus_url", OMNIBUS_URL) + retries = max(0, util.get_cfg_option_int(chef_cfg, + "omnibus_url_retries", + default=OMNIBUS_URL_RETRIES)) + content = url_helper.readurl(url=url, retries=retries) + with util.tempdir() as tmpd: + # Use tmpdir over tmpfile to avoid 'text file busy' on execute + tmpf = "%s/chef-omnibus-install" % tmpd + util.write_file(tmpf, str(content), mode=0700) + util.subp([tmpf], capture=False) + else: + log.warn("Unknown chef install type '%s'", install_type) + run = False + return run def get_ruby_packages(version): @@ -133,9 +323,9 @@ def install_chef_from_gems(ruby_version, chef_version, distro): util.sym_link('/usr/bin/ruby%s' % ruby_version, '/usr/bin/ruby') if chef_version: util.subp(['/usr/bin/gem', 'install', 'chef', - '-v %s' % chef_version, '--no-ri', - '--no-rdoc', '--bindir', '/usr/bin', '-q'], capture=False) + '-v %s' % chef_version, '--no-ri', + '--no-rdoc', '--bindir', '/usr/bin', '-q'], capture=False) else: util.subp(['/usr/bin/gem', 'install', 'chef', - '--no-ri', '--no-rdoc', '--bindir', - '/usr/bin', '-q'], capture=False) + '--no-ri', '--no-rdoc', '--bindir', + '/usr/bin', '-q'], capture=False) diff --git a/cloudinit/config/cc_debug.py b/cloudinit/config/cc_debug.py index 7219b0f8..8c489426 100644 --- a/cloudinit/config/cc_debug.py +++ b/cloudinit/config/cc_debug.py @@ -14,11 +14,33 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see <http://www.gnu.org/licenses/>. -from cloudinit import type_utils -from cloudinit import util +""" +**Summary:** helper to debug cloud-init *internal* datastructures. + +**Description:** This module will enable for outputting various internal +information that cloud-init sources provide to either a file or to the output +console/log location that this cloud-init has been configured with when +running. + +It can be configured with the following option structure:: + + debug: + verbose: (defaulting to true) + output: (location to write output, defaulting to console + log) + +.. note:: + + Log configurations are not output. +""" + import copy from StringIO import StringIO +from cloudinit import type_utils +from cloudinit import util + +SKIP_KEYS = frozenset(['log_cfgs']) + def _make_header(text): header = StringIO() @@ -31,7 +53,14 @@ def _make_header(text): return header.getvalue() +def _dumps(obj): + text = util.yaml_dumps(obj, explicit_start=False, explicit_end=False) + return text.rstrip() + + def handle(name, cfg, cloud, log, args): + """Handler method activated by cloud-init.""" + verbose = util.get_cfg_by_path(cfg, ('debug', 'verbose'), default=True) if args: # if args are provided (from cmdline) then explicitly set verbose @@ -46,7 +75,7 @@ def handle(name, cfg, cloud, log, args): return # Clean out some keys that we just don't care about showing... dump_cfg = copy.deepcopy(cfg) - for k in ['log_cfgs']: + for k in SKIP_KEYS: dump_cfg.pop(k, None) all_keys = list(dump_cfg.keys()) for k in all_keys: @@ -55,10 +84,10 @@ def handle(name, cfg, cloud, log, args): # Now dump it... to_print = StringIO() to_print.write(_make_header("Config")) - to_print.write(util.yaml_dumps(dump_cfg)) + to_print.write(_dumps(dump_cfg)) to_print.write("\n") to_print.write(_make_header("MetaData")) - to_print.write(util.yaml_dumps(cloud.datasource.metadata)) + to_print.write(_dumps(cloud.datasource.metadata)) to_print.write("\n") to_print.write(_make_header("Misc")) to_print.write("Datasource: %s\n" % diff --git a/cloudinit/config/cc_ubuntu_init_switch.py b/cloudinit/config/cc_ubuntu_init_switch.py index 6f994bff..7e88ed85 100644 --- a/cloudinit/config/cc_ubuntu_init_switch.py +++ b/cloudinit/config/cc_ubuntu_init_switch.py @@ -17,30 +17,27 @@ # along with this program. If not, see <http://www.gnu.org/licenses/>. """ -ubuntu_init_switch: reboot system into another init +**Summary:** reboot system into another init. -This provides a way for the user to boot with systemd even if the -image is set to boot with upstart. It should be run as one of the first -cloud_init_modules, and will switch the init system and then issue a reboot. -The next boot will come up in the target init system and no action will +**Description:** This module provides a way for the user to boot with systemd +even if the image is set to boot with upstart. It should be run as one of the +first ``cloud_init_modules``, and will switch the init system and then issue a +reboot. The next boot will come up in the target init system and no action will be taken. This should be inert on non-ubuntu systems, and also exit quickly. -config is comes under the top level 'init_switch' dictionary. +It can be configured with the following option structure:: -#cloud-config -init_switch: - target: systemd - reboot: true + init_switch: + target: systemd (can be 'systemd' or 'upstart') + reboot: true (reboot if a change was made, or false to not reboot) -'target' can be 'systemd' or 'upstart'. Best effort is made, but its possible -this system will break, and probably won't interact well with any other -mechanism you've used to switch the init system. +.. note:: -'reboot': [default=true]. - true: reboot if a change was made. - false: do not reboot. + Best effort is made, but it's possible + this system will break, and probably won't interact well with any other + mechanism you've used to switch the init system. """ from cloudinit.settings import PER_INSTANCE @@ -91,6 +88,7 @@ fi def handle(name, cfg, cloud, log, args): + """Handler method activated by cloud-init.""" if not isinstance(cloud.distro, ubuntu.Distro): log.debug("%s: distro is '%s', not ubuntu. returning", diff --git a/cloudinit/ssh_util.py b/cloudinit/ssh_util.py index 70a577bc..14d0cb0f 100644 --- a/cloudinit/ssh_util.py +++ b/cloudinit/ssh_util.py @@ -293,7 +293,10 @@ def parse_ssh_config(fname): if not line or line.startswith("#"): lines.append(SshdConfigLine(line)) continue - (key, val) = line.split(None, 1) + try: + key, val = line.split(None, 1) + except ValueError: + key, val = line.split('=', 1) lines.append(SshdConfigLine(line, key, val)) return lines diff --git a/cloudinit/templater.py b/cloudinit/templater.py index 02f6261d..4cd3f13d 100644 --- a/cloudinit/templater.py +++ b/cloudinit/templater.py @@ -89,9 +89,11 @@ def detect_template(text): return CTemplate(content, searchList=[params]).respond() def jinja_render(content, params): + # keep_trailing_newline is in jinja2 2.7+, not 2.6 + add = "\n" if content.endswith("\n") else "" return JTemplate(content, undefined=jinja2.StrictUndefined, - trim_blocks=True).render(**params) + trim_blocks=True).render(**params) + add if text.find("\n") != -1: ident, rest = text.split("\n", 1) diff --git a/cloudinit/util.py b/cloudinit/util.py index f236d0bf..ee5e5c0a 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -399,6 +399,10 @@ def get_cfg_option_str(yobj, key, default=None): return val +def get_cfg_option_int(yobj, key, default=0): + return int(get_cfg_option_str(yobj, key, default=default)) + + def system_info(): return { 'platform': platform.platform(), @@ -1146,7 +1150,7 @@ def chownbyname(fname, user=None, group=None): # this returns the specific 'mode' entry, cleanly formatted, with value def get_output_cfg(cfg, mode): ret = [None, None] - if cfg or 'output' not in cfg: + if not cfg or 'output' not in cfg: return ret outcfg = cfg['output'] @@ -1270,14 +1274,14 @@ def read_write_cmdline_url(target_fn): logexc(LOG, "Failed writing url content to %s", target_fn) -def yaml_dumps(obj): - formatted = yaml.dump(obj, - line_break="\n", - indent=4, - explicit_start=True, - explicit_end=True, - default_flow_style=False) - return formatted +def yaml_dumps(obj, explicit_start=True, explicit_end=True): + return yaml.safe_dump(obj, + line_break="\n", + indent=4, + explicit_start=explicit_start, + explicit_end=explicit_end, + default_flow_style=False, + allow_unicode=True) def ensure_dir(path, mode=None): diff --git a/doc/rtd/conf.py b/doc/rtd/conf.py index 52a8f92b..9be02766 100644 --- a/doc/rtd/conf.py +++ b/doc/rtd/conf.py @@ -27,6 +27,8 @@ project = 'Cloud-Init' # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom ones. extensions = [ 'sphinx.ext.intersphinx', + 'sphinx.ext.autodoc', + 'sphinx.ext.viewcode', ] intersphinx_mapping = { diff --git a/doc/rtd/topics/modules.rst b/doc/rtd/topics/modules.rst index d4dd55df..4202338b 100644 --- a/doc/rtd/topics/modules.rst +++ b/doc/rtd/topics/modules.rst @@ -1,3 +1,342 @@ -========= +======= Modules -========= +======= + +Apt Configure +------------- + +**Internal name:** ``cc_apt_configure`` + +.. automodule:: cloudinit.config.cc_apt_configure + +Apt Pipelining +-------------- + +**Internal name:** ``cc_apt_pipelining`` + +.. automodule:: cloudinit.config.cc_apt_pipelining + +Bootcmd +------- + +**Internal name:** ``cc_bootcmd`` + +.. automodule:: cloudinit.config.cc_bootcmd + +Byobu +----- + +**Internal name:** ``cc_byobu`` + +.. automodule:: cloudinit.config.cc_byobu + +Ca Certs +-------- + +**Internal name:** ``cc_ca_certs`` + +.. automodule:: cloudinit.config.cc_ca_certs + +Chef +---- + +**Internal name:** ``cc_chef`` + +.. automodule:: cloudinit.config.cc_chef + :members: + +Debug +----- + +**Internal name:** ``cc_debug`` + +.. automodule:: cloudinit.config.cc_debug + :members: + +Disable Ec2 Metadata +-------------------- + +**Internal name:** ``cc_disable_ec2_metadata`` + +.. automodule:: cloudinit.config.cc_disable_ec2_metadata + +Disk Setup +---------- + +**Internal name:** ``cc_disk_setup`` + +.. automodule:: cloudinit.config.cc_disk_setup + +Emit Upstart +------------ + +**Internal name:** ``cc_emit_upstart`` + +.. automodule:: cloudinit.config.cc_emit_upstart + +Final Message +------------- + +**Internal name:** ``cc_final_message`` + +.. automodule:: cloudinit.config.cc_final_message + +Foo +--- + +**Internal name:** ``cc_foo`` + +.. automodule:: cloudinit.config.cc_foo + +Growpart +-------- + +**Internal name:** ``cc_growpart`` + +.. automodule:: cloudinit.config.cc_growpart + +Grub Dpkg +--------- + +**Internal name:** ``cc_grub_dpkg`` + +.. automodule:: cloudinit.config.cc_grub_dpkg + +Keys To Console +--------------- + +**Internal name:** ``cc_keys_to_console`` + +.. automodule:: cloudinit.config.cc_keys_to_console + +Landscape +--------- + +**Internal name:** ``cc_landscape`` + +.. automodule:: cloudinit.config.cc_landscape + +Locale +------ + +**Internal name:** ``cc_locale`` + +.. automodule:: cloudinit.config.cc_locale + +Mcollective +----------- + +**Internal name:** ``cc_mcollective`` + +.. automodule:: cloudinit.config.cc_mcollective + +Migrator +-------- + +**Internal name:** ``cc_migrator`` + +.. automodule:: cloudinit.config.cc_migrator + +Mounts +------ + +**Internal name:** ``cc_mounts`` + +.. automodule:: cloudinit.config.cc_mounts + +Package Update Upgrade Install +------------------------------ + +**Internal name:** ``cc_package_update_upgrade_install`` + +.. automodule:: cloudinit.config.cc_package_update_upgrade_install + +Phone Home +---------- + +**Internal name:** ``cc_phone_home`` + +.. automodule:: cloudinit.config.cc_phone_home + +Power State Change +------------------ + +**Internal name:** ``cc_power_state_change`` + +.. automodule:: cloudinit.config.cc_power_state_change + +Puppet +------ + +**Internal name:** ``cc_puppet`` + +.. automodule:: cloudinit.config.cc_puppet + +Resizefs +-------- + +**Internal name:** ``cc_resizefs`` + +.. automodule:: cloudinit.config.cc_resizefs + +Resolv Conf +----------- + +**Internal name:** ``cc_resolv_conf`` + +.. automodule:: cloudinit.config.cc_resolv_conf + +Rightscale Userdata +------------------- + +**Internal name:** ``cc_rightscale_userdata`` + +.. automodule:: cloudinit.config.cc_rightscale_userdata + +Rsyslog +------- + +**Internal name:** ``cc_rsyslog`` + +.. automodule:: cloudinit.config.cc_rsyslog + +Runcmd +------ + +**Internal name:** ``cc_runcmd`` + +.. automodule:: cloudinit.config.cc_runcmd + +Salt Minion +----------- + +**Internal name:** ``cc_salt_minion`` + +.. automodule:: cloudinit.config.cc_salt_minion + +Scripts Per Boot +---------------- + +**Internal name:** ``cc_scripts_per_boot`` + +.. automodule:: cloudinit.config.cc_scripts_per_boot + +Scripts Per Instance +-------------------- + +**Internal name:** ``cc_scripts_per_instance`` + +.. automodule:: cloudinit.config.cc_scripts_per_instance + +Scripts Per Once +---------------- + +**Internal name:** ``cc_scripts_per_once`` + +.. automodule:: cloudinit.config.cc_scripts_per_once + +Scripts User +------------ + +**Internal name:** ``cc_scripts_user`` + +.. automodule:: cloudinit.config.cc_scripts_user + +Scripts Vendor +-------------- + +**Internal name:** ``cc_scripts_vendor`` + +.. automodule:: cloudinit.config.cc_scripts_vendor + +Seed Random +----------- + +**Internal name:** ``cc_seed_random`` + +.. automodule:: cloudinit.config.cc_seed_random + +Set Hostname +------------ + +**Internal name:** ``cc_set_hostname`` + +.. automodule:: cloudinit.config.cc_set_hostname + +Set Passwords +------------- + +**Internal name:** ``cc_set_passwords`` + +.. automodule:: cloudinit.config.cc_set_passwords + +Ssh +--- + +**Internal name:** ``cc_ssh`` + +.. automodule:: cloudinit.config.cc_ssh + +Ssh Authkey Fingerprints +------------------------ + +**Internal name:** ``cc_ssh_authkey_fingerprints`` + +.. automodule:: cloudinit.config.cc_ssh_authkey_fingerprints + +Ssh Import Id +------------- + +**Internal name:** ``cc_ssh_import_id`` + +.. automodule:: cloudinit.config.cc_ssh_import_id + +Timezone +-------- + +**Internal name:** ``cc_timezone`` + +.. automodule:: cloudinit.config.cc_timezone + +Ubuntu Init Switch +------------------ + +**Internal name:** ``cc_ubuntu_init_switch`` + +.. automodule:: cloudinit.config.cc_ubuntu_init_switch + :members: + +Update Etc Hosts +---------------- + +**Internal name:** ``cc_update_etc_hosts`` + +.. automodule:: cloudinit.config.cc_update_etc_hosts + +Update Hostname +--------------- + +**Internal name:** ``cc_update_hostname`` + +.. automodule:: cloudinit.config.cc_update_hostname + +Users Groups +------------ + +**Internal name:** ``cc_users_groups`` + +.. automodule:: cloudinit.config.cc_users_groups + +Write Files +----------- + +**Internal name:** ``cc_write_files`` + +.. automodule:: cloudinit.config.cc_write_files + +Yum Add Repo +------------ + +**Internal name:** ``cc_yum_add_repo`` + +.. automodule:: cloudinit.config.cc_yum_add_repo @@ -23,6 +23,7 @@ from glob import glob import os +import sys import setuptools from setuptools.command.install import install @@ -86,6 +87,17 @@ if os.uname()[0] == 'FreeBSD': ETC = "/usr/local/etc" +# Avoid having datafiles installed in a virtualenv... +def in_virtualenv(): + try: + if sys.real_prefix == sys.prefix: + return False + else: + return True + except AttributeError: + return False + + def get_version(): cmd = ['tools/read-version'] (ver, _e) = tiny_p(cmd) @@ -135,6 +147,29 @@ class InitsysInstallData(install): self.distribution.reinitialize_command('install_data', True) +if in_virtualenv(): + data_files = [] + cmdclass = {} +else: + data_files = [ + (ETC + '/cloud', glob('config/*.cfg')), + (ETC + '/cloud/cloud.cfg.d', glob('config/cloud.cfg.d/*')), + (ETC + '/cloud/templates', glob('templates/*')), + (USR + '/lib/cloud-init', ['tools/uncloud-init', + 'tools/write-ssh-key-fingerprints']), + (USR + '/share/doc/cloud-init', [f for f in glob('doc/*') if is_f(f)]), + (USR + '/share/doc/cloud-init/examples', + [f for f in glob('doc/examples/*') if is_f(f)]), + (USR + '/share/doc/cloud-init/examples/seed', + [f for f in glob('doc/examples/seed/*') if is_f(f)]), + ] + # Use a subclass for install that handles + # adding on the right init system configuration files + cmdclass = { + 'install': InitsysInstallData, + } + + setuptools.setup(name='cloud-init', version=get_version(), description='EC2 initialisation magic', @@ -146,23 +181,7 @@ setuptools.setup(name='cloud-init', 'tools/cloud-init-per', ], license='GPLv3', - data_files=[(ETC + '/cloud', glob('config/*.cfg')), - (ETC + '/cloud/cloud.cfg.d', glob('config/cloud.cfg.d/*')), - (ETC + '/cloud/templates', glob('templates/*')), - (USR + '/lib/cloud-init', - ['tools/uncloud-init', - 'tools/write-ssh-key-fingerprints']), - (USR + '/share/doc/cloud-init', - [f for f in glob('doc/*') if is_f(f)]), - (USR + '/share/doc/cloud-init/examples', - [f for f in glob('doc/examples/*') if is_f(f)]), - (USR + '/share/doc/cloud-init/examples/seed', - [f for f in glob('doc/examples/seed/*') if is_f(f)]), - ], + data_files=data_files, install_requires=read_requires(), - cmdclass={ - # Use a subclass for install that handles - # adding on the right init system configuration files - 'install': InitsysInstallData, - }, + cmdclass=cmdclass, ) diff --git a/templates/chef_client.rb.tmpl b/templates/chef_client.rb.tmpl index 538850ca..c4069d22 100644 --- a/templates/chef_client.rb.tmpl +++ b/templates/chef_client.rb.tmpl @@ -9,17 +9,50 @@ you need to add the following to config: validation_name: XYZ server_url: XYZ -#} -log_level :info -log_location "/var/log/chef/client.log" -ssl_verify_mode :verify_none +{{generated_by}} +{# +The reason these are not in quotes is because they are ruby +symbols that will be placed inside here, and not actual strings... +#} +{% if log_level %} +log_level {{log_level}} +{% endif %} +{% if ssl_verify_mode %} +ssl_verify_mode {{ssl_verify_mode}} +{% endif %} +{% if log_location %} +log_location "{{log_location}}" +{% endif %} +{% if validation_name %} validation_client_name "{{validation_name}}" -validation_key "/etc/chef/validation.pem" -client_key "/etc/chef/client.pem" +{% endif %} +{% if validation_key %} +validation_key "{{validation_key}}" +{% endif %} +{% if client_key %} +client_key "{{client_key}}" +{% endif %} +{% if server_url %} chef_server_url "{{server_url}}" +{% endif %} +{% if environment %} environment "{{environment}}" +{% endif %} +{% if node_name %} node_name "{{node_name}}" -json_attribs "/etc/chef/firstboot.json" -file_cache_path "/var/cache/chef" -file_backup_path "/var/backups/chef" -pid_file "/var/run/chef/client.pid" +{% endif %} +{% if json_attribs %} +json_attribs "{{json_attribs}}" +{% endif %} +{% if file_cache_path %} +file_cache_path "{{file_cache_path}}" +{% endif %} +{% if file_backup_path %} +file_backup_path "{{file_backup_path}}" +{% endif %} +{% if pid_file %} +pid_file "{{pid_file}}" +{% endif %} +{% if show_time %} Chef::Log::Formatter.show_time = true +{% endif %} diff --git a/test-requirements.txt b/test-requirements.txt index 2edb8066..230f0404 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -1,4 +1,5 @@ httpretty>=0.7.1 +mock mocker nose pep8==1.5.7 diff --git a/tests/unittests/helpers.py b/tests/unittests/helpers.py index 9700a4ca..52305397 100644 --- a/tests/unittests/helpers.py +++ b/tests/unittests/helpers.py @@ -35,6 +35,11 @@ else: if PY26: # For now add these on, taken from python 2.7 + slightly adjusted class TestCase(unittest.TestCase): + def assertIs(self, expr1, expr2, msg=None): + if expr1 is not expr2: + standardMsg = '%r is not %r' % (expr1, expr2) + self.fail(self._formatMessage(msg, standardMsg)) + def assertIn(self, member, container, msg=None): if member not in container: standardMsg = '%r not found in %r' % (member, container) diff --git a/tests/unittests/test_handler/test_handler_chef.py b/tests/unittests/test_handler/test_handler_chef.py new file mode 100644 index 00000000..ef1aa208 --- /dev/null +++ b/tests/unittests/test_handler/test_handler_chef.py @@ -0,0 +1,121 @@ +import json +import os + +from cloudinit.config import cc_chef + +from cloudinit import cloud +from cloudinit import distros +from cloudinit import helpers +from cloudinit import util +from cloudinit.sources import DataSourceNone + +from .. import helpers as t_help + +import logging + +LOG = logging.getLogger(__name__) + + +class TestChef(t_help.FilesystemMockingTestCase): + def setUp(self): + super(TestChef, self).setUp() + self.tmp = self.makeDir(prefix="unittest_") + + def fetch_cloud(self, distro_kind): + cls = distros.fetch(distro_kind) + paths = helpers.Paths({}) + distro = cls(distro_kind, {}, paths) + ds = DataSourceNone.DataSourceNone({}, distro, paths, None) + return cloud.Cloud(ds, paths, {}, distro, None) + + def test_no_config(self): + self.patchUtils(self.tmp) + self.patchOS(self.tmp) + + cfg = {} + cc_chef.handle('chef', cfg, self.fetch_cloud('ubuntu'), LOG, []) + for d in cc_chef.CHEF_DIRS: + self.assertFalse(os.path.isdir(d)) + + def test_basic_config(self): + # This should create a file of the format... + """ + # Created by cloud-init v. 0.7.6 on Sat, 11 Oct 2014 23:57:21 +0000 + log_level :info + ssl_verify_mode :verify_none + log_location "/var/log/chef/client.log" + validation_client_name "bob" + validation_key "/etc/chef/validation.pem" + client_key "/etc/chef/client.pem" + chef_server_url "localhost" + environment "_default" + node_name "iid-datasource-none" + json_attribs "/etc/chef/firstboot.json" + file_cache_path "/var/cache/chef" + file_backup_path "/var/backups/chef" + pid_file "/var/run/chef/client.pid" + Chef::Log::Formatter.show_time = true + """ + tpl_file = util.load_file('templates/chef_client.rb.tmpl') + self.patchUtils(self.tmp) + self.patchOS(self.tmp) + + util.write_file('/etc/cloud/templates/chef_client.rb.tmpl', tpl_file) + cfg = { + 'chef': { + 'server_url': 'localhost', + 'validation_name': 'bob', + }, + } + cc_chef.handle('chef', cfg, self.fetch_cloud('ubuntu'), LOG, []) + for d in cc_chef.CHEF_DIRS: + self.assertTrue(os.path.isdir(d)) + c = util.load_file(cc_chef.CHEF_RB_PATH) + for k, v in cfg['chef'].items(): + self.assertIn(v, c) + for k, v in cc_chef.CHEF_RB_TPL_DEFAULTS.items(): + if isinstance(v, basestring): + self.assertIn(v, c) + c = util.load_file(cc_chef.CHEF_FB_PATH) + self.assertEqual({}, json.loads(c)) + + def test_firstboot_json(self): + self.patchUtils(self.tmp) + self.patchOS(self.tmp) + + cfg = { + 'chef': { + 'server_url': 'localhost', + 'validation_name': 'bob', + 'run_list': ['a', 'b', 'c'], + 'initial_attributes': { + 'c': 'd', + } + }, + } + cc_chef.handle('chef', cfg, self.fetch_cloud('ubuntu'), LOG, []) + c = util.load_file(cc_chef.CHEF_FB_PATH) + self.assertEqual( + { + 'run_list': ['a', 'b', 'c'], + 'c': 'd', + }, json.loads(c)) + + def test_template_deletes(self): + tpl_file = util.load_file('templates/chef_client.rb.tmpl') + self.patchUtils(self.tmp) + self.patchOS(self.tmp) + + util.write_file('/etc/cloud/templates/chef_client.rb.tmpl', tpl_file) + cfg = { + 'chef': { + 'server_url': 'localhost', + 'validation_name': 'bob', + 'json_attribs': None, + 'show_time': None, + }, + } + cc_chef.handle('chef', cfg, self.fetch_cloud('ubuntu'), LOG, []) + c = util.load_file(cc_chef.CHEF_RB_PATH) + self.assertNotIn('json_attribs', c) + self.assertNotIn('Formatter.show_time', c) diff --git a/tests/unittests/test_handler/test_handler_debug.py b/tests/unittests/test_handler/test_handler_debug.py new file mode 100644 index 00000000..bd9e29d8 --- /dev/null +++ b/tests/unittests/test_handler/test_handler_debug.py @@ -0,0 +1,78 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2014 Yahoo! Inc. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. + +from cloudinit.config import cc_debug + +from cloudinit import cloud +from cloudinit import distros +from cloudinit import helpers +from cloudinit import util + +from cloudinit.sources import DataSourceNone + +from .. import helpers as t_help + +import logging + +LOG = logging.getLogger(__name__) + + +class TestDebug(t_help.FilesystemMockingTestCase): + def setUp(self): + super(TestDebug, self).setUp() + self.new_root = self.makeDir(prefix="unittest_") + + def _get_cloud(self, distro, metadata=None): + self.patchUtils(self.new_root) + paths = helpers.Paths({}) + cls = distros.fetch(distro) + d = cls(distro, {}, paths) + ds = DataSourceNone.DataSourceNone({}, d, paths) + if metadata: + ds.metadata.update(metadata) + return cloud.Cloud(ds, paths, {}, d, None) + + def test_debug_write(self): + cfg = { + 'abc': '123', + 'c': u'\u20a0', + 'debug': { + 'verbose': True, + # Does not actually write here due to mocking... + 'output': '/var/log/cloud-init-debug.log', + }, + } + cc = self._get_cloud('ubuntu') + cc_debug.handle('cc_debug', cfg, cc, LOG, []) + contents = util.load_file('/var/log/cloud-init-debug.log') + # Some basic sanity tests... + self.assertGreater(len(contents), 0) + for k in cfg.keys(): + self.assertIn(k, contents) + + def test_debug_no_write(self): + cfg = { + 'abc': '123', + 'debug': { + 'verbose': False, + # Does not actually write here due to mocking... + 'output': '/var/log/cloud-init-debug.log', + }, + } + cc = self._get_cloud('ubuntu') + cc_debug.handle('cc_debug', cfg, cc, LOG, []) + self.assertRaises(IOError, + util.load_file, '/var/log/cloud-init-debug.log') diff --git a/tests/unittests/test_sshutil.py b/tests/unittests/test_sshutil.py index d8662cac..3b317121 100644 --- a/tests/unittests/test_sshutil.py +++ b/tests/unittests/test_sshutil.py @@ -1,5 +1,7 @@ +from mock import patch + +from . import helpers as test_helpers from cloudinit import ssh_util -from unittest import TestCase VALID_CONTENT = { @@ -35,7 +37,7 @@ TEST_OPTIONS = ("no-port-forwarding,no-agent-forwarding,no-X11-forwarding," 'user \"root\".\';echo;sleep 10"') -class TestAuthKeyLineParser(TestCase): +class TestAuthKeyLineParser(test_helpers.TestCase): def test_simple_parse(self): # test key line with common 3 fields (keytype, base64, comment) parser = ssh_util.AuthKeyLineParser() @@ -98,4 +100,71 @@ class TestAuthKeyLineParser(TestCase): self.assertFalse(key.valid()) +class TestParseSSHConfig(test_helpers.TestCase): + + def setUp(self): + self.load_file_patch = patch('cloudinit.ssh_util.util.load_file') + self.load_file = self.load_file_patch.start() + self.isfile_patch = patch('cloudinit.ssh_util.os.path.isfile') + self.isfile = self.isfile_patch.start() + self.isfile.return_value = True + + def tearDown(self): + self.load_file_patch.stop() + self.isfile_patch.stop() + + def test_not_a_file(self): + self.isfile.return_value = False + self.load_file.side_effect = IOError + ret = ssh_util.parse_ssh_config('not a real file') + self.assertEqual([], ret) + + def test_empty_file(self): + self.load_file.return_value = '' + ret = ssh_util.parse_ssh_config('some real file') + self.assertEqual([], ret) + + def test_comment_line(self): + comment_line = '# This is a comment' + self.load_file.return_value = comment_line + ret = ssh_util.parse_ssh_config('some real file') + self.assertEqual(1, len(ret)) + self.assertEqual(comment_line, ret[0].line) + + def test_blank_lines(self): + lines = ['', '\t', ' '] + self.load_file.return_value = '\n'.join(lines) + ret = ssh_util.parse_ssh_config('some real file') + self.assertEqual(len(lines), len(ret)) + for line in ret: + self.assertEqual('', line.line) + + def test_lower_case_config(self): + self.load_file.return_value = 'foo bar' + ret = ssh_util.parse_ssh_config('some real file') + self.assertEqual(1, len(ret)) + self.assertEqual('foo', ret[0].key) + self.assertEqual('bar', ret[0].value) + + def test_upper_case_config(self): + self.load_file.return_value = 'Foo Bar' + ret = ssh_util.parse_ssh_config('some real file') + self.assertEqual(1, len(ret)) + self.assertEqual('foo', ret[0].key) + self.assertEqual('Bar', ret[0].value) + + def test_lower_case_with_equals(self): + self.load_file.return_value = 'foo=bar' + ret = ssh_util.parse_ssh_config('some real file') + self.assertEqual(1, len(ret)) + self.assertEqual('foo', ret[0].key) + self.assertEqual('bar', ret[0].value) + + def test_upper_case_with_equals(self): + self.load_file.return_value = 'Foo=bar' + ret = ssh_util.parse_ssh_config('some real file') + self.assertEqual(1, len(ret)) + self.assertEqual('foo', ret[0].key) + self.assertEqual('bar', ret[0].value) + # vi: ts=4 expandtab |