diff options
| author | Scott Moser <smoser@ubuntu.com> | 2016-04-04 12:07:19 -0400 |
|---|---|---|
| committer | Scott Moser <smoser@ubuntu.com> | 2016-04-04 12:07:19 -0400 |
| commit | 7d8a3194552387fa9e21216bcd9a3bfc76fa2b04 (patch) | |
| tree | c8dc45b013208a4e5e09e6ade63b3b5994f80aa3 /cloudinit/config | |
| parent | 93f5af9f5075a416c65c1d0350c374e16f32f0d5 (diff) | |
| parent | 210b041b2fead7a57af91f60a6f89d9e5aa1ed4a (diff) | |
| download | vyos-cloud-init-7d8a3194552387fa9e21216bcd9a3bfc76fa2b04.tar.gz vyos-cloud-init-7d8a3194552387fa9e21216bcd9a3bfc76fa2b04.zip | |
merge with trunk
Diffstat (limited to 'cloudinit/config')
38 files changed, 1608 insertions, 286 deletions
diff --git a/cloudinit/config/cc_apt_configure.py b/cloudinit/config/cc_apt_configure.py index f10b76a3..702977cb 100644 --- a/cloudinit/config/cc_apt_configure.py +++ b/cloudinit/config/cc_apt_configure.py @@ -51,6 +51,10 @@ EXPORT_GPG_KEYID = """ def handle(name, cfg, cloud, log, _args): + if util.is_false(cfg.get('apt_configure_enabled', True)): + log.debug("Skipping module named %s, disabled by config.", name) + return + release = get_release() mirrors = find_apt_mirror_info(cloud, cfg) if not mirrors or "primary" not in mirrors: @@ -87,7 +91,8 @@ def handle(name, cfg, cloud, log, _args): if matchcfg: matcher = re.compile(matchcfg).search else: - matcher = lambda f: False + def matcher(x): + return False errors = add_sources(cfg['apt_sources'], params, aa_repo_match=matcher) @@ -105,7 +110,7 @@ def handle(name, cfg, cloud, log, _args): # get gpg keyid from keyserver def getkeybyid(keyid, keyserver): - with util.ExtendedTemporaryFile(suffix='.sh') as fh: + with util.ExtendedTemporaryFile(suffix='.sh', mode="w+", ) as fh: fh.write(EXPORT_GPG_KEYID) fh.flush() cmd = ['/bin/sh', fh.name, keyid, keyserver] @@ -126,7 +131,7 @@ def mirror2lists_fileprefix(mirror): def rename_apt_lists(old_mirrors, new_mirrors, lists_d="/var/lib/apt/lists"): - for (name, omirror) in old_mirrors.iteritems(): + for (name, omirror) in old_mirrors.items(): nmirror = new_mirrors.get(name) if not nmirror: continue @@ -169,7 +174,8 @@ def add_sources(srclist, template_params=None, aa_repo_match=None): template_params = {} if aa_repo_match is None: - aa_repo_match = lambda f: False + def aa_repo_match(x): + return False errorlist = [] for ent in srclist: diff --git a/cloudinit/config/cc_apt_pipelining.py b/cloudinit/config/cc_apt_pipelining.py index e5629175..40c32c84 100644 --- a/cloudinit/config/cc_apt_pipelining.py +++ b/cloudinit/config/cc_apt_pipelining.py @@ -43,7 +43,7 @@ def handle(_name, cfg, _cloud, log, _args): write_apt_snippet("0", log, DEFAULT_FILE) elif apt_pipe_value_s in ("none", "unchanged", "os"): return - elif apt_pipe_value_s in [str(b) for b in xrange(0, 6)]: + elif apt_pipe_value_s in [str(b) for b in range(0, 6)]: write_apt_snippet(apt_pipe_value_s, log, DEFAULT_FILE) else: log.warn("Invalid option for apt_pipeling: %s", apt_pipe_value) diff --git a/cloudinit/config/cc_bootcmd.py b/cloudinit/config/cc_bootcmd.py index 3ac22967..a295cc4e 100644 --- a/cloudinit/config/cc_bootcmd.py +++ b/cloudinit/config/cc_bootcmd.py @@ -36,7 +36,7 @@ def handle(name, cfg, cloud, log, _args): with util.ExtendedTemporaryFile(suffix=".sh") as tmpf: try: content = util.shellify(cfg["bootcmd"]) - tmpf.write(content) + tmpf.write(util.encode_text(content)) tmpf.flush() except: util.logexc(log, "Failed to shellify bootcmd") diff --git a/cloudinit/config/cc_ca_certs.py b/cloudinit/config/cc_ca_certs.py index 4f2a46a1..8248b020 100644 --- a/cloudinit/config/cc_ca_certs.py +++ b/cloudinit/config/cc_ca_certs.py @@ -44,7 +44,7 @@ def add_ca_certs(certs): if certs: # First ensure they are strings... cert_file_contents = "\n".join([str(c) for c in certs]) - util.write_file(CA_CERT_FULL_PATH, cert_file_contents, mode=0644) + util.write_file(CA_CERT_FULL_PATH, cert_file_contents, mode=0o644) # Append cert filename to CA_CERT_CONFIG file. # We have to strip the content because blank lines in the file @@ -63,7 +63,7 @@ def remove_default_ca_certs(): """ util.delete_dir_contents(CA_CERT_PATH) util.delete_dir_contents(CA_CERT_SYSTEM_PATH) - util.write_file(CA_CERT_CONFIG, "", mode=0644) + util.write_file(CA_CERT_CONFIG, "", mode=0o644) debconf_sel = "ca-certificates ca-certificates/trust_new_crts select no" util.subp(('debconf-set-selections', '-'), debconf_sel) diff --git a/cloudinit/config/cc_chef.py b/cloudinit/config/cc_chef.py index fc837363..e18c5405 100644 --- a/cloudinit/config/cc_chef.py +++ b/cloudinit/config/cc_chef.py @@ -76,6 +76,8 @@ from cloudinit import templater from cloudinit import url_helper from cloudinit import util +import six + RUBY_VERSION_DEFAULT = "1.8" CHEF_DIRS = tuple([ @@ -261,7 +263,7 @@ def run_chef(chef_cfg, log): cmd_args = chef_cfg['exec_arguments'] if isinstance(cmd_args, (list, tuple)): cmd.extend(cmd_args) - elif isinstance(cmd_args, (str, basestring)): + elif isinstance(cmd_args, six.string_types): cmd.append(cmd_args) else: log.warn("Unknown type %s provided for chef" @@ -300,7 +302,7 @@ def install_chef(cloud, chef_cfg, log): with util.tempdir() as tmpd: # Use tmpdir over tmpfile to avoid 'text file busy' on execute tmpf = "%s/chef-omnibus-install" % tmpd - util.write_file(tmpf, str(content), mode=0700) + util.write_file(tmpf, content, mode=0o700) util.subp([tmpf], capture=False) else: log.warn("Unknown chef install type '%s'", install_type) diff --git a/cloudinit/config/cc_debug.py b/cloudinit/config/cc_debug.py index 8c489426..bdc32fe6 100644 --- a/cloudinit/config/cc_debug.py +++ b/cloudinit/config/cc_debug.py @@ -34,7 +34,8 @@ It can be configured with the following option structure:: """ import copy -from StringIO import StringIO + +from six import StringIO from cloudinit import type_utils from cloudinit import util @@ -77,7 +78,7 @@ def handle(name, cfg, cloud, log, args): dump_cfg = copy.deepcopy(cfg) for k in SKIP_KEYS: dump_cfg.pop(k, None) - all_keys = list(dump_cfg.keys()) + all_keys = list(dump_cfg) for k in all_keys: if k.startswith("_"): dump_cfg.pop(k, None) @@ -103,6 +104,6 @@ def handle(name, cfg, cloud, log, args): line = "ci-info: %s\n" % (line) content_to_file.append(line) if out_file: - util.write_file(out_file, "".join(content_to_file), 0644, "w") + util.write_file(out_file, "".join(content_to_file), 0o644, "w") else: util.multi_log("".join(content_to_file), console=True, stderr=False) diff --git a/cloudinit/config/cc_disk_setup.py b/cloudinit/config/cc_disk_setup.py index 1660832b..0ecc2e4c 100644 --- a/cloudinit/config/cc_disk_setup.py +++ b/cloudinit/config/cc_disk_setup.py @@ -27,6 +27,7 @@ frequency = PER_INSTANCE # Define the commands to use UDEVADM_CMD = util.which('udevadm') SFDISK_CMD = util.which("sfdisk") +SGDISK_CMD = util.which("sgdisk") LSBLK_CMD = util.which("lsblk") BLKID_CMD = util.which("blkid") BLKDEV_CMD = util.which("blockdev") @@ -151,7 +152,7 @@ def enumerate_disk(device, nodeps=False): name: the device name, i.e. sda """ - lsblk_cmd = [LSBLK_CMD, '--pairs', '--out', 'NAME,TYPE,FSTYPE,LABEL', + lsblk_cmd = [LSBLK_CMD, '--pairs', '--output', 'NAME,TYPE,FSTYPE,LABEL', device] if nodeps: @@ -166,11 +167,12 @@ def enumerate_disk(device, nodeps=False): parts = [x for x in (info.strip()).splitlines() if len(x.split()) > 0] for part in parts: - d = {'name': None, - 'type': None, - 'fstype': None, - 'label': None, - } + d = { + 'name': None, + 'type': None, + 'fstype': None, + 'label': None, + } for key, value in value_splitter(part): d[key.lower()] = value @@ -303,8 +305,7 @@ def is_disk_used(device): # If the child count is higher 1, then there are child nodes # such as partition or device mapper nodes - use_count = [x for x in enumerate_disk(device)] - if len(use_count.splitlines()) > 1: + if len(list(enumerate_disk(device))) > 1: return True # If we see a file system, then its used @@ -315,22 +316,6 @@ def is_disk_used(device): return False -def get_hdd_size(device): - """ - Returns the hard disk size. - This works with any disk type, including GPT. - """ - - size_cmd = [SFDISK_CMD, '--show-size', device] - size = None - try: - size, _err = util.subp(size_cmd) - except Exception as e: - raise Exception("Failed to get %s size\n%s" % (device, e)) - - return int(size.strip()) - - def get_dyn_func(*args): """ Call the appropriate function. @@ -358,6 +343,30 @@ def get_dyn_func(*args): raise Exception("No such function %s to call!" % func_name) +def get_mbr_hdd_size(device): + size_cmd = [SFDISK_CMD, '--show-size', device] + size = None + try: + size, _err = util.subp(size_cmd) + except Exception as e: + raise Exception("Failed to get %s size\n%s" % (device, e)) + + return int(size.strip()) + + +def get_gpt_hdd_size(device): + out, _ = util.subp([SGDISK_CMD, '-p', device]) + return out.splitlines()[0].split()[2] + + +def get_hdd_size(table_type, device): + """ + Returns the hard disk size. + This works with any disk type, including GPT. + """ + return get_dyn_func("get_%s_hdd_size", table_type, device) + + def check_partition_mbr_layout(device, layout): """ Returns true if the partition layout matches the one on the disk @@ -393,6 +402,36 @@ def check_partition_mbr_layout(device, layout): break found_layout.append(type_label) + return found_layout + + +def check_partition_gpt_layout(device, layout): + prt_cmd = [SGDISK_CMD, '-p', device] + try: + out, _err = util.subp(prt_cmd) + except Exception as e: + raise Exception("Error running partition command on %s\n%s" % ( + device, e)) + + out_lines = iter(out.splitlines()) + # Skip header + for line in out_lines: + if line.strip().startswith('Number'): + break + + return [line.strip().split()[-1] for line in out_lines] + + +def check_partition_layout(table_type, device, layout): + """ + See if the partition lay out matches. + + This is future a future proofing function. In order + to add support for other disk layout schemes, add a + function called check_partition_%s_layout + """ + found_layout = get_dyn_func( + "check_partition_%s_layout", table_type, device, layout) if isinstance(layout, bool): # if we are using auto partitioning, or "True" be happy @@ -417,18 +456,6 @@ def check_partition_mbr_layout(device, layout): return False -def check_partition_layout(table_type, device, layout): - """ - See if the partition lay out matches. - - This is future a future proofing function. In order - to add support for other disk layout schemes, add a - function called check_partition_%s_layout - """ - return get_dyn_func("check_partition_%s_layout", table_type, device, - layout) - - def get_partition_mbr_layout(size, layout): """ Calculate the layout of the partition table. Partition sizes @@ -481,6 +508,29 @@ def get_partition_mbr_layout(size, layout): return sfdisk_definition +def get_partition_gpt_layout(size, layout): + if isinstance(layout, bool): + return [(None, [0, 0])] + + partition_specs = [] + for partition in layout: + if isinstance(partition, list): + if len(partition) != 2: + raise Exception( + "Partition was incorrectly defined: %s" % partition) + percent, partition_type = partition + else: + percent = partition + partition_type = None + + part_size = int(float(size) * (float(percent) / 100)) + partition_specs.append((partition_type, [0, '+{}'.format(part_size)])) + + # The last partition should use up all remaining space + partition_specs[-1][-1][-1] = 0 + return partition_specs + + def purge_disk_ptable(device): # wipe the first and last megabyte of a disk (or file) # gpt stores partition table both at front and at end. @@ -556,6 +606,22 @@ def exec_mkpart_mbr(device, layout): read_parttbl(device) +def exec_mkpart_gpt(device, layout): + try: + util.subp([SGDISK_CMD, '-Z', device]) + for index, (partition_type, (start, end)) in enumerate(layout): + index += 1 + util.subp([SGDISK_CMD, + '-n', '{}:{}:{}'.format(index, start, end), device]) + if partition_type is not None: + util.subp( + [SGDISK_CMD, + '-t', '{}:{}'.format(index, partition_type), device]) + except Exception: + LOG.warn("Failed to partition device %s" % device) + raise + + def exec_mkpart(table_type, device, layout): """ Fetches the function for creating the table type. @@ -583,6 +649,8 @@ def mkpart(device, definition): table_type: Which partition table to use, defaults to MBR device: the device to work on. """ + # ensure that we get a real device rather than a symbolic link + device = os.path.realpath(device) LOG.debug("Checking values for %s definition" % device) overwrite = definition.get('overwrite', False) @@ -618,7 +686,7 @@ def mkpart(device, definition): return LOG.debug("Checking for device size") - device_size = get_hdd_size(device) + device_size = get_hdd_size(table_type, device) LOG.debug("Calculating partition layout") part_definition = get_partition_layout(table_type, device_size, layout) @@ -634,11 +702,12 @@ def lookup_force_flag(fs): """ A force flag might be -F or -F, this look it up """ - flags = {'ext': '-F', - 'btrfs': '-f', - 'xfs': '-f', - 'reiserfs': '-f', - } + flags = { + 'ext': '-F', + 'btrfs': '-f', + 'xfs': '-f', + 'reiserfs': '-f', + } if 'ext' in fs.lower(): fs = 'ext' @@ -680,6 +749,9 @@ def mkfs(fs_cfg): fs_replace = fs_cfg.get('replace_fs', False) overwrite = fs_cfg.get('overwrite', False) + # ensure that we get a real device rather than a symbolic link + device = os.path.realpath(device) + # This allows you to define the default ephemeral or swap LOG.debug("Checking %s against default devices", device) @@ -754,10 +826,11 @@ def mkfs(fs_cfg): # Create the commands if fs_cmd: - fs_cmd = fs_cfg['cmd'] % {'label': label, - 'filesystem': fs_type, - 'device': device, - } + fs_cmd = fs_cfg['cmd'] % { + 'label': label, + 'filesystem': fs_type, + 'device': device, + } else: # Find the mkfs command mkfs_cmd = util.which("mkfs.%s" % fs_type) diff --git a/cloudinit/config/cc_emit_upstart.py b/cloudinit/config/cc_emit_upstart.py index 6d376184..86ae97ab 100644 --- a/cloudinit/config/cc_emit_upstart.py +++ b/cloudinit/config/cc_emit_upstart.py @@ -21,11 +21,31 @@ import os from cloudinit.settings import PER_ALWAYS +from cloudinit import log as logging from cloudinit import util frequency = PER_ALWAYS distros = ['ubuntu', 'debian'] +LOG = logging.getLogger(__name__) + + +def is_upstart_system(): + if not os.path.isfile("/sbin/initctl"): + LOG.debug("no /sbin/initctl located") + return False + + myenv = os.environ.copy() + if 'UPSTART_SESSION' in myenv: + del myenv['UPSTART_SESSION'] + check_cmd = ['initctl', 'version'] + try: + (out, err) = util.subp(check_cmd, env=myenv) + return 'upstart' in out + except util.ProcessExecutionError as e: + LOG.debug("'%s' returned '%s', not using upstart", + ' '.join(check_cmd), e.exit_code) + return False def handle(name, _cfg, cloud, log, args): @@ -34,10 +54,11 @@ def handle(name, _cfg, cloud, log, args): # Default to the 'cloud-config' # event for backwards compat. event_names = ['cloud-config'] - if not os.path.isfile("/sbin/initctl"): - log.debug(("Skipping module named %s," - " no /sbin/initctl located"), name) + + if not is_upstart_system(): + log.debug("not upstart system, '%s' disabled") return + cfgpath = cloud.paths.get_ipath_cur("cloud_config") for n in event_names: cmd = ['initctl', 'emit', str(n), 'CLOUD_CFG=%s' % cfgpath] diff --git a/cloudinit/config/cc_fan.py b/cloudinit/config/cc_fan.py new file mode 100644 index 00000000..39e3850e --- /dev/null +++ b/cloudinit/config/cc_fan.py @@ -0,0 +1,101 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2015 Canonical Ltd. +# +# Author: Scott Moser <scott.moser@canonical.com> +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. +""" +fan module allows configuration of Ubuntu Fan + https://wiki.ubuntu.com/FanNetworking + +Example config: + #cloud-config + fan: + config: | + # fan 240 + 10.0.0.0/8 eth0/16 dhcp + 10.0.0.0/8 eth1/16 dhcp off + # fan 241 + 241.0.0.0/8 eth0/16 dhcp + config_path: /etc/network/fan + +If cloud-init sees a 'fan' entry in cloud-config it will + a.) write 'config_path' with the contents + b.) install the package 'ubuntu-fan' if it is not installed + c.) ensure the service is started (or restarted if was previously running) +""" + +from cloudinit import log as logging +from cloudinit import util +from cloudinit.settings import PER_INSTANCE + +LOG = logging.getLogger(__name__) + +frequency = PER_INSTANCE + +BUILTIN_CFG = { + 'config': None, + 'config_path': '/etc/network/fan', +} + + +def stop_update_start(service, config_file, content, systemd=False): + if systemd: + cmds = {'stop': ['systemctl', 'stop', service], + 'start': ['systemctl', 'start', service], + 'enable': ['systemctl', 'enable', service]} + else: + cmds = {'stop': ['service', 'stop'], + 'start': ['service', 'start']} + + def run(cmd, msg): + try: + return util.subp(cmd, capture=True) + except util.ProcessExecutionError as e: + LOG.warn("failed: %s (%s): %s", service, cmd, e) + return False + + stop_failed = not run(cmds['stop'], msg='stop %s' % service) + if not content.endswith('\n'): + content += '\n' + util.write_file(config_file, content, omode="w") + + ret = run(cmds['start'], msg='start %s' % service) + if ret and stop_failed: + LOG.warn("success: %s started", service) + + if 'enable' in cmds: + ret = run(cmds['enable'], msg='enable %s' % service) + + return ret + + +def handle(name, cfg, cloud, log, args): + cfgin = cfg.get('fan') + if not cfgin: + cfgin = {} + mycfg = util.mergemanydict([cfgin, BUILTIN_CFG]) + + if not mycfg.get('config'): + LOG.debug("%s: no 'fan' config entry. disabling", name) + return + + util.write_file(mycfg.get('config_path'), mycfg.get('config'), omode="w") + distro = cloud.distro + if not util.which('fanctl'): + distro.install_packages(['ubuntu-fan']) + + stop_update_start( + service='ubuntu-fan', config_file=mycfg.get('config_path'), + content=mycfg.get('config'), systemd=distro.uses_systemd()) diff --git a/cloudinit/config/cc_final_message.py b/cloudinit/config/cc_final_message.py index b24294e4..4a51476f 100644 --- a/cloudinit/config/cc_final_message.py +++ b/cloudinit/config/cc_final_message.py @@ -26,9 +26,12 @@ from cloudinit.settings import PER_ALWAYS frequency = PER_ALWAYS -# Cheetah formated default message -FINAL_MESSAGE_DEF = ("Cloud-init v. ${version} finished at ${timestamp}." - " Datasource ${datasource}. Up ${uptime} seconds") +# Jinja formated default message +FINAL_MESSAGE_DEF = ( + "## template: jinja\n" + "Cloud-init v. {{version}} finished at {{timestamp}}." + " Datasource {{datasource}}. Up {{uptime}} seconds" +) def handle(_name, cfg, cloud, log, args): diff --git a/cloudinit/config/cc_growpart.py b/cloudinit/config/cc_growpart.py index f52c41f0..859d69f1 100644 --- a/cloudinit/config/cc_growpart.py +++ b/cloudinit/config/cc_growpart.py @@ -276,7 +276,7 @@ def handle(_name, cfg, _cloud, log, _args): log.debug("use ignore_growroot_disabled to ignore") return - devices = util.get_cfg_option_list(cfg, "devices", ["/"]) + devices = util.get_cfg_option_list(mycfg, "devices", ["/"]) if not len(devices): log.debug("growpart: empty device list") return diff --git a/cloudinit/config/cc_grub_dpkg.py b/cloudinit/config/cc_grub_dpkg.py index e3219e81..3c2d9985 100644 --- a/cloudinit/config/cc_grub_dpkg.py +++ b/cloudinit/config/cc_grub_dpkg.py @@ -25,19 +25,23 @@ from cloudinit import util distros = ['ubuntu', 'debian'] -def handle(_name, cfg, _cloud, log, _args): - idevs = None - idevs_empty = None +def handle(name, cfg, _cloud, log, _args): - if "grub-dpkg" in cfg: - idevs = util.get_cfg_option_str(cfg["grub-dpkg"], - "grub-pc/install_devices", None) - idevs_empty = util.get_cfg_option_str(cfg["grub-dpkg"], - "grub-pc/install_devices_empty", None) + mycfg = cfg.get("grub_dpkg", cfg.get("grub-dpkg", {})) + if not mycfg: + mycfg = {} + + enabled = mycfg.get('enabled', True) + if util.is_false(enabled): + log.debug("%s disabled by config grub_dpkg/enabled=%s", name, enabled) + return + + idevs = util.get_cfg_option_str(mycfg, "grub-pc/install_devices", None) + idevs_empty = util.get_cfg_option_str( + mycfg, "grub-pc/install_devices_empty", None) if ((os.path.exists("/dev/sda1") and not os.path.exists("/dev/sda")) or - (os.path.exists("/dev/xvda1") - and not os.path.exists("/dev/xvda"))): + (os.path.exists("/dev/xvda1") and not os.path.exists("/dev/xvda"))): if idevs is None: idevs = "" if idevs_empty is None: @@ -61,7 +65,7 @@ def handle(_name, cfg, _cloud, log, _args): (idevs, idevs_empty)) log.debug("Setting grub debconf-set-selections with '%s','%s'" % - (idevs, idevs_empty)) + (idevs, idevs_empty)) try: util.subp(['debconf-set-selections'], dconf_sel) diff --git a/cloudinit/config/cc_keys_to_console.py b/cloudinit/config/cc_keys_to_console.py index f1c1adff..aa844ee9 100644 --- a/cloudinit/config/cc_keys_to_console.py +++ b/cloudinit/config/cc_keys_to_console.py @@ -48,7 +48,7 @@ def handle(name, cfg, cloud, log, _args): "ssh_fp_console_blacklist", []) key_blacklist = util.get_cfg_option_list(cfg, "ssh_key_console_blacklist", - ["ssh-dss"]) + ["ssh-dss"]) try: cmd = [helper_path] diff --git a/cloudinit/config/cc_landscape.py b/cloudinit/config/cc_landscape.py index 8a709677..68fcb27f 100644 --- a/cloudinit/config/cc_landscape.py +++ b/cloudinit/config/cc_landscape.py @@ -20,7 +20,7 @@ import os -from StringIO import StringIO +from six import StringIO from configobj import ConfigObj @@ -38,12 +38,12 @@ distros = ['ubuntu'] # defaults taken from stock client.conf in landscape-client 11.07.1.1-0ubuntu2 LSC_BUILTIN_CFG = { - 'client': { - 'log_level': "info", - 'url': "https://landscape.canonical.com/message-system", - 'ping_url': "http://landscape.canonical.com/ping", - 'data_path': "/var/lib/landscape/client", - } + 'client': { + 'log_level': "info", + 'url': "https://landscape.canonical.com/message-system", + 'ping_url': "http://landscape.canonical.com/ping", + 'data_path': "/var/lib/landscape/client", + } } diff --git a/cloudinit/config/cc_locale.py b/cloudinit/config/cc_locale.py index 6feaae9d..bbe5fcae 100644 --- a/cloudinit/config/cc_locale.py +++ b/cloudinit/config/cc_locale.py @@ -27,9 +27,9 @@ def handle(name, cfg, cloud, log, args): else: locale = util.get_cfg_option_str(cfg, "locale", cloud.get_locale()) - if not locale: - log.debug(("Skipping module named %s, " - "no 'locale' configuration found"), name) + if util.is_false(locale): + log.debug("Skipping module named %s, disabled by config: %s", + name, locale) return log.debug("Setting locale to %s", locale) diff --git a/cloudinit/config/cc_lxd.py b/cloudinit/config/cc_lxd.py new file mode 100644 index 00000000..63b8fb63 --- /dev/null +++ b/cloudinit/config/cc_lxd.py @@ -0,0 +1,85 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2016 Canonical Ltd. +# +# Author: Wesley Wiedenmeier <wesley.wiedenmeier@canonical.com> +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. + +""" +This module initializes lxd using 'lxd init' + +Example config: + #cloud-config + lxd: + init: + network_address: <ip addr> + network_port: <port> + storage_backend: <zfs/dir> + storage_create_device: <dev> + storage_create_loop: <size> + storage_pool: <name> + trust_password: <password> +""" + +from cloudinit import util + + +def handle(name, cfg, cloud, log, args): + # Get config + lxd_cfg = cfg.get('lxd') + if not lxd_cfg: + log.debug("Skipping module named %s, not present or disabled by cfg") + return + if not isinstance(lxd_cfg, dict): + log.warn("lxd config must be a dictionary. found a '%s'", + type(lxd_cfg)) + return + + init_cfg = lxd_cfg.get('init') + if not isinstance(init_cfg, dict): + log.warn("lxd/init config must be a dictionary. found a '%s'", + type(init_cfg)) + init_cfg = {} + + if not init_cfg: + log.debug("no lxd/init config. disabled.") + return + + packages = [] + # Ensure lxd is installed + if not util.which("lxd"): + packages.append('lxd') + + # if using zfs, get the utils + if init_cfg.get("storage_backend") == "zfs" and not util.which('zfs'): + packages.append('zfs') + + if len(packages): + try: + cloud.distro.install_packages(packages) + except util.ProcessExecutionError as exc: + log.warn("failed to install packages %s: %s", packages, exc) + return + + # Set up lxd if init config is given + init_keys = ( + 'network_address', 'network_port', 'storage_backend', + 'storage_create_device', 'storage_create_loop', + 'storage_pool', 'trust_password') + cmd = ['lxd', 'init', '--auto'] + for k in init_keys: + if init_cfg.get(k): + cmd.extend(["--%s=%s" % + (k.replace('_', '-'), str(init_cfg[k]))]) + util.subp(cmd) diff --git a/cloudinit/config/cc_mcollective.py b/cloudinit/config/cc_mcollective.py index b670390d..425420ae 100644 --- a/cloudinit/config/cc_mcollective.py +++ b/cloudinit/config/cc_mcollective.py @@ -19,7 +19,8 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see <http://www.gnu.org/licenses/>. -from StringIO import StringIO +import six +from six import StringIO # Used since this can maintain comments # and doesn't need a top level section @@ -51,17 +52,17 @@ def handle(name, cfg, cloud, log, _args): # original file in order to be able to mix the rest up mcollective_config = ConfigObj(SERVER_CFG) # See: http://tiny.cc/jh9agw - for (cfg_name, cfg) in mcollective_cfg['conf'].iteritems(): + for (cfg_name, cfg) in mcollective_cfg['conf'].items(): if cfg_name == 'public-cert': - util.write_file(PUBCERT_FILE, cfg, mode=0644) + util.write_file(PUBCERT_FILE, cfg, mode=0o644) mcollective_config['plugin.ssl_server_public'] = PUBCERT_FILE mcollective_config['securityprovider'] = 'ssl' elif cfg_name == 'private-cert': - util.write_file(PRICERT_FILE, cfg, mode=0600) + util.write_file(PRICERT_FILE, cfg, mode=0o600) mcollective_config['plugin.ssl_server_private'] = PRICERT_FILE mcollective_config['securityprovider'] = 'ssl' else: - if isinstance(cfg, (basestring, str)): + if isinstance(cfg, six.string_types): # Just set it in the 'main' section mcollective_config[cfg_name] = cfg elif isinstance(cfg, (dict)): @@ -69,7 +70,7 @@ def handle(name, cfg, cloud, log, _args): # if it is needed and then add/or create items as needed if cfg_name not in mcollective_config.sections: mcollective_config[cfg_name] = {} - for (o, v) in cfg.iteritems(): + for (o, v) in cfg.items(): mcollective_config[cfg_name][o] = v else: # Otherwise just try to convert it to a string @@ -81,7 +82,7 @@ def handle(name, cfg, cloud, log, _args): contents = StringIO() mcollective_config.write(contents) contents = contents.getvalue() - util.write_file(SERVER_CFG, contents, mode=0644) + util.write_file(SERVER_CFG, contents, mode=0o644) # Start mcollective util.subp(['service', 'mcollective', 'start'], capture=False) diff --git a/cloudinit/config/cc_mounts.py b/cloudinit/config/cc_mounts.py index 1cb1e839..4fe3ee21 100644 --- a/cloudinit/config/cc_mounts.py +++ b/cloudinit/config/cc_mounts.py @@ -28,15 +28,15 @@ from cloudinit import type_utils from cloudinit import util # Shortname matches 'sda', 'sda1', 'xvda', 'hda', 'sdb', xvdb, vda, vdd1, sr0 -SHORTNAME_FILTER = r"^([x]{0,1}[shv]d[a-z][0-9]*|sr[0-9]+)$" -SHORTNAME = re.compile(SHORTNAME_FILTER) +DEVICE_NAME_FILTER = r"^([x]{0,1}[shv]d[a-z][0-9]*|sr[0-9]+)$" +DEVICE_NAME_RE = re.compile(DEVICE_NAME_FILTER) WS = re.compile("[%s]+" % (whitespace)) FSTAB_PATH = "/etc/fstab" LOG = logging.getLogger(__name__) -def is_mdname(name): +def is_meta_device_name(name): # return true if this is a metadata service name if name in ["ami", "root", "swap"]: return True @@ -48,6 +48,25 @@ def is_mdname(name): return False +def _get_nth_partition_for_device(device_path, partition_number): + potential_suffixes = [str(partition_number), 'p%s' % (partition_number,), + '-part%s' % (partition_number,)] + for suffix in potential_suffixes: + potential_partition_device = '%s%s' % (device_path, suffix) + if os.path.exists(potential_partition_device): + return potential_partition_device + return None + + +def _is_block_device(device_path, partition_path=None): + device_name = os.path.realpath(device_path).split('/')[-1] + sys_path = os.path.join('/sys/block/', device_name) + if partition_path is not None: + sys_path = os.path.join( + sys_path, os.path.realpath(partition_path).split('/')[-1]) + return os.path.exists(sys_path) + + def sanitize_devname(startname, transformer, log): log.debug("Attempting to determine the real name of %s", startname) @@ -58,21 +77,34 @@ def sanitize_devname(startname, transformer, log): devname = "ephemeral0" log.debug("Adjusted mount option from ephemeral to ephemeral0") - (blockdev, part) = util.expand_dotted_devname(devname) + device_path, partition_number = util.expand_dotted_devname(devname) - if is_mdname(blockdev): - orig = blockdev - blockdev = transformer(blockdev) - if not blockdev: + if is_meta_device_name(device_path): + orig = device_path + device_path = transformer(device_path) + if not device_path: return None - if not blockdev.startswith("/"): - blockdev = "/dev/%s" % blockdev - log.debug("Mapped metadata name %s to %s", orig, blockdev) + if not device_path.startswith("/"): + device_path = "/dev/%s" % (device_path,) + log.debug("Mapped metadata name %s to %s", orig, device_path) + else: + if DEVICE_NAME_RE.match(startname): + device_path = "/dev/%s" % (device_path,) + + partition_path = None + if partition_number is None: + partition_path = _get_nth_partition_for_device(device_path, 1) else: - if SHORTNAME.match(startname): - blockdev = "/dev/%s" % blockdev + partition_path = _get_nth_partition_for_device(device_path, + partition_number) + if partition_path is None: + return None - return devnode_for_dev_part(blockdev, part) + if _is_block_device(device_path, partition_path): + if partition_path is not None: + return partition_path + return device_path + return None def suggested_swapsize(memsize=None, maxsize=None, fsys=None): @@ -172,11 +204,12 @@ def setup_swapfile(fname, size=None, maxsize=None): try: util.ensure_dir(tdir) util.log_time(LOG.debug, msg, func=util.subp, - args=[['sh', '-c', - ('rm -f "$1" && umask 0066 && ' - 'dd if=/dev/zero "of=$1" bs=1M "count=$2" && ' - 'mkswap "$1" || { r=$?; rm -f "$1"; exit $r; }'), - 'setup_swap', fname, mbsize]]) + args=[['sh', '-c', + ('rm -f "$1" && umask 0066 && ' + '{ fallocate -l "${2}M" "$1" || ' + ' dd if=/dev/zero "of=$1" bs=1M "count=$2"; } && ' + 'mkswap "$1" || { r=$?; rm -f "$1"; exit $r; }'), + 'setup_swap', fname, mbsize]]) except Exception as e: raise IOError("Failed %s: %s" % (msg, e)) @@ -230,7 +263,11 @@ def handle_swapcfg(swapcfg): def handle(_name, cfg, cloud, log, _args): # fs_spec, fs_file, fs_vfstype, fs_mntops, fs-freq, fs_passno - defvals = [None, None, "auto", "defaults,nobootwait", "0", "2"] + def_mnt_opts = "defaults,nobootwait" + if cloud.distro.uses_systemd(): + def_mnt_opts = "defaults,nofail" + + defvals = [None, None, "auto", def_mnt_opts, "0", "2"] defvals = cfg.get("mount_default_fields", defvals) # these are our default set of mounts @@ -366,49 +403,3 @@ def handle(_name, cfg, cloud, log, _args): util.subp(("mount", "-a")) except: util.logexc(log, "Activating mounts via 'mount -a' failed") - - -def devnode_for_dev_part(device, partition): - """ - Find the name of the partition. While this might seem rather - straight forward, its not since some devices are '<device><partition>' - while others are '<device>p<partition>'. For example, /dev/xvda3 on EC2 - will present as /dev/xvda3p1 for the first partition since /dev/xvda3 is - a block device. - """ - if not os.path.exists(device): - return None - - short_name = os.path.basename(device) - sys_path = "/sys/block/%s" % short_name - - if not os.path.exists(sys_path): - LOG.debug("did not find entry for %s in /sys/block", short_name) - return None - - sys_long_path = sys_path + "/" + short_name - - if partition is not None: - partition = str(partition) - - if partition is None: - valid_mappings = [sys_long_path + "1", sys_long_path + "p1"] - elif partition != "0": - valid_mappings = [sys_long_path + "%s" % partition, - sys_long_path + "p%s" % partition] - else: - valid_mappings = [] - - for cdisk in valid_mappings: - if not os.path.exists(cdisk): - continue - - dev_path = "/dev/%s" % os.path.basename(cdisk) - if os.path.exists(dev_path): - return dev_path - - if partition is None or partition == "0": - return device - - LOG.debug("Did not fine partition %s for device %s", partition, device) - return None diff --git a/cloudinit/config/cc_phone_home.py b/cloudinit/config/cc_phone_home.py index 5bc68b83..18a7ddad 100644 --- a/cloudinit/config/cc_phone_home.py +++ b/cloudinit/config/cc_phone_home.py @@ -81,7 +81,7 @@ def handle(name, cfg, cloud, log, args): 'pub_key_ecdsa': '/etc/ssh/ssh_host_ecdsa_key.pub', } - for (n, path) in pubkeys.iteritems(): + for (n, path) in pubkeys.items(): try: all_keys[n] = util.load_file(path) except: @@ -99,7 +99,7 @@ def handle(name, cfg, cloud, log, args): # Get them read to be posted real_submit_keys = {} - for (k, v) in submit_keys.iteritems(): + for (k, v) in submit_keys.items(): if v is None: real_submit_keys[k] = 'N/A' else: diff --git a/cloudinit/config/cc_power_state_change.py b/cloudinit/config/cc_power_state_change.py index 09d37371..cc3f7f70 100644 --- a/cloudinit/config/cc_power_state_change.py +++ b/cloudinit/config/cc_power_state_change.py @@ -22,6 +22,7 @@ from cloudinit import util import errno import os import re +import six import subprocess import time @@ -48,10 +49,40 @@ def givecmdline(pid): return None +def check_condition(cond, log=None): + if isinstance(cond, bool): + if log: + log.debug("Static Condition: %s" % cond) + return cond + + pre = "check_condition command (%s): " % cond + try: + proc = subprocess.Popen(cond, shell=not isinstance(cond, list)) + proc.communicate() + ret = proc.returncode + if ret == 0: + if log: + log.debug(pre + "exited 0. condition met.") + return True + elif ret == 1: + if log: + log.debug(pre + "exited 1. condition not met.") + return False + else: + if log: + log.warn(pre + "unexpected exit %s. " % ret + + "do not apply change.") + return False + except Exception as e: + if log: + log.warn(pre + "Unexpected error: %s" % e) + return False + + def handle(_name, cfg, _cloud, log, _args): try: - (args, timeout) = load_power_state(cfg) + (args, timeout, condition) = load_power_state(cfg) if args is None: log.debug("no power_state provided. doing nothing") return @@ -59,6 +90,10 @@ def handle(_name, cfg, _cloud, log, _args): log.warn("%s Not performing power state change!" % str(e)) return + if condition is False: + log.debug("Condition was false. Will not perform state change.") + return + mypid = os.getpid() cmdline = givecmdline(mypid) @@ -70,8 +105,8 @@ def handle(_name, cfg, _cloud, log, _args): log.debug("After pid %s ends, will execute: %s" % (mypid, ' '.join(args))) - util.fork_cb(run_after_pid_gone, mypid, cmdline, timeout, log, execmd, - [args, devnull_fp]) + util.fork_cb(run_after_pid_gone, mypid, cmdline, timeout, log, + condition, execmd, [args, devnull_fp]) def load_power_state(cfg): @@ -80,7 +115,7 @@ def load_power_state(cfg): pstate = cfg.get('power_state') if pstate is None: - return (None, None) + return (None, None, None) if not isinstance(pstate, dict): raise TypeError("power_state is not a dict.") @@ -115,7 +150,10 @@ def load_power_state(cfg): raise ValueError("failed to convert timeout '%s' to float." % pstate['timeout']) - return (args, timeout) + condition = pstate.get("condition", True) + if not isinstance(condition, six.string_types + (list, bool)): + raise TypeError("condition type %s invalid. must be list, bool, str") + return (args, timeout, condition) def doexit(sysexit): @@ -133,7 +171,7 @@ def execmd(exe_args, output=None, data_in=None): doexit(ret) -def run_after_pid_gone(pid, pidcmdline, timeout, log, func, args): +def run_after_pid_gone(pid, pidcmdline, timeout, log, condition, func, args): # wait until pid, with /proc/pid/cmdline contents of pidcmdline # is no longer alive. After it is gone, or timeout has passed # execute func(args) @@ -175,4 +213,11 @@ def run_after_pid_gone(pid, pidcmdline, timeout, log, func, args): if log: log.debug(msg) + + try: + if not check_condition(condition, log): + return + except Exception as e: + fatal("Unexpected Exception when checking condition: %s" % e) + func(*args) diff --git a/cloudinit/config/cc_puppet.py b/cloudinit/config/cc_puppet.py index 471a1a8a..774d3322 100644 --- a/cloudinit/config/cc_puppet.py +++ b/cloudinit/config/cc_puppet.py @@ -18,7 +18,7 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see <http://www.gnu.org/licenses/>. -from StringIO import StringIO +from six import StringIO import os import socket @@ -36,8 +36,8 @@ def _autostart_puppet(log): # Set puppet to automatically start if os.path.exists('/etc/default/puppet'): util.subp(['sed', '-i', - '-e', 's/^START=.*/START=yes/', - '/etc/default/puppet'], capture=False) + '-e', 's/^START=.*/START=yes/', + '/etc/default/puppet'], capture=False) elif os.path.exists('/bin/systemctl'): util.subp(['/bin/systemctl', 'enable', 'puppet.service'], capture=False) @@ -65,7 +65,7 @@ def handle(name, cfg, cloud, log, _args): " doing nothing.")) elif install: log.debug(("Attempting to install puppet %s,"), - version if version else 'latest') + version if version else 'latest') cloud.distro.install_packages(('puppet', version)) # ... and then update the puppet configuration @@ -81,22 +81,22 @@ def handle(name, cfg, cloud, log, _args): cleaned_contents = '\n'.join(cleaned_lines) puppet_config.readfp(StringIO(cleaned_contents), filename=PUPPET_CONF_PATH) - for (cfg_name, cfg) in puppet_cfg['conf'].iteritems(): + for (cfg_name, cfg) in puppet_cfg['conf'].items(): # Cert configuration is a special case # Dump the puppet master ca certificate in the correct place if cfg_name == 'ca_cert': # Puppet ssl sub-directory isn't created yet # Create it with the proper permissions and ownership - util.ensure_dir(PUPPET_SSL_DIR, 0771) + util.ensure_dir(PUPPET_SSL_DIR, 0o771) util.chownbyname(PUPPET_SSL_DIR, 'puppet', 'root') util.ensure_dir(PUPPET_SSL_CERT_DIR) util.chownbyname(PUPPET_SSL_CERT_DIR, 'puppet', 'root') - util.write_file(PUPPET_SSL_CERT_PATH, str(cfg)) + util.write_file(PUPPET_SSL_CERT_PATH, cfg) util.chownbyname(PUPPET_SSL_CERT_PATH, 'puppet', 'root') else: # Iterate throug the config items, we'll use ConfigParser.set # to overwrite or create new items as needed - for (o, v) in cfg.iteritems(): + for (o, v) in cfg.items(): if o == 'certname': # Expand %f as the fqdn # TODO(harlowja) should this use the cloud fqdn?? diff --git a/cloudinit/config/cc_resizefs.py b/cloudinit/config/cc_resizefs.py index cbc07853..2a2a9f59 100644 --- a/cloudinit/config/cc_resizefs.py +++ b/cloudinit/config/cc_resizefs.py @@ -166,7 +166,7 @@ def handle(name, cfg, _cloud, log, args): func=do_resize, args=(resize_cmd, log)) else: util.log_time(logfunc=log.debug, msg="Resizing", - func=do_resize, args=(resize_cmd, log)) + func=do_resize, args=(resize_cmd, log)) action = 'Resized' if resize_root == NOBLOCK: diff --git a/cloudinit/config/cc_resolv_conf.py b/cloudinit/config/cc_resolv_conf.py index bbaa6c63..71d9e3a7 100644 --- a/cloudinit/config/cc_resolv_conf.py +++ b/cloudinit/config/cc_resolv_conf.py @@ -66,8 +66,8 @@ def generate_resolv_conf(template_fn, params, target_fname="/etc/resolv.conf"): false_flags = [] if 'options' in params: - for key, val in params['options'].iteritems(): - if type(val) == bool: + for key, val in params['options'].items(): + if isinstance(val, bool): if val: flags.append(key) else: diff --git a/cloudinit/config/cc_rh_subscription.py b/cloudinit/config/cc_rh_subscription.py new file mode 100644 index 00000000..6087c45c --- /dev/null +++ b/cloudinit/config/cc_rh_subscription.py @@ -0,0 +1,402 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2015 Red Hat, Inc. +# +# Author: Brent Baude <bbaude@redhat.com> +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. + +from cloudinit import util + + +def handle(_name, cfg, _cloud, log, _args): + sm = SubscriptionManager(cfg) + sm.log = log + if not sm.is_registered: + try: + verify, verify_msg = sm._verify_keys() + if verify is not True: + raise SubscriptionError(verify_msg) + cont = sm.rhn_register() + if not cont: + raise SubscriptionError("Registration failed or did not " + "run completely") + + # Splitting up the registration, auto-attach, and servicelevel + # commands because the error codes, messages from subman are not + # specific enough. + + # Attempt to change the service level + if sm.auto_attach and sm.servicelevel is not None: + if not sm._set_service_level(): + raise SubscriptionError("Setting of service-level " + "failed") + else: + sm.log.debug("Completed auto-attach with service level") + elif sm.auto_attach: + if not sm._set_auto_attach(): + raise SubscriptionError("Setting auto-attach failed") + else: + sm.log.debug("Completed auto-attach") + + if sm.pools is not None: + if not isinstance(sm.pools, list): + pool_fail = "Pools must in the format of a list" + raise SubscriptionError(pool_fail) + + return_stat = sm.addPool(sm.pools) + if not return_stat: + raise SubscriptionError("Unable to attach pools {0}" + .format(sm.pools)) + if (sm.enable_repo is not None) or (sm.disable_repo is not None): + return_stat = sm.update_repos(sm.enable_repo, sm.disable_repo) + if not return_stat: + raise SubscriptionError("Unable to add or remove repos") + sm.log_success("rh_subscription plugin completed successfully") + except SubscriptionError as e: + sm.log_warn(str(e)) + sm.log_warn("rh_subscription plugin did not complete successfully") + else: + sm.log_success("System is already registered") + + +class SubscriptionError(Exception): + pass + + +class SubscriptionManager(object): + valid_rh_keys = ['org', 'activation-key', 'username', 'password', + 'disable-repo', 'enable-repo', 'add-pool', + 'rhsm-baseurl', 'server-hostname', + 'auto-attach', 'service-level'] + + def __init__(self, cfg): + self.cfg = cfg + self.rhel_cfg = self.cfg.get('rh_subscription', {}) + self.rhsm_baseurl = self.rhel_cfg.get('rhsm-baseurl') + self.server_hostname = self.rhel_cfg.get('server-hostname') + self.pools = self.rhel_cfg.get('add-pool') + self.activation_key = self.rhel_cfg.get('activation-key') + self.org = self.rhel_cfg.get('org') + self.userid = self.rhel_cfg.get('username') + self.password = self.rhel_cfg.get('password') + self.auto_attach = self.rhel_cfg.get('auto-attach') + self.enable_repo = self.rhel_cfg.get('enable-repo') + self.disable_repo = self.rhel_cfg.get('disable-repo') + self.servicelevel = self.rhel_cfg.get('service-level') + self.subman = ['subscription-manager'] + self.is_registered = self._is_registered() + + def log_success(self, msg): + '''Simple wrapper for logging info messages. Useful for unittests''' + self.log.info(msg) + + def log_warn(self, msg): + '''Simple wrapper for logging warning messages. Useful for unittests''' + self.log.warn(msg) + + def _verify_keys(self): + ''' + Checks that the keys in the rh_subscription dict from the user-data + are what we expect. + ''' + + for k in self.rhel_cfg: + if k not in self.valid_rh_keys: + bad_key = "{0} is not a valid key for rh_subscription. "\ + "Valid keys are: "\ + "{1}".format(k, ', '.join(self.valid_rh_keys)) + return False, bad_key + + # Check for bad auto-attach value + if (self.auto_attach is not None) and \ + not (util.is_true(self.auto_attach) or + util.is_false(self.auto_attach)): + not_bool = "The key auto-attach must be a boolean value "\ + "(True/False " + return False, not_bool + + if (self.servicelevel is not None) and ((not self.auto_attach) or + (util.is_false(str(self.auto_attach)))): + no_auto = ("The service-level key must be used in conjunction " + "with the auto-attach key. Please re-run with " + "auto-attach: True") + return False, no_auto + return True, None + + def _is_registered(self): + ''' + Checks if the system is already registered and returns + True if so, else False + ''' + cmd = ['identity'] + + try: + self._sub_man_cli(cmd) + except util.ProcessExecutionError: + return False + + return True + + def _sub_man_cli(self, cmd, logstring_val=False): + ''' + Uses the prefered cloud-init subprocess def of util.subp + and runs subscription-manager. Breaking this to a + separate function for later use in mocking and unittests + ''' + cmd = self.subman + cmd + return util.subp(cmd, logstring=logstring_val) + + def rhn_register(self): + ''' + Registers the system by userid and password or activation key + and org. Returns True when successful False when not. + ''' + + if (self.activation_key is not None) and (self.org is not None): + # register by activation key + cmd = ['register', '--activationkey={0}'. + format(self.activation_key), '--org={0}'.format(self.org)] + + # If the baseurl and/or server url are passed in, we register + # with them. + + if self.rhsm_baseurl is not None: + cmd.append("--baseurl={0}".format(self.rhsm_baseurl)) + + if self.server_hostname is not None: + cmd.append("--serverurl={0}".format(self.server_hostname)) + + try: + return_out, return_err = self._sub_man_cli(cmd, + logstring_val=True) + except util.ProcessExecutionError as e: + if e.stdout == "": + self.log_warn("Registration failed due " + "to: {0}".format(e.stderr)) + return False + + elif (self.userid is not None) and (self.password is not None): + # register by username and password + cmd = ['register', '--username={0}'.format(self.userid), + '--password={0}'.format(self.password)] + + # If the baseurl and/or server url are passed in, we register + # with them. + + if self.rhsm_baseurl is not None: + cmd.append("--baseurl={0}".format(self.rhsm_baseurl)) + + if self.server_hostname is not None: + cmd.append("--serverurl={0}".format(self.server_hostname)) + + # Attempting to register the system only + try: + return_out, return_err = self._sub_man_cli(cmd, + logstring_val=True) + except util.ProcessExecutionError as e: + if e.stdout == "": + self.log_warn("Registration failed due " + "to: {0}".format(e.stderr)) + return False + + else: + self.log_warn("Unable to register system due to incomplete " + "information.") + self.log_warn("Use either activationkey and org *or* userid " + "and password") + return False + + reg_id = return_out.split("ID: ")[1].rstrip() + self.log.debug("Registered successfully with ID {0}".format(reg_id)) + return True + + def _set_service_level(self): + cmd = ['attach', '--auto', '--servicelevel={0}' + .format(self.servicelevel)] + + try: + return_out, return_err = self._sub_man_cli(cmd) + except util.ProcessExecutionError as e: + if e.stdout.rstrip() != '': + for line in e.stdout.split("\n"): + if line is not '': + self.log_warn(line) + else: + self.log_warn("Setting the service level failed with: " + "{0}".format(e.stderr.strip())) + return False + for line in return_out.split("\n"): + if line is not "": + self.log.debug(line) + return True + + def _set_auto_attach(self): + cmd = ['attach', '--auto'] + try: + return_out, return_err = self._sub_man_cli(cmd) + except util.ProcessExecutionError: + self.log_warn("Auto-attach failed with: " + "{0}]".format(return_err.strip())) + return False + for line in return_out.split("\n"): + if line is not "": + self.log.debug(line) + return True + + def _getPools(self): + ''' + Gets the list pools for the active subscription and returns them + in list form. + ''' + available = [] + consumed = [] + + # Get all available pools + cmd = ['list', '--available', '--pool-only'] + results, errors = self._sub_man_cli(cmd) + available = (results.rstrip()).split("\n") + + # Get all consumed pools + cmd = ['list', '--consumed', '--pool-only'] + results, errors = self._sub_man_cli(cmd) + consumed = (results.rstrip()).split("\n") + + return available, consumed + + def _getRepos(self): + ''' + Obtains the current list of active yum repositories and returns + them in list form. + ''' + + cmd = ['repos', '--list-enabled'] + return_out, return_err = self._sub_man_cli(cmd) + active_repos = [] + for repo in return_out.split("\n"): + if "Repo ID:" in repo: + active_repos.append((repo.split(':')[1]).strip()) + + cmd = ['repos', '--list-disabled'] + return_out, return_err = self._sub_man_cli(cmd) + + inactive_repos = [] + for repo in return_out.split("\n"): + if "Repo ID:" in repo: + inactive_repos.append((repo.split(':')[1]).strip()) + return active_repos, inactive_repos + + def addPool(self, pools): + ''' + Takes a list of subscription pools and "attaches" them to the + current subscription + ''' + + # An empty list was passed + if len(pools) == 0: + self.log.debug("No pools to attach") + return True + + pool_available, pool_consumed = self._getPools() + pool_list = [] + cmd = ['attach'] + for pool in pools: + if (pool not in pool_consumed) and (pool in pool_available): + pool_list.append('--pool={0}'.format(pool)) + else: + self.log_warn("Pool {0} is not available".format(pool)) + if len(pool_list) > 0: + cmd.extend(pool_list) + try: + self._sub_man_cli(cmd) + self.log.debug("Attached the following pools to your " + "system: %s" % (", ".join(pool_list)) + .replace('--pool=', '')) + return True + except util.ProcessExecutionError as e: + self.log_warn("Unable to attach pool {0} " + "due to {1}".format(pool, e)) + return False + + def update_repos(self, erepos, drepos): + ''' + Takes a list of yum repo ids that need to be disabled or enabled; then + it verifies if they are already enabled or disabled and finally + executes the action to disable or enable + ''' + + if (erepos is not None) and (not isinstance(erepos, list)): + self.log_warn("Repo IDs must in the format of a list.") + return False + + if (drepos is not None) and (not isinstance(drepos, list)): + self.log_warn("Repo IDs must in the format of a list.") + return False + + # Bail if both lists are not populated + if (len(erepos) == 0) and (len(drepos) == 0): + self.log.debug("No repo IDs to enable or disable") + return True + + active_repos, inactive_repos = self._getRepos() + # Creating a list of repoids to be enabled + enable_list = [] + enable_list_fail = [] + for repoid in erepos: + if (repoid in inactive_repos): + enable_list.append("--enable={0}".format(repoid)) + else: + enable_list_fail.append(repoid) + + # Creating a list of repoids to be disabled + disable_list = [] + disable_list_fail = [] + for repoid in drepos: + if repoid in active_repos: + disable_list.append("--disable={0}".format(repoid)) + else: + disable_list_fail.append(repoid) + + # Logging any repos that are already enabled or disabled + if len(enable_list_fail) > 0: + for fail in enable_list_fail: + # Check if the repo exists or not + if fail in active_repos: + self.log.debug("Repo {0} is already enabled".format(fail)) + else: + self.log_warn("Repo {0} does not appear to " + "exist".format(fail)) + if len(disable_list_fail) > 0: + for fail in disable_list_fail: + self.log.debug("Repo {0} not disabled " + "because it is not enabled".format(fail)) + + cmd = ['repos'] + if len(enable_list) > 0: + cmd.extend(enable_list) + if len(disable_list) > 0: + cmd.extend(disable_list) + + try: + self._sub_man_cli(cmd) + except util.ProcessExecutionError as e: + self.log_warn("Unable to alter repos due to {0}".format(e)) + return False + + if len(enable_list) > 0: + self.log.debug("Enabled the following repos: %s" % + (", ".join(enable_list)).replace('--enable=', '')) + if len(disable_list) > 0: + self.log.debug("Disabled the following repos: %s" % + (", ".join(disable_list)).replace('--disable=', '')) + return True diff --git a/cloudinit/config/cc_rightscale_userdata.py b/cloudinit/config/cc_rightscale_userdata.py index 7d2ec10a..0ecf3a4d 100644 --- a/cloudinit/config/cc_rightscale_userdata.py +++ b/cloudinit/config/cc_rightscale_userdata.py @@ -41,7 +41,7 @@ from cloudinit.settings import PER_INSTANCE from cloudinit import url_helper as uhelp from cloudinit import util -from urlparse import parse_qs +from six.moves.urllib_parse import parse_qs frequency = PER_INSTANCE @@ -58,7 +58,7 @@ def handle(name, _cfg, cloud, log, _args): try: mdict = parse_qs(ud) - if mdict or MY_HOOKNAME not in mdict: + if not mdict or MY_HOOKNAME not in mdict: log.debug(("Skipping module %s, " "did not find %s in parsed" " raw userdata"), name, MY_HOOKNAME) @@ -82,7 +82,7 @@ def handle(name, _cfg, cloud, log, _args): resp = uhelp.readurl(url) # Ensure its a valid http response (and something gotten) if resp.ok() and resp.contents: - util.write_file(fname, str(resp), mode=0700) + util.write_file(fname, resp, mode=0o700) wrote_fns.append(fname) except Exception as e: captured_excps.append(e) diff --git a/cloudinit/config/cc_rsyslog.py b/cloudinit/config/cc_rsyslog.py index 57486edc..b8642d65 100644 --- a/cloudinit/config/cc_rsyslog.py +++ b/cloudinit/config/cc_rsyslog.py @@ -17,37 +17,166 @@ # # You should have received a copy of the GNU General Public License # along with this program. If not, see <http://www.gnu.org/licenses/>. +""" +rsyslog module allows configuration of syslog logging via rsyslog +Configuration is done under the cloud-config top level 'rsyslog'. + +Under 'rsyslog' you can define: + - configs: [default=[]] + this is a list. entries in it are a string or a dictionary. + each entry has 2 parts: + * content + * filename + if the entry is a string, then it is assigned to 'content'. + for each entry, content is written to the provided filename. + if filename is not provided, its default is read from 'config_filename' + + Content here can be any valid rsyslog configuration. No format + specific format is enforced. + + For simply logging to an existing remote syslog server, via udp: + configs: ["*.* @192.168.1.1"] + + - remotes: [default={}] + This is a dictionary of name / value pairs. + In comparison to 'config's, it is more focused in that it only supports + remote syslog configuration. It is not rsyslog specific, and could + convert to other syslog implementations. + + Each entry in remotes is a 'name' and a 'value'. + * name: an string identifying the entry. good practice would indicate + using a consistent and identifiable string for the producer. + For example, the MAAS service could use 'maas' as the key. + * value consists of the following parts: + * optional filter for log messages + default if not present: *.* + * optional leading '@' or '@@' (indicates udp or tcp respectively). + default if not present (udp): @ + This is rsyslog format for that. if not present, is '@'. + * ipv4 or ipv6 or hostname + ipv6 addresses must be in [::1] format. (@[fd00::1]:514) + * optional port + port defaults to 514 + + - config_filename: [default=20-cloud-config.conf] + this is the file name to use if none is provided in a config entry. + + - config_dir: [default=/etc/rsyslog.d] + this directory is used for filenames that are not absolute paths. + + - service_reload_command: [default="auto"] + this command is executed if files have been written and thus the syslog + daemon needs to be told. + +Note, since cloud-init 0.5 a legacy version of rsyslog config has been +present and is still supported. See below for the mappings between old +value and new value: + old value -> new value + 'rsyslog' -> rsyslog/configs + 'rsyslog_filename' -> rsyslog/config_filename + 'rsyslog_dir' -> rsyslog/config_dir + +the legacy config does not support 'service_reload_command'. + +Example config: + #cloud-config + rsyslog: + configs: + - "*.* @@192.158.1.1" + - content: "*.* @@192.0.2.1:10514" + filename: 01-example.conf + - content: | + *.* @@syslogd.example.com + remotes: + maas: "192.168.1.1" + juju: "10.0.4.1" + config_dir: config_dir + config_filename: config_filename + service_reload_command: [your, syslog, restart, command] + +Example Legacy config: + #cloud-config + rsyslog: + - "*.* @@192.158.1.1" + rsyslog_dir: /etc/rsyslog-config.d/ + rsyslog_filename: 99-local.conf +""" import os +import re +import six +from cloudinit import log as logging from cloudinit import util DEF_FILENAME = "20-cloud-config.conf" DEF_DIR = "/etc/rsyslog.d" +DEF_RELOAD = "auto" +DEF_REMOTES = {} +KEYNAME_CONFIGS = 'configs' +KEYNAME_FILENAME = 'config_filename' +KEYNAME_DIR = 'config_dir' +KEYNAME_RELOAD = 'service_reload_command' +KEYNAME_LEGACY_FILENAME = 'rsyslog_filename' +KEYNAME_LEGACY_DIR = 'rsyslog_dir' +KEYNAME_REMOTES = 'remotes' -def handle(name, cfg, cloud, log, _args): - # rsyslog: - # - "*.* @@192.158.1.1" - # - content: "*.* @@192.0.2.1:10514" - # - filename: 01-examplecom.conf - # content: | - # *.* @@syslogd.example.com - - # process 'rsyslog' - if 'rsyslog' not in cfg: - log.debug(("Skipping module named %s," - " no 'rsyslog' key in configuration"), name) - return +LOG = logging.getLogger(__name__) - def_dir = cfg.get('rsyslog_dir', DEF_DIR) - def_fname = cfg.get('rsyslog_filename', DEF_FILENAME) +COMMENT_RE = re.compile(r'[ ]*[#]+[ ]*') +HOST_PORT_RE = re.compile( + r'^(?P<proto>[@]{0,2})' + '(([[](?P<bracket_addr>[^\]]*)[\]])|(?P<addr>[^:]*))' + '([:](?P<port>[0-9]+))?$') + +def reload_syslog(command=DEF_RELOAD, systemd=False): + service = 'rsyslog' + if command == DEF_RELOAD: + if systemd: + cmd = ['systemctl', 'reload-or-try-restart', service] + else: + cmd = ['service', service, 'restart'] + else: + cmd = command + util.subp(cmd, capture=True) + + +def load_config(cfg): + # return an updated config with entries of the correct type + # support converting the old top level format into new format + mycfg = cfg.get('rsyslog', {}) + + if isinstance(cfg.get('rsyslog'), list): + mycfg = {KEYNAME_CONFIGS: cfg.get('rsyslog')} + if KEYNAME_LEGACY_FILENAME in cfg: + mycfg[KEYNAME_FILENAME] = cfg[KEYNAME_LEGACY_FILENAME] + if KEYNAME_LEGACY_DIR in cfg: + mycfg[KEYNAME_DIR] = cfg[KEYNAME_LEGACY_DIR] + + fillup = ( + (KEYNAME_CONFIGS, [], list), + (KEYNAME_DIR, DEF_DIR, six.string_types), + (KEYNAME_FILENAME, DEF_FILENAME, six.string_types), + (KEYNAME_RELOAD, DEF_RELOAD, six.string_types + (list,)), + (KEYNAME_REMOTES, DEF_REMOTES, dict)) + + for key, default, vtypes in fillup: + if key not in mycfg or not isinstance(mycfg[key], vtypes): + mycfg[key] = default + + return mycfg + + +def apply_rsyslog_changes(configs, def_fname, cfg_dir): + # apply the changes in 'configs' to the paths in def_fname and cfg_dir + # return a list of the files changed files = [] - for i, ent in enumerate(cfg['rsyslog']): + for cur_pos, ent in enumerate(configs): if isinstance(ent, dict): if "content" not in ent: - log.warn("No 'content' entry in config entry %s", i + 1) + LOG.warn("No 'content' entry in config entry %s", cur_pos + 1) continue content = ent['content'] filename = ent.get("filename", def_fname) @@ -57,11 +186,10 @@ def handle(name, cfg, cloud, log, _args): filename = filename.strip() if not filename: - log.warn("Entry %s has an empty filename", i + 1) + LOG.warn("Entry %s has an empty filename", cur_pos + 1) continue - if not filename.startswith("/"): - filename = os.path.join(def_dir, filename) + filename = os.path.join(cfg_dir, filename) # Truncate filename first time you see it omode = "ab" @@ -70,27 +198,164 @@ def handle(name, cfg, cloud, log, _args): files.append(filename) try: - contents = "%s\n" % (content) - util.write_file(filename, contents, omode=omode) + endl = "" + if not content.endswith("\n"): + endl = "\n" + util.write_file(filename, content + endl, omode=omode) except Exception: - util.logexc(log, "Failed to write to %s", filename) + util.logexc(LOG, "Failed to write to %s", filename) + + return files + + +def parse_remotes_line(line, name=None): + try: + data, comment = COMMENT_RE.split(line) + comment = comment.strip() + except ValueError: + data, comment = (line, None) + + toks = data.strip().split() + match = None + if len(toks) == 1: + host_port = data + elif len(toks) == 2: + match, host_port = toks + else: + raise ValueError("line had multiple spaces: %s" % data) + + toks = HOST_PORT_RE.match(host_port) + + if not toks: + raise ValueError("Invalid host specification '%s'" % host_port) + + proto = toks.group('proto') + addr = toks.group('addr') or toks.group('bracket_addr') + port = toks.group('port') + + if addr.startswith("[") and not addr.endswith("]"): + raise ValueError("host spec had invalid brackets: %s" % addr) + + if comment and not name: + name = comment + + t = SyslogRemotesLine(name=name, match=match, proto=proto, + addr=addr, port=port) + t.validate() + return t + + +class SyslogRemotesLine(object): + def __init__(self, name=None, match=None, proto=None, addr=None, + port=None): + if not match: + match = "*.*" + self.name = name + self.match = match + if not proto: + proto = "udp" + if proto == "@": + proto = "udp" + elif proto == "@@": + proto = "tcp" + self.proto = proto + + self.addr = addr + if port: + self.port = int(port) + else: + self.port = None + + def validate(self): + if self.port: + try: + int(self.port) + except ValueError: + raise ValueError("port '%s' is not an integer" % self.port) + + if not self.addr: + raise ValueError("address is required") + + def __repr__(self): + return "[name=%s match=%s proto=%s address=%s port=%s]" % ( + self.name, self.match, self.proto, self.addr, self.port + ) + + def __str__(self): + buf = self.match + " " + if self.proto == "udp": + buf += "@" + elif self.proto == "tcp": + buf += "@@" + + if ":" in self.addr: + buf += "[" + self.addr + "]" + else: + buf += self.addr + + if self.port: + buf += ":%s" % self.port + + if self.name: + buf += " # %s" % self.name + return buf + + +def remotes_to_rsyslog_cfg(remotes, header=None, footer=None): + if not remotes: + return None + lines = [] + if header is not None: + lines.append(header) + for name, line in remotes.items(): + if not line: + continue + try: + lines.append(str(parse_remotes_line(line, name=name))) + except ValueError as e: + LOG.warn("failed loading remote %s: %s [%s]", name, line, e) + if footer is not None: + lines.append(footer) + return '\n'.join(lines) + "\n" + + +def handle(name, cfg, cloud, log, _args): + if 'rsyslog' not in cfg: + log.debug(("Skipping module named %s," + " no 'rsyslog' key in configuration"), name) + return + + mycfg = load_config(cfg) + configs = mycfg[KEYNAME_CONFIGS] + + if mycfg[KEYNAME_REMOTES]: + configs.append( + remotes_to_rsyslog_cfg( + mycfg[KEYNAME_REMOTES], + header="# begin remotes", + footer="# end remotes", + )) + + if not mycfg['configs']: + log.debug("Empty config rsyslog['configs'], nothing to do") + return + + changes = apply_rsyslog_changes( + configs=mycfg[KEYNAME_CONFIGS], + def_fname=mycfg[KEYNAME_FILENAME], + cfg_dir=mycfg[KEYNAME_DIR]) + + if not changes: + log.debug("restart of syslog not necessary, no changes made") + return - # Attempt to restart syslogd - restarted = False try: - # If this config module is running at cloud-init time - # (before rsyslog is running) we don't actually have to - # restart syslog. - # - # Upstart actually does what we want here, in that it doesn't - # start a service that wasn't running already on 'restart' - # it will also return failure on the attempt, so 'restarted' - # won't get set. - log.debug("Restarting rsyslog") - util.subp(['service', 'rsyslog', 'restart']) - restarted = True - except Exception: - util.logexc(log, "Failed restarting rsyslog") + restarted = reload_syslog( + command=mycfg[KEYNAME_RELOAD], + systemd=cloud.distro.uses_systemd()), + except util.ProcessExecutionError as e: + restarted = False + log.warn("Failed to reload syslog", e) if restarted: # This only needs to run if we *actually* restarted @@ -98,4 +363,4 @@ def handle(name, cfg, cloud, log, _args): cloud.cycle_logging() # This should now use rsyslog if # the logging was setup to use it... - log.debug("%s configured %s files", name, files) + log.debug("%s configured %s files", name, changes) diff --git a/cloudinit/config/cc_runcmd.py b/cloudinit/config/cc_runcmd.py index 598c3a3e..66dc3363 100644 --- a/cloudinit/config/cc_runcmd.py +++ b/cloudinit/config/cc_runcmd.py @@ -33,6 +33,6 @@ def handle(name, cfg, cloud, log, _args): cmd = cfg["runcmd"] try: content = util.shellify(cmd) - util.write_file(out_fn, content, 0700) + util.write_file(out_fn, content, 0o700) except: util.logexc(log, "Failed to shellify %s into file %s", cmd, out_fn) diff --git a/cloudinit/config/cc_salt_minion.py b/cloudinit/config/cc_salt_minion.py index 53013dcb..f5786a31 100644 --- a/cloudinit/config/cc_salt_minion.py +++ b/cloudinit/config/cc_salt_minion.py @@ -47,7 +47,7 @@ def handle(name, cfg, cloud, log, _args): # ... copy the key pair if specified if 'public_key' in salt_cfg and 'private_key' in salt_cfg: pki_dir = salt_cfg.get('pki_dir', '/etc/salt/pki') - with util.umask(077): + with util.umask(0o77): util.ensure_dir(pki_dir) pub_name = os.path.join(pki_dir, 'minion.pub') pem_name = os.path.join(pki_dir, 'minion.pem') diff --git a/cloudinit/config/cc_seed_random.py b/cloudinit/config/cc_seed_random.py index 49a6b3e8..1b011216 100644 --- a/cloudinit/config/cc_seed_random.py +++ b/cloudinit/config/cc_seed_random.py @@ -21,7 +21,8 @@ import base64 import os -from StringIO import StringIO + +from six import BytesIO from cloudinit.settings import PER_INSTANCE from cloudinit import log as logging @@ -33,13 +34,13 @@ LOG = logging.getLogger(__name__) def _decode(data, encoding=None): if not data: - return '' + return b'' if not encoding or encoding.lower() in ['raw']: - return data + return util.encode_text(data) elif encoding.lower() in ['base64', 'b64']: return base64.b64decode(data) elif encoding.lower() in ['gzip', 'gz']: - return util.decomp_gzip(data, quiet=False) + return util.decomp_gzip(data, quiet=False, decode=None) else: raise IOError("Unknown random_seed encoding: %s" % (encoding)) @@ -64,9 +65,9 @@ def handle_random_seed_command(command, required, env=None): def handle(name, cfg, cloud, log, _args): mycfg = cfg.get('random_seed', {}) seed_path = mycfg.get('file', '/dev/urandom') - seed_data = mycfg.get('data', '') + seed_data = mycfg.get('data', b'') - seed_buf = StringIO() + seed_buf = BytesIO() if seed_data: seed_buf.write(_decode(seed_data, encoding=mycfg.get('encoding'))) @@ -74,7 +75,7 @@ def handle(name, cfg, cloud, log, _args): # openstack meta_data.json metadata = cloud.datasource.metadata if metadata and 'random_seed' in metadata: - seed_buf.write(metadata['random_seed']) + seed_buf.write(util.encode_text(metadata['random_seed'])) seed_data = seed_buf.getvalue() if len(seed_data): @@ -82,7 +83,7 @@ def handle(name, cfg, cloud, log, _args): len(seed_data), seed_path) util.append_file(seed_path, seed_data) - command = mycfg.get('command', ['pollinate', '-q']) + command = mycfg.get('command', None) req = mycfg.get('command_required', False) try: env = os.environ.copy() diff --git a/cloudinit/config/cc_set_hostname.py b/cloudinit/config/cc_set_hostname.py index 5d7f4331..f43d8d5a 100644 --- a/cloudinit/config/cc_set_hostname.py +++ b/cloudinit/config/cc_set_hostname.py @@ -24,7 +24,7 @@ from cloudinit import util def handle(name, cfg, cloud, log, _args): if util.get_cfg_option_bool(cfg, "preserve_hostname", False): log.debug(("Configuration option 'preserve_hostname' is set," - " not setting the hostname in module %s"), name) + " not setting the hostname in module %s"), name) return (hostname, fqdn) = util.get_hostname_fqdn(cfg, cloud) diff --git a/cloudinit/config/cc_set_passwords.py b/cloudinit/config/cc_set_passwords.py index 4ca85e21..58e1b713 100644 --- a/cloudinit/config/cc_set_passwords.py +++ b/cloudinit/config/cc_set_passwords.py @@ -28,11 +28,11 @@ from cloudinit import distros as ds from cloudinit import ssh_util from cloudinit import util -from string import letters, digits +from string import ascii_letters, digits # We are removing certain 'painful' letters/numbers -PW_SET = (letters.translate(None, 'loLOI') + - digits.translate(None, '01')) +PW_SET = (''.join([x for x in ascii_letters + digits + if x not in 'loLOI01'])) def handle(_name, cfg, cloud, log, args): @@ -45,8 +45,6 @@ def handle(_name, cfg, cloud, log, args): password = util.get_cfg_option_str(cfg, "password", None) expire = True - pw_auth = "no" - change_pwauth = False plist = None if 'chpasswd' in cfg: @@ -104,11 +102,24 @@ def handle(_name, cfg, cloud, log, args): change_pwauth = False pw_auth = None if 'ssh_pwauth' in cfg: - change_pwauth = True if util.is_true(cfg['ssh_pwauth']): + change_pwauth = True pw_auth = 'yes' - if util.is_false(cfg['ssh_pwauth']): + elif util.is_false(cfg['ssh_pwauth']): + change_pwauth = True pw_auth = 'no' + elif str(cfg['ssh_pwauth']).lower() == 'unchanged': + log.debug('Leaving auth line unchanged') + change_pwauth = False + elif not str(cfg['ssh_pwauth']).strip(): + log.debug('Leaving auth line unchanged') + change_pwauth = False + elif not cfg['ssh_pwauth']: + log.debug('Leaving auth line unchanged') + change_pwauth = False + else: + msg = 'Unrecognized value %s for ssh_pwauth' % cfg['ssh_pwauth'] + util.logexc(log, msg) if change_pwauth: replaced_auth = False diff --git a/cloudinit/config/cc_snappy.py b/cloudinit/config/cc_snappy.py new file mode 100644 index 00000000..fa9d54a0 --- /dev/null +++ b/cloudinit/config/cc_snappy.py @@ -0,0 +1,304 @@ +# vi: ts=4 expandtab +# +""" +snappy modules allows configuration of snappy. +Example config: + #cloud-config + snappy: + system_snappy: auto + ssh_enabled: auto + packages: [etcd, pkg2.smoser] + config: + pkgname: + key2: value2 + pkg2: + key1: value1 + packages_dir: '/writable/user-data/cloud-init/snaps' + + - ssh_enabled: + This controls the system's ssh service. The default value is 'auto'. + True: enable ssh service + False: disable ssh service + auto: enable ssh service if either ssh keys have been provided + or user has requested password authentication (ssh_pwauth). + + - snap installation and config + The above would install 'etcd', and then install 'pkg2.smoser' with a + '<config-file>' argument where 'config-file' has 'config-blob' inside it. + If 'pkgname' is installed already, then 'snappy config pkgname <file>' + will be called where 'file' has 'pkgname-config-blob' as its content. + + Entries in 'config' can be namespaced or non-namespaced for a package. + In either case, the config provided to snappy command is non-namespaced. + The package name is provided as it appears. + + If 'packages_dir' has files in it that end in '.snap', then they are + installed. Given 3 files: + <packages_dir>/foo.snap + <packages_dir>/foo.config + <packages_dir>/bar.snap + cloud-init will invoke: + snappy install <packages_dir>/foo.snap <packages_dir>/foo.config + snappy install <packages_dir>/bar.snap + + Note, that if provided a 'config' entry for 'ubuntu-core', then + cloud-init will invoke: snappy config ubuntu-core <config> + Allowing you to configure ubuntu-core in this way. +""" + +from cloudinit import log as logging +from cloudinit import util +from cloudinit.settings import PER_INSTANCE + +import glob +import tempfile +import os + +LOG = logging.getLogger(__name__) + +frequency = PER_INSTANCE +SNAPPY_CMD = "snappy" +NAMESPACE_DELIM = '.' + +BUILTIN_CFG = { + 'packages': [], + 'packages_dir': '/writable/user-data/cloud-init/snaps', + 'ssh_enabled': "auto", + 'system_snappy': "auto", + 'config': {}, +} + + +def parse_filename(fname): + fname = os.path.basename(fname) + fname_noext = fname.rpartition(".")[0] + name = fname_noext.partition("_")[0] + shortname = name.partition(".")[0] + return(name, shortname, fname_noext) + + +def get_fs_package_ops(fspath): + if not fspath: + return [] + ops = [] + for snapfile in sorted(glob.glob(os.path.sep.join([fspath, '*.snap']))): + (name, shortname, fname_noext) = parse_filename(snapfile) + cfg = None + for cand in (fname_noext, name, shortname): + fpcand = os.path.sep.join([fspath, cand]) + ".config" + if os.path.isfile(fpcand): + cfg = fpcand + break + ops.append(makeop('install', name, config=None, + path=snapfile, cfgfile=cfg)) + return ops + + +def makeop(op, name, config=None, path=None, cfgfile=None): + return({'op': op, 'name': name, 'config': config, 'path': path, + 'cfgfile': cfgfile}) + + +def get_package_config(configs, name): + # load the package's config from the configs dict. + # prefer full-name entry (config-example.canonical) + # over short name entry (config-example) + if name in configs: + return configs[name] + return configs.get(name.partition(NAMESPACE_DELIM)[0]) + + +def get_package_ops(packages, configs, installed=None, fspath=None): + # get the install an config operations that should be done + if installed is None: + installed = read_installed_packages() + short_installed = [p.partition(NAMESPACE_DELIM)[0] for p in installed] + + if not packages: + packages = [] + if not configs: + configs = {} + + ops = [] + ops += get_fs_package_ops(fspath) + + for name in packages: + ops.append(makeop('install', name, get_package_config(configs, name))) + + to_install = [f['name'] for f in ops] + short_to_install = [f['name'].partition(NAMESPACE_DELIM)[0] for f in ops] + + for name in configs: + if name in to_install: + continue + shortname = name.partition(NAMESPACE_DELIM)[0] + if shortname in short_to_install: + continue + if name in installed or shortname in short_installed: + ops.append(makeop('config', name, + config=get_package_config(configs, name))) + + # prefer config entries to filepath entries + for op in ops: + if op['op'] != 'install' or not op['cfgfile']: + continue + name = op['name'] + fromcfg = get_package_config(configs, op['name']) + if fromcfg: + LOG.debug("preferring configs[%(name)s] over '%(cfgfile)s'", op) + op['cfgfile'] = None + op['config'] = fromcfg + + return ops + + +def render_snap_op(op, name, path=None, cfgfile=None, config=None): + if op not in ('install', 'config'): + raise ValueError("cannot render op '%s'" % op) + + shortname = name.partition(NAMESPACE_DELIM)[0] + try: + cfg_tmpf = None + if config is not None: + # input to 'snappy config packagename' must have nested data. odd. + # config: + # packagename: + # config + # Note, however, we do not touch config files on disk. + nested_cfg = {'config': {shortname: config}} + (fd, cfg_tmpf) = tempfile.mkstemp() + os.write(fd, util.yaml_dumps(nested_cfg).encode()) + os.close(fd) + cfgfile = cfg_tmpf + + cmd = [SNAPPY_CMD, op] + if op == 'install': + if path: + cmd.append("--allow-unauthenticated") + cmd.append(path) + else: + cmd.append(name) + if cfgfile: + cmd.append(cfgfile) + elif op == 'config': + cmd += [name, cfgfile] + + util.subp(cmd) + + finally: + if cfg_tmpf: + os.unlink(cfg_tmpf) + + +def read_installed_packages(): + ret = [] + for (name, date, version, dev) in read_pkg_data(): + if dev: + ret.append(NAMESPACE_DELIM.join([name, dev])) + else: + ret.append(name) + return ret + + +def read_pkg_data(): + out, err = util.subp([SNAPPY_CMD, "list"]) + pkg_data = [] + for line in out.splitlines()[1:]: + toks = line.split(sep=None, maxsplit=3) + if len(toks) == 3: + (name, date, version) = toks + dev = None + else: + (name, date, version, dev) = toks + pkg_data.append((name, date, version, dev,)) + return pkg_data + + +def disable_enable_ssh(enabled): + LOG.debug("setting enablement of ssh to: %s", enabled) + # do something here that would enable or disable + not_to_be_run = "/etc/ssh/sshd_not_to_be_run" + if enabled: + util.del_file(not_to_be_run) + # this is an indempotent operation + util.subp(["systemctl", "start", "ssh"]) + else: + # this is an indempotent operation + util.subp(["systemctl", "stop", "ssh"]) + util.write_file(not_to_be_run, "cloud-init\n") + + +def system_is_snappy(): + # channel.ini is configparser loadable. + # snappy will move to using /etc/system-image/config.d/*.ini + # this is certainly not a perfect test, but good enough for now. + content = util.load_file("/etc/system-image/channel.ini", quiet=True) + if 'ubuntu-core' in content.lower(): + return True + if os.path.isdir("/etc/system-image/config.d/"): + return True + return False + + +def set_snappy_command(): + global SNAPPY_CMD + if util.which("snappy-go"): + SNAPPY_CMD = "snappy-go" + else: + SNAPPY_CMD = "snappy" + LOG.debug("snappy command is '%s'", SNAPPY_CMD) + + +def handle(name, cfg, cloud, log, args): + cfgin = cfg.get('snappy') + if not cfgin: + cfgin = {} + mycfg = util.mergemanydict([cfgin, BUILTIN_CFG]) + + sys_snappy = str(mycfg.get("system_snappy", "auto")) + if util.is_false(sys_snappy): + LOG.debug("%s: System is not snappy. disabling", name) + return + + if sys_snappy.lower() == "auto" and not(system_is_snappy()): + LOG.debug("%s: 'auto' mode, and system not snappy", name) + return + + set_snappy_command() + + pkg_ops = get_package_ops(packages=mycfg['packages'], + configs=mycfg['config'], + fspath=mycfg['packages_dir']) + + fails = [] + for pkg_op in pkg_ops: + try: + render_snap_op(**pkg_op) + except Exception as e: + fails.append((pkg_op, e,)) + LOG.warn("'%s' failed for '%s': %s", + pkg_op['op'], pkg_op['name'], e) + + # Default to disabling SSH + ssh_enabled = mycfg.get('ssh_enabled', "auto") + + # If the user has not explicitly enabled or disabled SSH, then enable it + # when password SSH authentication is requested or there are SSH keys + if ssh_enabled == "auto": + user_ssh_keys = cloud.get_public_ssh_keys() or None + password_auth_enabled = cfg.get('ssh_pwauth', False) + if user_ssh_keys: + LOG.debug("Enabling SSH, ssh keys found in datasource") + ssh_enabled = True + elif cfg.get('ssh_authorized_keys'): + LOG.debug("Enabling SSH, ssh keys found in config") + elif password_auth_enabled: + LOG.debug("Enabling SSH, password authentication requested") + ssh_enabled = True + elif ssh_enabled not in (True, False): + LOG.warn("Unknown value '%s' in ssh_enabled", ssh_enabled) + + disable_enable_ssh(ssh_enabled) + + if fails: + raise Exception("failed to install/configure snaps") diff --git a/cloudinit/config/cc_ssh.py b/cloudinit/config/cc_ssh.py index 4c76581c..d24e43c0 100644 --- a/cloudinit/config/cc_ssh.py +++ b/cloudinit/config/cc_ssh.py @@ -20,6 +20,7 @@ import glob import os +import sys # Ensure this is aliased to a name not 'distros' # since the module attribute 'distros' @@ -29,30 +30,23 @@ from cloudinit import distros as ds from cloudinit import ssh_util from cloudinit import util -DISABLE_ROOT_OPTS = ("no-port-forwarding,no-agent-forwarding," -"no-X11-forwarding,command=\"echo \'Please login as the user \\\"$USER\\\" " -"rather than the user \\\"root\\\".\';echo;sleep 10\"") - -KEY_2_FILE = { - "rsa_private": ("/etc/ssh/ssh_host_rsa_key", 0600), - "rsa_public": ("/etc/ssh/ssh_host_rsa_key.pub", 0644), - "dsa_private": ("/etc/ssh/ssh_host_dsa_key", 0600), - "dsa_public": ("/etc/ssh/ssh_host_dsa_key.pub", 0644), - "ecdsa_private": ("/etc/ssh/ssh_host_ecdsa_key", 0600), - "ecdsa_public": ("/etc/ssh/ssh_host_ecdsa_key.pub", 0644), -} - -PRIV_2_PUB = { - 'rsa_private': 'rsa_public', - 'dsa_private': 'dsa_public', - 'ecdsa_private': 'ecdsa_public', -} +DISABLE_ROOT_OPTS = ( + "no-port-forwarding,no-agent-forwarding," + "no-X11-forwarding,command=\"echo \'Please login as the user \\\"$USER\\\"" + " rather than the user \\\"root\\\".\';echo;sleep 10\"") -KEY_GEN_TPL = 'o=$(ssh-keygen -yf "%s") && echo "$o" root@localhost > "%s"' +GENERATE_KEY_NAMES = ['rsa', 'dsa', 'ecdsa', 'ed25519'] +KEY_FILE_TPL = '/etc/ssh/ssh_host_%s_key' -GENERATE_KEY_NAMES = ['rsa', 'dsa', 'ecdsa'] +CONFIG_KEY_TO_FILE = {} +PRIV_TO_PUB = {} +for k in GENERATE_KEY_NAMES: + CONFIG_KEY_TO_FILE.update({"%s_private" % k: (KEY_FILE_TPL % k, 0o600)}) + CONFIG_KEY_TO_FILE.update( + {"%s_public" % k: (KEY_FILE_TPL % k + ".pub", 0o600)}) + PRIV_TO_PUB["%s_private" % k] = "%s_public" % k -KEY_FILE_TPL = '/etc/ssh/ssh_host_%s_key' +KEY_GEN_TPL = 'o=$(ssh-keygen -yf "%s") && echo "$o" root@localhost > "%s"' def handle(_name, cfg, cloud, log, _args): @@ -68,16 +62,16 @@ def handle(_name, cfg, cloud, log, _args): if "ssh_keys" in cfg: # if there are keys in cloud-config, use them - for (key, val) in cfg["ssh_keys"].iteritems(): - if key in KEY_2_FILE: - tgt_fn = KEY_2_FILE[key][0] - tgt_perms = KEY_2_FILE[key][1] + for (key, val) in cfg["ssh_keys"].items(): + if key in CONFIG_KEY_TO_FILE: + tgt_fn = CONFIG_KEY_TO_FILE[key][0] + tgt_perms = CONFIG_KEY_TO_FILE[key][1] util.write_file(tgt_fn, val, tgt_perms) - for (priv, pub) in PRIV_2_PUB.iteritems(): + for (priv, pub) in PRIV_TO_PUB.items(): if pub in cfg['ssh_keys'] or priv not in cfg['ssh_keys']: continue - pair = (KEY_2_FILE[priv][0], KEY_2_FILE[pub][0]) + pair = (CONFIG_KEY_TO_FILE[priv][0], CONFIG_KEY_TO_FILE[pub][0]) cmd = ['sh', '-xc', KEY_GEN_TPL % pair] try: # TODO(harlowja): Is this guard needed? @@ -92,18 +86,28 @@ def handle(_name, cfg, cloud, log, _args): genkeys = util.get_cfg_option_list(cfg, 'ssh_genkeytypes', GENERATE_KEY_NAMES) + lang_c = os.environ.copy() + lang_c['LANG'] = 'C' for keytype in genkeys: keyfile = KEY_FILE_TPL % (keytype) + if os.path.exists(keyfile): + continue util.ensure_dir(os.path.dirname(keyfile)) - if not os.path.exists(keyfile): - cmd = ['ssh-keygen', '-t', keytype, '-N', '', '-f', keyfile] + cmd = ['ssh-keygen', '-t', keytype, '-N', '', '-f', keyfile] + + # TODO(harlowja): Is this guard needed? + with util.SeLinuxGuard("/etc/ssh", recursive=True): try: - # TODO(harlowja): Is this guard needed? - with util.SeLinuxGuard("/etc/ssh", recursive=True): - util.subp(cmd, capture=False) - except: - util.logexc(log, "Failed generating key type %s to " - "file %s", keytype, keyfile) + out, err = util.subp(cmd, capture=True, env=lang_c) + sys.stdout.write(util.decode_binary(out)) + except util.ProcessExecutionError as e: + err = util.decode_binary(e.stderr).lower() + if (e.exit_code == 1 and + err.lower().startswith("unknown key")): + log.debug("ssh-keygen: unknown key type '%s'", keytype) + else: + util.logexc(log, "Failed generating key type %s to " + "file %s", keytype, keyfile) try: (users, _groups) = ds.normalize_users_groups(cfg, cloud.distro) diff --git a/cloudinit/config/cc_ssh_authkey_fingerprints.py b/cloudinit/config/cc_ssh_authkey_fingerprints.py index 51580633..6ce831bc 100644 --- a/cloudinit/config/cc_ssh_authkey_fingerprints.py +++ b/cloudinit/config/cc_ssh_authkey_fingerprints.py @@ -32,7 +32,7 @@ from cloudinit import util def _split_hash(bin_hash): split_up = [] - for i in xrange(0, len(bin_hash), 2): + for i in range(0, len(bin_hash), 2): split_up.append(bin_hash[i:i + 2]) return split_up diff --git a/cloudinit/config/cc_update_etc_hosts.py b/cloudinit/config/cc_update_etc_hosts.py index d3dd1f32..15703efe 100644 --- a/cloudinit/config/cc_update_etc_hosts.py +++ b/cloudinit/config/cc_update_etc_hosts.py @@ -41,10 +41,10 @@ def handle(name, cfg, cloud, log, _args): if not tpl_fn_name: raise RuntimeError(("No hosts template could be" " found for distro %s") % - (cloud.distro.osfamily)) + (cloud.distro.osfamily)) templater.render_to_file(tpl_fn_name, '/etc/hosts', - {'hostname': hostname, 'fqdn': fqdn}) + {'hostname': hostname, 'fqdn': fqdn}) elif manage_hosts == "localhost": (hostname, fqdn) = util.get_hostname_fqdn(cfg, cloud) @@ -57,4 +57,4 @@ def handle(name, cfg, cloud, log, _args): cloud.distro.update_etc_hosts(hostname, fqdn) else: log.debug(("Configuration option 'manage_etc_hosts' is not set," - " not managing /etc/hosts in module %s"), name) + " not managing /etc/hosts in module %s"), name) diff --git a/cloudinit/config/cc_update_hostname.py b/cloudinit/config/cc_update_hostname.py index e396ba13..5b78afe1 100644 --- a/cloudinit/config/cc_update_hostname.py +++ b/cloudinit/config/cc_update_hostname.py @@ -29,7 +29,7 @@ frequency = PER_ALWAYS def handle(name, cfg, cloud, log, _args): if util.get_cfg_option_bool(cfg, "preserve_hostname", False): log.debug(("Configuration option 'preserve_hostname' is set," - " not updating the hostname in module %s"), name) + " not updating the hostname in module %s"), name) return (hostname, fqdn) = util.get_hostname_fqdn(cfg, cloud) diff --git a/cloudinit/config/cc_write_files.py b/cloudinit/config/cc_write_files.py index a73d6f4e..4b03ea91 100644 --- a/cloudinit/config/cc_write_files.py +++ b/cloudinit/config/cc_write_files.py @@ -18,6 +18,7 @@ import base64 import os +import six from cloudinit.settings import PER_INSTANCE from cloudinit import util @@ -25,7 +26,7 @@ from cloudinit import util frequency = PER_INSTANCE DEFAULT_OWNER = "root:root" -DEFAULT_PERMS = 0644 +DEFAULT_PERMS = 0o644 UNKNOWN_ENC = 'text/plain' @@ -79,7 +80,7 @@ def write_files(name, files, log): def decode_perms(perm, default, log): try: - if isinstance(perm, (int, long, float)): + if isinstance(perm, six.integer_types + (float,)): # Just 'downcast' it (if a float) return int(perm) else: diff --git a/cloudinit/config/cc_yum_add_repo.py b/cloudinit/config/cc_yum_add_repo.py index 0d836f28..64fba869 100644 --- a/cloudinit/config/cc_yum_add_repo.py +++ b/cloudinit/config/cc_yum_add_repo.py @@ -18,9 +18,10 @@ import os -from cloudinit import util - import configobj +import six + +from cloudinit import util def _canonicalize_id(repo_id): @@ -37,7 +38,7 @@ def _format_repo_value(val): # Can handle 'lists' in certain cases # See: http://bit.ly/Qqrf1t return "\n ".join([_format_repo_value(v) for v in val]) - if not isinstance(val, (basestring, str)): + if not isinstance(val, six.string_types): return str(val) return val @@ -91,7 +92,7 @@ def handle(name, cfg, _cloud, log, _args): for req_field in ['baseurl']: if req_field not in repo_config: log.warn(("Repository %s does not contain a %s" - " configuration 'required' entry"), + " configuration 'required' entry"), repo_id, req_field) missing_required += 1 if not missing_required: |