summaryrefslogtreecommitdiff
path: root/cloudinit
diff options
context:
space:
mode:
Diffstat (limited to 'cloudinit')
-rw-r--r--cloudinit/config/cc_debug.py4
-rw-r--r--cloudinit/config/cc_growpart.py46
-rw-r--r--cloudinit/config/cc_power_state_change.py29
-rw-r--r--cloudinit/config/cc_resizefs.py40
-rw-r--r--cloudinit/distros/__init__.py1
-rw-r--r--cloudinit/distros/freebsd.py259
-rw-r--r--cloudinit/distros/net_util.py163
-rw-r--r--cloudinit/distros/rhel.py3
-rw-r--r--cloudinit/distros/rhel_util.py88
-rw-r--r--cloudinit/distros/sles.py3
-rw-r--r--cloudinit/ec2_utils.py26
-rw-r--r--cloudinit/netinfo.py44
-rw-r--r--cloudinit/sources/DataSourceConfigDrive.py9
-rw-r--r--cloudinit/sources/DataSourceNoCloud.py92
-rw-r--r--cloudinit/sources/DataSourceSmartOS.py101
-rw-r--r--cloudinit/sources/__init__.py2
-rw-r--r--cloudinit/url_helper.py51
-rw-r--r--cloudinit/user_data.py3
-rw-r--r--cloudinit/util.py151
19 files changed, 901 insertions, 214 deletions
diff --git a/cloudinit/config/cc_debug.py b/cloudinit/config/cc_debug.py
index cfd31fa1..7219b0f8 100644
--- a/cloudinit/config/cc_debug.py
+++ b/cloudinit/config/cc_debug.py
@@ -14,10 +14,10 @@
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
-from StringIO import StringIO
-from cloudinit import util
from cloudinit import type_utils
+from cloudinit import util
import copy
+from StringIO import StringIO
def _make_header(text):
diff --git a/cloudinit/config/cc_growpart.py b/cloudinit/config/cc_growpart.py
index 6bddf847..f52c41f0 100644
--- a/cloudinit/config/cc_growpart.py
+++ b/cloudinit/config/cc_growpart.py
@@ -114,6 +114,41 @@ class ResizeGrowPart(object):
return (before, get_size(partdev))
+class ResizeGpart(object):
+ def available(self):
+ if not util.which('gpart'):
+ return False
+ return True
+
+ def resize(self, diskdev, partnum, partdev):
+ """
+ GPT disks store metadata at the beginning (primary) and at the
+ end (secondary) of the disk. When launching an image with a
+ larger disk compared to the original image, the secondary copy
+ is lost. Thus, the metadata will be marked CORRUPT, and need to
+ be recovered.
+ """
+ try:
+ util.subp(["gpart", "recover", diskdev])
+ except util.ProcessExecutionError as e:
+ if e.exit_code != 0:
+ util.logexc(LOG, "Failed: gpart recover %s", diskdev)
+ raise ResizeFailedException(e)
+
+ before = get_size(partdev)
+ try:
+ util.subp(["gpart", "resize", "-i", partnum, diskdev])
+ except util.ProcessExecutionError as e:
+ util.logexc(LOG, "Failed: gpart resize -i %s %s", partnum, diskdev)
+ raise ResizeFailedException(e)
+
+ # Since growing the FS requires a reboot, make sure we reboot
+ # first when this module has finished.
+ open('/var/run/reboot-required', 'a').close()
+
+ return (before, get_size(partdev))
+
+
def get_size(filename):
fd = os.open(filename, os.O_RDONLY)
try:
@@ -132,6 +167,12 @@ def device_part_info(devpath):
bname = os.path.basename(rpath)
syspath = "/sys/class/block/%s" % bname
+ # FreeBSD doesn't know of sysfs so just get everything we need from
+ # the device, like /dev/vtbd0p2.
+ if util.system_info()["platform"].startswith('FreeBSD'):
+ m = re.search('^(/dev/.+)p([0-9])$', devpath)
+ return (m.group(1), m.group(2))
+
if not os.path.exists(syspath):
raise ValueError("%s had no syspath (%s)" % (devpath, syspath))
@@ -182,7 +223,8 @@ def resize_devices(resizer, devices):
"stat of '%s' failed: %s" % (blockdev, e),))
continue
- if not stat.S_ISBLK(statret.st_mode):
+ if (not stat.S_ISBLK(statret.st_mode) and
+ not stat.S_ISCHR(statret.st_mode)):
info.append((devent, RESIZE.SKIPPED,
"device '%s' not a block device" % blockdev,))
continue
@@ -255,4 +297,4 @@ def handle(_name, cfg, _cloud, log, _args):
else:
log.debug("'%s' %s: %s" % (entry, action, msg))
-RESIZERS = (('growpart', ResizeGrowPart),)
+RESIZERS = (('growpart', ResizeGrowPart), ('gpart', ResizeGpart))
diff --git a/cloudinit/config/cc_power_state_change.py b/cloudinit/config/cc_power_state_change.py
index e3150808..561c5abd 100644
--- a/cloudinit/config/cc_power_state_change.py
+++ b/cloudinit/config/cc_power_state_change.py
@@ -22,6 +22,7 @@ from cloudinit import util
import errno
import os
import re
+import signal
import subprocess
import time
@@ -30,6 +31,24 @@ frequency = PER_INSTANCE
EXIT_FAIL = 254
+def givecmdline(pid):
+ # Returns the cmdline for the given process id. In Linux we can use procfs
+ # for this but on BSD there is /usr/bin/procstat.
+ try:
+ # Example output from procstat -c 1
+ # PID COMM ARGS
+ # 1 init /bin/init --
+ if util.system_info()["platform"].startswith('FreeBSD'):
+ (output, _err) = util.subp(['procstat', '-c', str(pid)])
+ line = output.splitlines()[1]
+ m = re.search('\d+ (\w|\.|-)+\s+(/\w.+)', line)
+ return m.group(2)
+ else:
+ return util.load_file("/proc/%s/cmdline" % pid)
+ except IOError:
+ return None
+
+
def handle(_name, cfg, _cloud, log, _args):
try:
@@ -42,8 +61,8 @@ def handle(_name, cfg, _cloud, log, _args):
return
mypid = os.getpid()
- cmdline = util.load_file("/proc/%s/cmdline" % mypid)
+ cmdline = givecmdline(mypid)
if not cmdline:
log.warn("power_state: failed to get cmdline of current process")
return
@@ -119,8 +138,6 @@ def run_after_pid_gone(pid, pidcmdline, timeout, log, func, args):
msg = None
end_time = time.time() + timeout
- cmdline_f = "/proc/%s/cmdline" % pid
-
def fatal(msg):
if log:
log.warn(msg)
@@ -134,16 +151,14 @@ def run_after_pid_gone(pid, pidcmdline, timeout, log, func, args):
break
try:
- cmdline = ""
- with open(cmdline_f) as fp:
- cmdline = fp.read()
+ cmdline = givecmdline(pid)
if cmdline != pidcmdline:
msg = "cmdline changed for %s [now: %s]" % (pid, cmdline)
break
except IOError as ioerr:
if ioerr.errno in known_errnos:
- msg = "pidfile '%s' gone [%d]" % (cmdline_f, ioerr.errno)
+ msg = "pidfile gone [%d]" % ioerr.errno
else:
fatal("IOError during wait: %s" % ioerr)
break
diff --git a/cloudinit/config/cc_resizefs.py b/cloudinit/config/cc_resizefs.py
index 56040fdd..be406034 100644
--- a/cloudinit/config/cc_resizefs.py
+++ b/cloudinit/config/cc_resizefs.py
@@ -39,6 +39,10 @@ def _resize_ext(mount_point, devpth): # pylint: disable=W0613
def _resize_xfs(mount_point, devpth): # pylint: disable=W0613
return ('xfs_growfs', devpth)
+
+def _resize_ufs(mount_point, devpth): # pylint: disable=W0613
+ return ('growfs', devpth)
+
# Do not use a dictionary as these commands should be able to be used
# for multiple filesystem types if possible, e.g. one command for
# ext2, ext3 and ext4.
@@ -46,11 +50,31 @@ RESIZE_FS_PREFIXES_CMDS = [
('btrfs', _resize_btrfs),
('ext', _resize_ext),
('xfs', _resize_xfs),
+ ('ufs', _resize_ufs),
]
NOBLOCK = "noblock"
+def rootdev_from_cmdline(cmdline):
+ found = None
+ for tok in cmdline.split():
+ if tok.startswith("root="):
+ found = tok[5:]
+ break
+ if found is None:
+ return None
+
+ if found.startswith("/dev/"):
+ return found
+ if found.startswith("LABEL="):
+ return "/dev/disk/by-label/" + found[len("LABEL="):]
+ if found.startswith("UUID="):
+ return "/dev/disk/by-uuid/" + found[len("UUID="):]
+
+ return "/dev/" + found
+
+
def handle(name, cfg, _cloud, log, args):
if len(args) != 0:
resize_root = args[0]
@@ -78,10 +102,20 @@ def handle(name, cfg, _cloud, log, args):
info = "dev=%s mnt_point=%s path=%s" % (devpth, mount_point, resize_what)
log.debug("resize_info: %s" % info)
+ container = util.is_container()
+
+ if (devpth == "/dev/root" and not os.path.exists(devpth) and
+ not container):
+ devpth = rootdev_from_cmdline(util.get_cmdline())
+ if devpth is None:
+ log.warn("Unable to find device '/dev/root'")
+ return
+ log.debug("Converted /dev/root to '%s' per kernel cmdline", devpth)
+
try:
statret = os.stat(devpth)
except OSError as exc:
- if util.is_container() and exc.errno == errno.ENOENT:
+ if container and exc.errno == errno.ENOENT:
log.debug("Device '%s' did not exist in container. "
"cannot resize: %s" % (devpth, info))
elif exc.errno == errno.ENOENT:
@@ -91,8 +125,8 @@ def handle(name, cfg, _cloud, log, args):
raise exc
return
- if not stat.S_ISBLK(statret.st_mode):
- if util.is_container():
+ if not stat.S_ISBLK(statret.st_mode) and not stat.S_ISCHR(statret.st_mode):
+ if container:
log.debug("device '%s' not a block device in container."
" cannot resize: %s" % (devpth, info))
else:
diff --git a/cloudinit/distros/__init__.py b/cloudinit/distros/__init__.py
index 74e95797..46b67fa3 100644
--- a/cloudinit/distros/__init__.py
+++ b/cloudinit/distros/__init__.py
@@ -39,6 +39,7 @@ from cloudinit.distros.parsers import hosts
OSFAMILIES = {
'debian': ['debian', 'ubuntu'],
'redhat': ['fedora', 'rhel'],
+ 'freebsd': ['freebsd'],
'suse': ['sles']
}
diff --git a/cloudinit/distros/freebsd.py b/cloudinit/distros/freebsd.py
new file mode 100644
index 00000000..afb502c9
--- /dev/null
+++ b/cloudinit/distros/freebsd.py
@@ -0,0 +1,259 @@
+# vi: ts=4 expandtab
+#
+# Copyright (C) 2014 Harm Weites
+#
+# Author: Harm Weites <harm@weites.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 3, as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+from StringIO import StringIO
+
+import re
+
+from cloudinit import distros
+from cloudinit import helpers
+from cloudinit import log as logging
+from cloudinit import ssh_util
+from cloudinit import util
+
+LOG = logging.getLogger(__name__)
+
+
+class Distro(distros.Distro):
+ rc_conf_fn = "/etc/rc.conf"
+ login_conf_fn = '/etc/login.conf'
+ login_conf_fn_bak = '/etc/login.conf.orig'
+
+ def __init__(self, name, cfg, paths):
+ distros.Distro.__init__(self, name, cfg, paths)
+ # This will be used to restrict certain
+ # calls from repeatly happening (when they
+ # should only happen say once per instance...)
+ self._runner = helpers.Runners(paths)
+ self.osfamily = 'freebsd'
+
+ # Updates a key in /etc/rc.conf.
+ def updatercconf(self, key, value):
+ LOG.debug("updatercconf: %s => %s", key, value)
+ conf = self.loadrcconf()
+ config_changed = False
+ for item in conf:
+ if item == key and conf[item] != value:
+ conf[item] = value
+ LOG.debug("[rc.conf]: Value %s for key %s needs to be changed",
+ value, key)
+ config_changed = True
+
+ if config_changed:
+ LOG.debug("Writing new %s file", self.rc_conf_fn)
+ buf = StringIO()
+ for keyval in conf.items():
+ buf.write("%s=%s\n" % keyval)
+ util.write_file(self.rc_conf_fn, buf.getvalue())
+
+ # Load the contents of /etc/rc.conf and store all keys in a dict.
+ def loadrcconf(self):
+ conf = {}
+ lines = util.load_file(self.rc_conf_fn).splitlines()
+ for line in lines:
+ tok = line.split('=')
+ conf[tok[0]] = tok[1].rstrip()
+ return conf
+
+ def readrcconf(self, key):
+ conf = self.loadrcconf()
+ try:
+ val = conf[key]
+ except KeyError:
+ val = None
+ return val
+
+ def _read_system_hostname(self):
+ sys_hostname = self._read_hostname()
+ return ('rc.conf', sys_hostname)
+
+ def _read_hostname(self, filename, default=None):
+ hostname = None
+ try:
+ hostname = self.readrcconf('hostname')
+ except IOError:
+ pass
+ if not hostname:
+ return default
+ return hostname
+
+ def _select_hostname(self, hostname, fqdn):
+ if not hostname:
+ return fqdn
+ return hostname
+
+ def _write_hostname(self, hostname, filename):
+ self.updatercconf('hostname', hostname)
+
+ def create_group(self, name, members):
+ group_add_cmd = ['pw', '-n', name]
+ if util.is_group(name):
+ LOG.warn("Skipping creation of existing group '%s'", name)
+ else:
+ try:
+ util.subp(group_add_cmd)
+ LOG.info("Created new group %s", name)
+ except Exception as e:
+ util.logexc(LOG, "Failed to create group %s", name)
+ raise e
+
+ if len(members) > 0:
+ for member in members:
+ if not util.is_user(member):
+ LOG.warn("Unable to add group member '%s' to group '%s'"
+ "; user does not exist.", member, name)
+ continue
+ try:
+ util.subp(['pw', 'usermod', '-n', name, '-G', member])
+ LOG.info("Added user '%s' to group '%s'", member, name)
+ except Exception:
+ util.logexc(LOG, "Failed to add user '%s' to group '%s'",
+ member, name)
+
+ def add_user(self, name, **kwargs):
+ if util.is_user(name):
+ LOG.info("User %s already exists, skipping.", name)
+ return False
+
+ adduser_cmd = ['pw', 'useradd', '-n', name]
+ log_adduser_cmd = ['pw', 'useradd', '-n', name]
+
+ adduser_opts = {
+ "homedir": '-d',
+ "gecos": '-c',
+ "primary_group": '-g',
+ "groups": '-G',
+ "passwd": '-h',
+ "shell": '-s',
+ "inactive": '-E',
+ }
+ adduser_flags = {
+ "no_user_group": '--no-user-group',
+ "system": '--system',
+ "no_log_init": '--no-log-init',
+ }
+
+ redact_opts = ['passwd']
+
+ for key, val in kwargs.iteritems():
+ if key in adduser_opts and val and isinstance(val, basestring):
+ adduser_cmd.extend([adduser_opts[key], val])
+
+ # Redact certain fields from the logs
+ if key in redact_opts:
+ log_adduser_cmd.extend([adduser_opts[key], 'REDACTED'])
+ else:
+ log_adduser_cmd.extend([adduser_opts[key], val])
+
+ elif key in adduser_flags and val:
+ adduser_cmd.append(adduser_flags[key])
+ log_adduser_cmd.append(adduser_flags[key])
+
+ if 'no_create_home' in kwargs or 'system' in kwargs:
+ adduser_cmd.append('-d/nonexistent')
+ log_adduser_cmd.append('-d/nonexistent')
+ else:
+ adduser_cmd.append('-d/usr/home/%s' % name)
+ adduser_cmd.append('-m')
+ log_adduser_cmd.append('-d/usr/home/%s' % name)
+ log_adduser_cmd.append('-m')
+
+ # Run the command
+ LOG.info("Adding user %s", name)
+ try:
+ util.subp(adduser_cmd, logstring=log_adduser_cmd)
+ except Exception as e:
+ util.logexc(LOG, "Failed to create user %s", name)
+ raise e
+
+ # TODO:
+ def set_passwd(self, user, passwd, hashed=False):
+ return False
+
+ def lock_passwd(self, name):
+ try:
+ util.subp(['pw', 'usermod', name, '-h', '-'])
+ except Exception as e:
+ util.logexc(LOG, "Failed to lock user %s", name)
+ raise e
+
+ # TODO:
+ def write_sudo_rules(self, name, rules, sudo_file=None):
+ LOG.debug("[write_sudo_rules] Name: %s", name)
+
+ def create_user(self, name, **kwargs):
+ self.add_user(name, **kwargs)
+
+ # Set password if plain-text password provided and non-empty
+ if 'plain_text_passwd' in kwargs and kwargs['plain_text_passwd']:
+ self.set_passwd(name, kwargs['plain_text_passwd'])
+
+ # Default locking down the account. 'lock_passwd' defaults to True.
+ # lock account unless lock_password is False.
+ if kwargs.get('lock_passwd', True):
+ self.lock_passwd(name)
+
+ # Configure sudo access
+ if 'sudo' in kwargs:
+ self.write_sudo_rules(name, kwargs['sudo'])
+
+ # Import SSH keys
+ if 'ssh_authorized_keys' in kwargs:
+ keys = set(kwargs['ssh_authorized_keys']) or []
+ ssh_util.setup_user_keys(keys, name, options=None)
+
+ def _write_network(self, settings):
+ return
+
+ def apply_locale(self, locale, out_fn=None):
+ # Adjust the locals value to the new value
+ newconf = StringIO()
+ for line in util.load_file(self.login_conf_fn).splitlines():
+ newconf.write(re.sub(r'^default:',
+ r'default:lang=%s:' % locale, line))
+ newconf.write("\n")
+
+ # Make a backup of login.conf.
+ util.copy(self.login_conf_fn, self.login_conf_fn_bak)
+
+ # And write the new login.conf.
+ util.write_file(self.login_conf_fn, newconf.getvalue())
+
+ try:
+ LOG.debug("Running cap_mkdb for %s", locale)
+ util.subp(['cap_mkdb', self.login_conf_fn])
+ except util.ProcessExecutionError:
+ # cap_mkdb failed, so restore the backup.
+ util.logexc(LOG, "Failed to apply locale %s", locale)
+ try:
+ util.copy(self.login_conf_fn_bak, self.login_conf_fn)
+ except IOError:
+ util.logexc(LOG, "Failed to restore %s backup",
+ self.login_conf_fn)
+
+ def install_packages(self, pkglist):
+ return
+
+ def package_command(self, cmd, args=None, pkgs=None):
+ return
+
+ def set_timezone(self, tz):
+ return
+
+ def update_package_sources(self):
+ return
diff --git a/cloudinit/distros/net_util.py b/cloudinit/distros/net_util.py
new file mode 100644
index 00000000..b9bcfd8b
--- /dev/null
+++ b/cloudinit/distros/net_util.py
@@ -0,0 +1,163 @@
+# vi: ts=4 expandtab
+#
+# Copyright (C) 2012 Canonical Ltd.
+# Copyright (C) 2012, 2013 Hewlett-Packard Development Company, L.P.
+# Copyright (C) 2012 Yahoo! Inc.
+#
+# Author: Scott Moser <scott.moser@canonical.com>
+# Author: Juerg Haefliger <juerg.haefliger@hp.com>
+# Author: Joshua Harlow <harlowja@yahoo-inc.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 3, as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+
+# This is a util function to translate debian based distro interface blobs as
+# given in /etc/network/interfaces to an *somewhat* agnostic format for
+# distributions that use other formats.
+#
+# TODO(harlowja) remove when we have python-netcf active...
+#
+# The format is the following:
+# {
+# <device-name>: {
+# # All optional (if not existent in original format)
+# "netmask": <ip>,
+# "broadcast": <ip>,
+# "gateway": <ip>,
+# "address": <ip>,
+# "bootproto": "static"|"dhcp",
+# "dns-search": <hostname>,
+# "hwaddress": <mac-address>,
+# "auto": True (or non-existent),
+# "dns-nameservers": [<ip/hostname>, ...],
+# }
+# }
+#
+# Things to note, comments are removed, if a ubuntu/debian interface is
+# marked as auto then only then first segment (?) is retained, ie
+# 'auto eth0 eth0:1' just marks eth0 as auto (not eth0:1).
+#
+# Example input:
+#
+# auto lo
+# iface lo inet loopback
+#
+# auto eth0
+# iface eth0 inet static
+# address 10.0.0.1
+# netmask 255.255.252.0
+# broadcast 10.0.0.255
+# gateway 10.0.0.2
+# dns-nameservers 98.0.0.1 98.0.0.2
+#
+# Example output:
+# {
+# "lo": {
+# "auto": true
+# },
+# "eth0": {
+# "auto": true,
+# "dns-nameservers": [
+# "98.0.0.1",
+# "98.0.0.2"
+# ],
+# "broadcast": "10.0.0.255",
+# "netmask": "255.255.252.0",
+# "bootproto": "static",
+# "address": "10.0.0.1",
+# "gateway": "10.0.0.2"
+# }
+# }
+
+def translate_network(settings):
+ # Get the standard cmd, args from the ubuntu format
+ entries = []
+ for line in settings.splitlines():
+ line = line.strip()
+ if not line or line.startswith("#"):
+ continue
+ split_up = line.split(None, 1)
+ if len(split_up) <= 1:
+ continue
+ entries.append(split_up)
+ # Figure out where each iface section is
+ ifaces = []
+ consume = {}
+ for (cmd, args) in entries:
+ if cmd == 'iface':
+ if consume:
+ ifaces.append(consume)
+ consume = {}
+ consume[cmd] = args
+ else:
+ consume[cmd] = args
+ # Check if anything left over to consume
+ absorb = False
+ for (cmd, args) in consume.iteritems():
+ if cmd == 'iface':
+ absorb = True
+ if absorb:
+ ifaces.append(consume)
+ # Now translate
+ real_ifaces = {}
+ for info in ifaces:
+ if 'iface' not in info:
+ continue
+ iface_details = info['iface'].split(None)
+ dev_name = None
+ if len(iface_details) >= 1:
+ dev = iface_details[0].strip().lower()
+ if dev:
+ dev_name = dev
+ if not dev_name:
+ continue
+ iface_info = {}
+ if len(iface_details) >= 3:
+ proto_type = iface_details[2].strip().lower()
+ # Seems like this can be 'loopback' which we don't
+ # really care about
+ if proto_type in ['dhcp', 'static']:
+ iface_info['bootproto'] = proto_type
+ # These can just be copied over
+ for k in ['netmask', 'address', 'gateway', 'broadcast']:
+ if k in info:
+ val = info[k].strip().lower()
+ if val:
+ iface_info[k] = val
+ # Name server info provided??
+ if 'dns-nameservers' in info:
+ iface_info['dns-nameservers'] = info['dns-nameservers'].split()
+ # Name server search info provided??
+ if 'dns-search' in info:
+ iface_info['dns-search'] = info['dns-search'].split()
+ # Is any mac address spoofing going on??
+ if 'hwaddress' in info:
+ hw_info = info['hwaddress'].lower().strip()
+ hw_split = hw_info.split(None, 1)
+ if len(hw_split) == 2 and hw_split[0].startswith('ether'):
+ hw_addr = hw_split[1]
+ if hw_addr:
+ iface_info['hwaddress'] = hw_addr
+ real_ifaces[dev_name] = iface_info
+ # Check for those that should be started on boot via 'auto'
+ for (cmd, args) in entries:
+ if cmd == 'auto':
+ # Seems like auto can be like 'auto eth0 eth0:1' so just get the
+ # first part out as the device name
+ args = args.split(None)
+ if not args:
+ continue
+ dev_name = args[0].strip().lower()
+ if dev_name in real_ifaces:
+ real_ifaces[dev_name]['auto'] = True
+ return real_ifaces
diff --git a/cloudinit/distros/rhel.py b/cloudinit/distros/rhel.py
index 30195384..e8abf111 100644
--- a/cloudinit/distros/rhel.py
+++ b/cloudinit/distros/rhel.py
@@ -25,6 +25,7 @@ from cloudinit import helpers
from cloudinit import log as logging
from cloudinit import util
+from cloudinit.distros import net_util
from cloudinit.distros import rhel_util
from cloudinit.settings import PER_INSTANCE
@@ -63,7 +64,7 @@ class Distro(distros.Distro):
def _write_network(self, settings):
# TODO(harlowja) fix this... since this is the ubuntu format
- entries = rhel_util.translate_network(settings)
+ entries = net_util.translate_network(settings)
LOG.debug("Translated ubuntu style network settings %s into %s",
settings, entries)
# Make the intermediate format as the rhel format...
diff --git a/cloudinit/distros/rhel_util.py b/cloudinit/distros/rhel_util.py
index 1aba58b8..063d536e 100644
--- a/cloudinit/distros/rhel_util.py
+++ b/cloudinit/distros/rhel_util.py
@@ -30,94 +30,6 @@ from cloudinit import util
LOG = logging.getLogger(__name__)
-# This is a util function to translate Debian based distro interface blobs as
-# given in /etc/network/interfaces to an equivalent format for distributions
-# that use ifcfg-* style (Red Hat and SUSE).
-# TODO(harlowja) remove when we have python-netcf active...
-def translate_network(settings):
- # Get the standard cmd, args from the ubuntu format
- entries = []
- for line in settings.splitlines():
- line = line.strip()
- if not line or line.startswith("#"):
- continue
- split_up = line.split(None, 1)
- if len(split_up) <= 1:
- continue
- entries.append(split_up)
- # Figure out where each iface section is
- ifaces = []
- consume = {}
- for (cmd, args) in entries:
- if cmd == 'iface':
- if consume:
- ifaces.append(consume)
- consume = {}
- consume[cmd] = args
- else:
- consume[cmd] = args
- # Check if anything left over to consume
- absorb = False
- for (cmd, args) in consume.iteritems():
- if cmd == 'iface':
- absorb = True
- if absorb:
- ifaces.append(consume)
- # Now translate
- real_ifaces = {}
- for info in ifaces:
- if 'iface' not in info:
- continue
- iface_details = info['iface'].split(None)
- dev_name = None
- if len(iface_details) >= 1:
- dev = iface_details[0].strip().lower()
- if dev:
- dev_name = dev
- if not dev_name:
- continue
- iface_info = {}
- if len(iface_details) >= 3:
- proto_type = iface_details[2].strip().lower()
- # Seems like this can be 'loopback' which we don't
- # really care about
- if proto_type in ['dhcp', 'static']:
- iface_info['bootproto'] = proto_type
- # These can just be copied over
- for k in ['netmask', 'address', 'gateway', 'broadcast']:
- if k in info:
- val = info[k].strip().lower()
- if val:
- iface_info[k] = val
- # Name server info provided??
- if 'dns-nameservers' in info:
- iface_info['dns-nameservers'] = info['dns-nameservers'].split()
- # Name server search info provided??
- if 'dns-search' in info:
- iface_info['dns-search'] = info['dns-search'].split()
- # Is any mac address spoofing going on??
- if 'hwaddress' in info:
- hw_info = info['hwaddress'].lower().strip()
- hw_split = hw_info.split(None, 1)
- if len(hw_split) == 2 and hw_split[0].startswith('ether'):
- hw_addr = hw_split[1]
- if hw_addr:
- iface_info['hwaddress'] = hw_addr
- real_ifaces[dev_name] = iface_info
- # Check for those that should be started on boot via 'auto'
- for (cmd, args) in entries:
- if cmd == 'auto':
- # Seems like auto can be like 'auto eth0 eth0:1' so just get the
- # first part out as the device name
- args = args.split(None)
- if not args:
- continue
- dev_name = args[0].strip().lower()
- if dev_name in real_ifaces:
- real_ifaces[dev_name]['auto'] = True
- return real_ifaces
-
-
# Helper function to update a RHEL/SUSE /etc/sysconfig/* file
def update_sysconfig_file(fn, adjustments, allow_empty=False):
if not adjustments:
diff --git a/cloudinit/distros/sles.py b/cloudinit/distros/sles.py
index f2ac4efc..9788a1ba 100644
--- a/cloudinit/distros/sles.py
+++ b/cloudinit/distros/sles.py
@@ -26,6 +26,7 @@ from cloudinit import helpers
from cloudinit import log as logging
from cloudinit import util
+from cloudinit.distros import net_util
from cloudinit.distros import rhel_util
from cloudinit.settings import PER_INSTANCE
@@ -54,7 +55,7 @@ class Distro(distros.Distro):
def _write_network(self, settings):
# Convert debian settings to ifcfg format
- entries = rhel_util.translate_network(settings)
+ entries = net_util.translate_network(settings)
LOG.debug("Translated ubuntu style network settings %s into %s",
settings, entries)
# Make the intermediate format as the suse format...
diff --git a/cloudinit/ec2_utils.py b/cloudinit/ec2_utils.py
index 80736a8f..91cba20f 100644
--- a/cloudinit/ec2_utils.py
+++ b/cloudinit/ec2_utils.py
@@ -16,6 +16,7 @@
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
+import httplib
from urlparse import (urlparse, urlunparse)
import functools
@@ -26,6 +27,7 @@ from cloudinit import url_helper
from cloudinit import util
LOG = logging.getLogger(__name__)
+SKIP_USERDATA_CODES = frozenset([httplib.NOT_FOUND])
def maybe_json_object(text):
@@ -128,20 +130,38 @@ class MetadataMaterializer(object):
return joined
+def _skip_retry_on_codes(status_codes, _request_args, cause):
+ """Returns if a request should retry based on a given set of codes that
+ case retrying to be stopped/skipped.
+ """
+ if cause.code in status_codes:
+ return False
+ return True
+
+
def get_instance_userdata(api_version='latest',
metadata_address='http://169.254.169.254',
ssl_details=None, timeout=5, retries=5):
ud_url = url_helper.combine_url(metadata_address, api_version)
ud_url = url_helper.combine_url(ud_url, 'user-data')
+ user_data = ''
try:
+ # It is ok for userdata to not exist (thats why we are stopping if
+ # NOT_FOUND occurs) and just in that case returning an empty string.
+ exception_cb = functools.partial(_skip_retry_on_codes,
+ SKIP_USERDATA_CODES)
response = util.read_file_or_url(ud_url,
ssl_details=ssl_details,
timeout=timeout,
- retries=retries)
- return str(response)
+ retries=retries,
+ exception_cb=exception_cb)
+ user_data = str(response)
+ except url_helper.UrlError as e:
+ if e.code not in SKIP_USERDATA_CODES:
+ util.logexc(LOG, "Failed fetching userdata from url %s", ud_url)
except Exception:
util.logexc(LOG, "Failed fetching userdata from url %s", ud_url)
- return ''
+ return user_data
def get_instance_metadata(api_version='latest',
diff --git a/cloudinit/netinfo.py b/cloudinit/netinfo.py
index feba5a62..ac3c011f 100644
--- a/cloudinit/netinfo.py
+++ b/cloudinit/netinfo.py
@@ -21,6 +21,7 @@
# along with this program. If not, see <http://www.gnu.org/licenses/>.
import cloudinit.util as util
+import re
from prettytable import PrettyTable
@@ -40,27 +41,40 @@ def netdev_info(empty=""):
toks = line.lower().strip().split()
if toks[0] == "up":
devs[curdev]['up'] = True
+ # If the output of ifconfig doesn't contain the required info in the
+ # obvious place, use a regex filter to be sure.
+ elif len(toks) > 1:
+ if re.search(r"flags=\d+<up,", toks[1]):
+ devs[curdev]['up'] = True
fieldpost = ""
if toks[0] == "inet6":
fieldpost = "6"
for i in range(len(toks)):
- if toks[i] == "hwaddr":
+ if toks[i] == "hwaddr" or toks[i] == "ether":
try:
devs[curdev]["hwaddr"] = toks[i + 1]
except IndexError:
pass
- for field in ("addr", "bcast", "mask"):
+
+ # Couple the different items we're interested in with the correct
+ # field since FreeBSD/CentOS/Fedora differ in the output.
+ ifconfigfields = {
+ "addr:": "addr", "inet": "addr",
+ "bcast:": "bcast", "broadcast": "bcast",
+ "mask:": "mask", "netmask": "mask"
+ }
+ for origfield, field in ifconfigfields.items():
target = "%s%s" % (field, fieldpost)
if devs[curdev].get(target, ""):
continue
- if toks[i] == "%s:" % field:
+ if toks[i] == "%s" % origfield:
try:
devs[curdev][target] = toks[i + 1]
except IndexError:
pass
- elif toks[i].startswith("%s:" % field):
+ elif toks[i].startswith("%s" % origfield):
devs[curdev][target] = toks[i][len(field) + 1:]
if empty != "":
@@ -73,15 +87,32 @@ def netdev_info(empty=""):
def route_info():
- (route_out, _err) = util.subp(["route", "-n"])
+ (route_out, _err) = util.subp(["netstat", "-rn"])
routes = []
entries = route_out.splitlines()[1:]
for line in entries:
if not line:
continue
toks = line.split()
- if len(toks) < 8 or toks[0] == "Kernel" or toks[0] == "Destination":
+
+ # FreeBSD shows 6 items in the routing table:
+ # Destination Gateway Flags Refs Use Netif Expire
+ # default 10.65.0.1 UGS 0 34920 vtnet0
+ #
+ # Linux netstat shows 2 more:
+ # Destination Gateway Genmask Flags MSS Window irtt Iface
+ # 0.0.0.0 10.65.0.1 0.0.0.0 UG 0 0 0 eth0
+ if (len(toks) < 6 or toks[0] == "Kernel" or
+ toks[0] == "Destination" or toks[0] == "Internet" or
+ toks[0] == "Internet6" or toks[0] == "Routing"):
continue
+
+ if len(toks) < 8:
+ toks.append("-")
+ toks.append("-")
+ toks[7] = toks[5]
+ toks[5] = "-"
+
entry = {
'destination': toks[0],
'gateway': toks[1],
@@ -92,6 +123,7 @@ def route_info():
'use': toks[6],
'iface': toks[7],
}
+
routes.append(entry)
return routes
diff --git a/cloudinit/sources/DataSourceConfigDrive.py b/cloudinit/sources/DataSourceConfigDrive.py
index c45a1119..1d30fe08 100644
--- a/cloudinit/sources/DataSourceConfigDrive.py
+++ b/cloudinit/sources/DataSourceConfigDrive.py
@@ -246,10 +246,13 @@ def find_candidate_devs(probe_optical=True):
# combine list of items by putting by-label items first
# followed by fstype items, but with dupes removed
- combined = (by_label + [d for d in by_fstype if d not in by_label])
+ candidates = (by_label + [d for d in by_fstype if d not in by_label])
- # we are looking for block device (sda, not sda1), ignore partitions
- return [d for d in combined if not util.is_partition(d)]
+ # We are looking for a block device or partition with necessary label or
+ # an unpartitioned block device (ex sda, not sda1)
+ devices = [d for d in candidates
+ if d in by_label or not util.is_partition(d)]
+ return devices
# Used to match classes to dependencies
diff --git a/cloudinit/sources/DataSourceNoCloud.py b/cloudinit/sources/DataSourceNoCloud.py
index 4ef92a56..cbaac29f 100644
--- a/cloudinit/sources/DataSourceNoCloud.py
+++ b/cloudinit/sources/DataSourceNoCloud.py
@@ -50,40 +50,47 @@ class DataSourceNoCloud(sources.DataSource):
}
found = []
- md = {}
- ud = ""
+ mydata = {'meta-data': {}, 'user-data': "", 'vendor-data': ""}
try:
# Parse the kernel command line, getting data passed in
+ md = {}
if parse_cmdline_data(self.cmdline_id, md):
found.append("cmdline")
+ mydata.update(md)
except:
util.logexc(LOG, "Unable to parse command line data")
return False
# Check to see if the seed dir has data.
- seedret = {}
- if util.read_optional_seed(seedret, base=self.seed_dir + "/"):
- md = util.mergemanydict([md, seedret['meta-data']])
- ud = seedret['user-data']
+ pp2d_kwargs = {'required': ['user-data', 'meta-data'],
+ 'optional': ['vendor-data']}
+
+ try:
+ seeded = util.pathprefix2dict(self.seed_dir, **pp2d_kwargs)
found.append(self.seed_dir)
- LOG.debug("Using seeded cache data from %s", self.seed_dir)
+ LOG.debug("Using seeded data from %s", self.seed_dir)
+ except ValueError as e:
+ pass
+
+ if self.seed_dir in found:
+ mydata = _merge_new_seed(mydata, seeded)
# If the datasource config had a 'seedfrom' entry, then that takes
# precedence over a 'seedfrom' that was found in a filesystem
# but not over external media
- if 'seedfrom' in self.ds_cfg and self.ds_cfg['seedfrom']:
- found.append("ds_config")
- md["seedfrom"] = self.ds_cfg['seedfrom']
+ if self.ds_cfg.get('seedfrom'):
+ found.append("ds_config_seedfrom")
+ mydata['meta-data']["seedfrom"] = self.ds_cfg['seedfrom']
- # if ds_cfg has 'user-data' and 'meta-data'
+ # fields appropriately named can also just come from the datasource
+ # config (ie, 'user-data', 'meta-data', 'vendor-data' there)
if 'user-data' in self.ds_cfg and 'meta-data' in self.ds_cfg:
- if self.ds_cfg['user-data']:
- ud = self.ds_cfg['user-data']
- if self.ds_cfg['meta-data'] is not False:
- md = util.mergemanydict([md, self.ds_cfg['meta-data']])
- if 'ds_config' not in found:
- found.append("ds_config")
+ mydata = _merge_new_seed(mydata, self.ds_cfg)
+ found.append("ds_config")
+
+ def _pp2d_callback(mp, data):
+ util.pathprefix2dict(mp, **data)
label = self.ds_cfg.get('fs_label', "cidata")
if label is not None:
@@ -102,15 +109,21 @@ class DataSourceNoCloud(sources.DataSource):
try:
LOG.debug("Attempting to use data from %s", dev)
- (newmd, newud) = util.mount_cb(dev, util.read_seeded)
- md = util.mergemanydict([newmd, md])
- ud = newud
+ try:
+ seeded = util.mount_cb(dev, _pp2d_callback)
+ except ValueError as e:
+ if dev in label_list:
+ LOG.warn("device %s with label=%s not a"
+ "valid seed.", dev, label)
+ continue
+
+ mydata = _merge_new_seed(mydata, seeded)
# For seed from a device, the default mode is 'net'.
# that is more likely to be what is desired. If they want
# dsmode of local, then they must specify that.
- if 'dsmode' not in md:
- md['dsmode'] = "net"
+ if 'dsmode' not in mydata['meta-data']:
+ mydata['meta-data'] = "net"
LOG.debug("Using data from %s", dev)
found.append(dev)
@@ -133,8 +146,8 @@ class DataSourceNoCloud(sources.DataSource):
# attempt to seed the userdata / metadata from its value
# its primarily value is in allowing the user to type less
# on the command line, ie: ds=nocloud;s=http://bit.ly/abcdefg
- if "seedfrom" in md:
- seedfrom = md["seedfrom"]
+ if "seedfrom" in mydata['meta-data']:
+ seedfrom = mydata['meta-data']["seedfrom"]
seedfound = False
for proto in self.supported_seed_starts:
if seedfrom.startswith(proto):
@@ -144,7 +157,7 @@ class DataSourceNoCloud(sources.DataSource):
LOG.debug("Seed from %s not supported by %s", seedfrom, self)
return False
- if 'network-interfaces' in md:
+ if 'network-interfaces' in mydata['meta-data']:
seeded_interfaces = self.dsmode
# This could throw errors, but the user told us to do it
@@ -153,25 +166,30 @@ class DataSourceNoCloud(sources.DataSource):
LOG.debug("Using seeded cache data from %s", seedfrom)
# Values in the command line override those from the seed
- md = util.mergemanydict([md, md_seed])
+ mydata['meta-data'] = util.mergemanydict([mydata['meta-data'],
+ md_seed])
+ mydata['user-data'] = ud
found.append(seedfrom)
# Now that we have exhausted any other places merge in the defaults
- md = util.mergemanydict([md, defaults])
+ mydata['meta-data'] = util.mergemanydict([mydata['meta-data'],
+ defaults])
# Update the network-interfaces if metadata had 'network-interfaces'
# entry and this is the local datasource, or 'seedfrom' was used
# and the source of the seed was self.dsmode
# ('local' for NoCloud, 'net' for NoCloudNet')
- if ('network-interfaces' in md and
+ if ('network-interfaces' in mydata['meta-data'] and
(self.dsmode in ("local", seeded_interfaces))):
LOG.debug("Updating network interfaces from %s", self)
- self.distro.apply_network(md['network-interfaces'])
+ self.distro.apply_network(
+ mydata['meta-data']['network-interfaces'])
- if md['dsmode'] == self.dsmode:
+ if mydata['meta-data']['dsmode'] == self.dsmode:
self.seed = ",".join(found)
- self.metadata = md
- self.userdata_raw = ud
+ self.metadata = mydata['meta-data']
+ self.userdata_raw = mydata['user-data']
+ self.vendordata = mydata['vendor-data']
return True
LOG.debug("%s: not claiming datasource, dsmode=%s", self, md['dsmode'])
@@ -222,6 +240,16 @@ def parse_cmdline_data(ds_id, fill, cmdline=None):
return True
+def _merge_new_seed(cur, seeded):
+ ret = cur.copy()
+ ret['meta-data'] = util.mergemanydict([cur['meta-data'],
+ util.load_yaml(seeded['meta-data'])])
+ ret['user-data'] = seeded['user-data']
+ if 'vendor-data' in seeded:
+ ret['vendor-data'] = seeded['vendor-data']
+ return ret
+
+
class DataSourceNoCloudNet(DataSourceNoCloud):
def __init__(self, sys_cfg, distro, paths):
DataSourceNoCloud.__init__(self, sys_cfg, distro, paths)
diff --git a/cloudinit/sources/DataSourceSmartOS.py b/cloudinit/sources/DataSourceSmartOS.py
index 6593ce6e..140c7814 100644
--- a/cloudinit/sources/DataSourceSmartOS.py
+++ b/cloudinit/sources/DataSourceSmartOS.py
@@ -25,7 +25,9 @@
# requests on the console. For example, to get the hostname, you
# would send "GET hostname" on /dev/ttyS1.
#
-
+# Certain behavior is defined by the DataDictionary
+# http://us-east.manta.joyent.com/jmc/public/mdata/datadict.html
+# Comments with "@datadictionary" are snippets of the definition
import base64
from cloudinit import log as logging
@@ -43,10 +45,11 @@ SMARTOS_ATTRIB_MAP = {
'local-hostname': ('hostname', True),
'public-keys': ('root_authorized_keys', True),
'user-script': ('user-script', False),
- 'user-data': ('user-data', False),
+ 'legacy-user-data': ('user-data', False),
+ 'user-data': ('cloud-init:user-data', False),
'iptables_disable': ('iptables_disable', True),
'motd_sys_info': ('motd_sys_info', True),
- 'availability_zone': ('datacenter_name', True),
+ 'availability_zone': ('sdc:datacenter_name', True),
'vendordata': ('sdc:operator-script', False),
}
@@ -71,7 +74,11 @@ BUILTIN_DS_CONFIG = {
'seed_timeout': 60,
'no_base64_decode': ['root_authorized_keys',
'motd_sys_info',
- 'iptables_disable'],
+ 'iptables_disable',
+ 'user-data',
+ 'user-script',
+ 'sdc:datacenter_name',
+ ],
'base64_keys': [],
'base64_all': False,
'disk_aliases': {'ephemeral0': '/dev/vdb'},
@@ -88,6 +95,11 @@ BUILTIN_CLOUD_CONFIG = {
'device': 'ephemeral0'}],
}
+# @datadictionary: this is legacy path for placing files from metadata
+# per the SmartOS location. It is not preferable, but is done for
+# legacy reasons
+LEGACY_USER_D = "/var/db"
+
class DataSourceSmartOS(sources.DataSource):
def __init__(self, sys_cfg, distro, paths):
@@ -107,6 +119,9 @@ class DataSourceSmartOS(sources.DataSource):
self.smartos_no_base64 = self.ds_cfg.get('no_base64_decode')
self.b64_keys = self.ds_cfg.get('base64_keys')
self.b64_all = self.ds_cfg.get('base64_all')
+ self.script_base_d = os.path.join(self.paths.get_cpath("scripts"))
+ self.user_script_d = os.path.join(self.paths.get_cpath("scripts"),
+ 'per-boot')
def __str__(self):
root = sources.DataSource.__str__(self)
@@ -144,14 +159,32 @@ class DataSourceSmartOS(sources.DataSource):
smartos_noun, strip = attribute
md[ci_noun] = self.query(smartos_noun, strip=strip)
+ # @datadictionary: This key may contain a program that is written
+ # to a file in the filesystem of the guest on each boot and then
+ # executed. It may be of any format that would be considered
+ # executable in the guest instance.
+ u_script = md.get('user-script')
+ u_script_f = "%s/99_user_script" % self.user_script_d
+ u_script_l = "%s/user-script" % LEGACY_USER_D
+ write_boot_content(u_script, u_script_f, link=u_script_l, shebang=True,
+ mode=0700)
+
+ # @datadictionary: This key has no defined format, but its value
+ # is written to the file /var/db/mdata-user-data on each boot prior
+ # to the phase that runs user-script. This file is not to be executed.
+ # This allows a configuration file of some kind to be injected into
+ # the machine to be consumed by the user-script when it runs.
+ u_data = md.get('legacy-user-data')
+ u_data_f = "%s/mdata-user-data" % LEGACY_USER_D
+ write_boot_content(u_data, u_data_f)
+
+ # Handle the cloud-init regular meta
if not md['local-hostname']:
md['local-hostname'] = system_uuid
ud = None
if md['user-data']:
ud = md['user-data']
- elif md['user-script']:
- ud = md['user-script']
self.metadata = util.mergemanydict([md, self.metadata])
self.userdata_raw = ud
@@ -279,6 +312,62 @@ def dmi_data():
return (sys_uuid.lower().strip(), sys_type.strip())
+def write_boot_content(content, content_f, link=None, shebang=False,
+ mode=0400):
+ """
+ Write the content to content_f. Under the following rules:
+ 1. If no content, remove the file
+ 2. Write the content
+ 3. If executable and no file magic, add it
+ 4. If there is a link, create it
+
+ @param content: what to write
+ @param content_f: the file name
+ @param backup_d: the directory to save the backup at
+ @param link: if defined, location to create a symlink to
+ @param shebang: if no file magic, set shebang
+ @param mode: file mode
+
+ Becuase of the way that Cloud-init executes scripts (no shell),
+ a script will fail to execute if does not have a magic bit (shebang) set
+ for the file. If shebang=True, then the script will be checked for a magic
+ bit and to the SmartOS default of assuming that bash.
+ """
+
+ if not content and os.path.exists(content_f):
+ os.unlink(content_f)
+ if link and os.path.islink(link):
+ os.unlink(link)
+ if not content:
+ return
+
+ util.write_file(content_f, content, mode=mode)
+
+ if shebang and not content.startswith("#!"):
+ try:
+ cmd = ["file", "--brief", "--mime-type", content_f]
+ (f_type, _err) = util.subp(cmd)
+ LOG.debug("script %s mime type is %s", content_f, f_type)
+ if f_type.strip() == "text/plain":
+ new_content = "\n".join(["#!/bin/bash", content])
+ util.write_file(content_f, new_content, mode=mode)
+ LOG.debug("added shebang to file %s", content_f)
+
+ except Exception as e:
+ util.logexc(LOG, ("Failed to identify script type for %s" %
+ content_f, e))
+
+ if link:
+ try:
+ if os.path.islink(link):
+ os.unlink(link)
+ if content and os.path.exists(content_f):
+ util.ensure_dir(os.path.dirname(link))
+ os.symlink(content_f, link)
+ except IOError as e:
+ util.logexc(LOG, "failed establishing content link", e)
+
+
# Used to match classes to dependencies
datasources = [
(DataSourceSmartOS, (sources.DEP_FILESYSTEM, sources.DEP_NETWORK)),
diff --git a/cloudinit/sources/__init__.py b/cloudinit/sources/__init__.py
index 4b3bf62f..fef4d460 100644
--- a/cloudinit/sources/__init__.py
+++ b/cloudinit/sources/__init__.py
@@ -129,7 +129,7 @@ class DataSource(object):
# when the kernel named them 'vda' or 'xvda'
# we want to return the correct value for what will actually
# exist in this instance
- mappings = {"sd": ("vd", "xvd")}
+ mappings = {"sd": ("vd", "xvd", "vtb")}
for (nfrom, tlist) in mappings.iteritems():
if not short_name.startswith(nfrom):
continue
diff --git a/cloudinit/url_helper.py b/cloudinit/url_helper.py
index 5c33d1e4..76a8e29b 100644
--- a/cloudinit/url_helper.py
+++ b/cloudinit/url_helper.py
@@ -20,6 +20,7 @@
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
+import httplib
import time
import urllib
@@ -33,6 +34,8 @@ from cloudinit import version
LOG = logging.getLogger(__name__)
+NOT_FOUND = httplib.NOT_FOUND
+
# Check if requests has ssl support (added in requests >= 0.8.8)
SSL_ENABLED = False
CONFIG_ENABLED = False # This was added in 0.7 (but taken out in >=1.0)
@@ -76,6 +79,31 @@ def combine_url(base, *add_ons):
return url
+# Made to have same accessors as UrlResponse so that the
+# read_file_or_url can return this or that object and the
+# 'user' of those objects will not need to know the difference.
+class StringResponse(object):
+ def __init__(self, contents, code=200):
+ self.code = code
+ self.headers = {}
+ self.contents = contents
+ self.url = None
+
+ def ok(self, *args, **kwargs): # pylint: disable=W0613
+ if self.code != 200:
+ return False
+ return True
+
+ def __str__(self):
+ return self.contents
+
+
+class FileResponse(StringResponse):
+ def __init__(self, path, contents, code=200):
+ StringResponse.__init__(self, contents, code=code)
+ self.url = path
+
+
class UrlResponse(object):
def __init__(self, response):
self._response = response
@@ -146,17 +174,19 @@ def existsurl(url, ssl_details=None, timeout=None):
def readurl(url, data=None, timeout=None, retries=0, sec_between=1,
headers=None, headers_cb=None, ssl_details=None,
- check_status=True, allow_redirects=True):
+ check_status=True, allow_redirects=True, exception_cb=None):
return _readurl(url, data=data, timeout=timeout, retries=retries,
sec_between=sec_between, headers=headers,
headers_cb=headers_cb, ssl_details=ssl_details,
check_status=check_status,
- allow_redirects=allow_redirects)
+ allow_redirects=allow_redirects,
+ exception_cb=exception_cb)
def _readurl(url, data=None, timeout=None, retries=0, sec_between=1,
headers=None, headers_cb=None, ssl_details=None,
- check_status=True, allow_redirects=True, method='GET'):
+ check_status=True, allow_redirects=True, exception_cb=None,
+ method='GET'):
url = _cleanurl(url)
req_args = {
'url': url,
@@ -203,14 +233,13 @@ def _readurl(url, data=None, timeout=None, retries=0, sec_between=1,
# Handle retrying ourselves since the built-in support
# doesn't handle sleeping between tries...
for i in range(0, manual_tries):
+ req_args['headers'] = headers_cb(url)
+ filtered_req_args = {}
+ for (k, v) in req_args.items():
+ if k == 'data':
+ continue
+ filtered_req_args[k] = v
try:
- req_args['headers'] = headers_cb(url)
- filtered_req_args = {}
- for (k, v) in req_args.items():
- if k == 'data':
- continue
- filtered_req_args[k] = v
-
LOG.debug("[%s/%s] open '%s' with %s configuration", i,
manual_tries, url, filtered_req_args)
@@ -236,6 +265,8 @@ def _readurl(url, data=None, timeout=None, retries=0, sec_between=1,
# ssl exceptions are not going to get fixed by waiting a
# few seconds
break
+ if exception_cb and not exception_cb(req_args.copy(), excps[-1]):
+ break
if i + 1 < manual_tries and sec_between > 0:
LOG.debug("Please wait %s seconds while we wait to try again",
sec_between)
diff --git a/cloudinit/user_data.py b/cloudinit/user_data.py
index 3032ef70..de6487d8 100644
--- a/cloudinit/user_data.py
+++ b/cloudinit/user_data.py
@@ -311,7 +311,8 @@ class UserDataProcessor(object):
def _attach_part(self, outer_msg, part):
"""
Attach a message to an outer message. outermsg must be a MIMEMultipart.
- Modifies a header in the outer message to keep track of number of attachments.
+ Modifies a header in the outer message to keep track of number of
+ attachments.
"""
part_count = self._multi_part_count(outer_msg)
self._process_before_attach(part, part_count + 1)
diff --git a/cloudinit/util.py b/cloudinit/util.py
index 9bafd5b3..87b0c853 100644
--- a/cloudinit/util.py
+++ b/cloudinit/util.py
@@ -26,6 +26,7 @@ from StringIO import StringIO
import contextlib
import copy as obj_copy
+import ctypes
import errno
import glob
import grp
@@ -37,6 +38,7 @@ import os.path
import platform
import pwd
import random
+import re
import shutil
import socket
import stat
@@ -73,31 +75,6 @@ FN_ALLOWED = ('_-.()' + string.digits + string.ascii_letters)
CONTAINER_TESTS = ['running-in-container', 'lxc-is-container']
-# Made to have same accessors as UrlResponse so that the
-# read_file_or_url can return this or that object and the
-# 'user' of those objects will not need to know the difference.
-class StringResponse(object):
- def __init__(self, contents, code=200):
- self.code = code
- self.headers = {}
- self.contents = contents
- self.url = None
-
- def ok(self, *args, **kwargs): # pylint: disable=W0613
- if self.code != 200:
- return False
- return True
-
- def __str__(self):
- return self.contents
-
-
-class FileResponse(StringResponse):
- def __init__(self, path, contents, code=200):
- StringResponse.__init__(self, contents, code=code)
- self.url = path
-
-
class ProcessExecutionError(IOError):
MESSAGE_TMPL = ('%(description)s\n'
@@ -402,11 +379,11 @@ def is_ipv4(instr):
return False
try:
- toks = [x for x in toks if (int(x) < 256 and int(x) > 0)]
+ toks = [x for x in toks if int(x) < 256 and int(x) >= 0]
except:
return False
- return (len(toks) == 4)
+ return len(toks) == 4
def get_cfg_option_bool(yobj, key, default=False):
@@ -659,8 +636,8 @@ def read_optional_seed(fill, base="", ext="", timeout=5):
fill['user-data'] = ud
fill['meta-data'] = md
return True
- except IOError as e:
- if e.errno == errno.ENOENT:
+ except url_helper.UrlError as e:
+ if e.code == url_helper.NOT_FOUND:
return False
raise
@@ -699,7 +676,7 @@ def fetch_ssl_details(paths=None):
def read_file_or_url(url, timeout=5, retries=10,
headers=None, data=None, sec_between=1, ssl_details=None,
- headers_cb=None):
+ headers_cb=None, exception_cb=None):
url = url.lstrip()
if url.startswith("/"):
url = "file://%s" % url
@@ -707,7 +684,14 @@ def read_file_or_url(url, timeout=5, retries=10,
if data:
LOG.warn("Unable to post data to file resource %s", url)
file_path = url[len("file://"):]
- return FileResponse(file_path, contents=load_file(file_path))
+ try:
+ contents = load_file(file_path)
+ except IOError as e:
+ code = e.errno
+ if e.errno == errno.ENOENT:
+ code = url_helper.NOT_FOUND
+ raise url_helper.UrlError(cause=e, code=code, headers=None)
+ return url_helper.FileResponse(file_path, contents=contents)
else:
return url_helper.readurl(url,
timeout=timeout,
@@ -716,7 +700,8 @@ def read_file_or_url(url, timeout=5, retries=10,
headers_cb=headers_cb,
data=data,
sec_between=sec_between,
- ssl_details=ssl_details)
+ ssl_details=ssl_details,
+ exception_cb=exception_cb)
def load_yaml(blob, default=None, allowed=(dict,)):
@@ -885,8 +870,8 @@ def get_fqdn_from_hosts(hostname, filename="/etc/hosts"):
IP_address canonical_hostname [aliases...]
Fields of the entry are separated by any number of blanks and/or tab
- characters. Text from a "#" character until the end of the line is a
- comment, and is ignored. Host names may contain only alphanumeric
+ characters. Text from a "#" character until the end of the line is a
+ comment, and is ignored. Host names may contain only alphanumeric
characters, minus signs ("-"), and periods ("."). They must begin with
an alphabetic character and end with an alphanumeric character.
Optional aliases provide for name changes, alternate spellings, shorter
@@ -970,7 +955,7 @@ def is_resolvable(name):
pass
_DNS_REDIRECT_IP = badips
if badresults:
- LOG.debug("detected dns redirection: %s" % badresults)
+ LOG.debug("detected dns redirection: %s", badresults)
try:
result = socket.getaddrinfo(name, None)
@@ -997,7 +982,7 @@ def gethostbyaddr(ip):
def is_resolvable_url(url):
"""determine if this url is resolvable (existing or ip)."""
- return (is_resolvable(urlparse.urlparse(url).hostname))
+ return is_resolvable(urlparse.urlparse(url).hostname)
def search_for_mirror(candidates):
@@ -1322,11 +1307,26 @@ def mounts():
mounted = {}
try:
# Go through mounts to see what is already mounted
- mount_locs = load_file("/proc/mounts").splitlines()
+ if os.path.exists("/proc/mounts"):
+ mount_locs = load_file("/proc/mounts").splitlines()
+ method = 'proc'
+ else:
+ (mountoutput, _err) = subp("mount")
+ mount_locs = mountoutput.splitlines()
+ method = 'mount'
+ mountre = r'^(/dev/[\S]+) on (/.*) \((.+), .+, (.+)\)$'
for mpline in mount_locs:
- # Format at: man fstab
+ # Linux: /dev/sda1 on /boot type ext4 (rw,relatime,data=ordered)
+ # FreeBSD: /dev/vtbd0p2 on / (ufs, local, journaled soft-updates)
try:
- (dev, mp, fstype, opts, _freq, _passno) = mpline.split()
+ if method == 'proc':
+ (dev, mp, fstype, opts, _freq, _passno) = mpline.split()
+ else:
+ m = re.search(mountre, mpline)
+ dev = m.group(1)
+ mp = m.group(2)
+ fstype = m.group(3)
+ opts = m.group(4)
except:
continue
# If the name of the mount point contains spaces these
@@ -1337,9 +1337,9 @@ def mounts():
'mountpoint': mp,
'opts': opts,
}
- LOG.debug("Fetched %s mounts from %s", mounted, "/proc/mounts")
+ LOG.debug("Fetched %s mounts from %s", mounted, method)
except (IOError, OSError):
- logexc(LOG, "Failed fetching mount points from /proc/mounts")
+ logexc(LOG, "Failed fetching mount points")
return mounted
@@ -1396,7 +1396,7 @@ def get_builtin_cfg():
def sym_link(source, link):
- LOG.debug("Creating symbolic link from %r => %r" % (link, source))
+ LOG.debug("Creating symbolic link from %r => %r", link, source)
os.symlink(source, link)
@@ -1424,12 +1424,27 @@ def time_rfc2822():
def uptime():
uptime_str = '??'
+ method = 'unknown'
try:
- contents = load_file("/proc/uptime").strip()
- if contents:
- uptime_str = contents.split()[0]
+ if os.path.exists("/proc/uptime"):
+ method = '/proc/uptime'
+ contents = load_file("/proc/uptime").strip()
+ if contents:
+ uptime_str = contents.split()[0]
+ else:
+ method = 'ctypes'
+ libc = ctypes.CDLL('/lib/libc.so.7')
+ size = ctypes.c_size_t()
+ buf = ctypes.c_int()
+ size.value = ctypes.sizeof(buf)
+ libc.sysctlbyname("kern.boottime", ctypes.byref(buf),
+ ctypes.byref(size), None, 0)
+ now = time.time()
+ bootup = buf.value
+ uptime_str = now - bootup
+
except:
- logexc(LOG, "Unable to read uptime from /proc/uptime")
+ logexc(LOG, "Unable to read uptime using method: %s" % method)
return uptime_str
@@ -1768,6 +1783,19 @@ def parse_mtab(path):
return None
+def parse_mount(path):
+ (mountoutput, _err) = subp("mount")
+ mount_locs = mountoutput.splitlines()
+ for line in mount_locs:
+ m = re.search(r'^(/dev/[\S]+) on (/.*) \((.+), .+, (.+)\)$', line)
+ devpth = m.group(1)
+ mount_point = m.group(2)
+ fs_type = m.group(3)
+ if mount_point == path:
+ return devpth, fs_type, mount_point
+ return None
+
+
def get_mount_info(path, log=LOG):
# Use /proc/$$/mountinfo to find the device where path is mounted.
# This is done because with a btrfs filesystem using os.stat(path)
@@ -1801,8 +1829,10 @@ def get_mount_info(path, log=LOG):
if os.path.exists(mountinfo_path):
lines = load_file(mountinfo_path).splitlines()
return parse_mount_info(path, lines, log)
- else:
+ elif os.path.exists("/etc/mtab"):
return parse_mtab(path)
+ else:
+ return parse_mount(path)
def which(program):
@@ -1815,7 +1845,7 @@ def which(program):
if is_exe(program):
return program
else:
- for path in os.environ["PATH"].split(os.pathsep):
+ for path in os.environ.get("PATH", "").split(os.pathsep):
path = path.strip('"')
exe_file = os.path.join(path, program)
if is_exe(exe_file):
@@ -1869,3 +1899,28 @@ def expand_dotted_devname(dotted):
return toks
else:
return (dotted, None)
+
+
+def pathprefix2dict(base, required=None, optional=None, delim=os.path.sep):
+ # return a dictionary populated with keys in 'required' and 'optional'
+ # by reading files in prefix + delim + entry
+ if required is None:
+ required = []
+ if optional is None:
+ optional = []
+
+ missing = []
+ ret = {}
+ for f in required + optional:
+ try:
+ ret[f] = load_file(base + delim + f, quiet=False)
+ except IOError as e:
+ if e.errno != errno.ENOENT:
+ raise
+ if f in required:
+ missing.append(f)
+
+ if len(missing):
+ raise ValueError("Missing required files: %s", ','.join(missing))
+
+ return ret
generated by cgit v1.2.3 (git 2.39.1) at 2024-09-20 09:49:06 +0000