diff options
| -rw-r--r-- | ChangeLog | 2 | ||||
| -rw-r--r-- | cloudinit/config/cc_mounts.py | 157 | ||||
| -rw-r--r-- | cloudinit/distros/freebsd.py | 79 | ||||
| -rw-r--r-- | cloudinit/sources/DataSourceConfigDrive.py | 14 | ||||
| -rw-r--r-- | cloudinit/sources/DataSourceOVF.py | 3 | ||||
| -rw-r--r-- | cloudinit/util.py | 131 | ||||
| -rw-r--r-- | config/cloud.cfg-freebsd | 2 | ||||
| -rw-r--r-- | doc/examples/cloud-config-mount-points.txt | 7 | ||||
| -rwxr-xr-x | sysvinit/freebsd/cloudconfig | 1 | ||||
| -rwxr-xr-x | sysvinit/freebsd/cloudfinal | 1 | ||||
| -rwxr-xr-x | sysvinit/freebsd/cloudinit | 1 | ||||
| -rwxr-xr-x | sysvinit/freebsd/cloudinitlocal | 1 | ||||
| -rw-r--r-- | templates/hosts.freebsd.tmpl | 24 | ||||
| -rw-r--r-- | tests/unittests/test_distros/test_netconfig.py | 4 | ||||
| -rwxr-xr-x | tools/build-on-freebsd | 22 |
15 files changed, 395 insertions, 54 deletions
@@ -37,6 +37,8 @@ - resizefs: fix broken background resizing [Jay Faulkner] (LP: #1338614) - cc_grub_dpkg: fix EC2 hvm instances to avoid prompt on grub update. (LP: #1336855) + - FreeBsd: support config drive datasource [Joseph bajin] + - cc_mounts: support creating a swap file 0.7.5: - open 0.7.5 - Add a debug log message around import failures diff --git a/cloudinit/config/cc_mounts.py b/cloudinit/config/cc_mounts.py index ba1303d1..1cb1e839 100644 --- a/cloudinit/config/cc_mounts.py +++ b/cloudinit/config/cc_mounts.py @@ -75,6 +75,159 @@ def sanitize_devname(startname, transformer, log): return devnode_for_dev_part(blockdev, part) +def suggested_swapsize(memsize=None, maxsize=None, fsys=None): + # make a suggestion on the size of swap for this system. + if memsize is None: + memsize = util.read_meminfo()['total'] + + GB = 2 ** 30 + sugg_max = 8 * GB + + info = {'avail': 'na', 'max_in': maxsize, 'mem': memsize} + + if fsys is None and maxsize is None: + # set max to 8GB default if no filesystem given + maxsize = sugg_max + elif fsys: + statvfs = os.statvfs(fsys) + avail = statvfs.f_frsize * statvfs.f_bfree + info['avail'] = avail + + if maxsize is None: + # set to 25% of filesystem space + maxsize = min(int(avail / 4), sugg_max) + elif maxsize > ((avail * .9)): + # set to 90% of available disk space + maxsize = int(avail * .9) + elif maxsize is None: + maxsize = sugg_max + + info['max'] = maxsize + + formulas = [ + # < 1G: swap = double memory + (1 * GB, lambda x: x * 2), + # < 2G: swap = 2G + (2 * GB, lambda x: 2 * GB), + # < 4G: swap = memory + (4 * GB, lambda x: x), + # < 16G: 4G + (16 * GB, lambda x: 4 * GB), + # < 64G: 1/2 M up to max + (64 * GB, lambda x: x / 2), + ] + + size = None + for top, func in formulas: + if memsize <= top: + size = min(func(memsize), maxsize) + # if less than 1/2 memory and not much, return 0 + if size < (memsize / 2) and size < 4 * GB: + size = 0 + break + break + + if size is not None: + size = maxsize + + info['size'] = size + + MB = 2 ** 20 + pinfo = {} + for k, v in info.items(): + if isinstance(v, int): + pinfo[k] = "%s MB" % (v / MB) + else: + pinfo[k] = v + + LOG.debug("suggest %(size)s swap for %(mem)s memory with '%(avail)s'" + " disk given max=%(max_in)s [max=%(max)s]'" % pinfo) + return size + + +def setup_swapfile(fname, size=None, maxsize=None): + """ + fname: full path string of filename to setup + size: the size to create. set to "auto" for recommended + maxsize: the maximum size + """ + tdir = os.path.dirname(fname) + if str(size).lower() == "auto": + try: + memsize = util.read_meminfo()['total'] + except IOError as e: + LOG.debug("Not creating swap. failed to read meminfo") + return + + util.ensure_dir(tdir) + size = suggested_swapsize(fsys=tdir, maxsize=maxsize, + memsize=memsize) + + if not size: + LOG.debug("Not creating swap: suggested size was 0") + return + + mbsize = str(int(size / (2 ** 20))) + msg = "creating swap file '%s' of %sMB" % (fname, mbsize) + try: + util.ensure_dir(tdir) + util.log_time(LOG.debug, msg, func=util.subp, + args=[['sh', '-c', + ('rm -f "$1" && umask 0066 && ' + 'dd if=/dev/zero "of=$1" bs=1M "count=$2" && ' + 'mkswap "$1" || { r=$?; rm -f "$1"; exit $r; }'), + 'setup_swap', fname, mbsize]]) + + except Exception as e: + raise IOError("Failed %s: %s" % (msg, e)) + + return fname + + +def handle_swapcfg(swapcfg): + """handle the swap config, calling setup_swap if necessary. + return None or (filename, size) + """ + if not isinstance(swapcfg, dict): + LOG.warn("input for swap config was not a dict.") + return None + + fname = swapcfg.get('filename', '/swap.img') + size = swapcfg.get('size', 0) + maxsize = swapcfg.get('maxsize', None) + + if not (size and fname): + LOG.debug("no need to setup swap") + return + + if os.path.exists(fname): + if not os.path.exists("/proc/swaps"): + LOG.debug("swap file %s existed. no /proc/swaps. Being safe.", + fname) + return fname + try: + for line in util.load_file("/proc/swaps").splitlines(): + if line.startswith(fname + " "): + LOG.debug("swap file %s already in use.", fname) + return fname + LOG.debug("swap file %s existed, but not in /proc/swaps", fname) + except: + LOG.warn("swap file %s existed. Error reading /proc/swaps", fname) + return fname + + try: + if isinstance(size, str) and size != "auto": + size = util.human2bytes(size) + if isinstance(maxsize, str): + maxsize = util.human2bytes(maxsize) + return setup_swapfile(fname=fname, size=size, maxsize=maxsize) + + except Exception as e: + LOG.warn("failed to setup swap: %s", e) + + return None + + def handle(_name, cfg, cloud, log, _args): # fs_spec, fs_file, fs_vfstype, fs_mntops, fs-freq, fs_passno defvals = [None, None, "auto", "defaults,nobootwait", "0", "2"] @@ -162,6 +315,10 @@ def handle(_name, cfg, cloud, log, _args): else: actlist.append(x) + swapret = handle_swapcfg(cfg.get('swap', {})) + if swapret: + actlist.append([swapret, "none", "swap", "sw", "0", "0"]) + if len(actlist) == 0: log.debug("No modifications to fstab needed.") return diff --git a/cloudinit/distros/freebsd.py b/cloudinit/distros/freebsd.py index cff10387..ee23fd20 100644 --- a/cloudinit/distros/freebsd.py +++ b/cloudinit/distros/freebsd.py @@ -106,6 +106,35 @@ class Distro(distros.Distro): val = None return val + # NOVA will inject something like eth0, rewrite that to use the FreeBSD + # adapter. Since this adapter is based on the used driver, we need to + # figure out which interfaces are available. On KVM platforms this is + # vtnet0, where Xen would use xn0. + def getnetifname(self, dev): + LOG.debug("Translating network interface %s", dev) + if dev.startswith('lo'): + return dev + + n = re.search('\d+$', dev) + index = n.group(0) + + (out, err) = util.subp(['ifconfig', '-a']) + ifconfigoutput = [x for x in (out.strip()).splitlines() if len(x.split()) > 0] + for line in ifconfigoutput: + m = re.match('^\w+', line) + if m: + if m.group(0).startswith('lo'): + continue + # Just settle with the first non-lo adapter we find, since it's + # rather unlikely there will be multiple nicdrivers involved. + bsddev = m.group(0) + break + + # Replace the index with the one we're after. + bsddev = re.sub('\d+$', index, bsddev) + LOG.debug("Using network interface %s", bsddev) + return bsddev + def _read_system_hostname(self): sys_hostname = self._read_hostname(filename=None) return ('rc.conf', sys_hostname) @@ -162,18 +191,18 @@ class Distro(distros.Distro): log_adduser_cmd = ['pw', 'useradd', '-n', name] adduser_opts = { - "homedir": '-d', - "gecos": '-c', - "primary_group": '-g', - "groups": '-G', - "passwd": '-h', - "shell": '-s', - "inactive": '-E', + "homedir": '-d', + "gecos": '-c', + "primary_group": '-g', + "groups": '-G', + "passwd": '-h', + "shell": '-s', + "inactive": '-E', } adduser_flags = { - "no_user_group": '--no-user-group', - "system": '--system', - "no_log_init": '--no-log-init', + "no_user_group": '--no-user-group', + "system": '--system', + "no_log_init": '--no-log-init', } redact_opts = ['passwd'] @@ -246,17 +275,21 @@ class Distro(distros.Distro): nameservers = [] searchdomains = [] dev_names = entries.keys() - for (dev, info) in entries.iteritems(): + for (device, info) in entries.iteritems(): # Skip the loopback interface. - if dev.startswith('lo'): + if device.startswith('lo'): continue + dev = self.getnetifname(device) + LOG.info('Configuring interface %s', dev) if info.get('bootproto') == 'static': - LOG.debug('Configuring dev %s with %s / %s', dev, info.get('address'), info.get('netmask')) + LOG.debug('Configuring dev %s with %s / %s', dev, + info.get('address'), info.get('netmask')) # Configure an ipv4 address. - ifconfig = info.get('address') + ' netmask ' + info.get('netmask') + ifconfig = (info.get('address') + ' netmask ' + + info.get('netmask')) # Configure the gateway. self.updatercconf('defaultrouter', info.get('gateway')) @@ -267,7 +300,7 @@ class Distro(distros.Distro): searchdomains.extend(info['dns-search']) else: ifconfig = 'DHCP' - + self.updatercconf('ifconfig_' + dev, ifconfig) # Try to read the /etc/resolv.conf or just start from scratch if that @@ -276,7 +309,8 @@ class Distro(distros.Distro): resolvconf = ResolvConf(util.load_file(self.resolv_conf_fn)) resolvconf.parse() except IOError: - util.logexc(LOG, "Failed to parse %s, use new empty file", self.resolv_conf_fn) + util.logexc(LOG, "Failed to parse %s, use new empty file", + self.resolv_conf_fn) resolvconf = ResolvConf('') resolvconf.parse() @@ -323,6 +357,19 @@ class Distro(distros.Distro): util.logexc(LOG, "Failed to restore %s backup", self.login_conf_fn) + def _bring_up_interface(self, device_name): + if device_name.startswith('lo'): + return + dev = self.getnetifname(device_name) + cmd = ['/etc/rc.d/netif', 'start', dev] + LOG.debug("Attempting to bring up interface %s using command %s", + dev, cmd) + # This could return 1 when the interface has already been put UP by the + # OS. This is just fine. + (_out, err) = util.subp(cmd, rcs=[0, 1]) + if len(err): + LOG.warn("Error running %s: %s", cmd, err) + def install_packages(self, pkglist): return diff --git a/cloudinit/sources/DataSourceConfigDrive.py b/cloudinit/sources/DataSourceConfigDrive.py index 4e5d90de..27658073 100644 --- a/cloudinit/sources/DataSourceConfigDrive.py +++ b/cloudinit/sources/DataSourceConfigDrive.py @@ -37,7 +37,9 @@ DEFAULT_METADATA = { VALID_DSMODES = ("local", "net", "pass", "disabled") FS_TYPES = ('vfat', 'iso9660') LABEL_TYPES = ('config-2',) -OPTICAL_DEVICES = tuple(('/dev/sr%s' % i for i in range(0, 2))) +POSSIBLE_MOUNTS = ('sr', 'cd') +OPTICAL_DEVICES = tuple(('/dev/%s%s' % (z, i) for z in POSSIBLE_MOUNTS + for i in range(0, 2))) class DataSourceConfigDrive(openstack.SourceMixin, sources.DataSource): @@ -70,7 +72,15 @@ class DataSourceConfigDrive(openstack.SourceMixin, sources.DataSource): if not found: for dev in find_candidate_devs(): try: - results = util.mount_cb(dev, read_config_drive) + # Set mtype if freebsd and turn off sync + if dev.startswith("/dev/cd"): + mtype = "cd9660" + sync = False + else: + mtype = None + sync = True + results = util.mount_cb(dev, read_config_drive, mtype=mtype, + sync=sync) found = dev except openstack.NonReadable: pass diff --git a/cloudinit/sources/DataSourceOVF.py b/cloudinit/sources/DataSourceOVF.py index 2f53c1ba..7ba60735 100644 --- a/cloudinit/sources/DataSourceOVF.py +++ b/cloudinit/sources/DataSourceOVF.py @@ -215,8 +215,7 @@ def transport_iso9660(require_iso=True): continue try: - (fname, contents) = util.mount_cb(fullp, - get_ovf_env, mtype=mtype) + (fname, contents) = util.mount_cb(fullp, get_ovf_env, mtype=mtype) except util.MountFailedError: LOG.debug("%s not mountable as iso9660" % fullp) continue diff --git a/cloudinit/util.py b/cloudinit/util.py index 946059e9..f236d0bf 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -1297,7 +1297,7 @@ def unmounter(umount): yield umount finally: if umount: - umount_cmd = ["umount", '-l', umount] + umount_cmd = ["umount", umount] subp(umount_cmd) @@ -1346,37 +1346,70 @@ def mount_cb(device, callback, data=None, rw=False, mtype=None, sync=True): Mount the device, call method 'callback' passing the directory in which it was mounted, then unmount. Return whatever 'callback' returned. If data != None, also pass data to callback. + + mtype is a filesystem type. it may be a list, string (a single fsname) + or a list of fsnames. """ + + if isinstance(mtype, str): + mtypes = [mtype] + elif isinstance(mtype, (list, tuple)): + mtypes = list(mtype) + elif mtype is None: + mtypes = None + + # clean up 'mtype' input a bit based on platform. + platsys = platform.system().lower() + if platsys == "linux": + if mtypes is None: + mtypes = ["auto"] + elif platsys.endswith("bsd"): + if mtypes is None: + mtypes = ['ufs', 'cd9660', 'vfat'] + for index, mtype in enumerate(mtypes): + if mtype == "iso9660": + mtypes[index] = "cd9660" + else: + # we cannot do a smart "auto", so just call 'mount' once with no -t + mtypes = [''] + mounted = mounts() with tempdir() as tmpd: umount = False if device in mounted: mountpoint = mounted[device]['mountpoint'] else: - try: - mountcmd = ['mount'] - mountopts = [] - if rw: - mountopts.append('rw') - else: - mountopts.append('ro') - if sync: - # This seems like the safe approach to do - # (ie where this is on by default) - mountopts.append("sync") - if mountopts: - mountcmd.extend(["-o", ",".join(mountopts)]) - if mtype: - mountcmd.extend(['-t', mtype]) - mountcmd.append(device) - mountcmd.append(tmpd) - subp(mountcmd) - umount = tmpd # This forces it to be unmounted (when set) - mountpoint = tmpd - except (IOError, OSError) as exc: - raise MountFailedError(("Failed mounting %s " - "to %s due to: %s") % + for mtype in mtypes: + mountpoint = None + try: + mountcmd = ['mount'] + mountopts = [] + if rw: + mountopts.append('rw') + else: + mountopts.append('ro') + if sync: + # This seems like the safe approach to do + # (ie where this is on by default) + mountopts.append("sync") + if mountopts: + mountcmd.extend(["-o", ",".join(mountopts)]) + if mtype: + mountcmd.extend(['-t', mtype]) + mountcmd.append(device) + mountcmd.append(tmpd) + subp(mountcmd) + umount = tmpd # This forces it to be unmounted (when set) + mountpoint = tmpd + break + except (IOError, OSError) as exc: + LOG.debug("Failed mount of '%s' as '%s': %s", + device, mtype, exc) + pass + if not mountpoint: + raise MountFailedError("Failed mounting %s to %s due to: %s" % (device, tmpd, exc)) + # Be nice and ensure it ends with a slash if not mountpoint.endswith("/"): mountpoint += "/" @@ -1924,3 +1957,53 @@ def pathprefix2dict(base, required=None, optional=None, delim=os.path.sep): raise ValueError("Missing required files: %s", ','.join(missing)) return ret + + +def read_meminfo(meminfo="/proc/meminfo", raw=False): + # read a /proc/meminfo style file and return + # a dict with 'total', 'free', and 'available' + mpliers = {'kB': 2**10, 'mB': 2 ** 20, 'B': 1, 'gB': 2 ** 30} + kmap = {'MemTotal:': 'total', 'MemFree:': 'free', + 'MemAvailable:': 'available'} + ret = {} + for line in load_file(meminfo).splitlines(): + try: + key, value, unit = line.split() + except ValueError: + key, value = line.split() + unit = 'B' + if raw: + ret[key] = int(value) * mpliers[unit] + elif key in kmap: + ret[kmap[key]] = int(value) * mpliers[unit] + + return ret + + +def human2bytes(size): + """Convert human string or integer to size in bytes + 10M => 10485760 + .5G => 536870912 + """ + size_in = size + if size.endswith("B"): + size = size[:-1] + + mpliers = {'B': 1, 'K': 2 ** 10, 'M': 2 ** 20, 'G': 2 ** 30, 'T': 2 ** 40} + + num = size + mplier = 'B' + for m in mpliers: + if size.endswith(m): + mplier = m + num = size[0:-len(m)] + + try: + num = float(num) + except ValueError: + raise ValueError("'%s' is not valid input." % size_in) + + if num < 0: + raise ValueError("'%s': cannot be negative" % size_in) + + return int(num * mpliers[mplier]) diff --git a/config/cloud.cfg-freebsd b/config/cloud.cfg-freebsd index bb3a4a51..5ac181ff 100644 --- a/config/cloud.cfg-freebsd +++ b/config/cloud.cfg-freebsd @@ -5,7 +5,7 @@ syslog_fix_perms: root:wheel # This should not be required, but leave it in place until the real cause of # not beeing able to find -any- datasources is resolved. -datasource_list: ['OpenStack'] +datasource_list: ['ConfigDrive', 'OpenStack', 'Ec2'] # A set of users which may be applied and/or used by various modules # when a 'default' entry is found it will reference the 'default_user' diff --git a/doc/examples/cloud-config-mount-points.txt b/doc/examples/cloud-config-mount-points.txt index 416006db..3b45b47f 100644 --- a/doc/examples/cloud-config-mount-points.txt +++ b/doc/examples/cloud-config-mount-points.txt @@ -37,3 +37,10 @@ mounts: # complete. This must be an array, and must have 7 fields. mount_default_fields: [ None, None, "auto", "defaults,nobootwait", "0", "2" ] + +# swap can also be set up by the 'mounts' module +# default is to not create any swap files, because 'size' is set to 0 +swap: + filename: /swap.img + size: "auto" or size in bytes + maxsize: size in bytes diff --git a/sysvinit/freebsd/cloudconfig b/sysvinit/freebsd/cloudconfig index 44c216b3..01bc061e 100755 --- a/sysvinit/freebsd/cloudconfig +++ b/sysvinit/freebsd/cloudconfig @@ -6,6 +6,7 @@ . /etc/rc.subr +PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" export CLOUD_CFG=/usr/local/etc/cloud/cloud.cfg name="cloudconfig" diff --git a/sysvinit/freebsd/cloudfinal b/sysvinit/freebsd/cloudfinal index f668e036..1b487aa0 100755 --- a/sysvinit/freebsd/cloudfinal +++ b/sysvinit/freebsd/cloudfinal @@ -6,6 +6,7 @@ . /etc/rc.subr +PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" export CLOUD_CFG=/usr/local/etc/cloud/cloud.cfg name="cloudfinal" diff --git a/sysvinit/freebsd/cloudinit b/sysvinit/freebsd/cloudinit index c5478678..862eeab4 100755 --- a/sysvinit/freebsd/cloudinit +++ b/sysvinit/freebsd/cloudinit @@ -6,6 +6,7 @@ . /etc/rc.subr +PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" export CLOUD_CFG=/usr/local/etc/cloud/cloud.cfg name="cloudinit" diff --git a/sysvinit/freebsd/cloudinitlocal b/sysvinit/freebsd/cloudinitlocal index c340d5d0..fb342a0f 100755 --- a/sysvinit/freebsd/cloudinitlocal +++ b/sysvinit/freebsd/cloudinitlocal @@ -6,6 +6,7 @@ . /etc/rc.subr +PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" export CLOUD_CFG=/usr/local/etc/cloud/cloud.cfg name="cloudinitlocal" diff --git a/templates/hosts.freebsd.tmpl b/templates/hosts.freebsd.tmpl new file mode 100644 index 00000000..7ded762f --- /dev/null +++ b/templates/hosts.freebsd.tmpl @@ -0,0 +1,24 @@ +## template:jinja +{# +This file /etc/cloud/templates/hosts.freebsd.tmpl is only utilized +if enabled in cloud-config. Specifically, in order to enable it +you need to add the following to config: + manage_etc_hosts: True +-#} +# Your system has configured 'manage_etc_hosts' as True. +# As a result, if you wish for changes to this file to persist +# then you will need to either +# a.) make changes to the master file in /etc/cloud/templates/hosts.freebsd.tmpl +# b.) change or remove the value of 'manage_etc_hosts' in +# /etc/cloud/cloud.cfg or cloud-config from user-data +# +# The following lines are desirable for IPv4 capable hosts +127.0.0.1 {{fqdn}} {{hostname}} +127.0.0.1 localhost.localdomain localhost +127.0.0.1 localhost4.localdomain4 localhost4 + +# The following lines are desirable for IPv6 capable hosts +::1 {{fqdn}} {{hostname}} +::1 localhost.localdomain localhost +::1 localhost6.localdomain6 localhost6 + diff --git a/tests/unittests/test_distros/test_netconfig.py b/tests/unittests/test_distros/test_netconfig.py index ed997a1d..fbdb7b3f 100644 --- a/tests/unittests/test_distros/test_netconfig.py +++ b/tests/unittests/test_distros/test_netconfig.py @@ -223,8 +223,8 @@ NETWORKING=yes self.assertIn('/etc/rc.conf', write_bufs) write_buf = write_bufs['/etc/rc.conf'] expected_buf = ''' -ifconfig_eth0="192.168.1.5 netmask 255.255.255.0" -ifconfig_eth1="DHCP" +ifconfig_vtnet0="192.168.1.5 netmask 255.255.255.0" +ifconfig_vtnet1="DHCP" defaultrouter="192.168.1.254" ''' self.assertCfgEquals(expected_buf, str(write_buf)) diff --git a/tools/build-on-freebsd b/tools/build-on-freebsd index 65d783f7..8436498e 100755 --- a/tools/build-on-freebsd +++ b/tools/build-on-freebsd @@ -9,15 +9,23 @@ fail() { echo "FAILED:" "$@" 1>&2; exit 1; } depschecked=/tmp/c-i.dependencieschecked pkgs=" dmidecode - py27-argparse - py27-boto gpart sudo - py27-configobj py27-yaml + e2fsprogs + gpart py27-Jinja2 - py27-oauth py27-serial + py27-argparse + py27-boto + py27-cheetah + py27-configobj + py27-jsonpatch + py27-jsonpointer + py27-oauth py27-prettytable - py27-requests py27-six - python py27-cheetah - py27-jsonpointer py27-jsonpatch + py27-requests + py27-serial + py27-six + py27-yaml + python + sudo " [ -f "$depschecked" ] || pkg install ${pkgs} || fail "install packages" touch $depschecked |