From 923a7d0efc3cf3224b891dc783321018d59ce33e Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Sat, 14 Dec 2013 14:17:31 -0500 Subject: support calling apt with eatmydata, enable by default if available. This allows a general config option to prefix apt-get commands via 'apt_get_wrapper'. By default, the command is set to 'eatmydata', and the mode set to 'auto'. That means if eatmydata is available (via which), it will use it. The 'command' can be either a array or a string. LP: #1236531 --- cloudinit/distros/debian.py | 28 ++++++++++++++++++++++++++-- 1 file changed, 26 insertions(+), 2 deletions(-) (limited to 'cloudinit/distros') diff --git a/cloudinit/distros/debian.py b/cloudinit/distros/debian.py index 8fe49cbe..1ae232fd 100644 --- a/cloudinit/distros/debian.py +++ b/cloudinit/distros/debian.py @@ -36,6 +36,10 @@ LOG = logging.getLogger(__name__) APT_GET_COMMAND = ('apt-get', '--option=Dpkg::Options::=--force-confold', '--option=Dpkg::options::=--force-unsafe-io', '--assume-yes', '--quiet') +APT_GET_WRAPPER = { + 'command': 'eatmydata', + 'enabled': 'auto', +} class Distro(distros.Distro): @@ -148,7 +152,13 @@ class Distro(distros.Distro): # See: http://tiny.cc/kg91fw # Or: http://tiny.cc/mh91fw e['DEBIAN_FRONTEND'] = 'noninteractive' - cmd = list(self.get_option("apt_get_command", APT_GET_COMMAND)) + + wcfg = self.get_option("apt_get_wrapper", APT_GET_WRAPPER) + cmd = _get_wrapper_prefix( + wcfg.get('command', APT_GET_WRAPPER['command']), + wcfg.get('enabled', APT_GET_WRAPPER['enabled'])) + + cmd.extend(list(self.get_option("apt_get_command", APT_GET_COMMAND))) if args and isinstance(args, str): cmd.append(args) @@ -166,7 +176,9 @@ class Distro(distros.Distro): cmd.extend(pkglist) # Allow the output of this to flow outwards (ie not be captured) - util.subp(cmd, env=e, capture=False) + util.log_time(logfunc=LOG.debug, + msg="apt-%s [%s]" % (command, ' '.join(cmd)), func=util.subp, + args=(cmd,), kwargs={'env': e, 'capture': False}) def update_package_sources(self): self._runner.run("update-sources", self.package_command, @@ -175,3 +187,15 @@ class Distro(distros.Distro): def get_primary_arch(self): (arch, _err) = util.subp(['dpkg', '--print-architecture']) return str(arch).strip() + + +def _get_wrapper_prefix(cmd, mode): + if isinstance(cmd, str): + cmd = [str(cmd)] + + if (util.is_true(mode) or + (str(mode).lower() == "auto" and cmd[0] and + util.which(cmd[0]))): + return cmd + else: + return [] -- cgit v1.2.3 From 31ece5e92797bf20141878de5dd7cd91559cb336 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Wed, 22 Jan 2014 12:04:39 -0800 Subject: Split net-parsing into own module The ubuntu/debian networking file parsing function really is more generic than just a rhel utility function and can be used by others that want to use this functionality for there own purposes (say in writing down a freebsd network format instead) so moving this to its own module to encourage its usage outside of rhel. --- cloudinit/distros/net_util.py | 110 +++++++++++++++++++++++++++++++++++++++++ cloudinit/distros/rhel.py | 4 +- cloudinit/distros/rhel_util.py | 88 --------------------------------- cloudinit/distros/sles.py | 4 +- 4 files changed, 116 insertions(+), 90 deletions(-) create mode 100644 cloudinit/distros/net_util.py (limited to 'cloudinit/distros') diff --git a/cloudinit/distros/net_util.py b/cloudinit/distros/net_util.py new file mode 100644 index 00000000..4c9095be --- /dev/null +++ b/cloudinit/distros/net_util.py @@ -0,0 +1,110 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2012 Canonical Ltd. +# Copyright (C) 2012, 2013 Hewlett-Packard Development Company, L.P. +# Copyright (C) 2012 Yahoo! Inc. +# +# Author: Scott Moser +# Author: Juerg Haefliger +# Author: Joshua Harlow +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + + +# This is a util function to translate debian based distro interface blobs as +# given in /etc/network/interfaces to an *somewhat* agnostic format for +# distributions that use other formats. +# +# TODO(harlowja) remove when we have python-netcf active... +def translate_network(settings): + # Get the standard cmd, args from the ubuntu format + entries = [] + for line in settings.splitlines(): + line = line.strip() + if not line or line.startswith("#"): + continue + split_up = line.split(None, 1) + if len(split_up) <= 1: + continue + entries.append(split_up) + # Figure out where each iface section is + ifaces = [] + consume = {} + for (cmd, args) in entries: + if cmd == 'iface': + if consume: + ifaces.append(consume) + consume = {} + consume[cmd] = args + else: + consume[cmd] = args + # Check if anything left over to consume + absorb = False + for (cmd, args) in consume.iteritems(): + if cmd == 'iface': + absorb = True + if absorb: + ifaces.append(consume) + # Now translate + real_ifaces = {} + for info in ifaces: + if 'iface' not in info: + continue + iface_details = info['iface'].split(None) + dev_name = None + if len(iface_details) >= 1: + dev = iface_details[0].strip().lower() + if dev: + dev_name = dev + if not dev_name: + continue + iface_info = {} + if len(iface_details) >= 3: + proto_type = iface_details[2].strip().lower() + # Seems like this can be 'loopback' which we don't + # really care about + if proto_type in ['dhcp', 'static']: + iface_info['bootproto'] = proto_type + # These can just be copied over + for k in ['netmask', 'address', 'gateway', 'broadcast']: + if k in info: + val = info[k].strip().lower() + if val: + iface_info[k] = val + # Name server info provided?? + if 'dns-nameservers' in info: + iface_info['dns-nameservers'] = info['dns-nameservers'].split() + # Name server search info provided?? + if 'dns-search' in info: + iface_info['dns-search'] = info['dns-search'].split() + # Is any mac address spoofing going on?? + if 'hwaddress' in info: + hw_info = info['hwaddress'].lower().strip() + hw_split = hw_info.split(None, 1) + if len(hw_split) == 2 and hw_split[0].startswith('ether'): + hw_addr = hw_split[1] + if hw_addr: + iface_info['hwaddress'] = hw_addr + real_ifaces[dev_name] = iface_info + # Check for those that should be started on boot via 'auto' + for (cmd, args) in entries: + if cmd == 'auto': + # Seems like auto can be like 'auto eth0 eth0:1' so just get the + # first part out as the device name + args = args.split(None) + if not args: + continue + dev_name = args[0].strip().lower() + if dev_name in real_ifaces: + real_ifaces[dev_name]['auto'] = True + return real_ifaces diff --git a/cloudinit/distros/rhel.py b/cloudinit/distros/rhel.py index 30195384..6087929e 100644 --- a/cloudinit/distros/rhel.py +++ b/cloudinit/distros/rhel.py @@ -25,7 +25,9 @@ from cloudinit import helpers from cloudinit import log as logging from cloudinit import util +from cloudinit.distros import net_util from cloudinit.distros import rhel_util + from cloudinit.settings import PER_INSTANCE LOG = logging.getLogger(__name__) @@ -63,7 +65,7 @@ class Distro(distros.Distro): def _write_network(self, settings): # TODO(harlowja) fix this... since this is the ubuntu format - entries = rhel_util.translate_network(settings) + entries = net_util.translate_network(settings) LOG.debug("Translated ubuntu style network settings %s into %s", settings, entries) # Make the intermediate format as the rhel format... diff --git a/cloudinit/distros/rhel_util.py b/cloudinit/distros/rhel_util.py index 1aba58b8..063d536e 100644 --- a/cloudinit/distros/rhel_util.py +++ b/cloudinit/distros/rhel_util.py @@ -30,94 +30,6 @@ from cloudinit import util LOG = logging.getLogger(__name__) -# This is a util function to translate Debian based distro interface blobs as -# given in /etc/network/interfaces to an equivalent format for distributions -# that use ifcfg-* style (Red Hat and SUSE). -# TODO(harlowja) remove when we have python-netcf active... -def translate_network(settings): - # Get the standard cmd, args from the ubuntu format - entries = [] - for line in settings.splitlines(): - line = line.strip() - if not line or line.startswith("#"): - continue - split_up = line.split(None, 1) - if len(split_up) <= 1: - continue - entries.append(split_up) - # Figure out where each iface section is - ifaces = [] - consume = {} - for (cmd, args) in entries: - if cmd == 'iface': - if consume: - ifaces.append(consume) - consume = {} - consume[cmd] = args - else: - consume[cmd] = args - # Check if anything left over to consume - absorb = False - for (cmd, args) in consume.iteritems(): - if cmd == 'iface': - absorb = True - if absorb: - ifaces.append(consume) - # Now translate - real_ifaces = {} - for info in ifaces: - if 'iface' not in info: - continue - iface_details = info['iface'].split(None) - dev_name = None - if len(iface_details) >= 1: - dev = iface_details[0].strip().lower() - if dev: - dev_name = dev - if not dev_name: - continue - iface_info = {} - if len(iface_details) >= 3: - proto_type = iface_details[2].strip().lower() - # Seems like this can be 'loopback' which we don't - # really care about - if proto_type in ['dhcp', 'static']: - iface_info['bootproto'] = proto_type - # These can just be copied over - for k in ['netmask', 'address', 'gateway', 'broadcast']: - if k in info: - val = info[k].strip().lower() - if val: - iface_info[k] = val - # Name server info provided?? - if 'dns-nameservers' in info: - iface_info['dns-nameservers'] = info['dns-nameservers'].split() - # Name server search info provided?? - if 'dns-search' in info: - iface_info['dns-search'] = info['dns-search'].split() - # Is any mac address spoofing going on?? - if 'hwaddress' in info: - hw_info = info['hwaddress'].lower().strip() - hw_split = hw_info.split(None, 1) - if len(hw_split) == 2 and hw_split[0].startswith('ether'): - hw_addr = hw_split[1] - if hw_addr: - iface_info['hwaddress'] = hw_addr - real_ifaces[dev_name] = iface_info - # Check for those that should be started on boot via 'auto' - for (cmd, args) in entries: - if cmd == 'auto': - # Seems like auto can be like 'auto eth0 eth0:1' so just get the - # first part out as the device name - args = args.split(None) - if not args: - continue - dev_name = args[0].strip().lower() - if dev_name in real_ifaces: - real_ifaces[dev_name]['auto'] = True - return real_ifaces - - # Helper function to update a RHEL/SUSE /etc/sysconfig/* file def update_sysconfig_file(fn, adjustments, allow_empty=False): if not adjustments: diff --git a/cloudinit/distros/sles.py b/cloudinit/distros/sles.py index f2ac4efc..239e51b5 100644 --- a/cloudinit/distros/sles.py +++ b/cloudinit/distros/sles.py @@ -26,7 +26,9 @@ from cloudinit import helpers from cloudinit import log as logging from cloudinit import util +from cloudinit.distros import net_util from cloudinit.distros import rhel_util + from cloudinit.settings import PER_INSTANCE LOG = logging.getLogger(__name__) @@ -54,7 +56,7 @@ class Distro(distros.Distro): def _write_network(self, settings): # Convert debian settings to ifcfg format - entries = rhel_util.translate_network(settings) + entries = net_util.translate_network(settings) LOG.debug("Translated ubuntu style network settings %s into %s", settings, entries) # Make the intermediate format as the suse format... -- cgit v1.2.3 From 5f14b91c2a17101e845ee18068c7bb8c3c7a0556 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Wed, 22 Jan 2014 12:39:37 -0800 Subject: Add comments as to format with example in/out --- cloudinit/distros/net_util.py | 52 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) (limited to 'cloudinit/distros') diff --git a/cloudinit/distros/net_util.py b/cloudinit/distros/net_util.py index 4c9095be..8c781f9a 100644 --- a/cloudinit/distros/net_util.py +++ b/cloudinit/distros/net_util.py @@ -26,6 +26,58 @@ # distributions that use other formats. # # TODO(harlowja) remove when we have python-netcf active... +# +# The format is the following: +# { +# : { +# # All optional (if not existent in original format) +# "netmask": , +# "broadcast": , +# "gateway": , +# "address": , +# "bootproto": "static"|"dhcp", +# "dns-search": , +# "hwaddress": , +# "auto": True (or non-existent), +# "dns-nameservers": [, ...], +# } +# } +# +# Things to note, comments are removed, if a ubuntu/debian interface is +# marked as auto then only then first segment (?) is retained, ie +# 'auto eth0 eth0:1' just marks eth0 as auto (not eth0:1). +# +# Example input: +# +# auto lo +# iface lo inet loopback +# +# auto eth0 +# iface eth0 inet static +# address 10.0.0.1 +# netmask 255.255.252.0 +# broadcast 10.0.0.255 +# gateway 10.0.0.2 +# dns-nameservers 98.0.0.1 98.0.0.2 +# +# Example output: +# { +# "lo": { +# "auto": true +# }, +# "eth0": { +# "auto": true, +# "dns-nameservers": [ +# "98.0.0.1", +# "98.0.0.2" +# ], +# "broadcast": "10.0.0.255", +# "netmask": "255.255.252.0", +# "bootproto": "static", +# "address": "10.0.0.1", +# "gateway": "10.0.0.2" +# } +# } def translate_network(settings): # Get the standard cmd, args from the ubuntu format entries = [] -- cgit v1.2.3 From efe969756d0208433e953d1692ea85006a56abe3 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Wed, 22 Jan 2014 12:47:51 -0800 Subject: Add a new line --- cloudinit/distros/net_util.py | 1 + 1 file changed, 1 insertion(+) (limited to 'cloudinit/distros') diff --git a/cloudinit/distros/net_util.py b/cloudinit/distros/net_util.py index 8c781f9a..5f60666d 100644 --- a/cloudinit/distros/net_util.py +++ b/cloudinit/distros/net_util.py @@ -78,6 +78,7 @@ # "gateway": "10.0.0.2" # } # } + def translate_network(settings): # Get the standard cmd, args from the ubuntu format entries = [] -- cgit v1.2.3 From c0ee33fc1d70b272a96430def658afd6f1867afa Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Thu, 23 Jan 2014 14:06:13 -0500 Subject: remove some white space --- cloudinit/distros/rhel.py | 1 - cloudinit/distros/sles.py | 1 - 2 files changed, 2 deletions(-) (limited to 'cloudinit/distros') diff --git a/cloudinit/distros/rhel.py b/cloudinit/distros/rhel.py index 6087929e..e8abf111 100644 --- a/cloudinit/distros/rhel.py +++ b/cloudinit/distros/rhel.py @@ -27,7 +27,6 @@ from cloudinit import util from cloudinit.distros import net_util from cloudinit.distros import rhel_util - from cloudinit.settings import PER_INSTANCE LOG = logging.getLogger(__name__) diff --git a/cloudinit/distros/sles.py b/cloudinit/distros/sles.py index 239e51b5..9788a1ba 100644 --- a/cloudinit/distros/sles.py +++ b/cloudinit/distros/sles.py @@ -28,7 +28,6 @@ from cloudinit import util from cloudinit.distros import net_util from cloudinit.distros import rhel_util - from cloudinit.settings import PER_INSTANCE LOG = logging.getLogger(__name__) -- cgit v1.2.3