From a800da2371b85b2ece1a30de00f988035819c958 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Mon, 9 May 2016 16:36:04 -0700 Subject: Use the new renderer in the debian and stages files --- cloudinit/stages.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'cloudinit/stages.py') diff --git a/cloudinit/stages.py b/cloudinit/stages.py index ffb15165..e6bd34fe 100644 --- a/cloudinit/stages.py +++ b/cloudinit/stages.py @@ -43,7 +43,7 @@ from cloudinit import distros from cloudinit import helpers from cloudinit import importer from cloudinit import log as logging -from cloudinit import net +from cloudinit.net import klibc from cloudinit import sources from cloudinit import type_utils from cloudinit import util @@ -579,7 +579,7 @@ class Init(object): if os.path.exists(disable_file): return (None, disable_file) - cmdline_cfg = ('cmdline', net.read_kernel_cmdline_config()) + cmdline_cfg = ('cmdline', klibc.read_kernel_cmdline_config()) dscfg = ('ds', None) if self.datasource and hasattr(self.datasource, 'network_config'): dscfg = ('ds', self.datasource.network_config) -- cgit v1.2.3 From 483545957dca362452b94fc49064e298fc07dd71 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Tue, 10 May 2016 13:02:29 -0700 Subject: Rename to net/renderers and klibc -> cmdline.py --- cloudinit/net/cmdline.py | 191 +++++++++++++++++ cloudinit/net/distros/__init__.py | 0 cloudinit/net/distros/debian.py | 401 ------------------------------------ cloudinit/net/klibc.py | 191 ----------------- cloudinit/net/renderers/__init__.py | 0 cloudinit/net/renderers/eni.py | 399 +++++++++++++++++++++++++++++++++++ cloudinit/stages.py | 4 +- 7 files changed, 592 insertions(+), 594 deletions(-) create mode 100644 cloudinit/net/cmdline.py delete mode 100644 cloudinit/net/distros/__init__.py delete mode 100644 cloudinit/net/distros/debian.py delete mode 100644 cloudinit/net/klibc.py create mode 100644 cloudinit/net/renderers/__init__.py create mode 100644 cloudinit/net/renderers/eni.py (limited to 'cloudinit/stages.py') diff --git a/cloudinit/net/cmdline.py b/cloudinit/net/cmdline.py new file mode 100644 index 00000000..958c264b --- /dev/null +++ b/cloudinit/net/cmdline.py @@ -0,0 +1,191 @@ +# Copyright (C) 2013-2014 Canonical Ltd. +# +# Author: Scott Moser +# Author: Blake Rouse +# +# Curtin is free software: you can redistribute it and/or modify it under +# the terms of the GNU Affero General Public License as published by the +# Free Software Foundation, either version 3 of the License, or (at your +# option) any later version. +# +# Curtin is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +# FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for +# more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Curtin. If not, see . + +import base64 +import glob +import gzip +import io +import shlex + +from cloudinit.net import get_devicelist +from cloudinit.net import sys_netdev_info + +from cloudinit import util + + +def _load_shell_content(content, add_empty=False, empty_val=None): + """Given shell like syntax (key=value\nkey2=value2\n) in content + return the data in dictionary form. If 'add_empty' is True + then add entries in to the returned dictionary for 'VAR=' + variables. Set their value to empty_val.""" + data = {} + for line in shlex.split(content): + key, value = line.split("=", 1) + if not value: + value = empty_val + if add_empty or value: + data[key] = value + + return data + + +def _klibc_to_config_entry(content, mac_addrs=None): + """Convert a klibc writtent shell content file to a 'config' entry + When ip= is seen on the kernel command line in debian initramfs + and networking is brought up, ipconfig will populate + /run/net-.cfg. + + The files are shell style syntax, and examples are in the tests + provided here. There is no good documentation on this unfortunately. + + DEVICE= is expected/required and PROTO should indicate if + this is 'static' or 'dhcp'. + """ + + if mac_addrs is None: + mac_addrs = {} + + data = _load_shell_content(content) + try: + name = data['DEVICE'] + except KeyError: + raise ValueError("no 'DEVICE' entry in data") + + # ipconfig on precise does not write PROTO + proto = data.get('PROTO') + if not proto: + if data.get('filename'): + proto = 'dhcp' + else: + proto = 'static' + + if proto not in ('static', 'dhcp'): + raise ValueError("Unexpected value for PROTO: %s" % proto) + + iface = { + 'type': 'physical', + 'name': name, + 'subnets': [], + } + + if name in mac_addrs: + iface['mac_address'] = mac_addrs[name] + + # originally believed there might be IPV6* values + for v, pre in (('ipv4', 'IPV4'),): + # if no IPV4ADDR or IPV6ADDR, then go on. + if pre + "ADDR" not in data: + continue + subnet = {'type': proto, 'control': 'manual'} + + # these fields go right on the subnet + for key in ('NETMASK', 'BROADCAST', 'GATEWAY'): + if pre + key in data: + subnet[key.lower()] = data[pre + key] + + dns = [] + # handle IPV4DNS0 or IPV6DNS0 + for nskey in ('DNS0', 'DNS1'): + ns = data.get(pre + nskey) + # verify it has something other than 0.0.0.0 (or ipv6) + if ns and len(ns.strip(":.0")): + dns.append(data[pre + nskey]) + if dns: + subnet['dns_nameservers'] = dns + # add search to both ipv4 and ipv6, as it has no namespace + search = data.get('DOMAINSEARCH') + if search: + if ',' in search: + subnet['dns_search'] = search.split(",") + else: + subnet['dns_search'] = search.split() + + iface['subnets'].append(subnet) + + return name, iface + + +def config_from_klibc_net_cfg(files=None, mac_addrs=None): + if files is None: + files = glob.glob('/run/net*.conf') + + entries = [] + names = {} + for cfg_file in files: + name, entry = _klibc_to_config_entry(util.load_file(cfg_file), + mac_addrs=mac_addrs) + if name in names: + raise ValueError( + "device '%s' defined multiple times: %s and %s" % ( + name, names[name], cfg_file)) + + names[name] = cfg_file + entries.append(entry) + return {'config': entries, 'version': 1} + + +def _decomp_gzip(blob, strict=True): + # decompress blob. raise exception if not compressed unless strict=False. + with io.BytesIO(blob) as iobuf: + gzfp = None + try: + gzfp = gzip.GzipFile(mode="rb", fileobj=iobuf) + return gzfp.read() + except IOError: + if strict: + raise + return blob + finally: + if gzfp: + gzfp.close() + + +def _b64dgz(b64str, gzipped="try"): + # decode a base64 string. If gzipped is true, transparently uncompresss + # if gzipped is 'try', then try gunzip, returning the original on fail. + try: + blob = base64.b64decode(b64str) + except TypeError: + raise ValueError("Invalid base64 text: %s" % b64str) + + if not gzipped: + return blob + + return _decomp_gzip(blob, strict=gzipped != "try") + + +def read_kernel_cmdline_config(files=None, mac_addrs=None, cmdline=None): + if cmdline is None: + cmdline = util.get_cmdline() + + if 'network-config=' in cmdline: + data64 = None + for tok in cmdline.split(): + if tok.startswith("network-config="): + data64 = tok.split("=", 1)[1] + if data64: + return util.load_yaml(_b64dgz(data64)) + + if 'ip=' not in cmdline: + return None + + if mac_addrs is None: + mac_addrs = {k: sys_netdev_info(k, 'address') + for k in get_devicelist()} + + return config_from_klibc_net_cfg(files=files, mac_addrs=mac_addrs) diff --git a/cloudinit/net/distros/__init__.py b/cloudinit/net/distros/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/cloudinit/net/distros/debian.py b/cloudinit/net/distros/debian.py deleted file mode 100644 index 4bf34fd7..00000000 --- a/cloudinit/net/distros/debian.py +++ /dev/null @@ -1,401 +0,0 @@ -# vi: ts=4 expandtab -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -import glob -import os -import re - -from cloudinit.net import LINKS_FNAME_PREFIX -from cloudinit.net import ParserError -from cloudinit.net.udev import generate_udev_rule -from cloudinit import util - - -NET_CONFIG_COMMANDS = [ - "pre-up", "up", "post-up", "down", "pre-down", "post-down", -] - -NET_CONFIG_BRIDGE_OPTIONS = [ - "bridge_ageing", "bridge_bridgeprio", "bridge_fd", "bridge_gcinit", - "bridge_hello", "bridge_maxage", "bridge_maxwait", "bridge_stp", -] - -NET_CONFIG_OPTIONS = [ - "address", "netmask", "broadcast", "network", "metric", "gateway", - "pointtopoint", "media", "mtu", "hostname", "leasehours", "leasetime", - "vendor", "client", "bootfile", "server", "hwaddr", "provider", "frame", - "netnum", "endpoint", "local", "ttl", -] - - -# TODO: switch valid_map based on mode inet/inet6 -def _iface_add_subnet(iface, subnet): - content = "" - valid_map = [ - 'address', - 'netmask', - 'broadcast', - 'metric', - 'gateway', - 'pointopoint', - 'mtu', - 'scope', - 'dns_search', - 'dns_nameservers', - ] - for key, value in subnet.items(): - if value and key in valid_map: - if type(value) == list: - value = " ".join(value) - if '_' in key: - key = key.replace('_', '-') - content += " {} {}\n".format(key, value) - - return content - - -# TODO: switch to valid_map for attrs -def _iface_add_attrs(iface): - content = "" - ignore_map = [ - 'control', - 'index', - 'inet', - 'mode', - 'name', - 'subnets', - 'type', - ] - if iface['type'] not in ['bond', 'bridge', 'vlan']: - ignore_map.append('mac_address') - - for key, value in iface.items(): - if value and key not in ignore_map: - if type(value) == list: - value = " ".join(value) - content += " {} {}\n".format(key, value) - - return content - - -def _iface_start_entry(iface, index): - fullname = iface['name'] - if index != 0: - fullname += ":%s" % index - - control = iface['control'] - if control == "auto": - cverb = "auto" - elif control in ("hotplug",): - cverb = "allow-" + control - else: - cverb = "# control-" + control - - subst = iface.copy() - subst.update({'fullname': fullname, 'cverb': cverb}) - - return ("{cverb} {fullname}\n" - "iface {fullname} {inet} {mode}\n").format(**subst) - - -def _parse_deb_config_data(ifaces, contents, src_dir, src_path): - """Parses the file contents, placing result into ifaces. - - '_source_path' is added to every dictionary entry to define which file - the configration information came from. - - :param ifaces: interface dictionary - :param contents: contents of interfaces file - :param src_dir: directory interfaces file was located - :param src_path: file path the `contents` was read - """ - currif = None - for line in contents.splitlines(): - line = line.strip() - if line.startswith('#'): - continue - split = line.split(' ') - option = split[0] - if option == "source-directory": - parsed_src_dir = split[1] - if not parsed_src_dir.startswith("/"): - parsed_src_dir = os.path.join(src_dir, parsed_src_dir) - for expanded_path in glob.glob(parsed_src_dir): - dir_contents = os.listdir(expanded_path) - dir_contents = [ - os.path.join(expanded_path, path) - for path in dir_contents - if (os.path.isfile(os.path.join(expanded_path, path)) and - re.match("^[a-zA-Z0-9_-]+$", path) is not None) - ] - for entry in dir_contents: - with open(entry, "r") as fp: - src_data = fp.read().strip() - abs_entry = os.path.abspath(entry) - _parse_deb_config_data( - ifaces, src_data, - os.path.dirname(abs_entry), abs_entry) - elif option == "source": - new_src_path = split[1] - if not new_src_path.startswith("/"): - new_src_path = os.path.join(src_dir, new_src_path) - for expanded_path in glob.glob(new_src_path): - with open(expanded_path, "r") as fp: - src_data = fp.read().strip() - abs_path = os.path.abspath(expanded_path) - _parse_deb_config_data( - ifaces, src_data, - os.path.dirname(abs_path), abs_path) - elif option == "auto": - for iface in split[1:]: - if iface not in ifaces: - ifaces[iface] = { - # Include the source path this interface was found in. - "_source_path": src_path - } - ifaces[iface]['auto'] = True - elif option == "iface": - iface, family, method = split[1:4] - if iface not in ifaces: - ifaces[iface] = { - # Include the source path this interface was found in. - "_source_path": src_path - } - elif 'family' in ifaces[iface]: - raise ParserError( - "Interface %s can only be defined once. " - "Re-defined in '%s'." % (iface, src_path)) - ifaces[iface]['family'] = family - ifaces[iface]['method'] = method - currif = iface - elif option == "hwaddress": - ifaces[currif]['hwaddress'] = split[1] - elif option in NET_CONFIG_OPTIONS: - ifaces[currif][option] = split[1] - elif option in NET_CONFIG_COMMANDS: - if option not in ifaces[currif]: - ifaces[currif][option] = [] - ifaces[currif][option].append(' '.join(split[1:])) - elif option.startswith('dns-'): - if 'dns' not in ifaces[currif]: - ifaces[currif]['dns'] = {} - if option == 'dns-search': - ifaces[currif]['dns']['search'] = [] - for domain in split[1:]: - ifaces[currif]['dns']['search'].append(domain) - elif option == 'dns-nameservers': - ifaces[currif]['dns']['nameservers'] = [] - for server in split[1:]: - ifaces[currif]['dns']['nameservers'].append(server) - elif option.startswith('bridge_'): - if 'bridge' not in ifaces[currif]: - ifaces[currif]['bridge'] = {} - if option in NET_CONFIG_BRIDGE_OPTIONS: - bridge_option = option.replace('bridge_', '', 1) - ifaces[currif]['bridge'][bridge_option] = split[1] - elif option == "bridge_ports": - ifaces[currif]['bridge']['ports'] = [] - for iface in split[1:]: - ifaces[currif]['bridge']['ports'].append(iface) - elif option == "bridge_hw" and split[1].lower() == "mac": - ifaces[currif]['bridge']['mac'] = split[2] - elif option == "bridge_pathcost": - if 'pathcost' not in ifaces[currif]['bridge']: - ifaces[currif]['bridge']['pathcost'] = {} - ifaces[currif]['bridge']['pathcost'][split[1]] = split[2] - elif option == "bridge_portprio": - if 'portprio' not in ifaces[currif]['bridge']: - ifaces[currif]['bridge']['portprio'] = {} - ifaces[currif]['bridge']['portprio'][split[1]] = split[2] - elif option.startswith('bond-'): - if 'bond' not in ifaces[currif]: - ifaces[currif]['bond'] = {} - bond_option = option.replace('bond-', '', 1) - ifaces[currif]['bond'][bond_option] = split[1] - for iface in ifaces.keys(): - if 'auto' not in ifaces[iface]: - ifaces[iface]['auto'] = False - - -def _parse_deb_config(path): - """Parses a debian network configuration file.""" - ifaces = {} - with open(path, "r") as fp: - contents = fp.read().strip() - abs_path = os.path.abspath(path) - _parse_deb_config_data( - ifaces, contents, - os.path.dirname(abs_path), abs_path) - return ifaces - - -class Renderer(object): - """Renders network information in a debian format.""" - - def render_persistent_net(self, network_state): - ''' Given state, emit udev rules to map - mac to ifname - ''' - content = "" - interfaces = network_state.get('interfaces') - for iface in interfaces.values(): - # for physical interfaces write out a persist net udev rule - if iface['type'] == 'physical' and \ - 'name' in iface and iface.get('mac_address'): - content += generate_udev_rule(iface['name'], - iface['mac_address']) - - return content - - def render_route(self, route, indent=""): - """ When rendering routes for an iface, in some cases applying a route - may result in the route command returning non-zero which produces - some confusing output for users manually using ifup/ifdown[1]. To - that end, we will optionally include an '|| true' postfix to each - route line allowing users to work with ifup/ifdown without using - --force option. - - We may at somepoint not want to emit this additional postfix, and - add a 'strict' flag to this function. When called with strict=True, - then we will not append the postfix. - - 1. http://askubuntu.com/questions/168033/ - how-to-set-static-routes-in-ubuntu-server - """ - content = "" - up = indent + "post-up route add" - down = indent + "pre-down route del" - eol = " || true\n" - mapping = { - 'network': '-net', - 'netmask': 'netmask', - 'gateway': 'gw', - 'metric': 'metric', - } - if route['network'] == '0.0.0.0' and route['netmask'] == '0.0.0.0': - default_gw = " default gw %s" % route['gateway'] - content += up + default_gw + eol - content += down + default_gw + eol - elif route['network'] == '::' and route['netmask'] == 0: - # ipv6! - default_gw = " -A inet6 default gw %s" % route['gateway'] - content += up + default_gw + eol - content += down + default_gw + eol - else: - route_line = "" - for k in ['network', 'netmask', 'gateway', 'metric']: - if k in route: - route_line += " %s %s" % (mapping[k], route[k]) - content += up + route_line + eol - content += down + route_line + eol - - return content - - def render_interfaces(self, network_state): - ''' Given state, emit etc/network/interfaces content ''' - - content = "" - interfaces = network_state.get('interfaces') - ''' Apply a sort order to ensure that we write out - the physical interfaces first; this is critical for - bonding - ''' - order = { - 'physical': 0, - 'bond': 1, - 'bridge': 2, - 'vlan': 3, - } - content += "auto lo\niface lo inet loopback\n" - for dnskey, value in network_state.get('dns', {}).items(): - if len(value): - content += " dns-{} {}\n".format(dnskey, " ".join(value)) - - for iface in sorted(interfaces.values(), - key=lambda k: (order[k['type']], k['name'])): - - if content[-2:] != "\n\n": - content += "\n" - subnets = iface.get('subnets', {}) - if subnets: - for index, subnet in zip(range(0, len(subnets)), subnets): - if content[-2:] != "\n\n": - content += "\n" - iface['index'] = index - iface['mode'] = subnet['type'] - iface['control'] = subnet.get('control', 'auto') - if iface['mode'].endswith('6'): - iface['inet'] += '6' - elif iface['mode'] == 'static' \ - and ":" in subnet['address']: - iface['inet'] += '6' - if iface['mode'].startswith('dhcp'): - iface['mode'] = 'dhcp' - - content += _iface_start_entry(iface, index) - content += _iface_add_subnet(iface, subnet) - content += _iface_add_attrs(iface) - else: - # ifenslave docs say to auto the slave devices - if 'bond-master' in iface: - content += "auto {name}\n".format(**iface) - content += "iface {name} {inet} {mode}\n".format(**iface) - content += _iface_add_attrs(iface) - - for route in network_state.get('routes'): - content += self.render_route(route) - - # global replacements until v2 format - content = content.replace('mac_address', 'hwaddress') - return content - - def render_network_state(self, - target, network_state, eni="etc/network/interfaces", - links_prefix=LINKS_FNAME_PREFIX, - netrules='etc/udev/rules.d/70-persistent-net.rules'): - - fpeni = os.path.sep.join((target, eni,)) - util.ensure_dir(os.path.dirname(fpeni)) - with open(fpeni, 'w+') as f: - f.write(self.render_interfaces(network_state)) - - if netrules: - netrules = os.path.sep.join((target, netrules,)) - util.ensure_dir(os.path.dirname(netrules)) - with open(netrules, 'w+') as f: - f.write(self.render_persistent_net(network_state)) - - if links_prefix: - self.render_systemd_links(target, network_state, links_prefix) - - def render_systemd_links(self, target, network_state, - links_prefix=LINKS_FNAME_PREFIX): - fp_prefix = os.path.sep.join((target, links_prefix)) - for f in glob.glob(fp_prefix + "*"): - os.unlink(f) - - interfaces = network_state.get('interfaces') - for iface in interfaces.values(): - if (iface['type'] == 'physical' and 'name' in iface and - iface.get('mac_address')): - fname = fp_prefix + iface['name'] + ".link" - with open(fname, "w") as fp: - fp.write("\n".join([ - "[Match]", - "MACAddress=" + iface['mac_address'], - "", - "[Link]", - "Name=" + iface['name'], - "" - ])) diff --git a/cloudinit/net/klibc.py b/cloudinit/net/klibc.py deleted file mode 100644 index 958c264b..00000000 --- a/cloudinit/net/klibc.py +++ /dev/null @@ -1,191 +0,0 @@ -# Copyright (C) 2013-2014 Canonical Ltd. -# -# Author: Scott Moser -# Author: Blake Rouse -# -# Curtin is free software: you can redistribute it and/or modify it under -# the terms of the GNU Affero General Public License as published by the -# Free Software Foundation, either version 3 of the License, or (at your -# option) any later version. -# -# Curtin is distributed in the hope that it will be useful, but WITHOUT ANY -# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS -# FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for -# more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with Curtin. If not, see . - -import base64 -import glob -import gzip -import io -import shlex - -from cloudinit.net import get_devicelist -from cloudinit.net import sys_netdev_info - -from cloudinit import util - - -def _load_shell_content(content, add_empty=False, empty_val=None): - """Given shell like syntax (key=value\nkey2=value2\n) in content - return the data in dictionary form. If 'add_empty' is True - then add entries in to the returned dictionary for 'VAR=' - variables. Set their value to empty_val.""" - data = {} - for line in shlex.split(content): - key, value = line.split("=", 1) - if not value: - value = empty_val - if add_empty or value: - data[key] = value - - return data - - -def _klibc_to_config_entry(content, mac_addrs=None): - """Convert a klibc writtent shell content file to a 'config' entry - When ip= is seen on the kernel command line in debian initramfs - and networking is brought up, ipconfig will populate - /run/net-.cfg. - - The files are shell style syntax, and examples are in the tests - provided here. There is no good documentation on this unfortunately. - - DEVICE= is expected/required and PROTO should indicate if - this is 'static' or 'dhcp'. - """ - - if mac_addrs is None: - mac_addrs = {} - - data = _load_shell_content(content) - try: - name = data['DEVICE'] - except KeyError: - raise ValueError("no 'DEVICE' entry in data") - - # ipconfig on precise does not write PROTO - proto = data.get('PROTO') - if not proto: - if data.get('filename'): - proto = 'dhcp' - else: - proto = 'static' - - if proto not in ('static', 'dhcp'): - raise ValueError("Unexpected value for PROTO: %s" % proto) - - iface = { - 'type': 'physical', - 'name': name, - 'subnets': [], - } - - if name in mac_addrs: - iface['mac_address'] = mac_addrs[name] - - # originally believed there might be IPV6* values - for v, pre in (('ipv4', 'IPV4'),): - # if no IPV4ADDR or IPV6ADDR, then go on. - if pre + "ADDR" not in data: - continue - subnet = {'type': proto, 'control': 'manual'} - - # these fields go right on the subnet - for key in ('NETMASK', 'BROADCAST', 'GATEWAY'): - if pre + key in data: - subnet[key.lower()] = data[pre + key] - - dns = [] - # handle IPV4DNS0 or IPV6DNS0 - for nskey in ('DNS0', 'DNS1'): - ns = data.get(pre + nskey) - # verify it has something other than 0.0.0.0 (or ipv6) - if ns and len(ns.strip(":.0")): - dns.append(data[pre + nskey]) - if dns: - subnet['dns_nameservers'] = dns - # add search to both ipv4 and ipv6, as it has no namespace - search = data.get('DOMAINSEARCH') - if search: - if ',' in search: - subnet['dns_search'] = search.split(",") - else: - subnet['dns_search'] = search.split() - - iface['subnets'].append(subnet) - - return name, iface - - -def config_from_klibc_net_cfg(files=None, mac_addrs=None): - if files is None: - files = glob.glob('/run/net*.conf') - - entries = [] - names = {} - for cfg_file in files: - name, entry = _klibc_to_config_entry(util.load_file(cfg_file), - mac_addrs=mac_addrs) - if name in names: - raise ValueError( - "device '%s' defined multiple times: %s and %s" % ( - name, names[name], cfg_file)) - - names[name] = cfg_file - entries.append(entry) - return {'config': entries, 'version': 1} - - -def _decomp_gzip(blob, strict=True): - # decompress blob. raise exception if not compressed unless strict=False. - with io.BytesIO(blob) as iobuf: - gzfp = None - try: - gzfp = gzip.GzipFile(mode="rb", fileobj=iobuf) - return gzfp.read() - except IOError: - if strict: - raise - return blob - finally: - if gzfp: - gzfp.close() - - -def _b64dgz(b64str, gzipped="try"): - # decode a base64 string. If gzipped is true, transparently uncompresss - # if gzipped is 'try', then try gunzip, returning the original on fail. - try: - blob = base64.b64decode(b64str) - except TypeError: - raise ValueError("Invalid base64 text: %s" % b64str) - - if not gzipped: - return blob - - return _decomp_gzip(blob, strict=gzipped != "try") - - -def read_kernel_cmdline_config(files=None, mac_addrs=None, cmdline=None): - if cmdline is None: - cmdline = util.get_cmdline() - - if 'network-config=' in cmdline: - data64 = None - for tok in cmdline.split(): - if tok.startswith("network-config="): - data64 = tok.split("=", 1)[1] - if data64: - return util.load_yaml(_b64dgz(data64)) - - if 'ip=' not in cmdline: - return None - - if mac_addrs is None: - mac_addrs = {k: sys_netdev_info(k, 'address') - for k in get_devicelist()} - - return config_from_klibc_net_cfg(files=files, mac_addrs=mac_addrs) diff --git a/cloudinit/net/renderers/__init__.py b/cloudinit/net/renderers/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/cloudinit/net/renderers/eni.py b/cloudinit/net/renderers/eni.py new file mode 100644 index 00000000..b427012e --- /dev/null +++ b/cloudinit/net/renderers/eni.py @@ -0,0 +1,399 @@ +# vi: ts=4 expandtab +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import glob +import os +import re + +from cloudinit.net import LINKS_FNAME_PREFIX +from cloudinit.net import ParserError +from cloudinit.net.udev import generate_udev_rule +from cloudinit import util + + +NET_CONFIG_COMMANDS = [ + "pre-up", "up", "post-up", "down", "pre-down", "post-down", +] + +NET_CONFIG_BRIDGE_OPTIONS = [ + "bridge_ageing", "bridge_bridgeprio", "bridge_fd", "bridge_gcinit", + "bridge_hello", "bridge_maxage", "bridge_maxwait", "bridge_stp", +] + +NET_CONFIG_OPTIONS = [ + "address", "netmask", "broadcast", "network", "metric", "gateway", + "pointtopoint", "media", "mtu", "hostname", "leasehours", "leasetime", + "vendor", "client", "bootfile", "server", "hwaddr", "provider", "frame", + "netnum", "endpoint", "local", "ttl", +] + + +# TODO: switch valid_map based on mode inet/inet6 +def _iface_add_subnet(iface, subnet): + content = "" + valid_map = [ + 'address', + 'netmask', + 'broadcast', + 'metric', + 'gateway', + 'pointopoint', + 'mtu', + 'scope', + 'dns_search', + 'dns_nameservers', + ] + for key, value in subnet.items(): + if value and key in valid_map: + if type(value) == list: + value = " ".join(value) + if '_' in key: + key = key.replace('_', '-') + content += " {} {}\n".format(key, value) + + return content + + +# TODO: switch to valid_map for attrs +def _iface_add_attrs(iface): + content = "" + ignore_map = [ + 'control', + 'index', + 'inet', + 'mode', + 'name', + 'subnets', + 'type', + ] + if iface['type'] not in ['bond', 'bridge', 'vlan']: + ignore_map.append('mac_address') + + for key, value in iface.items(): + if value and key not in ignore_map: + if type(value) == list: + value = " ".join(value) + content += " {} {}\n".format(key, value) + + return content + + +def _iface_start_entry(iface, index): + fullname = iface['name'] + if index != 0: + fullname += ":%s" % index + + control = iface['control'] + if control == "auto": + cverb = "auto" + elif control in ("hotplug",): + cverb = "allow-" + control + else: + cverb = "# control-" + control + + subst = iface.copy() + subst.update({'fullname': fullname, 'cverb': cverb}) + + return ("{cverb} {fullname}\n" + "iface {fullname} {inet} {mode}\n").format(**subst) + + +def _parse_deb_config_data(ifaces, contents, src_dir, src_path): + """Parses the file contents, placing result into ifaces. + + '_source_path' is added to every dictionary entry to define which file + the configration information came from. + + :param ifaces: interface dictionary + :param contents: contents of interfaces file + :param src_dir: directory interfaces file was located + :param src_path: file path the `contents` was read + """ + currif = None + for line in contents.splitlines(): + line = line.strip() + if line.startswith('#'): + continue + split = line.split(' ') + option = split[0] + if option == "source-directory": + parsed_src_dir = split[1] + if not parsed_src_dir.startswith("/"): + parsed_src_dir = os.path.join(src_dir, parsed_src_dir) + for expanded_path in glob.glob(parsed_src_dir): + dir_contents = os.listdir(expanded_path) + dir_contents = [ + os.path.join(expanded_path, path) + for path in dir_contents + if (os.path.isfile(os.path.join(expanded_path, path)) and + re.match("^[a-zA-Z0-9_-]+$", path) is not None) + ] + for entry in dir_contents: + with open(entry, "r") as fp: + src_data = fp.read().strip() + abs_entry = os.path.abspath(entry) + _parse_deb_config_data( + ifaces, src_data, + os.path.dirname(abs_entry), abs_entry) + elif option == "source": + new_src_path = split[1] + if not new_src_path.startswith("/"): + new_src_path = os.path.join(src_dir, new_src_path) + for expanded_path in glob.glob(new_src_path): + with open(expanded_path, "r") as fp: + src_data = fp.read().strip() + abs_path = os.path.abspath(expanded_path) + _parse_deb_config_data( + ifaces, src_data, + os.path.dirname(abs_path), abs_path) + elif option == "auto": + for iface in split[1:]: + if iface not in ifaces: + ifaces[iface] = { + # Include the source path this interface was found in. + "_source_path": src_path + } + ifaces[iface]['auto'] = True + elif option == "iface": + iface, family, method = split[1:4] + if iface not in ifaces: + ifaces[iface] = { + # Include the source path this interface was found in. + "_source_path": src_path + } + elif 'family' in ifaces[iface]: + raise ParserError( + "Interface %s can only be defined once. " + "Re-defined in '%s'." % (iface, src_path)) + ifaces[iface]['family'] = family + ifaces[iface]['method'] = method + currif = iface + elif option == "hwaddress": + ifaces[currif]['hwaddress'] = split[1] + elif option in NET_CONFIG_OPTIONS: + ifaces[currif][option] = split[1] + elif option in NET_CONFIG_COMMANDS: + if option not in ifaces[currif]: + ifaces[currif][option] = [] + ifaces[currif][option].append(' '.join(split[1:])) + elif option.startswith('dns-'): + if 'dns' not in ifaces[currif]: + ifaces[currif]['dns'] = {} + if option == 'dns-search': + ifaces[currif]['dns']['search'] = [] + for domain in split[1:]: + ifaces[currif]['dns']['search'].append(domain) + elif option == 'dns-nameservers': + ifaces[currif]['dns']['nameservers'] = [] + for server in split[1:]: + ifaces[currif]['dns']['nameservers'].append(server) + elif option.startswith('bridge_'): + if 'bridge' not in ifaces[currif]: + ifaces[currif]['bridge'] = {} + if option in NET_CONFIG_BRIDGE_OPTIONS: + bridge_option = option.replace('bridge_', '', 1) + ifaces[currif]['bridge'][bridge_option] = split[1] + elif option == "bridge_ports": + ifaces[currif]['bridge']['ports'] = [] + for iface in split[1:]: + ifaces[currif]['bridge']['ports'].append(iface) + elif option == "bridge_hw" and split[1].lower() == "mac": + ifaces[currif]['bridge']['mac'] = split[2] + elif option == "bridge_pathcost": + if 'pathcost' not in ifaces[currif]['bridge']: + ifaces[currif]['bridge']['pathcost'] = {} + ifaces[currif]['bridge']['pathcost'][split[1]] = split[2] + elif option == "bridge_portprio": + if 'portprio' not in ifaces[currif]['bridge']: + ifaces[currif]['bridge']['portprio'] = {} + ifaces[currif]['bridge']['portprio'][split[1]] = split[2] + elif option.startswith('bond-'): + if 'bond' not in ifaces[currif]: + ifaces[currif]['bond'] = {} + bond_option = option.replace('bond-', '', 1) + ifaces[currif]['bond'][bond_option] = split[1] + for iface in ifaces.keys(): + if 'auto' not in ifaces[iface]: + ifaces[iface]['auto'] = False + + +def _parse_deb_config(path): + """Parses a debian network configuration file.""" + ifaces = {} + with open(path, "r") as fp: + contents = fp.read().strip() + abs_path = os.path.abspath(path) + _parse_deb_config_data( + ifaces, contents, + os.path.dirname(abs_path), abs_path) + return ifaces + + +class Renderer(object): + """Renders network information in a /etc/network/interfaces format.""" + + def render_persistent_net(self, network_state): + """Given state, emit udev rules to map mac to ifname.""" + content = "" + interfaces = network_state.get('interfaces') + for iface in interfaces.values(): + # for physical interfaces write out a persist net udev rule + if iface['type'] == 'physical' and \ + 'name' in iface and iface.get('mac_address'): + content += generate_udev_rule(iface['name'], + iface['mac_address']) + + return content + + def render_route(self, route, indent=""): + """ When rendering routes for an iface, in some cases applying a route + may result in the route command returning non-zero which produces + some confusing output for users manually using ifup/ifdown[1]. To + that end, we will optionally include an '|| true' postfix to each + route line allowing users to work with ifup/ifdown without using + --force option. + + We may at somepoint not want to emit this additional postfix, and + add a 'strict' flag to this function. When called with strict=True, + then we will not append the postfix. + + 1. http://askubuntu.com/questions/168033/ + how-to-set-static-routes-in-ubuntu-server + """ + content = "" + up = indent + "post-up route add" + down = indent + "pre-down route del" + eol = " || true\n" + mapping = { + 'network': '-net', + 'netmask': 'netmask', + 'gateway': 'gw', + 'metric': 'metric', + } + if route['network'] == '0.0.0.0' and route['netmask'] == '0.0.0.0': + default_gw = " default gw %s" % route['gateway'] + content += up + default_gw + eol + content += down + default_gw + eol + elif route['network'] == '::' and route['netmask'] == 0: + # ipv6! + default_gw = " -A inet6 default gw %s" % route['gateway'] + content += up + default_gw + eol + content += down + default_gw + eol + else: + route_line = "" + for k in ['network', 'netmask', 'gateway', 'metric']: + if k in route: + route_line += " %s %s" % (mapping[k], route[k]) + content += up + route_line + eol + content += down + route_line + eol + + return content + + def render_interfaces(self, network_state): + ''' Given state, emit etc/network/interfaces content ''' + + content = "" + interfaces = network_state.get('interfaces') + ''' Apply a sort order to ensure that we write out + the physical interfaces first; this is critical for + bonding + ''' + order = { + 'physical': 0, + 'bond': 1, + 'bridge': 2, + 'vlan': 3, + } + content += "auto lo\niface lo inet loopback\n" + for dnskey, value in network_state.get('dns', {}).items(): + if len(value): + content += " dns-{} {}\n".format(dnskey, " ".join(value)) + + for iface in sorted(interfaces.values(), + key=lambda k: (order[k['type']], k['name'])): + + if content[-2:] != "\n\n": + content += "\n" + subnets = iface.get('subnets', {}) + if subnets: + for index, subnet in zip(range(0, len(subnets)), subnets): + if content[-2:] != "\n\n": + content += "\n" + iface['index'] = index + iface['mode'] = subnet['type'] + iface['control'] = subnet.get('control', 'auto') + if iface['mode'].endswith('6'): + iface['inet'] += '6' + elif iface['mode'] == 'static' \ + and ":" in subnet['address']: + iface['inet'] += '6' + if iface['mode'].startswith('dhcp'): + iface['mode'] = 'dhcp' + + content += _iface_start_entry(iface, index) + content += _iface_add_subnet(iface, subnet) + content += _iface_add_attrs(iface) + else: + # ifenslave docs say to auto the slave devices + if 'bond-master' in iface: + content += "auto {name}\n".format(**iface) + content += "iface {name} {inet} {mode}\n".format(**iface) + content += _iface_add_attrs(iface) + + for route in network_state.get('routes'): + content += self.render_route(route) + + # global replacements until v2 format + content = content.replace('mac_address', 'hwaddress') + return content + + def render_network_state(self, + target, network_state, eni="etc/network/interfaces", + links_prefix=LINKS_FNAME_PREFIX, + netrules='etc/udev/rules.d/70-persistent-net.rules'): + + fpeni = os.path.sep.join((target, eni,)) + util.ensure_dir(os.path.dirname(fpeni)) + with open(fpeni, 'w+') as f: + f.write(self.render_interfaces(network_state)) + + if netrules: + netrules = os.path.sep.join((target, netrules,)) + util.ensure_dir(os.path.dirname(netrules)) + with open(netrules, 'w+') as f: + f.write(self.render_persistent_net(network_state)) + + if links_prefix: + self.render_systemd_links(target, network_state, links_prefix) + + def render_systemd_links(self, target, network_state, + links_prefix=LINKS_FNAME_PREFIX): + fp_prefix = os.path.sep.join((target, links_prefix)) + for f in glob.glob(fp_prefix + "*"): + os.unlink(f) + + interfaces = network_state.get('interfaces') + for iface in interfaces.values(): + if (iface['type'] == 'physical' and 'name' in iface and + iface.get('mac_address')): + fname = fp_prefix + iface['name'] + ".link" + with open(fname, "w") as fp: + fp.write("\n".join([ + "[Match]", + "MACAddress=" + iface['mac_address'], + "", + "[Link]", + "Name=" + iface['name'], + "" + ])) diff --git a/cloudinit/stages.py b/cloudinit/stages.py index e6bd34fe..1112bf0d 100644 --- a/cloudinit/stages.py +++ b/cloudinit/stages.py @@ -43,7 +43,7 @@ from cloudinit import distros from cloudinit import helpers from cloudinit import importer from cloudinit import log as logging -from cloudinit.net import klibc +from cloudinit.net import cmdline from cloudinit import sources from cloudinit import type_utils from cloudinit import util @@ -579,7 +579,7 @@ class Init(object): if os.path.exists(disable_file): return (None, disable_file) - cmdline_cfg = ('cmdline', klibc.read_kernel_cmdline_config()) + cmdline_cfg = ('cmdline', cmdline.read_kernel_cmdline_config()) dscfg = ('ds', None) if self.datasource and hasattr(self.datasource, 'network_config'): dscfg = ('ds', self.datasource.network_config) -- cgit v1.2.3 From f11827aee59cf2678c3d2a157218a3fe2831f5c2 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Thu, 12 May 2016 10:56:26 -0700 Subject: Fix up a ton of flake8 issues --- cloudinit/config/cc_apt_configure.py | 6 +++--- cloudinit/config/cc_bootcmd.py | 4 ++-- cloudinit/config/cc_disk_setup.py | 2 +- cloudinit/config/cc_emit_upstart.py | 2 +- cloudinit/config/cc_fan.py | 2 +- cloudinit/config/cc_final_message.py | 2 +- cloudinit/config/cc_grub_dpkg.py | 2 +- cloudinit/config/cc_keys_to_console.py | 2 +- cloudinit/config/cc_lxd.py | 2 +- cloudinit/config/cc_mounts.py | 10 ++++----- cloudinit/config/cc_phone_home.py | 6 +++--- cloudinit/config/cc_rightscale_userdata.py | 4 ++-- cloudinit/config/cc_runcmd.py | 2 +- cloudinit/config/cc_scripts_per_boot.py | 2 +- cloudinit/config/cc_scripts_per_instance.py | 2 +- cloudinit/config/cc_scripts_per_once.py | 2 +- cloudinit/config/cc_scripts_user.py | 2 +- cloudinit/config/cc_scripts_vendor.py | 2 +- cloudinit/config/cc_seed_random.py | 2 +- cloudinit/config/cc_set_passwords.py | 2 +- cloudinit/config/cc_snappy.py | 4 ++-- cloudinit/config/cc_ssh.py | 6 +++--- cloudinit/config/cc_ssh_import_id.py | 4 ++-- cloudinit/config/cc_ubuntu_init_switch.py | 4 ++-- cloudinit/distros/__init__.py | 4 ++-- cloudinit/distros/debian.py | 4 ++-- cloudinit/handlers/__init__.py | 6 +++--- cloudinit/handlers/cloud_config.py | 2 +- cloudinit/handlers/upstart_job.py | 2 +- cloudinit/helpers.py | 10 ++++----- cloudinit/net/__init__.py | 18 +++++++--------- cloudinit/net/network_state.py | 7 +++--- cloudinit/net/udev.py | 2 +- cloudinit/netinfo.py | 7 +++--- cloudinit/reporting/handlers.py | 10 ++++----- cloudinit/sources/DataSourceAzure.py | 12 ++++++----- cloudinit/sources/DataSourceBigstep.py | 4 ++-- cloudinit/sources/DataSourceCloudSigma.py | 5 +++-- cloudinit/sources/DataSourceCloudStack.py | 5 +++-- cloudinit/sources/DataSourceDigitalOcean.py | 4 ++-- cloudinit/sources/DataSourceGCE.py | 4 ++-- cloudinit/sources/DataSourceNoCloud.py | 4 ++-- cloudinit/sources/DataSourceOVF.py | 25 ++++++++++++++-------- cloudinit/sources/__init__.py | 3 +-- cloudinit/sources/helpers/azure.py | 1 + cloudinit/sources/helpers/openstack.py | 4 ++-- cloudinit/sources/helpers/vmware/imc/boot_proto.py | 2 +- cloudinit/sources/helpers/vmware/imc/config.py | 2 +- cloudinit/sources/helpers/vmware/imc/config_nic.py | 2 +- .../sources/helpers/vmware/imc/config_source.py | 2 +- .../sources/helpers/vmware/imc/guestcust_error.py | 2 +- .../sources/helpers/vmware/imc/guestcust_event.py | 2 +- .../sources/helpers/vmware/imc/guestcust_state.py | 2 +- .../sources/helpers/vmware/imc/guestcust_util.py | 2 +- cloudinit/sources/helpers/vmware/imc/ipv4_mode.py | 2 +- cloudinit/sources/helpers/vmware/imc/nic_base.py | 6 +++--- cloudinit/stages.py | 4 ++-- cloudinit/url_helper.py | 5 +++-- cloudinit/user_data.py | 2 +- cloudinit/util.py | 18 ++++++++-------- test-requirements.txt | 3 ++- tox.ini | 5 +++++ 62 files changed, 150 insertions(+), 132 deletions(-) (limited to 'cloudinit/stages.py') diff --git a/cloudinit/config/cc_apt_configure.py b/cloudinit/config/cc_apt_configure.py index 702977cb..e3fadc12 100644 --- a/cloudinit/config/cc_apt_configure.py +++ b/cloudinit/config/cc_apt_configure.py @@ -207,20 +207,20 @@ def add_sources(srclist, template_params=None, aa_repo_match=None): ks = ent['keyserver'] try: ent['key'] = getkeybyid(ent['keyid'], ks) - except: + except Exception: errorlist.append([source, "failed to get key from %s" % ks]) continue if 'key' in ent: try: util.subp(('apt-key', 'add', '-'), ent['key']) - except: + except Exception: errorlist.append([source, "failed add key"]) try: contents = "%s\n" % (source) util.write_file(ent['filename'], contents, omode="ab") - except: + except Exception: errorlist.append([source, "failed write to file %s" % ent['filename']]) diff --git a/cloudinit/config/cc_bootcmd.py b/cloudinit/config/cc_bootcmd.py index a295cc4e..b763a3c3 100644 --- a/cloudinit/config/cc_bootcmd.py +++ b/cloudinit/config/cc_bootcmd.py @@ -38,7 +38,7 @@ def handle(name, cfg, cloud, log, _args): content = util.shellify(cfg["bootcmd"]) tmpf.write(util.encode_text(content)) tmpf.flush() - except: + except Exception: util.logexc(log, "Failed to shellify bootcmd") raise @@ -49,6 +49,6 @@ def handle(name, cfg, cloud, log, _args): env['INSTANCE_ID'] = str(iid) cmd = ['/bin/sh', tmpf.name] util.subp(cmd, env=env, capture=False) - except: + except Exception: util.logexc(log, "Failed to run bootcmd module %s", name) raise diff --git a/cloudinit/config/cc_disk_setup.py b/cloudinit/config/cc_disk_setup.py index bbaf9646..b642f1f8 100644 --- a/cloudinit/config/cc_disk_setup.py +++ b/cloudinit/config/cc_disk_setup.py @@ -198,7 +198,7 @@ def is_device_valid(name, partition=False): d_type = "" try: d_type = device_type(name) - except: + except Exception: LOG.warn("Query against device %s failed" % name) return False diff --git a/cloudinit/config/cc_emit_upstart.py b/cloudinit/config/cc_emit_upstart.py index 86ae97ab..06c53272 100644 --- a/cloudinit/config/cc_emit_upstart.py +++ b/cloudinit/config/cc_emit_upstart.py @@ -20,8 +20,8 @@ import os -from cloudinit.settings import PER_ALWAYS from cloudinit import log as logging +from cloudinit.settings import PER_ALWAYS from cloudinit import util frequency = PER_ALWAYS diff --git a/cloudinit/config/cc_fan.py b/cloudinit/config/cc_fan.py index 39e3850e..545fee22 100644 --- a/cloudinit/config/cc_fan.py +++ b/cloudinit/config/cc_fan.py @@ -37,8 +37,8 @@ If cloud-init sees a 'fan' entry in cloud-config it will """ from cloudinit import log as logging -from cloudinit import util from cloudinit.settings import PER_INSTANCE +from cloudinit import util LOG = logging.getLogger(__name__) diff --git a/cloudinit/config/cc_final_message.py b/cloudinit/config/cc_final_message.py index 4a51476f..c9021eb1 100644 --- a/cloudinit/config/cc_final_message.py +++ b/cloudinit/config/cc_final_message.py @@ -66,7 +66,7 @@ def handle(_name, cfg, cloud, log, args): try: contents = "%s - %s - v. %s\n" % (uptime, ts, cver) util.write_file(boot_fin_fn, contents) - except: + except Exception: util.logexc(log, "Failed to write boot finished file %s", boot_fin_fn) if cloud.datasource.is_disconnected: diff --git a/cloudinit/config/cc_grub_dpkg.py b/cloudinit/config/cc_grub_dpkg.py index 3c2d9985..156722d9 100644 --- a/cloudinit/config/cc_grub_dpkg.py +++ b/cloudinit/config/cc_grub_dpkg.py @@ -69,5 +69,5 @@ def handle(name, cfg, _cloud, log, _args): try: util.subp(['debconf-set-selections'], dconf_sel) - except: + except Exception: util.logexc(log, "Failed to run debconf-set-selections for grub-dpkg") diff --git a/cloudinit/config/cc_keys_to_console.py b/cloudinit/config/cc_keys_to_console.py index aa844ee9..9a02f056 100644 --- a/cloudinit/config/cc_keys_to_console.py +++ b/cloudinit/config/cc_keys_to_console.py @@ -57,6 +57,6 @@ def handle(name, cfg, cloud, log, _args): (stdout, _stderr) = util.subp(cmd) util.multi_log("%s\n" % (stdout.strip()), stderr=False, console=True) - except: + except Exception: log.warn("Writing keys to the system console failed!") raise diff --git a/cloudinit/config/cc_lxd.py b/cloudinit/config/cc_lxd.py index bf735648..b1de8f84 100644 --- a/cloudinit/config/cc_lxd.py +++ b/cloudinit/config/cc_lxd.py @@ -111,7 +111,7 @@ def handle(name, cfg, cloud, log, args): data = "\n".join(["set %s %s" % (k, v) for k, v in debconf.items()]) + "\n" util.subp(['debconf-communicate'], data) - except: + except Exception: util.logexc(log, "Failed to run '%s' for lxd with" % dconf_comm) # Remove the existing configuration file (forces re-generation) diff --git a/cloudinit/config/cc_mounts.py b/cloudinit/config/cc_mounts.py index 4fe3ee21..2b981935 100644 --- a/cloudinit/config/cc_mounts.py +++ b/cloudinit/config/cc_mounts.py @@ -244,7 +244,7 @@ def handle_swapcfg(swapcfg): LOG.debug("swap file %s already in use.", fname) return fname LOG.debug("swap file %s existed, but not in /proc/swaps", fname) - except: + except Exception: LOG.warn("swap file %s existed. Error reading /proc/swaps", fname) return fname @@ -379,7 +379,7 @@ def handle(_name, cfg, cloud, log, _args): toks = WS.split(line) if toks[3].find(comment) != -1: continue - except: + except Exception: pass fstab_lines.append(line) @@ -390,16 +390,16 @@ def handle(_name, cfg, cloud, log, _args): if needswap: try: util.subp(("swapon", "-a")) - except: + except Exception: util.logexc(log, "Activating swap via 'swapon -a' failed") for d in dirs: try: util.ensure_dir(d) - except: + except Exception: util.logexc(log, "Failed to make '%s' config-mount", d) try: util.subp(("mount", "-a")) - except: + except Exception: util.logexc(log, "Activating mounts via 'mount -a' failed") diff --git a/cloudinit/config/cc_phone_home.py b/cloudinit/config/cc_phone_home.py index 3dcc9459..72176d42 100644 --- a/cloudinit/config/cc_phone_home.py +++ b/cloudinit/config/cc_phone_home.py @@ -65,7 +65,7 @@ def handle(name, cfg, cloud, log, args): tries = ph_cfg.get('tries') try: tries = int(tries) - except: + except Exception: tries = 10 util.logexc(log, "Configuration entry 'tries' is not an integer, " "using %s instead", tries) @@ -87,7 +87,7 @@ def handle(name, cfg, cloud, log, args): for (n, path) in pubkeys.items(): try: all_keys[n] = util.load_file(path) - except: + except Exception: util.logexc(log, "%s: failed to open, can not phone home that " "data!", path) @@ -117,6 +117,6 @@ def handle(name, cfg, cloud, log, args): util.read_file_or_url(url, data=real_submit_keys, retries=tries, sec_between=3, ssl_details=util.fetch_ssl_details(cloud.paths)) - except: + except Exception: util.logexc(log, "Failed to post phone home data to %s in %s tries", url, tries) diff --git a/cloudinit/config/cc_rightscale_userdata.py b/cloudinit/config/cc_rightscale_userdata.py index 0ecf3a4d..8118fac4 100644 --- a/cloudinit/config/cc_rightscale_userdata.py +++ b/cloudinit/config/cc_rightscale_userdata.py @@ -52,7 +52,7 @@ MY_HOOKNAME = 'CLOUD_INIT_REMOTE_HOOK' def handle(name, _cfg, cloud, log, _args): try: ud = cloud.get_userdata_raw() - except: + except Exception: log.debug("Failed to get raw userdata in module %s", name) return @@ -63,7 +63,7 @@ def handle(name, _cfg, cloud, log, _args): "did not find %s in parsed" " raw userdata"), name, MY_HOOKNAME) return - except: + except Exception: util.logexc(log, "Failed to parse query string %s into a dictionary", ud) raise diff --git a/cloudinit/config/cc_runcmd.py b/cloudinit/config/cc_runcmd.py index 66dc3363..bc09d38c 100644 --- a/cloudinit/config/cc_runcmd.py +++ b/cloudinit/config/cc_runcmd.py @@ -34,5 +34,5 @@ def handle(name, cfg, cloud, log, _args): try: content = util.shellify(cmd) util.write_file(out_fn, content, 0o700) - except: + except Exception: util.logexc(log, "Failed to shellify %s into file %s", cmd, out_fn) diff --git a/cloudinit/config/cc_scripts_per_boot.py b/cloudinit/config/cc_scripts_per_boot.py index 42b987eb..ee3b6c9f 100644 --- a/cloudinit/config/cc_scripts_per_boot.py +++ b/cloudinit/config/cc_scripts_per_boot.py @@ -35,7 +35,7 @@ def handle(name, _cfg, cloud, log, _args): runparts_path = os.path.join(cloud.get_cpath(), 'scripts', SCRIPT_SUBDIR) try: util.runparts(runparts_path) - except: + except Exception: log.warn("Failed to run module %s (%s in %s)", name, SCRIPT_SUBDIR, runparts_path) raise diff --git a/cloudinit/config/cc_scripts_per_instance.py b/cloudinit/config/cc_scripts_per_instance.py index b5d71c13..c0d62b12 100644 --- a/cloudinit/config/cc_scripts_per_instance.py +++ b/cloudinit/config/cc_scripts_per_instance.py @@ -35,7 +35,7 @@ def handle(name, _cfg, cloud, log, _args): runparts_path = os.path.join(cloud.get_cpath(), 'scripts', SCRIPT_SUBDIR) try: util.runparts(runparts_path) - except: + except Exception: log.warn("Failed to run module %s (%s in %s)", name, SCRIPT_SUBDIR, runparts_path) raise diff --git a/cloudinit/config/cc_scripts_per_once.py b/cloudinit/config/cc_scripts_per_once.py index d77d36d5..ecb527f6 100644 --- a/cloudinit/config/cc_scripts_per_once.py +++ b/cloudinit/config/cc_scripts_per_once.py @@ -35,7 +35,7 @@ def handle(name, _cfg, cloud, log, _args): runparts_path = os.path.join(cloud.get_cpath(), 'scripts', SCRIPT_SUBDIR) try: util.runparts(runparts_path) - except: + except Exception: log.warn("Failed to run module %s (%s in %s)", name, SCRIPT_SUBDIR, runparts_path) raise diff --git a/cloudinit/config/cc_scripts_user.py b/cloudinit/config/cc_scripts_user.py index 5c53014f..699857d1 100644 --- a/cloudinit/config/cc_scripts_user.py +++ b/cloudinit/config/cc_scripts_user.py @@ -36,7 +36,7 @@ def handle(name, _cfg, cloud, log, _args): runparts_path = os.path.join(cloud.get_ipath_cur(), SCRIPT_SUBDIR) try: util.runparts(runparts_path) - except: + except Exception: log.warn("Failed to run module %s (%s in %s)", name, SCRIPT_SUBDIR, runparts_path) raise diff --git a/cloudinit/config/cc_scripts_vendor.py b/cloudinit/config/cc_scripts_vendor.py index 0c9e504e..80bf10ff 100644 --- a/cloudinit/config/cc_scripts_vendor.py +++ b/cloudinit/config/cc_scripts_vendor.py @@ -37,7 +37,7 @@ def handle(name, cfg, cloud, log, _args): try: util.runparts(runparts_path, exe_prefix=prefix) - except: + except Exception: log.warn("Failed to run module %s (%s in %s)", name, SCRIPT_SUBDIR, runparts_path) raise diff --git a/cloudinit/config/cc_seed_random.py b/cloudinit/config/cc_seed_random.py index 1b011216..5085c23a 100644 --- a/cloudinit/config/cc_seed_random.py +++ b/cloudinit/config/cc_seed_random.py @@ -24,8 +24,8 @@ import os from six import BytesIO -from cloudinit.settings import PER_INSTANCE from cloudinit import log as logging +from cloudinit.settings import PER_INSTANCE from cloudinit import util frequency = PER_INSTANCE diff --git a/cloudinit/config/cc_set_passwords.py b/cloudinit/config/cc_set_passwords.py index 58e1b713..5c8c23b8 100644 --- a/cloudinit/config/cc_set_passwords.py +++ b/cloudinit/config/cc_set_passwords.py @@ -155,7 +155,7 @@ def handle(_name, cfg, cloud, log, args): cmd = filter(None, cmd) # Remove empty arguments util.subp(cmd) log.debug("Restarted the ssh daemon") - except: + except Exception: util.logexc(log, "Restarting of the ssh daemon failed") if len(errors): diff --git a/cloudinit/config/cc_snappy.py b/cloudinit/config/cc_snappy.py index fa9d54a0..1a485ee6 100644 --- a/cloudinit/config/cc_snappy.py +++ b/cloudinit/config/cc_snappy.py @@ -47,12 +47,12 @@ Example config: """ from cloudinit import log as logging -from cloudinit import util from cloudinit.settings import PER_INSTANCE +from cloudinit import util import glob -import tempfile import os +import tempfile LOG = logging.getLogger(__name__) diff --git a/cloudinit/config/cc_ssh.py b/cloudinit/config/cc_ssh.py index d24e43c0..cb9b70aa 100644 --- a/cloudinit/config/cc_ssh.py +++ b/cloudinit/config/cc_ssh.py @@ -57,7 +57,7 @@ def handle(_name, cfg, cloud, log, _args): for f in glob.glob(key_pth): try: util.del_file(f) - except: + except Exception: util.logexc(log, "Failed deleting key file %s", f) if "ssh_keys" in cfg: @@ -78,7 +78,7 @@ def handle(_name, cfg, cloud, log, _args): with util.SeLinuxGuard("/etc/ssh", recursive=True): util.subp(cmd, capture=False) log.debug("Generated a key for %s from %s", pair[0], pair[1]) - except: + except Exception: util.logexc(log, "Failed generated a key for %s from %s", pair[0], pair[1]) else: @@ -122,7 +122,7 @@ def handle(_name, cfg, cloud, log, _args): keys.extend(cfgkeys) apply_credentials(keys, user, disable_root, disable_root_opts) - except: + except Exception: util.logexc(log, "Applying ssh credentials failed!") diff --git a/cloudinit/config/cc_ssh_import_id.py b/cloudinit/config/cc_ssh_import_id.py index 2d480d7e..28c4585b 100644 --- a/cloudinit/config/cc_ssh_import_id.py +++ b/cloudinit/config/cc_ssh_import_id.py @@ -52,14 +52,14 @@ def handle(_name, cfg, cloud, log, args): else: try: import_ids = user_cfg['ssh_import_id'] - except: + except Exception: log.debug("User %s is not configured for ssh_import_id", user) continue try: import_ids = util.uniq_merge(import_ids) import_ids = [str(i) for i in import_ids] - except: + except Exception: log.debug("User %s is not correctly configured for ssh_import_id", user) continue diff --git a/cloudinit/config/cc_ubuntu_init_switch.py b/cloudinit/config/cc_ubuntu_init_switch.py index 7e88ed85..884d79f1 100644 --- a/cloudinit/config/cc_ubuntu_init_switch.py +++ b/cloudinit/config/cc_ubuntu_init_switch.py @@ -40,10 +40,10 @@ It can be configured with the following option structure:: mechanism you've used to switch the init system. """ -from cloudinit.settings import PER_INSTANCE +from cloudinit.distros import ubuntu from cloudinit import log as logging +from cloudinit.settings import PER_INSTANCE from cloudinit import util -from cloudinit.distros import ubuntu import os import time diff --git a/cloudinit/distros/__init__.py b/cloudinit/distros/__init__.py index 5879dabf..0f222c8c 100644 --- a/cloudinit/distros/__init__.py +++ b/cloudinit/distros/__init__.py @@ -50,8 +50,8 @@ OSFAMILIES = { LOG = logging.getLogger(__name__) +@six.add_metaclass(abc.ABCMeta) class Distro(object): - __metaclass__ = abc.ABCMeta usr_lib_exec = "/usr/lib" hosts_fn = "/etc/hosts" @@ -97,7 +97,7 @@ class Distro(object): try: res = os.lstat('/run/systemd/system') return stat.S_ISDIR(res.st_mode) - except: + except Exception: return False @abc.abstractmethod diff --git a/cloudinit/distros/debian.py b/cloudinit/distros/debian.py index 75ab340f..32bef1cd 100644 --- a/cloudinit/distros/debian.py +++ b/cloudinit/distros/debian.py @@ -25,8 +25,8 @@ import os from cloudinit import distros from cloudinit import helpers from cloudinit import log as logging -from cloudinit import util from cloudinit import net +from cloudinit import util from cloudinit.distros.parsers.hostname import HostnameConf @@ -221,7 +221,7 @@ def _maybe_remove_legacy_eth0(path="/etc/network/interfaces.d/eth0.cfg"): msg = "removed %s with known contents" % path else: msg = (bmsg + " '%s' exists with user configured content." % path) - except: + except Exception: msg = bmsg + " %s exists, but could not be read." % path LOG.warn(msg) diff --git a/cloudinit/handlers/__init__.py b/cloudinit/handlers/__init__.py index 53d5604a..b6c43ce8 100644 --- a/cloudinit/handlers/__init__.py +++ b/cloudinit/handlers/__init__.py @@ -71,8 +71,8 @@ INCLUSION_SRCH = sorted(list(INCLUSION_TYPES_MAP.keys()), key=(lambda e: 0 - len(e))) +@six.add_metaclass(abc.ABCMeta) class Handler(object): - __metaclass__ = abc.ABCMeta def __init__(self, frequency, version=2): self.handler_version = version @@ -118,7 +118,7 @@ def run_part(mod, data, filename, payload, frequency, headers): mod.handle_part(data, content_type, filename, payload) else: raise ValueError("Unknown module version %s" % (mod_ver)) - except: + except Exception: util.logexc(LOG, "Failed calling handler %s (%s, %s, %s) with " "frequency %s", mod, content_type, filename, mod_ver, frequency) @@ -157,7 +157,7 @@ def walker_handle_handler(pdata, _ctype, _filename, payload): # register if it fails starting. handlers.register(mod, initialized=True) pdata['handlercount'] = curcount + 1 - except: + except Exception: util.logexc(LOG, "Failed at registering python file: %s (part " "handler %s)", modfname, curcount) diff --git a/cloudinit/handlers/cloud_config.py b/cloudinit/handlers/cloud_config.py index 07b6d0e0..cad4dc0f 100644 --- a/cloudinit/handlers/cloud_config.py +++ b/cloudinit/handlers/cloud_config.py @@ -158,6 +158,6 @@ class CloudConfigPartHandler(handlers.Handler): for i in ("\n", "\r", "\t"): filename = filename.replace(i, " ") self.file_names.append(filename.strip()) - except: + except Exception: util.logexc(LOG, "Failed at merging in cloud config part from %s", filename) diff --git a/cloudinit/handlers/upstart_job.py b/cloudinit/handlers/upstart_job.py index c5bea711..ab381e00 100644 --- a/cloudinit/handlers/upstart_job.py +++ b/cloudinit/handlers/upstart_job.py @@ -80,7 +80,7 @@ def _has_suitable_upstart(): return False try: (version_out, _err) = util.subp(["initctl", "version"]) - except: + except Exception: util.logexc(LOG, "initctl version failed") return False diff --git a/cloudinit/helpers.py b/cloudinit/helpers.py index 7f00bf1f..09d75e65 100644 --- a/cloudinit/helpers.py +++ b/cloudinit/helpers.py @@ -86,7 +86,7 @@ class FileSemaphores(object): name = canon_sem_name(name) try: yield self._acquire(name, freq) - except: + except Exception: if clear_on_fail: self.clear(name, freq) raise @@ -219,7 +219,7 @@ class ConfigMerger(object): ds_cfg = self._ds.get_config_obj() if ds_cfg and isinstance(ds_cfg, (dict)): d_cfgs.append(ds_cfg) - except: + except Exception: util.logexc(LOG, "Failed loading of datasource config object " "from %s", self._ds) return d_cfgs @@ -230,7 +230,7 @@ class ConfigMerger(object): e_fn = os.environ[CFG_ENV_NAME] try: e_cfgs.append(util.read_conf(e_fn)) - except: + except Exception: util.logexc(LOG, 'Failed loading of env. config from %s', e_fn) return e_cfgs @@ -251,7 +251,7 @@ class ConfigMerger(object): if cc_fn and os.path.isfile(cc_fn): try: i_cfgs.append(util.read_conf(cc_fn)) - except: + except Exception: util.logexc(LOG, 'Failed loading of cloud-config from %s', cc_fn) return i_cfgs @@ -268,7 +268,7 @@ class ConfigMerger(object): for c_fn in self._fns: try: cfgs.append(util.read_conf(c_fn)) - except: + except Exception: util.logexc(LOG, "Failed loading of configuration from %s", c_fn) diff --git a/cloudinit/net/__init__.py b/cloudinit/net/__init__.py index 31544fd8..91e36aca 100644 --- a/cloudinit/net/__init__.py +++ b/cloudinit/net/__init__.py @@ -26,9 +26,9 @@ import re import shlex from cloudinit import log as logging +from cloudinit.net import network_state +from cloudinit.net.udev import generate_udev_rule from cloudinit import util -from .udev import generate_udev_rule -from . import network_state LOG = logging.getLogger(__name__) @@ -40,16 +40,16 @@ NET_CONFIG_OPTIONS = [ "pointtopoint", "media", "mtu", "hostname", "leasehours", "leasetime", "vendor", "client", "bootfile", "server", "hwaddr", "provider", "frame", "netnum", "endpoint", "local", "ttl", - ] +] NET_CONFIG_COMMANDS = [ "pre-up", "up", "post-up", "down", "pre-down", "post-down", - ] +] NET_CONFIG_BRIDGE_OPTIONS = [ "bridge_ageing", "bridge_bridgeprio", "bridge_fd", "bridge_gcinit", "bridge_hello", "bridge_maxage", "bridge_maxwait", "bridge_stp", - ] +] DEFAULT_PRIMARY_INTERFACE = 'eth0' @@ -399,9 +399,7 @@ def config_from_klibc_net_cfg(files=None, mac_addrs=None): def render_persistent_net(network_state): - ''' Given state, emit udev rules to map - mac to ifname - ''' + '''Given state, emit udev rules to map mac to ifname.''' content = "" interfaces = network_state.get('interfaces') for iface in interfaces.values(): @@ -465,7 +463,7 @@ def iface_add_attrs(iface): def render_route(route, indent=""): - """ When rendering routes for an iface, in some cases applying a route + """When rendering routes for an iface, in some cases applying a route may result in the route command returning non-zero which produces some confusing output for users manually using ifup/ifdown[1]. To that end, we will optionally include an '|| true' postfix to each @@ -530,7 +528,7 @@ def iface_start_entry(iface, index): def render_interfaces(network_state): - ''' Given state, emit etc/network/interfaces content ''' + '''Given state, emit etc/network/interfaces content.''' content = "" interfaces = network_state.get('interfaces') diff --git a/cloudinit/net/network_state.py b/cloudinit/net/network_state.py index e32d2cdf..4c726ab4 100644 --- a/cloudinit/net/network_state.py +++ b/cloudinit/net/network_state.py @@ -36,7 +36,7 @@ def from_state_file(state_file): return network_state -class NetworkState: +class NetworkState(object): def __init__(self, version=NETWORK_STATE_VERSION, config=None): self.version = version self.config = config @@ -53,7 +53,7 @@ class NetworkState: def get_command_handlers(self): METHOD_PREFIX = 'handle_' methods = filter(lambda x: callable(getattr(self, x)) and - x.startswith(METHOD_PREFIX), dir(self)) + x.startswith(METHOD_PREFIX), dir(self)) handlers = {} for m in methods: key = m.replace(METHOD_PREFIX, '') @@ -379,8 +379,9 @@ def mask2cidr(mask): if __name__ == '__main__': - import sys import random + import sys + from cloudinit import net def load_config(nc): diff --git a/cloudinit/net/udev.py b/cloudinit/net/udev.py index 6435ace0..09188295 100644 --- a/cloudinit/net/udev.py +++ b/cloudinit/net/udev.py @@ -48,7 +48,7 @@ def generate_udev_rule(interface, mac): compose_udev_equality('DRIVERS', '?*'), compose_udev_attr_equality('address', mac), compose_udev_setting('NAME', interface), - ]) + ]) return '%s\n' % rule # vi: ts=4 expandtab syntax=python diff --git a/cloudinit/netinfo.py b/cloudinit/netinfo.py index e30d6fb5..5d24062f 100644 --- a/cloudinit/netinfo.py +++ b/cloudinit/netinfo.py @@ -20,10 +20,11 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -import cloudinit.util as util -from cloudinit.log import logging import re +from cloudinit import logging +from cloudinit import util + from prettytable import PrettyTable LOG = logging.getLogger() @@ -163,7 +164,7 @@ def route_info(): def getgateway(): try: routes = route_info() - except: + except Exception: pass else: for r in routes.get('ipv4', []): diff --git a/cloudinit/reporting/handlers.py b/cloudinit/reporting/handlers.py index 3212d173..dff20ecb 100644 --- a/cloudinit/reporting/handlers.py +++ b/cloudinit/reporting/handlers.py @@ -4,9 +4,9 @@ import abc import json import six -from ..registry import DictRegistry -from .. import (url_helper, util) -from .. import log as logging +from cloudinit import log as logging +from cloudinit.registry import DictRegistry +from cloudinit import (url_helper, util) LOG = logging.getLogger(__name__) @@ -36,7 +36,7 @@ class LogHandler(ReportingHandler): input_level = level try: level = getattr(logging, level.upper()) - except: + except Exception: LOG.warn("invalid level '%s', using WARN", input_level) level = logging.WARN self.level = level @@ -81,7 +81,7 @@ class WebHookHandler(ReportingHandler): self.endpoint, data=json.dumps(event.as_dict()), timeout=self.timeout, retries=self.retries, ssl_details=self.ssl_details) - except: + except Exception: LOG.warn("failed posting event: %s" % event.as_string()) diff --git a/cloudinit/sources/DataSourceAzure.py b/cloudinit/sources/DataSourceAzure.py index 698f4cac..2d046600 100644 --- a/cloudinit/sources/DataSourceAzure.py +++ b/cloudinit/sources/DataSourceAzure.py @@ -27,11 +27,12 @@ import xml.etree.ElementTree as ET from xml.dom import minidom +from cloudinit.sources.helpers.azure import get_metadata_from_fabric + from cloudinit import log as logging from cloudinit.settings import PER_ALWAYS from cloudinit import sources from cloudinit import util -from cloudinit.sources.helpers.azure import get_metadata_from_fabric LOG = logging.getLogger(__name__) @@ -40,7 +41,8 @@ DEFAULT_METADATA = {"instance-id": "iid-AZURE-NODE"} AGENT_START = ['service', 'walinuxagent', 'start'] BOUNCE_COMMAND = [ 'sh', '-xc', - "i=$interface; x=0; ifdown $i || x=$?; ifup $i || x=$?; exit $x"] + "i=$interface; x=0; ifdown $i || x=$?; ifup $i || x=$?; exit $x" +] BUILTIN_DS_CONFIG = { 'agent_command': AGENT_START, @@ -51,7 +53,7 @@ BUILTIN_DS_CONFIG = { 'policy': True, 'command': BOUNCE_COMMAND, 'hostname_command': 'hostname', - }, + }, 'disk_aliases': {'ephemeral0': '/dev/sdb'}, } @@ -60,7 +62,7 @@ BUILTIN_CLOUD_CONFIG = { 'ephemeral0': {'table_type': 'gpt', 'layout': [100], 'overwrite': True}, - }, + }, 'fs_setup': [{'filesystem': 'ext4', 'device': 'ephemeral0.1', 'replace_fs': 'ntfs'}], @@ -312,7 +314,7 @@ def support_new_ephemeral(cfg): file_count = 0 try: file_count = util.mount_cb(device, count_files) - except: + except Exception: return None LOG.debug("fabric prepared ephmeral0.1 has %s files on it", file_count) diff --git a/cloudinit/sources/DataSourceBigstep.py b/cloudinit/sources/DataSourceBigstep.py index b5ee4129..f80956a5 100644 --- a/cloudinit/sources/DataSourceBigstep.py +++ b/cloudinit/sources/DataSourceBigstep.py @@ -4,13 +4,13 @@ # Author: Alexandru Sirbu # -import json import errno +import json from cloudinit import log as logging from cloudinit import sources -from cloudinit import util from cloudinit import url_helper +from cloudinit import util LOG = logging.getLogger(__name__) diff --git a/cloudinit/sources/DataSourceCloudSigma.py b/cloudinit/sources/DataSourceCloudSigma.py index d7d4e844..33fe78b9 100644 --- a/cloudinit/sources/DataSourceCloudSigma.py +++ b/cloudinit/sources/DataSourceCloudSigma.py @@ -19,10 +19,11 @@ from base64 import b64decode import os import re +from cloudinit.cs_utils import Cepko + from cloudinit import log as logging from cloudinit import sources from cloudinit import util -from cloudinit.cs_utils import Cepko LOG = logging.getLogger(__name__) @@ -77,7 +78,7 @@ class DataSourceCloudSigma(sources.DataSource): try: server_context = self.cepko.all().result server_meta = server_context['meta'] - except: + except Exception: # TODO: check for explicit "config on", and then warn # but since no explicit config is available now, just debug. LOG.debug("CloudSigma: Unable to read from serial port") diff --git a/cloudinit/sources/DataSourceCloudStack.py b/cloudinit/sources/DataSourceCloudStack.py index 4ba019df..4de1f563 100644 --- a/cloudinit/sources/DataSourceCloudStack.py +++ b/cloudinit/sources/DataSourceCloudStack.py @@ -25,14 +25,15 @@ # along with this program. If not, see . import os -import time from socket import inet_ntoa from struct import pack +import time from cloudinit import ec2_utils as ec2 from cloudinit import log as logging +from cloudinit import sources from cloudinit import url_helper as uhelp -from cloudinit import sources, util +from cloudinit import util LOG = logging.getLogger(__name__) diff --git a/cloudinit/sources/DataSourceDigitalOcean.py b/cloudinit/sources/DataSourceDigitalOcean.py index 12e863d2..44a17a00 100644 --- a/cloudinit/sources/DataSourceDigitalOcean.py +++ b/cloudinit/sources/DataSourceDigitalOcean.py @@ -14,10 +14,10 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . +from cloudinit import ec2_utils from cloudinit import log as logging -from cloudinit import util from cloudinit import sources -from cloudinit import ec2_utils +from cloudinit import util import functools diff --git a/cloudinit/sources/DataSourceGCE.py b/cloudinit/sources/DataSourceGCE.py index 7e7fc033..9234d1f8 100644 --- a/cloudinit/sources/DataSourceGCE.py +++ b/cloudinit/sources/DataSourceGCE.py @@ -18,9 +18,9 @@ from base64 import b64decode from cloudinit import log as logging -from cloudinit import util from cloudinit import sources from cloudinit import url_helper +from cloudinit import util LOG = logging.getLogger(__name__) @@ -71,7 +71,7 @@ class DataSourceGCE(sources.DataSource): index = public_key.index(':') if index > 0: return public_key[(index + 1):] - except: + except Exception: return public_key def get_data(self): diff --git a/cloudinit/sources/DataSourceNoCloud.py b/cloudinit/sources/DataSourceNoCloud.py index 74d0e5ec..48c61a90 100644 --- a/cloudinit/sources/DataSourceNoCloud.py +++ b/cloudinit/sources/DataSourceNoCloud.py @@ -61,7 +61,7 @@ class DataSourceNoCloud(sources.DataSource): if parse_cmdline_data(self.cmdline_id, md): found.append("cmdline") mydata = _merge_new_seed(mydata, {'meta-data': md}) - except: + except Exception: util.logexc(LOG, "Unable to parse command line data") return False @@ -288,7 +288,7 @@ def parse_cmdline_data(ds_id, fill, cmdline=None): continue try: (k, v) = item.split("=", 1) - except: + except Exception: k = item v = None if k in s2l: diff --git a/cloudinit/sources/DataSourceOVF.py b/cloudinit/sources/DataSourceOVF.py index 2a6cd050..f2bb9366 100644 --- a/cloudinit/sources/DataSourceOVF.py +++ b/cloudinit/sources/DataSourceOVF.py @@ -30,13 +30,20 @@ import time from cloudinit import log as logging from cloudinit import sources from cloudinit import util -from .helpers.vmware.imc.config import Config -from .helpers.vmware.imc.config_file import ConfigFile -from .helpers.vmware.imc.config_nic import NicConfigurator -from .helpers.vmware.imc.guestcust_event import GuestCustEventEnum -from .helpers.vmware.imc.guestcust_state import GuestCustStateEnum -from .helpers.vmware.imc.guestcust_error import GuestCustErrorEnum -from .helpers.vmware.imc.guestcust_util import ( + +from cloudinit.sources.helpers.vmware.imc.config \ + import Config +from cloudinit.sources.helpers.vmware.imc.config_file \ + import ConfigFile +from cloudinit.sources.helpers.vmware.imc.config_nic \ + import NicConfigurator +from cloudinit.sources.helpers.vmware.imc.guestcust_event \ + import GuestCustEventEnum +from cloudinit.sources.helpers.vmware.imc.guestcust_state \ + import GuestCustStateEnum +from cloudinit.sourceshelpers.vmware.imc.guestcust_error \ + import GuestCustErrorEnum +from cloudinit.sourceshelpers.vmware.imc.guestcust_util import ( set_customization_status, get_nics_to_enable, enable_nics @@ -262,7 +269,7 @@ def read_ovf_environment(contents): elif prop == "user-data": try: ud = base64.decodestring(val) - except: + except Exception: ud = val return (md, ud, cfg) @@ -277,7 +284,7 @@ def get_ovf_env(dirname): try: contents = util.load_file(full_fn) return (fname, contents) - except: + except Exception: util.logexc(LOG, "Failed loading ovf file %s", full_fn) return (None, False) diff --git a/cloudinit/sources/__init__.py b/cloudinit/sources/__init__.py index 6bf2c33b..43e4fd57 100644 --- a/cloudinit/sources/__init__.py +++ b/cloudinit/sources/__init__.py @@ -45,10 +45,9 @@ class DataSourceNotFoundException(Exception): pass +@six.add_metaclass(abc.ABCMeta) class DataSource(object): - __metaclass__ = abc.ABCMeta - def __init__(self, sys_cfg, distro, paths, ud_proc=None): self.sys_cfg = sys_cfg self.distro = distro diff --git a/cloudinit/sources/helpers/azure.py b/cloudinit/sources/helpers/azure.py index 018cac6d..bbfc283c 100644 --- a/cloudinit/sources/helpers/azure.py +++ b/cloudinit/sources/helpers/azure.py @@ -5,6 +5,7 @@ import socket import struct import tempfile import time + from contextlib import contextmanager from xml.etree import ElementTree diff --git a/cloudinit/sources/helpers/openstack.py b/cloudinit/sources/helpers/openstack.py index 1aa6bbae..156aba6c 100644 --- a/cloudinit/sources/helpers/openstack.py +++ b/cloudinit/sources/helpers/openstack.py @@ -145,8 +145,8 @@ class SourceMixin(object): return device +@six.add_metaclass(abc.ABCMeta) class BaseReader(object): - __metaclass__ = abc.ABCMeta def __init__(self, base_path): self.base_path = base_path @@ -475,7 +475,7 @@ class MetadataReader(BaseReader): def convert_vendordata_json(data, recurse=True): - """ data: a loaded json *object* (strings, arrays, dicts). + """data: a loaded json *object* (strings, arrays, dicts). return something suitable for cloudinit vendordata_raw. if data is: diff --git a/cloudinit/sources/helpers/vmware/imc/boot_proto.py b/cloudinit/sources/helpers/vmware/imc/boot_proto.py index faba5887..204fbcc4 100644 --- a/cloudinit/sources/helpers/vmware/imc/boot_proto.py +++ b/cloudinit/sources/helpers/vmware/imc/boot_proto.py @@ -18,7 +18,7 @@ # along with this program. If not, see . -class BootProtoEnum: +class BootProtoEnum(object): """Specifies the NIC Boot Settings.""" DHCP = 'dhcp' diff --git a/cloudinit/sources/helpers/vmware/imc/config.py b/cloudinit/sources/helpers/vmware/imc/config.py index aebc12a0..1dcd053a 100644 --- a/cloudinit/sources/helpers/vmware/imc/config.py +++ b/cloudinit/sources/helpers/vmware/imc/config.py @@ -20,7 +20,7 @@ from .nic import Nic -class Config: +class Config(object): """ Stores the Contents specified in the Customization Specification file. diff --git a/cloudinit/sources/helpers/vmware/imc/config_nic.py b/cloudinit/sources/helpers/vmware/imc/config_nic.py index 77098a05..511cc918 100644 --- a/cloudinit/sources/helpers/vmware/imc/config_nic.py +++ b/cloudinit/sources/helpers/vmware/imc/config_nic.py @@ -26,7 +26,7 @@ from cloudinit import util logger = logging.getLogger(__name__) -class NicConfigurator: +class NicConfigurator(object): def __init__(self, nics): """ Initialize the Nic Configurator diff --git a/cloudinit/sources/helpers/vmware/imc/config_source.py b/cloudinit/sources/helpers/vmware/imc/config_source.py index a367e476..8a650871 100644 --- a/cloudinit/sources/helpers/vmware/imc/config_source.py +++ b/cloudinit/sources/helpers/vmware/imc/config_source.py @@ -18,6 +18,6 @@ # along with this program. If not, see . -class ConfigSource: +class ConfigSource(object): """Specifies a source for the Config Content.""" pass diff --git a/cloudinit/sources/helpers/vmware/imc/guestcust_error.py b/cloudinit/sources/helpers/vmware/imc/guestcust_error.py index 1b04161f..750be1e3 100644 --- a/cloudinit/sources/helpers/vmware/imc/guestcust_error.py +++ b/cloudinit/sources/helpers/vmware/imc/guestcust_error.py @@ -18,7 +18,7 @@ # along with this program. If not, see . -class GuestCustErrorEnum: +class GuestCustErrorEnum(object): """Specifies different errors of Guest Customization engine""" GUESTCUST_ERROR_SUCCESS = 0 diff --git a/cloudinit/sources/helpers/vmware/imc/guestcust_event.py b/cloudinit/sources/helpers/vmware/imc/guestcust_event.py index fc22568f..e13b791d 100644 --- a/cloudinit/sources/helpers/vmware/imc/guestcust_event.py +++ b/cloudinit/sources/helpers/vmware/imc/guestcust_event.py @@ -18,7 +18,7 @@ # along with this program. If not, see . -class GuestCustEventEnum: +class GuestCustEventEnum(object): """Specifies different types of Guest Customization Events""" GUESTCUST_EVENT_CUSTOMIZE_FAILED = 100 diff --git a/cloudinit/sources/helpers/vmware/imc/guestcust_state.py b/cloudinit/sources/helpers/vmware/imc/guestcust_state.py index f255be5f..b9ddf513 100644 --- a/cloudinit/sources/helpers/vmware/imc/guestcust_state.py +++ b/cloudinit/sources/helpers/vmware/imc/guestcust_state.py @@ -18,7 +18,7 @@ # along with this program. If not, see . -class GuestCustStateEnum: +class GuestCustStateEnum(object): """Specifies different states of Guest Customization engine""" GUESTCUST_STATE_RUNNING = 4 diff --git a/cloudinit/sources/helpers/vmware/imc/guestcust_util.py b/cloudinit/sources/helpers/vmware/imc/guestcust_util.py index d39f0a65..020ab613 100644 --- a/cloudinit/sources/helpers/vmware/imc/guestcust_util.py +++ b/cloudinit/sources/helpers/vmware/imc/guestcust_util.py @@ -23,8 +23,8 @@ import time from cloudinit import util -from .guestcust_state import GuestCustStateEnum from .guestcust_event import GuestCustEventEnum +from .guestcust_state import GuestCustStateEnum logger = logging.getLogger(__name__) diff --git a/cloudinit/sources/helpers/vmware/imc/ipv4_mode.py b/cloudinit/sources/helpers/vmware/imc/ipv4_mode.py index 33f88726..873ddc3b 100644 --- a/cloudinit/sources/helpers/vmware/imc/ipv4_mode.py +++ b/cloudinit/sources/helpers/vmware/imc/ipv4_mode.py @@ -18,7 +18,7 @@ # along with this program. If not, see . -class Ipv4ModeEnum: +class Ipv4ModeEnum(object): """ The IPv4 configuration mode which directly represents the user's goal. diff --git a/cloudinit/sources/helpers/vmware/imc/nic_base.py b/cloudinit/sources/helpers/vmware/imc/nic_base.py index 030ba311..3c892db0 100644 --- a/cloudinit/sources/helpers/vmware/imc/nic_base.py +++ b/cloudinit/sources/helpers/vmware/imc/nic_base.py @@ -18,7 +18,7 @@ # along with this program. If not, see . -class NicBase: +class NicBase(object): """ Define what are expected of each nic. The following properties should be provided in an implementation class. @@ -93,7 +93,7 @@ class NicBase: raise NotImplementedError('Check constraints on properties') -class StaticIpv4Base: +class StaticIpv4Base(object): """ Define what are expected of a static IPv4 setting The following properties should be provided in an implementation class. @@ -124,7 +124,7 @@ class StaticIpv4Base: raise NotImplementedError('Ipv4 GATEWAY') -class StaticIpv6Base: +class StaticIpv6Base(object): """Define what are expected of a static IPv6 setting The following properties should be provided in an implementation class. """ diff --git a/cloudinit/stages.py b/cloudinit/stages.py index ffb15165..62d066de 100644 --- a/cloudinit/stages.py +++ b/cloudinit/stages.py @@ -44,10 +44,10 @@ from cloudinit import helpers from cloudinit import importer from cloudinit import log as logging from cloudinit import net +from cloudinit.reporting import events from cloudinit import sources from cloudinit import type_utils from cloudinit import util -from cloudinit.reporting import events LOG = logging.getLogger(__name__) @@ -483,7 +483,7 @@ class Init(object): c_handlers.initialized.remove(mod) try: handlers.call_end(mod, data, frequency) - except: + except Exception: util.logexc(LOG, "Failed to finalize handler: %s", mod) try: diff --git a/cloudinit/url_helper.py b/cloudinit/url_helper.py index 936f7da5..c05e9d90 100644 --- a/cloudinit/url_helper.py +++ b/cloudinit/url_helper.py @@ -28,8 +28,9 @@ import time from email.utils import parsedate from functools import partial -from requests import exceptions + import oauthlib.oauth1 as oauth1 +from requests import exceptions from six.moves.urllib.parse import ( urlparse, urlunparse, @@ -61,7 +62,7 @@ try: SSL_ENABLED = True if _REQ_VER >= LooseVersion('0.7.0') and _REQ_VER < LooseVersion('1.0.0'): CONFIG_ENABLED = True -except: +except ImportError: pass diff --git a/cloudinit/user_data.py b/cloudinit/user_data.py index f7c5787c..f0631906 100644 --- a/cloudinit/user_data.py +++ b/cloudinit/user_data.py @@ -178,7 +178,7 @@ class UserDataProcessor(object): payload = util.load_yaml(msg.get_payload(decode=True)) if payload: payload_idx = payload.get('launch-index') - except: + except Exception: pass # Header overrides contents, for now (?) or the other way around? if header_idx is not None: diff --git a/cloudinit/util.py b/cloudinit/util.py index 0d21e11b..8d6cbb4b 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -288,7 +288,7 @@ def fork_cb(child_cb, *args, **kwargs): try: child_cb(*args, **kwargs) os._exit(0) - except: + except Exception: logexc(LOG, "Failed forking and calling callback %s", type_utils.obj_name(child_cb)) os._exit(1) @@ -472,7 +472,7 @@ def is_ipv4(instr): try: toks = [x for x in toks if int(x) < 256 and int(x) >= 0] - except: + except Exception: return False return len(toks) == 4 @@ -1210,7 +1210,7 @@ def get_cmdline(): else: try: cmdline = load_file("/proc/cmdline").strip() - except: + except Exception: cmdline = "" PROC_CMDLINE = cmdline @@ -1380,7 +1380,7 @@ def read_write_cmdline_url(target_fn): if not os.path.exists(target_fn): try: (key, url, content) = get_cmdline_url() - except: + except Exception: logexc(LOG, "Failed fetching command line url") return try: @@ -1391,7 +1391,7 @@ def read_write_cmdline_url(target_fn): elif key and not content: LOG.debug(("Command line key %s with url" " %s had no contents"), key, url) - except: + except Exception: logexc(LOG, "Failed writing url content to %s", target_fn) @@ -1449,7 +1449,7 @@ def mounts(): mp = m.group(2) fstype = m.group(3) opts = m.group(4) - except: + except Exception: continue # If the name of the mount point contains spaces these # can be escaped as '\040', so undo that.. @@ -1575,7 +1575,7 @@ def copy(src, dest): def time_rfc2822(): try: ts = time.strftime("%a, %d %b %Y %H:%M:%S %z", time.gmtime()) - except: + except Exception: ts = "??" return ts @@ -1601,7 +1601,7 @@ def uptime(): bootup = buf.value uptime_str = now - bootup - except: + except Exception: logexc(LOG, "Unable to read uptime using method: %s" % method) return uptime_str @@ -2055,7 +2055,7 @@ def log_time(logfunc, msg, func, args=None, kwargs=None, get_uptime=False): tmsg += " (N/A)" try: logfunc(msg + tmsg) - except: + except Exception: pass return ret diff --git a/test-requirements.txt b/test-requirements.txt index 9b3d07c5..170856ad 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -1,7 +1,8 @@ httpretty>=0.7.1 mock nose -pep8==1.5.7 +pep8>=1.5.7 pyflakes contextlib2 setuptools +flake8>=2.5.4 diff --git a/tox.ini b/tox.ini index bd7c27dd..7a144efe 100644 --- a/tox.ini +++ b/tox.ini @@ -30,3 +30,8 @@ deps = pyflakes setenv = LC_ALL = C + +[flake8] + +ignore=H404,H405,H105,H301,H104,H403,H101 +exclude = .venv,.tox,dist,doc,*egg,.git,build,tools,tests -- cgit v1.2.3 From 2fb5af62229b8975910bf0ef63731047bd8d7e63 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Thu, 19 May 2016 15:33:15 -0700 Subject: Fix up tests and flake8 warnings --- cloudinit/net/__init__.py | 2 +- cloudinit/net/eni.py | 16 ++++----- cloudinit/sources/DataSourceConfigDrive.py | 1 - cloudinit/stages.py | 2 +- tests/unittests/helpers.py | 1 + tests/unittests/test__init__.py | 1 - tests/unittests/test_datasource/test_cloudsigma.py | 1 - .../unittests/test_datasource/test_configdrive.py | 41 ++++++++++++---------- tests/unittests/test_datasource/test_nocloud.py | 2 ++ tests/unittests/test_datasource/test_smartos.py | 2 +- tests/unittests/test_net.py | 6 ++-- tests/unittests/test_rh_subscription.py | 5 +-- tox.ini | 4 +-- 13 files changed, 41 insertions(+), 43 deletions(-) (limited to 'cloudinit/stages.py') diff --git a/cloudinit/net/__init__.py b/cloudinit/net/__init__.py index ad44911b..ba0e39ae 100644 --- a/cloudinit/net/__init__.py +++ b/cloudinit/net/__init__.py @@ -49,8 +49,8 @@ DEFAULT_PRIMARY_INTERFACE = 'eth0' # whole module can be easily extracted and placed into other # code-bases (curtin for example). + def write_file(path, content): - """Simple writing a file helper.""" base_path = os.path.dirname(path) if not os.path.isdir(base_path): os.makedirs(base_path) diff --git a/cloudinit/net/eni.py b/cloudinit/net/eni.py index adb31c22..18bae97a 100644 --- a/cloudinit/net/eni.py +++ b/cloudinit/net/eni.py @@ -258,7 +258,7 @@ class Renderer(object): return content def _render_route(self, route, indent=""): - """ When rendering routes for an iface, in some cases applying a route + """When rendering routes for an iface, in some cases applying a route may result in the route command returning non-zero which produces some confusing output for users manually using ifup/ifdown[1]. To that end, we will optionally include an '|| true' postfix to each @@ -302,7 +302,7 @@ class Renderer(object): return content def _render_interfaces(self, network_state): - ''' Given state, emit etc/network/interfaces content ''' + '''Given state, emit etc/network/interfaces content''' content = "" interfaces = network_state.get('interfaces') @@ -336,8 +336,8 @@ class Renderer(object): iface['control'] = subnet.get('control', 'auto') if iface['mode'].endswith('6'): iface['inet'] += '6' - elif iface['mode'] == 'static' \ - and ":" in subnet['address']: + elif (iface['mode'] == 'static' + and ":" in subnet['address']): iface['inet'] += '6' if iface['mode'].startswith('dhcp'): iface['mode'] = 'dhcp' @@ -359,10 +359,10 @@ class Renderer(object): content = content.replace('mac_address', 'hwaddress') return content - def render_network_state(self, - target, network_state, eni="etc/network/interfaces", - links_prefix=LINKS_FNAME_PREFIX, - netrules='etc/udev/rules.d/70-persistent-net.rules'): + def render_network_state( + self, target, network_state, + eni="etc/network/interfaces", links_prefix=LINKS_FNAME_PREFIX, + netrules='etc/udev/rules.d/70-persistent-net.rules'): fpeni = os.path.join(target, eni) net.write_file(fpeni, self._render_interfaces(network_state)) diff --git a/cloudinit/sources/DataSourceConfigDrive.py b/cloudinit/sources/DataSourceConfigDrive.py index 70373b43..4478c4e2 100644 --- a/cloudinit/sources/DataSourceConfigDrive.py +++ b/cloudinit/sources/DataSourceConfigDrive.py @@ -18,7 +18,6 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -import copy import os from cloudinit import log as logging diff --git a/cloudinit/stages.py b/cloudinit/stages.py index 5dd31539..b837009a 100644 --- a/cloudinit/stages.py +++ b/cloudinit/stages.py @@ -44,8 +44,8 @@ from cloudinit import helpers from cloudinit import importer from cloudinit import log as logging from cloudinit import net -from cloudinit.reporting import events from cloudinit.net import cmdline +from cloudinit.reporting import events from cloudinit import sources from cloudinit import type_utils from cloudinit import util diff --git a/tests/unittests/helpers.py b/tests/unittests/helpers.py index 7b4d44e8..8d46a8bf 100644 --- a/tests/unittests/helpers.py +++ b/tests/unittests/helpers.py @@ -45,6 +45,7 @@ else: if _PY_MINOR == 4 and _PY_MICRO < 3: FIX_HTTPRETTY = True + # Makes the old path start # with new base instead of whatever # it previously had diff --git a/tests/unittests/test__init__.py b/tests/unittests/test__init__.py index a9b35afe..0154784a 100644 --- a/tests/unittests/test__init__.py +++ b/tests/unittests/test__init__.py @@ -1,7 +1,6 @@ import os import shutil import tempfile -import unittest2 from cloudinit import handlers from cloudinit import helpers diff --git a/tests/unittests/test_datasource/test_cloudsigma.py b/tests/unittests/test_datasource/test_cloudsigma.py index 7950fc52..2a42ce0c 100644 --- a/tests/unittests/test_datasource/test_cloudsigma.py +++ b/tests/unittests/test_datasource/test_cloudsigma.py @@ -6,7 +6,6 @@ from cloudinit.cs_utils import Cepko from cloudinit.sources import DataSourceCloudSigma from .. import helpers as test_helpers -from ..helpers import SkipTest SERVER_CONTEXT = { "cpu": 1000, diff --git a/tests/unittests/test_datasource/test_configdrive.py b/tests/unittests/test_datasource/test_configdrive.py index 1db50798..5395e544 100644 --- a/tests/unittests/test_datasource/test_configdrive.py +++ b/tests/unittests/test_datasource/test_configdrive.py @@ -368,30 +368,32 @@ class TestNetJson(TestCase): self.assertEqual(myds.network_config, network_config) def test_network_config_conversions(self): - """Tests a bunch of input network json and checks the expected conversions.""" + """Tests a bunch of input network json and checks the + expected conversions.""" in_datas = [ NETWORK_DATA, { 'services': [{'type': 'dns', 'address': '172.19.0.12'}], - 'networks': [ - {'network_id': 'dacd568d-5be6-4786-91fe-750c374b78b4', - 'type': 'ipv4', 'netmask': '255.255.252.0', - 'link': 'tap1a81968a-79', - 'routes': [ - { - 'netmask': '0.0.0.0', - 'network': '0.0.0.0', - 'gateway': '172.19.3.254' - }, - ], - 'ip_address': '172.19.1.34', - 'id': 'network0', + 'networks': [{ + 'network_id': 'dacd568d-5be6-4786-91fe-750c374b78b4', + 'type': 'ipv4', + 'netmask': '255.255.252.0', + 'link': 'tap1a81968a-79', + 'routes': [{ + 'netmask': '0.0.0.0', + 'network': '0.0.0.0', + 'gateway': '172.19.3.254', + }], + 'ip_address': '172.19.1.34', + 'id': 'network0', + }], + 'links': [{ + 'type': 'bridge', + 'vif_id': '1a81968a-797a-400f-8a80-567f997eb93f', + 'ethernet_mac_address': 'fa:16:3e:ed:9a:59', + 'id': 'tap1a81968a-79', + 'mtu': None, }], - 'links': [ - {'type': 'bridge', - 'vif_id': '1a81968a-797a-400f-8a80-567f997eb93f', - 'ethernet_mac_address': 'fa:16:3e:ed:9a:59', - 'id': 'tap1a81968a-79', 'mtu': None}] }, ] out_datas = [ @@ -440,6 +442,7 @@ class TestNetJson(TestCase): 'address': '172.19.1.34', 'netmask': '255.255.252.0', 'type': 'static', + 'ipv4': True, 'routes': [{ 'gateway': '172.19.3.254', 'netmask': '0.0.0.0', diff --git a/tests/unittests/test_datasource/test_nocloud.py b/tests/unittests/test_datasource/test_nocloud.py index 077603b4..b0fa1130 100644 --- a/tests/unittests/test_datasource/test_nocloud.py +++ b/tests/unittests/test_datasource/test_nocloud.py @@ -7,6 +7,8 @@ import os import shutil import tempfile +import yaml + class TestNoCloudDataSource(TestCase): diff --git a/tests/unittests/test_datasource/test_smartos.py b/tests/unittests/test_datasource/test_smartos.py index 2f159ac4..28f56039 100644 --- a/tests/unittests/test_datasource/test_smartos.py +++ b/tests/unittests/test_datasource/test_smartos.py @@ -42,7 +42,7 @@ from cloudinit import helpers as c_helpers from cloudinit.util import b64e from .. import helpers -from ..helpers import mock, SkipTest +from ..helpers import mock MOCK_RETURNS = { 'hostname': 'test-host', diff --git a/tests/unittests/test_net.py b/tests/unittests/test_net.py index faf0f0fb..7998111a 100644 --- a/tests/unittests/test_net.py +++ b/tests/unittests/test_net.py @@ -1,13 +1,11 @@ -from cloudinit import net -from cloudinit import util - from cloudinit import net from cloudinit.net import cmdline from cloudinit.net import eni from cloudinit.net import network_state +from cloudinit import util -from .helpers import TestCase from .helpers import mock +from .helpers import TestCase import base64 import copy diff --git a/tests/unittests/test_rh_subscription.py b/tests/unittests/test_rh_subscription.py index e4dcc58b..891dbe77 100644 --- a/tests/unittests/test_rh_subscription.py +++ b/tests/unittests/test_rh_subscription.py @@ -10,13 +10,10 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . - import logging -import mock -import unittest - from cloudinit.config import cc_rh_subscription +from cloudinit import util from .helpers import TestCase, mock diff --git a/tox.ini b/tox.ini index b92ebac1..7802a291 100644 --- a/tox.ini +++ b/tox.ini @@ -1,11 +1,11 @@ [tox] -envlist = py27,py3,flake8 +envlist = py26,py27,py3,flake8 recreate = True usedevelop = True [testenv] -commands = python -m nose {posargs:tests} +commands = nosetests {posargs:tests} deps = -r{toxinidir}/test-requirements.txt -r{toxinidir}/requirements.txt setenv = -- cgit v1.2.3 From 7f2e99f5345c227d07849da68acdf8562b44c3e1 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Wed, 25 May 2016 17:05:09 -0400 Subject: commit to push for fear of loss. == background == DataSource Mode (dsmode) is present in many datasources in cloud-init. dsmode was originally added to cloud-init to specify when this datasource should be 'realized'. cloud-init has 4 stages of boot. a.) cloud-init --local . network is guaranteed not present. b.) cloud-init (--network). network is guaranteed present. c.) cloud-config d.) cloud-init final 'init_modules' [1] are run "as early as possible". And as such, are executed in either 'a' or 'b' based on the datasource. However, executing them means that user-data has been fully consumed. User-data and vendor-data may have '#include http://...' which then rely on the network being present. boothooks are an example of the things run in init_modules. The 'dsmode' was a way for a user to indicate that init_modules should run at 'a' (dsmode=local) or 'b' (dsmode=net) directly. Things were further confused when a datasource could provide networking configuration. Then, we needed to apply the networking config at 'a' but if the user had provided boothooks that expected networking, then the init_modules would need to be executed at 'b'. The config drive datasource hacked its way through this and applies networking if *it* detects it is a new instance. == Suggested Change == The plan is to 1. incorporate 'dsmode' into DataSource superclass 2. make all existing datasources default to network 3. apply any networking configuration from a datasource on first boot only apply_networking will always rename network devices when it runs. for bug 1579130. 4. run init_modules at cloud-init (network) time frame unless datasource is 'local'. 5. Datasources can provide a 'first_boot' method that will be called when a new instance_id is found. This will allow the config drive's write_files to be applied once. Over all, this will very much simplify things. We'll no longer have 2 sources like DataSourceNoCloud and DataSourceNoCloudNet, but would just have one source with a dsmode. == Concerns == Some things have odd reliance on dsmode. For example, OpenNebula's get_hostname uses it to determine if it should do a lookup of an ip address. == Bugs to fix here == http://pad.lv/1577982 ConfigDrive: cloud-init fails to configure network from network_data.json http://pad.lv/1579130 need to support systemd.link renaming of devices in container http://pad.lv/1577844 Drop unnecessary blocking of all net udev rules --- bin/cloud-init | 20 ++++--- cloudinit/distros/__init__.py | 2 + cloudinit/helpers.py | 23 ++++---- cloudinit/net/__init__.py | 45 +++++++++++++++ cloudinit/sources/DataSourceCloudSigma.py | 18 +----- cloudinit/sources/DataSourceConfigDrive.py | 90 ++++++++---------------------- cloudinit/sources/DataSourceNoCloud.py | 78 +++++++++++--------------- cloudinit/sources/DataSourceOpenNebula.py | 39 ++----------- cloudinit/sources/DataSourceOpenStack.py | 9 +-- cloudinit/sources/__init__.py | 33 +++++++++++ cloudinit/stages.py | 66 ++++++++++++++++------ tox.ini | 3 +- 12 files changed, 223 insertions(+), 203 deletions(-) (limited to 'cloudinit/stages.py') diff --git a/bin/cloud-init b/bin/cloud-init index 5857af32..482b8402 100755 --- a/bin/cloud-init +++ b/bin/cloud-init @@ -236,6 +236,7 @@ def main_init(name, args): else: LOG.debug("Execution continuing, no previous run detected that" " would allow us to stop early.") + else: existing = "check" if util.get_cfg_option_bool(init.cfg, 'manual_cache_clean', False): @@ -265,17 +266,20 @@ def main_init(name, args): else: return (None, ["No instance datasource found."]) - if args.local: - if not init.ds_restored: - # if local mode and the datasource was not restored from cache - # (this is not first boot) then apply networking. - init.apply_network_config() - else: - LOG.debug("skipping networking config from restored datasource.") - # Stage 6 iid = init.instancify() LOG.debug("%s will now be targeting instance id: %s", name, iid) + + if init.is_new_instance(): + # on new instance, apply network config. if not in local mode, + # then we just bring up new networking as the OS has already + # brought up the configured networking. + init.apply_network_config(bringup=not args.local) + + if args.local and init.datasource.dsmode != sources.DSMODE_LOCAL: + return (init.datasource, []) + + # update fully realizes user-data (pulling in #include if necessary) init.update() # Stage 7 try: diff --git a/cloudinit/distros/__init__.py b/cloudinit/distros/__init__.py index 0f222c8c..3bfbc484 100644 --- a/cloudinit/distros/__init__.py +++ b/cloudinit/distros/__init__.py @@ -128,6 +128,8 @@ class Distro(object): mirror_info=arch_info) def apply_network(self, settings, bring_up=True): + # this applies network where 'settings' is interfaces(5) style + # it is obsolete compared to apply_network_config # Write it out dev_names = self._write_network(settings) # Now try to bring them up diff --git a/cloudinit/helpers.py b/cloudinit/helpers.py index 09d75e65..abfb0cbb 100644 --- a/cloudinit/helpers.py +++ b/cloudinit/helpers.py @@ -328,6 +328,7 @@ class Paths(object): self.cfgs = path_cfgs # Populate all the initial paths self.cloud_dir = path_cfgs.get('cloud_dir', '/var/lib/cloud') + self.run_dir = path_cfgs.get('run_dir', '/run/cloud-init') self.instance_link = os.path.join(self.cloud_dir, 'instance') self.boot_finished = os.path.join(self.instance_link, "boot-finished") self.upstart_conf_d = path_cfgs.get('upstart_dir') @@ -349,26 +350,19 @@ class Paths(object): "data": "data", "vendordata_raw": "vendor-data.txt", "vendordata": "vendor-data.txt.i", + "instance_id": "instance-id", } # Set when a datasource becomes active self.datasource = ds # get_ipath_cur: get the current instance path for an item def get_ipath_cur(self, name=None): - ipath = self.instance_link - add_on = self.lookups.get(name) - if add_on: - ipath = os.path.join(ipath, add_on) - return ipath + return self._get_path(self.instance_link, name) # get_cpath : get the "clouddir" (/var/lib/cloud/) # for a name in dirmap def get_cpath(self, name=None): - cpath = self.cloud_dir - add_on = self.lookups.get(name) - if add_on: - cpath = os.path.join(cpath, add_on) - return cpath + return self._get_path(self.cloud_dir, name) # _get_ipath : get the instance path for a name in pathmap # (/var/lib/cloud/instances//) @@ -397,6 +391,15 @@ class Paths(object): else: return ipath + def _get_path(self, base, name=None): + add_on = self.lookups.get(name) + if not add_on: + return base + return os.path.join(base, add_on) + + def get_runpath(self, name=None): + return self._get_path(self.run_dir, name) + # This config parser will not throw when sections don't exist # and you are setting values on those sections which is useful diff --git a/cloudinit/net/__init__.py b/cloudinit/net/__init__.py index 91e36aca..40d330b5 100644 --- a/cloudinit/net/__init__.py +++ b/cloudinit/net/__init__.py @@ -768,4 +768,49 @@ def read_kernel_cmdline_config(files=None, mac_addrs=None, cmdline=None): return config_from_klibc_net_cfg(files=files, mac_addrs=mac_addrs) +def convert_eni_data(eni_data): + # return a network config representation of what is in eni_data + ifaces = {} + parse_deb_config_data(ifaces, eni_data, src_dir=None, src_path=None) + return _ifaces_to_net_config_data(ifaces) + + +def _ifaces_to_net_config_data(ifaces): + """Return network config that represents the ifaces data provided. + ifaces = parse_deb_config("/etc/network/interfaces") + config = ifaces_to_net_config_data(ifaces) + state = parse_net_config_data(config).""" + devs = {} + for name, data in ifaces.items(): + # devname is 'eth0' for name='eth0:1' + devname = name.partition(":")[0] + if devname not in devs: + devs[devname] = {'type': 'physical', 'name': devname, + 'subnets': []} + # this isnt strictly correct, but some might specify + # hwaddress on a nic for matching / declaring name. + if 'hwaddress' in data: + devs[devname]['mac_address'] = data['hwaddress'] + subnet = {'_orig_eni_name': name, 'type': data['method']} + if data.get('auto'): + subnet['control'] = 'auto' + else: + subnet['control'] = 'manual' + + if data.get('method') == 'static': + subnet['address'] = data['address'] + + if 'gateway' in data: + subnet['gateway'] = data['gateway'] + + if 'dns' in data: + for n in ('nameservers', 'search'): + if n in data['dns'] and data['dns'][n]: + subnet['dns_' + n] = data['dns'][n] + devs[devname]['subnets'].append(subnet) + + return {'version': 1, + 'config': [devs[d] for d in sorted(devs)]} + + # vi: ts=4 expandtab syntax=python diff --git a/cloudinit/sources/DataSourceCloudSigma.py b/cloudinit/sources/DataSourceCloudSigma.py index 33fe78b9..07e8ae11 100644 --- a/cloudinit/sources/DataSourceCloudSigma.py +++ b/cloudinit/sources/DataSourceCloudSigma.py @@ -27,8 +27,6 @@ from cloudinit import util LOG = logging.getLogger(__name__) -VALID_DSMODES = ("local", "net", "disabled") - class DataSourceCloudSigma(sources.DataSource): """ @@ -38,7 +36,6 @@ class DataSourceCloudSigma(sources.DataSource): http://cloudsigma-docs.readthedocs.org/en/latest/server_context.html """ def __init__(self, sys_cfg, distro, paths): - self.dsmode = 'local' self.cepko = Cepko() self.ssh_public_key = '' sources.DataSource.__init__(self, sys_cfg, distro, paths) @@ -84,11 +81,9 @@ class DataSourceCloudSigma(sources.DataSource): LOG.debug("CloudSigma: Unable to read from serial port") return False - dsmode = server_meta.get('cloudinit-dsmode', self.dsmode) - if dsmode not in VALID_DSMODES: - LOG.warn("Invalid dsmode %s, assuming default of 'net'", dsmode) - dsmode = 'net' - if dsmode == "disabled" or dsmode != self.dsmode: + self.dsmode = self._determine_dsmode( + [server_meta.get('cloudinit-dsmode')]) + if dsmode == sources.DSMODE_DISABLED: return False base64_fields = server_meta.get('base64_fields', '').split(',') @@ -120,17 +115,10 @@ class DataSourceCloudSigma(sources.DataSource): return self.metadata['uuid'] -class DataSourceCloudSigmaNet(DataSourceCloudSigma): - def __init__(self, sys_cfg, distro, paths): - DataSourceCloudSigma.__init__(self, sys_cfg, distro, paths) - self.dsmode = 'net' - - # Used to match classes to dependencies. Since this datasource uses the serial # port network is not really required, so it's okay to load without it, too. datasources = [ (DataSourceCloudSigma, (sources.DEP_FILESYSTEM)), - (DataSourceCloudSigmaNet, (sources.DEP_FILESYSTEM, sources.DEP_NETWORK)), ] diff --git a/cloudinit/sources/DataSourceConfigDrive.py b/cloudinit/sources/DataSourceConfigDrive.py index 52a9f543..20df5fcd 100644 --- a/cloudinit/sources/DataSourceConfigDrive.py +++ b/cloudinit/sources/DataSourceConfigDrive.py @@ -22,6 +22,7 @@ import copy import os from cloudinit import log as logging +from cloudinit import net from cloudinit import sources from cloudinit import util @@ -35,7 +36,6 @@ DEFAULT_MODE = 'pass' DEFAULT_METADATA = { "instance-id": DEFAULT_IID, } -VALID_DSMODES = ("local", "net", "pass", "disabled") FS_TYPES = ('vfat', 'iso9660') LABEL_TYPES = ('config-2',) POSSIBLE_MOUNTS = ('sr', 'cd') @@ -47,12 +47,12 @@ class DataSourceConfigDrive(openstack.SourceMixin, sources.DataSource): def __init__(self, sys_cfg, distro, paths): super(DataSourceConfigDrive, self).__init__(sys_cfg, distro, paths) self.source = None - self.dsmode = 'local' self.seed_dir = os.path.join(paths.seed_dir, 'config_drive') self.version = None self.ec2_metadata = None self._network_config = None self.network_json = None + self.network_eni = None self.files = {} def __str__(self): @@ -98,38 +98,22 @@ class DataSourceConfigDrive(openstack.SourceMixin, sources.DataSource): md = results.get('metadata', {}) md = util.mergemanydict([md, DEFAULT_METADATA]) - user_dsmode = results.get('dsmode', None) - if user_dsmode not in VALID_DSMODES + (None,): - LOG.warn("User specified invalid mode: %s", user_dsmode) - user_dsmode = None - dsmode = get_ds_mode(cfgdrv_ver=results['version'], - ds_cfg=self.ds_cfg.get('dsmode'), - user=user_dsmode) + self.dsmode = self._determine_dsmode( + [results.get('dsmode'), self.ds_cfg.get('dsmode'), + sources.DSMODE_PASS if results['version'] == 1 else None]) - if dsmode == "disabled": - # most likely user specified + if self.dsmode == sources.DSMODE_DISABLED: return False - # TODO(smoser): fix this, its dirty. - # we want to do some things (writing files and network config) - # only on first boot, and even then, we want to do so in the - # local datasource (so they happen earlier) even if the configured - # dsmode is 'net' or 'pass'. To do this, we check the previous - # instance-id + # This is legacy and sneaky. If dsmode is 'pass' then write + # 'injected files' and apply legacy ENI network format. prev_iid = get_previous_iid(self.paths) cur_iid = md['instance-id'] - if prev_iid != cur_iid and self.dsmode == "local": + if prev_iid != cur_iid and self.dsmode == sources.DSMODE_PASS: on_first_boot(results, distro=self.distro) - - # dsmode != self.dsmode here if: - # * dsmode = "pass", pass means it should only copy files and then - # pass to another datasource - # * dsmode = "net" and self.dsmode = "local" - # so that user boothooks would be applied with network, the - # local datasource just gets out of the way, and lets the net claim - if dsmode != self.dsmode: - LOG.debug("%s: not claiming datasource, dsmode=%s", self, dsmode) + LOG.debug("%s: not claiming datasource, dsmode=%s", self, + self.dsmode) return False self.source = found @@ -147,12 +131,11 @@ class DataSourceConfigDrive(openstack.SourceMixin, sources.DataSource): LOG.warn("Invalid content in vendor-data: %s", e) self.vendordata_raw = None - try: - self.network_json = results.get('networkdata') - except ValueError as e: - LOG.warn("Invalid content in network-data: %s", e) - self.network_json = None - + # network_config is an /etc/network/interfaces formated file and is + # obsolete compared to networkdata (from network_data.json) but both + # might be present. + self.network_eni = results.get("network_config") + self.network_json = results.get('networkdata') return True def check_instance_id(self, sys_cfg): @@ -164,40 +147,11 @@ class DataSourceConfigDrive(openstack.SourceMixin, sources.DataSource): if self._network_config is None: if self.network_json is not None: self._network_config = convert_network_data(self.network_json) + elif self.network_eni is not None: + self._network_config = net.convert_eni_data(self.network_eni) return self._network_config -class DataSourceConfigDriveNet(DataSourceConfigDrive): - def __init__(self, sys_cfg, distro, paths): - DataSourceConfigDrive.__init__(self, sys_cfg, distro, paths) - self.dsmode = 'net' - - -def get_ds_mode(cfgdrv_ver, ds_cfg=None, user=None): - """Determine what mode should be used. - valid values are 'pass', 'disabled', 'local', 'net' - """ - # user passed data trumps everything - if user is not None: - return user - - if ds_cfg is not None: - return ds_cfg - - # at config-drive version 1, the default behavior was pass. That - # meant to not use use it as primary data source, but expect a ec2 metadata - # source. for version 2, we default to 'net', which means - # the DataSourceConfigDriveNet, would be used. - # - # this could change in the future. If there was definitive metadata - # that indicated presense of an openstack metadata service, then - # we could change to 'pass' by default also. The motivation for that - # would be 'cloud-init query' as the web service could be more dynamic - if cfgdrv_ver == 1: - return "pass" - return "net" - - def read_config_drive(source_dir): reader = openstack.ConfigDriveReader(source_dir) finders = [ @@ -231,9 +185,12 @@ def on_first_boot(data, distro=None): % (type(data))) net_conf = data.get("network_config", '') if net_conf and distro: - LOG.debug("Updating network interfaces from config drive") + LOG.warn("Updating network interfaces from config drive") distro.apply_network(net_conf) - files = data.get('files', {}) + write_injected_files(data.get('files')) + + +def write_injected_files(files): if files: LOG.debug("Writing %s injected files", len(files)) for (filename, content) in files.items(): @@ -296,7 +253,6 @@ def find_candidate_devs(probe_optical=True): # Used to match classes to dependencies datasources = [ (DataSourceConfigDrive, (sources.DEP_FILESYSTEM, )), - (DataSourceConfigDriveNet, (sources.DEP_FILESYSTEM, sources.DEP_NETWORK)), ] diff --git a/cloudinit/sources/DataSourceNoCloud.py b/cloudinit/sources/DataSourceNoCloud.py index 48c61a90..7e30118c 100644 --- a/cloudinit/sources/DataSourceNoCloud.py +++ b/cloudinit/sources/DataSourceNoCloud.py @@ -24,6 +24,7 @@ import errno import os from cloudinit import log as logging +from cloudinit import net from cloudinit import sources from cloudinit import util @@ -35,7 +36,6 @@ class DataSourceNoCloud(sources.DataSource): sources.DataSource.__init__(self, sys_cfg, distro, paths) self.dsmode = 'local' self.seed = None - self.cmdline_id = "ds=nocloud" self.seed_dirs = [os.path.join(paths.seed_dir, 'nocloud'), os.path.join(paths.seed_dir, 'nocloud-net')] self.seed_dir = None @@ -58,7 +58,7 @@ class DataSourceNoCloud(sources.DataSource): try: # Parse the kernel command line, getting data passed in md = {} - if parse_cmdline_data(self.cmdline_id, md): + if load_cmdline_data(md): found.append("cmdline") mydata = _merge_new_seed(mydata, {'meta-data': md}) except Exception: @@ -123,12 +123,6 @@ class DataSourceNoCloud(sources.DataSource): mydata = _merge_new_seed(mydata, seeded) - # For seed from a device, the default mode is 'net'. - # that is more likely to be what is desired. If they want - # dsmode of local, then they must specify that. - if 'dsmode' not in mydata['meta-data']: - mydata['meta-data']['dsmode'] = "net" - LOG.debug("Using data from %s", dev) found.append(dev) break @@ -144,7 +138,6 @@ class DataSourceNoCloud(sources.DataSource): if len(found) == 0: return False - seeded_network = None # The special argument "seedfrom" indicates we should # attempt to seed the userdata / metadata from its value # its primarily value is in allowing the user to type less @@ -160,10 +153,6 @@ class DataSourceNoCloud(sources.DataSource): LOG.debug("Seed from %s not supported by %s", seedfrom, self) return False - if (mydata['meta-data'].get('network-interfaces') or - mydata.get('network-config')): - seeded_network = self.dsmode - # This could throw errors, but the user told us to do it # so if errors are raised, let them raise (md_seed, ud) = util.read_seeded(seedfrom, timeout=None) @@ -179,35 +168,21 @@ class DataSourceNoCloud(sources.DataSource): mydata['meta-data'] = util.mergemanydict([mydata['meta-data'], defaults]) - netdata = {'format': None, 'data': None} - if mydata['meta-data'].get('network-interfaces'): - netdata['format'] = 'interfaces' - netdata['data'] = mydata['meta-data']['network-interfaces'] - elif mydata.get('network-config'): - netdata['format'] = 'network-config' - netdata['data'] = mydata['network-config'] - - # if this is the local datasource or 'seedfrom' was used - # and the source of the seed was self.dsmode. - # Then see if there is network config to apply. - # note this is obsolete network-interfaces style seeding. - if self.dsmode in ("local", seeded_network): - if mydata['meta-data'].get('network-interfaces'): - LOG.debug("Updating network interfaces from %s", self) - self.distro.apply_network( - mydata['meta-data']['network-interfaces']) - - if mydata['meta-data']['dsmode'] == self.dsmode: - self.seed = ",".join(found) - self.metadata = mydata['meta-data'] - self.userdata_raw = mydata['user-data'] - self.vendordata_raw = mydata['vendor-data'] - self._network_config = mydata['network-config'] - return True + self.dsmode = self._determine_dsmode( + [mydata['meta-data'].get('dsmode')]) - LOG.debug("%s: not claiming datasource, dsmode=%s", self, - mydata['meta-data']['dsmode']) - return False + if self.dsmode == sources.DSMODE_DISABLED: + LOG.debug("%s: not claiming datasource, dsmode=%s", self, + self.dsmode) + return False + + self.seed = ",".join(found) + self.metadata = mydata['meta-data'] + self.userdata_raw = mydata['user-data'] + self.vendordata_raw = mydata['vendor-data'] + self._network_config = mydata['network-config'] + self._network_eni = mydata['meta-data'].get('network-interfaces') + return True def check_instance_id(self, sys_cfg): # quickly (local check only) if self.instance_id is still valid @@ -227,6 +202,9 @@ class DataSourceNoCloud(sources.DataSource): @property def network_config(self): + if self._network_config is None: + if self.network_eni is not None: + self._network_config = net.convert_eni_data(self.network_eni) return self._network_config @@ -254,8 +232,22 @@ def _quick_read_instance_id(cmdline_id, dirs=None): return None +def load_cmdline_data(fill, cmdline=None): + pairs = [("ds=nocloud", sources.DSMODE_LOCAL), + ("ds=nocloud-net", sources.DSMODE_NETWORK)] + for idstr, dsmode in pairs: + if parse_cmdline_data(idstr, fill, cmdline): + # if dsmode was explicitly in the commanad line, then + # prefer it to the dsmode based on the command line id + if 'dsmode' not in fill: + fill['dsmode'] = dsmode + return True + return False + + # Returns true or false indicating if cmdline indicated -# that this module should be used +# that this module should be used. Updates dictionary 'fill' +# with data that was found. # Example cmdline: # root=LABEL=uec-rootfs ro ds=nocloud def parse_cmdline_data(ds_id, fill, cmdline=None): @@ -319,9 +311,7 @@ def _merge_new_seed(cur, seeded): class DataSourceNoCloudNet(DataSourceNoCloud): def __init__(self, sys_cfg, distro, paths): DataSourceNoCloud.__init__(self, sys_cfg, distro, paths) - self.cmdline_id = "ds=nocloud-net" self.supported_seed_starts = ("http://", "https://", "ftp://") - self.dsmode = "net" # Used to match classes to dependencies diff --git a/cloudinit/sources/DataSourceOpenNebula.py b/cloudinit/sources/DataSourceOpenNebula.py index 681f3a96..15819a4f 100644 --- a/cloudinit/sources/DataSourceOpenNebula.py +++ b/cloudinit/sources/DataSourceOpenNebula.py @@ -37,16 +37,13 @@ from cloudinit import util LOG = logging.getLogger(__name__) DEFAULT_IID = "iid-dsopennebula" -DEFAULT_MODE = 'net' DEFAULT_PARSEUSER = 'nobody' CONTEXT_DISK_FILES = ["context.sh"] -VALID_DSMODES = ("local", "net", "disabled") class DataSourceOpenNebula(sources.DataSource): def __init__(self, sys_cfg, distro, paths): sources.DataSource.__init__(self, sys_cfg, distro, paths) - self.dsmode = 'local' self.seed = None self.seed_dir = os.path.join(paths.seed_dir, 'opennebula') @@ -93,52 +90,27 @@ class DataSourceOpenNebula(sources.DataSource): md = util.mergemanydict([md, defaults]) # check for valid user specified dsmode - user_dsmode = results['metadata'].get('DSMODE', None) - if user_dsmode not in VALID_DSMODES + (None,): - LOG.warn("user specified invalid mode: %s", user_dsmode) - user_dsmode = None - - # decide dsmode - if user_dsmode: - dsmode = user_dsmode - elif self.ds_cfg.get('dsmode'): - dsmode = self.ds_cfg.get('dsmode') - else: - dsmode = DEFAULT_MODE - - if dsmode == "disabled": - # most likely user specified - return False - - # apply static network configuration only in 'local' dsmode - if ('network-interfaces' in results and self.dsmode == "local"): - LOG.debug("Updating network interfaces from %s", self) - self.distro.apply_network(results['network-interfaces']) + self.dsmode = self._determine_dsmode( + [results.get('DSMODE'), self.ds_cfg.get('dsmode')]) - if dsmode != self.dsmode: - LOG.debug("%s: not claiming datasource, dsmode=%s", self, dsmode) + if self.dsmode == sources.DSMODE_DISABLED: return False self.seed = seed + self.network_eni = results.get("network_config") self.metadata = md self.userdata_raw = results.get('userdata') return True def get_hostname(self, fqdn=False, resolve_ip=None): if resolve_ip is None: - if self.dsmode == 'net': + if self.dsmode == sources.DSMODE_NET: resolve_ip = True else: resolve_ip = False return sources.DataSource.get_hostname(self, fqdn, resolve_ip) -class DataSourceOpenNebulaNet(DataSourceOpenNebula): - def __init__(self, sys_cfg, distro, paths): - DataSourceOpenNebula.__init__(self, sys_cfg, distro, paths) - self.dsmode = 'net' - - class NonContextDiskDir(Exception): pass @@ -446,7 +418,6 @@ def read_context_disk_dir(source_dir, asuser=None): # Used to match classes to dependencies datasources = [ (DataSourceOpenNebula, (sources.DEP_FILESYSTEM, )), - (DataSourceOpenNebulaNet, (sources.DEP_FILESYSTEM, sources.DEP_NETWORK)), ] diff --git a/cloudinit/sources/DataSourceOpenStack.py b/cloudinit/sources/DataSourceOpenStack.py index dfd96035..c06d17f3 100644 --- a/cloudinit/sources/DataSourceOpenStack.py +++ b/cloudinit/sources/DataSourceOpenStack.py @@ -33,13 +33,11 @@ DEFAULT_IID = "iid-dsopenstack" DEFAULT_METADATA = { "instance-id": DEFAULT_IID, } -VALID_DSMODES = ("net", "disabled") class DataSourceOpenStack(openstack.SourceMixin, sources.DataSource): def __init__(self, sys_cfg, distro, paths): super(DataSourceOpenStack, self).__init__(sys_cfg, distro, paths) - self.dsmode = 'net' self.metadata_address = None self.ssl_details = util.fetch_ssl_details(self.paths) self.version = None @@ -125,11 +123,8 @@ class DataSourceOpenStack(openstack.SourceMixin, sources.DataSource): self.metadata_address) return False - user_dsmode = results.get('dsmode', None) - if user_dsmode not in VALID_DSMODES + (None,): - LOG.warn("User specified invalid mode: %s", user_dsmode) - user_dsmode = None - if user_dsmode == 'disabled': + self.dsmode = self._determine_dsmode([results.get('dsmode')]) + if self.dsmode == sources.DSMODE_DISABLED: return False md = results.get('metadata', {}) diff --git a/cloudinit/sources/__init__.py b/cloudinit/sources/__init__.py index 43e4fd57..e0171e8c 100644 --- a/cloudinit/sources/__init__.py +++ b/cloudinit/sources/__init__.py @@ -34,6 +34,13 @@ from cloudinit import util from cloudinit.filters import launch_index from cloudinit.reporting import events +DSMODE_DISABLED = "disabled" +DSMODE_LOCAL = "net" +DSMODE_NETWORK = "local" +DSMODE_PASS = "pass" + +VALID_DSMODES = [DSMODE_DISABLED, DSMODE_LOCAL, DSMODE_NETWORK] + DEP_FILESYSTEM = "FILESYSTEM" DEP_NETWORK = "NETWORK" DS_PREFIX = 'DataSource' @@ -57,6 +64,7 @@ class DataSource(object): self.userdata_raw = None self.vendordata = None self.vendordata_raw = None + self.dsmode = DSMODE_NETWORK # find the datasource config name. # remove 'DataSource' from classname on front, and remove 'Net' on end. @@ -223,10 +231,35 @@ class DataSource(object): # quickly (local check only) if self.instance_id is still return False + @staticmethod + def _determine_dsmode(candidates, default=None, valid=None): + # return the first candidate that is non None, warn if not valid + if default is None: + default = DSMODE_NETWORK + + if valid is None: + valid = VALID_DSMODES + + for candidate in candidates: + if candidate is None: + continue + if candidate in valid: + return candidate + else: + LOG.warn("invalid dsmode '%s', using default=%s", + candidate, default) + return default + + return default + @property def network_config(self): return None + @property + def first_instance_boot(self): + return + def normalize_pubkey_data(pubkey_data): keys = [] diff --git a/cloudinit/stages.py b/cloudinit/stages.py index 62d066de..53ebcb45 100644 --- a/cloudinit/stages.py +++ b/cloudinit/stages.py @@ -67,6 +67,7 @@ class Init(object): # Changed only when a fetch occurs self.datasource = NULL_DATA_SOURCE self.ds_restored = False + self._previous_iid = None if reporter is None: reporter = events.ReportEventStack( @@ -213,6 +214,31 @@ class Init(object): cfg_list = self.cfg.get('datasource_list') or [] return (cfg_list, pkg_list) + def _restore_from_checked_cache(self, existing): + if existing not in ("check", "trust"): + raise ValueError("Unexpected value for existing: %s" % existing) + + ds = self._restore_from_cache() + if not ds: + return (None, "no cache found") + + run_iid_fn = self.paths.get_runpath('instance-id') + if os.path.exists(run_iid_fn): + run_iid = util.load_file(run_iid_fn).strip() + else: + run_iid = None + + if run_iid == ds.get_instance_id: + return (ds, "restored from cache with run check: %s" % ds) + elif existing == "trust": + return (ds, "restored from cache: %s" % ds) + else: + if (hasattr(ds, 'check_instance_id') and + ds.check_instance_id(self.cfg)): + return (ds, "restored from checked cache: %s" % ds) + else: + return (None, "cache invalid in datasource: %s" % ds) + def _get_data_source(self, existing): if self.datasource is not NULL_DATA_SOURCE: return self.datasource @@ -221,19 +247,9 @@ class Init(object): name="check-cache", description="attempting to read from cache [%s]" % existing, parent=self.reporter) as myrep: - ds = self._restore_from_cache() - if ds and existing == "trust": - myrep.description = "restored from cache: %s" % ds - elif ds and existing == "check": - if (hasattr(ds, 'check_instance_id') and - ds.check_instance_id(self.cfg)): - myrep.description = "restored from checked cache: %s" % ds - else: - myrep.description = "cache invalid in datasource: %s" % ds - ds = None - else: - myrep.description = "no cache found" + ds, desc = self._restore_from_checked_cache(existing) + myrep.description = desc self.ds_restored = bool(ds) LOG.debug(myrep.description) @@ -301,15 +317,15 @@ class Init(object): # What the instance id was and is... iid = self.datasource.get_instance_id() - previous_iid = None iid_fn = os.path.join(dp, 'instance-id') try: previous_iid = util.load_file(iid_fn).strip() except Exception: - pass + previous_iid = None if not previous_iid: previous_iid = iid util.write_file(iid_fn, "%s\n" % iid) + util.write_file(self.paths.get_runpath('instance-id'), "%s\n" % iid) util.write_file(os.path.join(dp, 'previous-instance-id'), "%s\n" % (previous_iid)) # Ensure needed components are regenerated @@ -318,6 +334,21 @@ class Init(object): self._reset() return iid + def previous_iid(self): + if self._previous_iid is not None: + return self._previous_iid + + dp = self.paths.get_cpath('data') + iid_fn = os.path.join(dp, 'instance-id') + try: + self._previous_iid = util.load_file(iid_fn).strip() + except Exception: + pass + return self._previous_iid + + def is_new_instance(self): + return self.datasource.get_instance_id() == self.previous_iid() + def fetch(self, existing="check"): return self._get_data_source(existing=existing) @@ -593,15 +624,16 @@ class Init(object): return (ncfg, loc) return (net.generate_fallback_config(), "fallback") - def apply_network_config(self): + def apply_network_config(self, bringup): netcfg, src = self._find_networking_config() if netcfg is None: LOG.info("network config is disabled by %s", src) return - LOG.info("Applying network configuration from %s: %s", src, netcfg) + LOG.info("Applying network configuration from %s bringup=%s: %s", + src, bringup, netcfg) try: - return self.distro.apply_network_config(netcfg) + return self.distro.apply_network_config(netcfg, bringup=bringup) except NotImplementedError: LOG.warn("distro '%s' does not implement apply_network_config. " "networking may not be configured properly." % diff --git a/tox.ini b/tox.ini index dafaaf6d..18d059df 100644 --- a/tox.ini +++ b/tox.ini @@ -1,6 +1,7 @@ [tox] envlist = py27,py3,flake8 -recreate = True +recreate = False +skip_install = True [testenv] commands = python -m nose {posargs:tests} -- cgit v1.2.3 From 443df65a45e42c354b8eb7638528970adf036290 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Wed, 25 May 2016 20:36:40 -0400 Subject: fix bring_up --- bin/cloud-init | 4 ++-- cloudinit/stages.py | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) (limited to 'cloudinit/stages.py') diff --git a/bin/cloud-init b/bin/cloud-init index 482b8402..1d421acf 100755 --- a/bin/cloud-init +++ b/bin/cloud-init @@ -260,7 +260,7 @@ def main_init(name, args): util.logexc(LOG, ("No instance datasource found!" " Likely bad things to come!")) if not args.force: - init.apply_network_config() + init.apply_network_config(bring_up=not args.local) if args.local: return (None, []) else: @@ -274,7 +274,7 @@ def main_init(name, args): # on new instance, apply network config. if not in local mode, # then we just bring up new networking as the OS has already # brought up the configured networking. - init.apply_network_config(bringup=not args.local) + init.apply_network_config(bring_up=not args.local) if args.local and init.datasource.dsmode != sources.DSMODE_LOCAL: return (init.datasource, []) diff --git a/cloudinit/stages.py b/cloudinit/stages.py index 53ebcb45..35eddc3b 100644 --- a/cloudinit/stages.py +++ b/cloudinit/stages.py @@ -624,16 +624,16 @@ class Init(object): return (ncfg, loc) return (net.generate_fallback_config(), "fallback") - def apply_network_config(self, bringup): + def apply_network_config(self, bring_up): netcfg, src = self._find_networking_config() if netcfg is None: LOG.info("network config is disabled by %s", src) return LOG.info("Applying network configuration from %s bringup=%s: %s", - src, bringup, netcfg) + src, bring_up, netcfg) try: - return self.distro.apply_network_config(netcfg, bringup=bringup) + return self.distro.apply_network_config(netcfg, bring_up=bring_up) except NotImplementedError: LOG.warn("distro '%s' does not implement apply_network_config. " "networking may not be configured properly." % -- cgit v1.2.3 From 63501f44eff7ef2d6083900c47180faf444662fc Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Thu, 26 May 2016 09:02:17 -0400 Subject: kernel command line: override all local settings settings on the kernel command line (cc:) were documented to override all local settings, but a bug in implementation meant they would only override those that are in /etc/cloud/cloud.cfg, not any found in /etc/cloud/cloud.cfg.d. LP: #1582323 --- ChangeLog | 2 ++ cloudinit/stages.py | 10 +++++----- 2 files changed, 7 insertions(+), 5 deletions(-) (limited to 'cloudinit/stages.py') diff --git a/ChangeLog b/ChangeLog index 6748e8fa..8db29e2e 100644 --- a/ChangeLog +++ b/ChangeLog @@ -110,6 +110,8 @@ - Paths: fix instance path if datasource's id has a '/'. (LP: #1575938) [Robert Jennings] - Ec2: do not retry requests for user-data path on 404. + - settings on the kernel command line (cc:) override all local settings + rather than only those in /etc/cloud/cloud.cfg (LP: #1582323) 0.7.6: - open 0.7.6 diff --git a/cloudinit/stages.py b/cloudinit/stages.py index 62d066de..002e5832 100644 --- a/cloudinit/stages.py +++ b/cloudinit/stages.py @@ -794,16 +794,16 @@ class Modules(object): def fetch_base_config(): base_cfgs = [] default_cfg = util.get_builtin_cfg() - kern_contents = util.read_cc_from_cmdline() - - # Kernel/cmdline parameters override system config - if kern_contents: - base_cfgs.append(util.load_yaml(kern_contents, default={})) # Anything in your conf.d location?? # or the 'default' cloud.cfg location??? base_cfgs.append(util.read_conf_with_confd(CLOUD_CONFIG)) + # Kernel/cmdline parameters override system config + kern_contents = util.read_cc_from_cmdline() + if kern_contents: + base_cfgs.append(util.load_yaml(kern_contents, default={})) + # And finally the default gets to play if default_cfg: base_cfgs.append(default_cfg) -- cgit v1.2.3 From ac851b8ff106abb4314d2b71f2a7dc194b259899 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Thu, 26 May 2016 11:24:28 -0400 Subject: fix usage of instance-id to instance_id in runpath, update cache on instancify the fix for instance_id is clear and necessary. making instancify write the cache is required for how we are having the local datasource be relevant. --- cloudinit/stages.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'cloudinit/stages.py') diff --git a/cloudinit/stages.py b/cloudinit/stages.py index 35eddc3b..5141612f 100644 --- a/cloudinit/stages.py +++ b/cloudinit/stages.py @@ -222,7 +222,7 @@ class Init(object): if not ds: return (None, "no cache found") - run_iid_fn = self.paths.get_runpath('instance-id') + run_iid_fn = self.paths.get_runpath('instance_id') if os.path.exists(run_iid_fn): run_iid = util.load_file(run_iid_fn).strip() else: @@ -325,9 +325,11 @@ class Init(object): if not previous_iid: previous_iid = iid util.write_file(iid_fn, "%s\n" % iid) - util.write_file(self.paths.get_runpath('instance-id'), "%s\n" % iid) + util.write_file(self.paths.get_runpath('instance_id'), "%s\n" % iid) util.write_file(os.path.join(dp, 'previous-instance-id'), "%s\n" % (previous_iid)) + + self._write_to_cache() # Ensure needed components are regenerated # after change of instance which may cause # change of configuration @@ -363,8 +365,6 @@ class Init(object): reporter=self.reporter) def update(self): - if not self._write_to_cache(): - return self._store_userdata() self._store_vendordata() -- cgit v1.2.3 From e86794ba646863ac54dcb348c016d2ae0be2c180 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Thu, 26 May 2016 15:50:40 -0400 Subject: compare instance id to get_instance_id() not the function itself. --- cloudinit/stages.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'cloudinit/stages.py') diff --git a/cloudinit/stages.py b/cloudinit/stages.py index 90c9016c..cbf4df8b 100644 --- a/cloudinit/stages.py +++ b/cloudinit/stages.py @@ -228,7 +228,7 @@ class Init(object): else: run_iid = None - if run_iid == ds.get_instance_id: + if run_iid == ds.get_instance_id(): return (ds, "restored from cache with run check: %s" % ds) elif existing == "trust": return (ds, "restored from cache: %s" % ds) -- cgit v1.2.3 From 1174e19116f2934f927d89fe00ed8d6e22983d75 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Thu, 26 May 2016 16:02:53 -0400 Subject: fix logic in is_new_instance --- cloudinit/stages.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'cloudinit/stages.py') diff --git a/cloudinit/stages.py b/cloudinit/stages.py index cbf4df8b..e34662b1 100644 --- a/cloudinit/stages.py +++ b/cloudinit/stages.py @@ -349,7 +349,7 @@ class Init(object): return self._previous_iid def is_new_instance(self): - return self.datasource.get_instance_id() == self.previous_iid() + return self.datasource.get_instance_id() != self.previous_iid() def fetch(self, existing="check"): return self._get_data_source(existing=existing) -- cgit v1.2.3 From 3ce4cebd417043e1d0a3bb387998f9b71bb76e03 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Thu, 26 May 2016 21:54:33 -0400 Subject: fix is_new_instance() to work better --- cloudinit/stages.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) (limited to 'cloudinit/stages.py') diff --git a/cloudinit/stages.py b/cloudinit/stages.py index e34662b1..20c334b7 100644 --- a/cloudinit/stages.py +++ b/cloudinit/stages.py @@ -52,6 +52,7 @@ from cloudinit import util LOG = logging.getLogger(__name__) NULL_DATA_SOURCE = None +NO_PREVIOUS_INSTANCE_ID = "NO_PREVIOUS_INSTANCE_ID" class Init(object): @@ -68,6 +69,8 @@ class Init(object): self.datasource = NULL_DATA_SOURCE self.ds_restored = False self._previous_iid = None + # simply ensure this gets set + self.previous_iid() if reporter is None: reporter = events.ReportEventStack( @@ -345,11 +348,15 @@ class Init(object): try: self._previous_iid = util.load_file(iid_fn).strip() except Exception: + self._previous_iid = NO_PREVIOUS_INSTANCE_ID pass return self._previous_iid def is_new_instance(self): - return self.datasource.get_instance_id() != self.previous_iid() + previous = self.previous_iid() + ret = (previous == NO_PREVIOUS_INSTANCE_ID or + previous != self.datasource.get_instance_id()) + return ret def fetch(self, existing="check"): return self._get_data_source(existing=existing) -- cgit v1.2.3 From 653676831e70192d4a0322ee453d6a4c3e6541da Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Fri, 27 May 2016 16:57:18 -0400 Subject: fix test cases by avoiding rendering paths to early --- cloudinit/stages.py | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) (limited to 'cloudinit/stages.py') diff --git a/cloudinit/stages.py b/cloudinit/stages.py index 20c334b7..f164d6f6 100644 --- a/cloudinit/stages.py +++ b/cloudinit/stages.py @@ -69,8 +69,6 @@ class Init(object): self.datasource = NULL_DATA_SOURCE self.ds_restored = False self._previous_iid = None - # simply ensure this gets set - self.previous_iid() if reporter is None: reporter = events.ReportEventStack( @@ -321,12 +319,8 @@ class Init(object): # What the instance id was and is... iid = self.datasource.get_instance_id() iid_fn = os.path.join(dp, 'instance-id') - try: - previous_iid = util.load_file(iid_fn).strip() - except Exception: - previous_iid = None - if not previous_iid: - previous_iid = iid + + previous_iid = self.previous_iid() util.write_file(iid_fn, "%s\n" % iid) util.write_file(self.paths.get_runpath('instance_id'), "%s\n" % iid) util.write_file(os.path.join(dp, 'previous-instance-id'), @@ -349,7 +343,8 @@ class Init(object): self._previous_iid = util.load_file(iid_fn).strip() except Exception: self._previous_iid = NO_PREVIOUS_INSTANCE_ID - pass + + LOG.debug("previous iid found to be %s", self._previous_iid) return self._previous_iid def is_new_instance(self): -- cgit v1.2.3 From 1b8a09389654a29af7e618b803bffaed0185e9e8 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Tue, 31 May 2016 17:17:39 -0400 Subject: add renaming code for renaming interfaces currently does not work in lxc https://github.com/lxc/lxd/issues/2063 --- bin/cloud-init | 6 +-- cloudinit/distros/__init__.py | 4 ++ cloudinit/net/__init__.py | 86 +++++++++++++++++++++++++++++++++++++++++++ cloudinit/stages.py | 10 +++++ 4 files changed, 101 insertions(+), 5 deletions(-) (limited to 'cloudinit/stages.py') diff --git a/bin/cloud-init b/bin/cloud-init index 29e9b521..21c3a684 100755 --- a/bin/cloud-init +++ b/bin/cloud-init @@ -280,11 +280,7 @@ def main_init(name, args): LOG.debug("[%s] %s will now be targeting instance id: %s. new=%s", mode, name, iid, init.is_new_instance()) - if init.is_new_instance(): - # on new instance, apply network config. - # in network mode 'bring_up' must be passed in as the OS - # has already brought up networking. - init.apply_network_config(bring_up=bool(mode != sources.DSMODE_LOCAL)) + init.apply_network_config(bring_up=bool(mode != sources.DSMODE_LOCAL)) if mode == sources.DSMODE_LOCAL: if init.datasource.dsmode != mode: diff --git a/cloudinit/distros/__init__.py b/cloudinit/distros/__init__.py index 3bfbc484..5c29c804 100644 --- a/cloudinit/distros/__init__.py +++ b/cloudinit/distros/__init__.py @@ -31,6 +31,7 @@ import stat from cloudinit import importer from cloudinit import log as logging +from cloudinit import net from cloudinit import ssh_util from cloudinit import type_utils from cloudinit import util @@ -145,6 +146,9 @@ class Distro(object): return self._bring_up_interfaces(dev_names) return False + def apply_network_config_names(self, netconfig): + net.apply_network_config_names(netconfig) + @abc.abstractmethod def apply_locale(self, locale, out_fn=None): raise NotImplementedError() diff --git a/cloudinit/net/__init__.py b/cloudinit/net/__init__.py index 40d330b5..ec1b3835 100644 --- a/cloudinit/net/__init__.py +++ b/cloudinit/net/__init__.py @@ -813,4 +813,90 @@ def _ifaces_to_net_config_data(ifaces): 'config': [devs[d] for d in sorted(devs)]} +def apply_network_config_names(netcfg, strict_present=True, strict_busy=True): + """read the network config and rename devices accordingly. + if strict_present is false, then do not raise exception if no devices + match. if strict_busy is false, then do not raise exception if the + device cannot be renamed because it is currently configured.""" + renames = [] + for ent in netcfg.get('config', {}): + if ent.get('type') != 'physical': + continue + mac = ent.get('mac_address') + name = ent.get('name') + if not mac: + continue + renames.append([mac, name]) + + return rename_interfaces(renames) + + +def rename_interfaces(renames, strict_present=True, strict_busy=True): + cur_bymac = {get_interface_mac(n): n for n in get_devicelist()} + expected = {mac: name for mac, name in renames} + cur_byname = {v: k for k, v in cur_bymac.items()} + + tmpname_fmt = "cirename%d" + tmpi = -1 + + moves = [] + changes = [] + errors = [] + for mac, new_name in expected.items(): + cur_name = cur_bymac.get(mac) + if cur_name == new_name: + # nothing to do + continue + + if not cur_name: + if strict_present: + errors.append( + "[nic not present] Cannot rename mac=%s to %s" + ", not available." % (mac, new_name)) + elif is_up(cur_name): + if strict_busy: + errors.append("[busy] Error renaming mac=%s from %s to %s." % + (mac, cur_name, new_name)) + elif new_name in cur_byname: + if is_up(new_name): + if strict_busy: + errors.append( + "[busy-target] Error renaming mac=%s from %s to %s." % + (mac, cur_name, new_name)) + else: + tmp_name = None + while tmp_name is None or tmp_name in cur_byname: + tmpi += 1 + tmp_name = tmpname_fmt % tmpi + moves.append((mac, cur_name, tmp_name)) + changes.append((mac, tmp_name, new_name)) + else: + changes.append((mac, cur_name, new_name)) + + def rename_dev(cur, new): + cmd = ["ip", "link", "set", cur, "name", new] + util.subp(cmd) + + for mac, cur, new in moves + changes: + try: + rename_dev(cur, new) + except util.ProcessExecutionError as e: + errors.append( + "[unknown] Error renaming mac=%s from %s to %s. (%s)" % + (mac, cur, new, e)) + + if len(errors): + raise Exception('\n'.join(errors)) + + +def get_interface_mac(ifname): + """Returns the string value of an interface's MAC Address""" + return read_sys_net(ifname, "address", enoent=False) + + +def get_ifname_mac_pairs(): + """Build a list of tuples (ifname, mac)""" + return [(ifname, get_interface_mac(ifname)) for ifname in get_devicelist()] + + # vi: ts=4 expandtab syntax=python diff --git a/cloudinit/stages.py b/cloudinit/stages.py index f164d6f6..211d2286 100644 --- a/cloudinit/stages.py +++ b/cloudinit/stages.py @@ -632,6 +632,16 @@ class Init(object): LOG.info("network config is disabled by %s", src) return + try: + LOG.debug("applying net config names for %s" % netcfg) + self.distro.apply_network_config_names(netcfg) + except Exception as e: + LOG.warn("Failed to rename devices: %s", e) + + if not self.is_new_instance(): + LOG.debug("not a new instance. network config is not applied.") + return + LOG.info("Applying network configuration from %s bringup=%s: %s", src, bring_up, netcfg) try: -- cgit v1.2.3 From 0fc5c222d2ff335b9a1de3e910bb8822b6d73031 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Wed, 1 Jun 2016 17:17:11 -0400 Subject: cloudinit/stages.py: if no datasource found, do not attempt is_new_instance if local does not find a datasource, then we try to apply networking. but that would then hit the NULL_DATA_SOURCE which does not work with is_new_instance. avoid that. --- cloudinit/stages.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'cloudinit/stages.py') diff --git a/cloudinit/stages.py b/cloudinit/stages.py index 211d2286..5756e74d 100644 --- a/cloudinit/stages.py +++ b/cloudinit/stages.py @@ -638,7 +638,8 @@ class Init(object): except Exception as e: LOG.warn("Failed to rename devices: %s", e) - if not self.is_new_instance(): + if (self.datasource is not NULL_DATA_SOURCE and + not self.is_new_instance()): LOG.debug("not a new instance. network config is not applied.") return -- cgit v1.2.3 From dbeff5c80281ee1c236583d523feb323c7518c19 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Tue, 7 Jun 2016 14:34:57 -0400 Subject: system config of networking should override datasource provided while datasource provided networking is more dynamic in most cases, preference should still be given to networking configuration provided in the system. This is because the user of the image should be ultimately in control of the networking configuration if they so choose. LP: #1590104 --- cloudinit/stages.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'cloudinit/stages.py') diff --git a/cloudinit/stages.py b/cloudinit/stages.py index 5756e74d..4ba95318 100644 --- a/cloudinit/stages.py +++ b/cloudinit/stages.py @@ -618,7 +618,7 @@ class Init(object): dscfg = ('ds', self.datasource.network_config) sys_cfg = ('system_cfg', self.cfg.get('network')) - for loc, ncfg in (cmdline_cfg, dscfg, sys_cfg): + for loc, ncfg in (cmdline_cfg, sys_cfg, dscfg): if net.is_disabled_cfg(ncfg): LOG.debug("network config disabled by %s", loc) return (None, loc) -- cgit v1.2.3