From 7479ffefc3c31805e2b96b94852610dd7211e8a8 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Thu, 10 Mar 2016 16:37:30 -0500 Subject: initial copy of curtin net just add curtin/net as cloudinit/net and then copy curtin/udev.py as cloudinit/net/udev.py --- cloudinit/net/__init__.py | 436 +++++++++++++++++++++++++++++++++++++++++ cloudinit/net/network_state.py | 400 +++++++++++++++++++++++++++++++++++++ cloudinit/net/udev.py | 54 +++++ 3 files changed, 890 insertions(+) create mode 100644 cloudinit/net/__init__.py create mode 100644 cloudinit/net/network_state.py create mode 100644 cloudinit/net/udev.py diff --git a/cloudinit/net/__init__.py b/cloudinit/net/__init__.py new file mode 100644 index 00000000..e5ca050e --- /dev/null +++ b/cloudinit/net/__init__.py @@ -0,0 +1,436 @@ +# Copyright (C) 2013-2014 Canonical Ltd. +# +# Author: Scott Moser +# Author: Blake Rouse +# +# Curtin is free software: you can redistribute it and/or modify it under +# the terms of the GNU Affero General Public License as published by the +# Free Software Foundation, either version 3 of the License, or (at your +# option) any later version. +# +# Curtin is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +# FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for +# more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Curtin. If not, see . + +import errno +import glob +import os +import re + +from curtin.log import LOG +from curtin.udev import generate_udev_rule +import curtin.util as util +import curtin.config as config +from . import network_state + +SYS_CLASS_NET = "/sys/class/net/" + +NET_CONFIG_OPTIONS = [ + "address", "netmask", "broadcast", "network", "metric", "gateway", + "pointtopoint", "media", "mtu", "hostname", "leasehours", "leasetime", + "vendor", "client", "bootfile", "server", "hwaddr", "provider", "frame", + "netnum", "endpoint", "local", "ttl", + ] + +NET_CONFIG_COMMANDS = [ + "pre-up", "up", "post-up", "down", "pre-down", "post-down", + ] + +NET_CONFIG_BRIDGE_OPTIONS = [ + "bridge_ageing", "bridge_bridgeprio", "bridge_fd", "bridge_gcinit", + "bridge_hello", "bridge_maxage", "bridge_maxwait", "bridge_stp", + ] + + +def sys_dev_path(devname, path=""): + return SYS_CLASS_NET + devname + "/" + path + + +def read_sys_net(devname, path, translate=None, enoent=None, keyerror=None): + try: + contents = "" + with open(sys_dev_path(devname, path), "r") as fp: + contents = fp.read().strip() + if translate is None: + return contents + + try: + return translate.get(contents) + except KeyError: + LOG.debug("found unexpected value '%s' in '%s/%s'", contents, + devname, path) + if keyerror is not None: + return keyerror + raise + except OSError as e: + if e.errno == errno.ENOENT and enoent is not None: + return enoent + raise + + +def is_up(devname): + # The linux kernel says to consider devices in 'unknown' + # operstate as up for the purposes of network configuration. See + # Documentation/networking/operstates.txt in the kernel source. + translate = {'up': True, 'unknown': True, 'down': False} + return read_sys_net(devname, "operstate", enoent=False, keyerror=False, + translate=translate) + + +def is_wireless(devname): + return os.path.exists(sys_dev_path(devname, "wireless")) + + +def is_connected(devname): + # is_connected isn't really as simple as that. 2 is + # 'physically connected'. 3 is 'not connected'. but a wlan interface will + # always show 3. + try: + iflink = read_sys_net(devname, "iflink", enoent=False) + if iflink == "2": + return True + if not is_wireless(devname): + return False + LOG.debug("'%s' is wireless, basing 'connected' on carrier", devname) + + return read_sys_net(devname, "carrier", enoent=False, keyerror=False, + translate={'0': False, '1': True}) + + except IOError as e: + if e.errno == errno.EINVAL: + return False + raise + + +def is_physical(devname): + return os.path.exists(sys_dev_path(devname, "device")) + + +def is_present(devname): + return os.path.exists(sys_dev_path(devname)) + + +def get_devicelist(): + return os.listdir(SYS_CLASS_NET) + + +class ParserError(Exception): + """Raised when parser has issue parsing the interfaces file.""" + + +def parse_deb_config_data(ifaces, contents, src_dir, src_path): + """Parses the file contents, placing result into ifaces. + + '_source_path' is added to every dictionary entry to define which file + the configration information came from. + + :param ifaces: interface dictionary + :param contents: contents of interfaces file + :param src_dir: directory interfaces file was located + :param src_path: file path the `contents` was read + """ + currif = None + for line in contents.splitlines(): + line = line.strip() + if line.startswith('#'): + continue + split = line.split(' ') + option = split[0] + if option == "source-directory": + parsed_src_dir = split[1] + if not parsed_src_dir.startswith("/"): + parsed_src_dir = os.path.join(src_dir, parsed_src_dir) + for expanded_path in glob.glob(parsed_src_dir): + dir_contents = os.listdir(expanded_path) + dir_contents = [ + os.path.join(expanded_path, path) + for path in dir_contents + if (os.path.isfile(os.path.join(expanded_path, path)) and + re.match("^[a-zA-Z0-9_-]+$", path) is not None) + ] + for entry in dir_contents: + with open(entry, "r") as fp: + src_data = fp.read().strip() + abs_entry = os.path.abspath(entry) + parse_deb_config_data( + ifaces, src_data, + os.path.dirname(abs_entry), abs_entry) + elif option == "source": + new_src_path = split[1] + if not new_src_path.startswith("/"): + new_src_path = os.path.join(src_dir, new_src_path) + for expanded_path in glob.glob(new_src_path): + with open(expanded_path, "r") as fp: + src_data = fp.read().strip() + abs_path = os.path.abspath(expanded_path) + parse_deb_config_data( + ifaces, src_data, + os.path.dirname(abs_path), abs_path) + elif option == "auto": + for iface in split[1:]: + if iface not in ifaces: + ifaces[iface] = { + # Include the source path this interface was found in. + "_source_path": src_path + } + ifaces[iface]['auto'] = True + elif option == "iface": + iface, family, method = split[1:4] + if iface not in ifaces: + ifaces[iface] = { + # Include the source path this interface was found in. + "_source_path": src_path + } + elif 'family' in ifaces[iface]: + raise ParserError( + "Interface %s can only be defined once. " + "Re-defined in '%s'." % (iface, src_path)) + ifaces[iface]['family'] = family + ifaces[iface]['method'] = method + currif = iface + elif option == "hwaddress": + ifaces[currif]['hwaddress'] = split[1] + elif option in NET_CONFIG_OPTIONS: + ifaces[currif][option] = split[1] + elif option in NET_CONFIG_COMMANDS: + if option not in ifaces[currif]: + ifaces[currif][option] = [] + ifaces[currif][option].append(' '.join(split[1:])) + elif option.startswith('dns-'): + if 'dns' not in ifaces[currif]: + ifaces[currif]['dns'] = {} + if option == 'dns-search': + ifaces[currif]['dns']['search'] = [] + for domain in split[1:]: + ifaces[currif]['dns']['search'].append(domain) + elif option == 'dns-nameservers': + ifaces[currif]['dns']['nameservers'] = [] + for server in split[1:]: + ifaces[currif]['dns']['nameservers'].append(server) + elif option.startswith('bridge_'): + if 'bridge' not in ifaces[currif]: + ifaces[currif]['bridge'] = {} + if option in NET_CONFIG_BRIDGE_OPTIONS: + bridge_option = option.replace('bridge_', '', 1) + ifaces[currif]['bridge'][bridge_option] = split[1] + elif option == "bridge_ports": + ifaces[currif]['bridge']['ports'] = [] + for iface in split[1:]: + ifaces[currif]['bridge']['ports'].append(iface) + elif option == "bridge_hw" and split[1].lower() == "mac": + ifaces[currif]['bridge']['mac'] = split[2] + elif option == "bridge_pathcost": + if 'pathcost' not in ifaces[currif]['bridge']: + ifaces[currif]['bridge']['pathcost'] = {} + ifaces[currif]['bridge']['pathcost'][split[1]] = split[2] + elif option == "bridge_portprio": + if 'portprio' not in ifaces[currif]['bridge']: + ifaces[currif]['bridge']['portprio'] = {} + ifaces[currif]['bridge']['portprio'][split[1]] = split[2] + elif option.startswith('bond-'): + if 'bond' not in ifaces[currif]: + ifaces[currif]['bond'] = {} + bond_option = option.replace('bond-', '', 1) + ifaces[currif]['bond'][bond_option] = split[1] + for iface in ifaces.keys(): + if 'auto' not in ifaces[iface]: + ifaces[iface]['auto'] = False + + +def parse_deb_config(path): + """Parses a debian network configuration file.""" + ifaces = {} + with open(path, "r") as fp: + contents = fp.read().strip() + abs_path = os.path.abspath(path) + parse_deb_config_data( + ifaces, contents, + os.path.dirname(abs_path), abs_path) + return ifaces + + +def parse_net_config_data(net_config): + """Parses the config, returns NetworkState dictionary + + :param net_config: curtin network config dict + """ + state = None + if 'version' in net_config and 'config' in net_config: + ns = network_state.NetworkState(version=net_config.get('version'), + config=net_config.get('config')) + ns.parse_config() + state = ns.network_state + + return state + + +def parse_net_config(path): + """Parses a curtin network configuration file and + return network state""" + ns = None + net_config = config.load_config(path) + if 'network' in net_config: + ns = parse_net_config_data(net_config.get('network')) + + return ns + + +def render_persistent_net(network_state): + ''' Given state, emit udev rules to map + mac to ifname + ''' + content = "" + interfaces = network_state.get('interfaces') + for iface in interfaces.values(): + # for physical interfaces write out a persist net udev rule + if iface['type'] == 'physical' and \ + 'name' in iface and 'mac_address' in iface: + content += generate_udev_rule(iface['name'], + iface['mac_address']) + + return content + + +# TODO: switch valid_map based on mode inet/inet6 +def iface_add_subnet(iface, subnet): + content = "" + valid_map = [ + 'address', + 'netmask', + 'broadcast', + 'metric', + 'gateway', + 'pointopoint', + 'mtu', + 'scope', + 'dns_search', + 'dns_nameservers', + ] + for key, value in subnet.items(): + if value and key in valid_map: + if type(value) == list: + value = " ".join(value) + if '_' in key: + key = key.replace('_', '-') + content += " {} {}\n".format(key, value) + + return content + + +# TODO: switch to valid_map for attrs +def iface_add_attrs(iface): + content = "" + ignore_map = [ + 'type', + 'name', + 'inet', + 'mode', + 'index', + 'subnets', + ] + if iface['type'] not in ['bond', 'bridge']: + ignore_map.append('mac_address') + + for key, value in iface.items(): + if value and key not in ignore_map: + if type(value) == list: + value = " ".join(value) + content += " {} {}\n".format(key, value) + + return content + + +def render_route(route): + content = "up route add" + mapping = { + 'network': '-net', + 'netmask': 'netmask', + 'gateway': 'gw', + 'metric': 'metric', + } + for k in ['network', 'netmask', 'gateway', 'metric']: + if k in route: + content += " %s %s" % (mapping[k], route[k]) + + content += '\n' + return content + + +def render_interfaces(network_state): + ''' Given state, emit etc/network/interfaces content ''' + + content = "" + interfaces = network_state.get('interfaces') + ''' Apply a sort order to ensure that we write out + the physical interfaces first; this is critical for + bonding + ''' + order = { + 'physical': 0, + 'bond': 1, + 'bridge': 2, + 'vlan': 3, + } + content += "auto lo\niface lo inet loopback\n" + for dnskey, value in network_state.get('dns', {}).items(): + if len(value): + content += " dns-{} {}\n".format(dnskey, " ".join(value)) + + for iface in sorted(interfaces.values(), + key=lambda k: (order[k['type']], k['name'])): + content += "auto {name}\n".format(**iface) + + subnets = iface.get('subnets', {}) + if subnets: + for index, subnet in zip(range(0, len(subnets)), subnets): + iface['index'] = index + iface['mode'] = subnet['type'] + if iface['mode'].endswith('6'): + iface['inet'] += '6' + elif iface['mode'] == 'static' and ":" in subnet['address']: + iface['inet'] += '6' + if iface['mode'].startswith('dhcp'): + iface['mode'] = 'dhcp' + + if index == 0: + content += "iface {name} {inet} {mode}\n".format(**iface) + else: + content += "auto {name}:{index}\n".format(**iface) + content += \ + "iface {name}:{index} {inet} {mode}\n".format(**iface) + + content += iface_add_subnet(iface, subnet) + content += iface_add_attrs(iface) + content += "\n" + else: + content += "iface {name} {inet} {mode}\n".format(**iface) + content += iface_add_attrs(iface) + content += "\n" + + for route in network_state.get('routes'): + content += render_route(route) + + # global replacements until v2 format + content = content.replace('mac_address', 'hwaddress') + return content + + +def render_network_state(target, network_state): + eni = 'etc/network/interfaces' + netrules = 'etc/udev/rules.d/70-persistent-net.rules' + + eni = os.path.sep.join((target, eni,)) + util.ensure_dir(os.path.dirname(eni)) + with open(eni, 'w+') as f: + f.write(render_interfaces(network_state)) + + netrules = os.path.sep.join((target, netrules,)) + util.ensure_dir(os.path.dirname(netrules)) + with open(netrules, 'w+') as f: + f.write(render_persistent_net(network_state)) + +# vi: ts=4 expandtab syntax=python diff --git a/cloudinit/net/network_state.py b/cloudinit/net/network_state.py new file mode 100644 index 00000000..83303317 --- /dev/null +++ b/cloudinit/net/network_state.py @@ -0,0 +1,400 @@ +# Copyright (C) 2013-2014 Canonical Ltd. +# +# Author: Ryan Harper +# +# Curtin is free software: you can redistribute it and/or modify it under +# the terms of the GNU Affero General Public License as published by the +# Free Software Foundation, either version 3 of the License, or (at your +# option) any later version. +# +# Curtin is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +# FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for +# more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Curtin. If not, see . + +from curtin.log import LOG +import curtin.config as curtin_config + +NETWORK_STATE_VERSION = 1 +NETWORK_STATE_REQUIRED_KEYS = { + 1: ['version', 'config', 'network_state'], +} + + +def from_state_file(state_file): + network_state = None + state = curtin_config.load_config(state_file) + network_state = NetworkState() + network_state.load(state) + + return network_state + + +class NetworkState: + def __init__(self, version=NETWORK_STATE_VERSION, config=None): + self.version = version + self.config = config + self.network_state = { + 'interfaces': {}, + 'routes': [], + 'dns': { + 'nameservers': [], + 'search': [], + } + } + self.command_handlers = self.get_command_handlers() + + def get_command_handlers(self): + METHOD_PREFIX = 'handle_' + methods = filter(lambda x: callable(getattr(self, x)) and + x.startswith(METHOD_PREFIX), dir(self)) + handlers = {} + for m in methods: + key = m.replace(METHOD_PREFIX, '') + handlers[key] = getattr(self, m) + + return handlers + + def dump(self): + state = { + 'version': self.version, + 'config': self.config, + 'network_state': self.network_state, + } + return curtin_config.dump_config(state) + + def load(self, state): + if 'version' not in state: + LOG.error('Invalid state, missing version field') + raise Exception('Invalid state, missing version field') + + required_keys = NETWORK_STATE_REQUIRED_KEYS[state['version']] + if not self.valid_command(state, required_keys): + msg = 'Invalid state, missing keys: {}'.format(required_keys) + LOG.error(msg) + raise Exception(msg) + + # v1 - direct attr mapping, except version + for key in [k for k in required_keys if k not in ['version']]: + setattr(self, key, state[key]) + self.command_handlers = self.get_command_handlers() + + def dump_network_state(self): + return curtin_config.dump_config(self.network_state) + + def parse_config(self): + # rebuild network state + for command in self.config: + handler = self.command_handlers.get(command['type']) + handler(command) + + def valid_command(self, command, required_keys): + if not required_keys: + return False + + found_keys = [key for key in command.keys() if key in required_keys] + return len(found_keys) == len(required_keys) + + def handle_physical(self, command): + ''' + command = { + 'type': 'physical', + 'mac_address': 'c0:d6:9f:2c:e8:80', + 'name': 'eth0', + 'subnets': [ + {'type': 'dhcp4'} + ] + } + ''' + required_keys = [ + 'name', + ] + if not self.valid_command(command, required_keys): + LOG.warn('Skipping Invalid command: {}'.format(command)) + LOG.debug(self.dump_network_state()) + return + + interfaces = self.network_state.get('interfaces') + iface = interfaces.get(command['name'], {}) + for param, val in command.get('params', {}).items(): + iface.update({param: val}) + iface.update({ + 'name': command.get('name'), + 'type': command.get('type'), + 'mac_address': command.get('mac_address'), + 'inet': 'inet', + 'mode': 'manual', + 'mtu': command.get('mtu'), + 'address': None, + 'gateway': None, + 'subnets': command.get('subnets'), + }) + self.network_state['interfaces'].update({command.get('name'): iface}) + self.dump_network_state() + + def handle_vlan(self, command): + ''' + auto eth0.222 + iface eth0.222 inet static + address 10.10.10.1 + netmask 255.255.255.0 + vlan-raw-device eth0 + ''' + required_keys = [ + 'name', + 'vlan_link', + 'vlan_id', + ] + if not self.valid_command(command, required_keys): + print('Skipping Invalid command: {}'.format(command)) + print(self.dump_network_state()) + return + + interfaces = self.network_state.get('interfaces') + self.handle_physical(command) + iface = interfaces.get(command.get('name'), {}) + iface['vlan-raw-device'] = command.get('vlan_link') + iface['vlan_id'] = command.get('vlan_id') + interfaces.update({iface['name']: iface}) + + def handle_bond(self, command): + ''' + #/etc/network/interfaces + auto eth0 + iface eth0 inet manual + bond-master bond0 + bond-mode 802.3ad + + auto eth1 + iface eth1 inet manual + bond-master bond0 + bond-mode 802.3ad + + auto bond0 + iface bond0 inet static + address 192.168.0.10 + gateway 192.168.0.1 + netmask 255.255.255.0 + bond-slaves none + bond-mode 802.3ad + bond-miimon 100 + bond-downdelay 200 + bond-updelay 200 + bond-lacp-rate 4 + ''' + required_keys = [ + 'name', + 'bond_interfaces', + 'params', + ] + if not self.valid_command(command, required_keys): + print('Skipping Invalid command: {}'.format(command)) + print(self.dump_network_state()) + return + + self.handle_physical(command) + interfaces = self.network_state.get('interfaces') + iface = interfaces.get(command.get('name'), {}) + for param, val in command.get('params').items(): + iface.update({param: val}) + iface.update({'bond-slaves': 'none'}) + self.network_state['interfaces'].update({iface['name']: iface}) + + # handle bond slaves + for ifname in command.get('bond_interfaces'): + if ifname not in interfaces: + cmd = { + 'name': ifname, + 'type': 'bond', + } + # inject placeholder + self.handle_physical(cmd) + + interfaces = self.network_state.get('interfaces') + bond_if = interfaces.get(ifname) + bond_if['bond-master'] = command.get('name') + # copy in bond config into slave + for param, val in command.get('params').items(): + bond_if.update({param: val}) + self.network_state['interfaces'].update({ifname: bond_if}) + + def handle_bridge(self, command): + ''' + auto br0 + iface br0 inet static + address 10.10.10.1 + netmask 255.255.255.0 + bridge_ports eth0 eth1 + bridge_stp off + bridge_fd 0 + bridge_maxwait 0 + + bridge_params = [ + "bridge_ports", + "bridge_ageing", + "bridge_bridgeprio", + "bridge_fd", + "bridge_gcint", + "bridge_hello", + "bridge_hw", + "bridge_maxage", + "bridge_maxwait", + "bridge_pathcost", + "bridge_portprio", + "bridge_stp", + "bridge_waitport", + ] + ''' + required_keys = [ + 'name', + 'bridge_interfaces', + 'params', + ] + if not self.valid_command(command, required_keys): + print('Skipping Invalid command: {}'.format(command)) + print(self.dump_network_state()) + return + + # find one of the bridge port ifaces to get mac_addr + # handle bridge_slaves + interfaces = self.network_state.get('interfaces') + for ifname in command.get('bridge_interfaces'): + if ifname in interfaces: + continue + + cmd = { + 'name': ifname, + } + # inject placeholder + self.handle_physical(cmd) + + interfaces = self.network_state.get('interfaces') + self.handle_physical(command) + iface = interfaces.get(command.get('name'), {}) + iface['bridge_ports'] = command['bridge_interfaces'] + for param, val in command.get('params').items(): + iface.update({param: val}) + + interfaces.update({iface['name']: iface}) + + def handle_nameserver(self, command): + required_keys = [ + 'address', + ] + if not self.valid_command(command, required_keys): + print('Skipping Invalid command: {}'.format(command)) + print(self.dump_network_state()) + return + + dns = self.network_state.get('dns') + if 'address' in command: + addrs = command['address'] + if not type(addrs) == list: + addrs = [addrs] + for addr in addrs: + dns['nameservers'].append(addr) + if 'search' in command: + paths = command['search'] + if not isinstance(paths, list): + paths = [paths] + for path in paths: + dns['search'].append(path) + + def handle_route(self, command): + required_keys = [ + 'destination', + ] + if not self.valid_command(command, required_keys): + print('Skipping Invalid command: {}'.format(command)) + print(self.dump_network_state()) + return + + routes = self.network_state.get('routes') + network, cidr = command['destination'].split("/") + netmask = cidr2mask(int(cidr)) + route = { + 'network': network, + 'netmask': netmask, + 'gateway': command.get('gateway'), + 'metric': command.get('metric'), + } + routes.append(route) + + +def cidr2mask(cidr): + mask = [0, 0, 0, 0] + for i in list(range(0, cidr)): + idx = int(i / 8) + mask[idx] = mask[idx] + (1 << (7 - i % 8)) + return ".".join([str(x) for x in mask]) + + +if __name__ == '__main__': + import sys + import random + from curtin import net + + def load_config(nc): + version = nc.get('version') + config = nc.get('config') + return (version, config) + + def test_parse(network_config): + (version, config) = load_config(network_config) + ns1 = NetworkState(version=version, config=config) + ns1.parse_config() + random.shuffle(config) + ns2 = NetworkState(version=version, config=config) + ns2.parse_config() + print("----NS1-----") + print(ns1.dump_network_state()) + print() + print("----NS2-----") + print(ns2.dump_network_state()) + print("NS1 == NS2 ?=> {}".format( + ns1.network_state == ns2.network_state)) + eni = net.render_interfaces(ns2.network_state) + print(eni) + udev_rules = net.render_persistent_net(ns2.network_state) + print(udev_rules) + + def test_dump_and_load(network_config): + print("Loading network_config into NetworkState") + (version, config) = load_config(network_config) + ns1 = NetworkState(version=version, config=config) + ns1.parse_config() + print("Dumping state to file") + ns1_dump = ns1.dump() + ns1_state = "/tmp/ns1.state" + with open(ns1_state, "w+") as f: + f.write(ns1_dump) + + print("Loading state from file") + ns2 = from_state_file(ns1_state) + print("NS1 == NS2 ?=> {}".format( + ns1.network_state == ns2.network_state)) + + def test_output(network_config): + (version, config) = load_config(network_config) + ns1 = NetworkState(version=version, config=config) + ns1.parse_config() + random.shuffle(config) + ns2 = NetworkState(version=version, config=config) + ns2.parse_config() + print("NS1 == NS2 ?=> {}".format( + ns1.network_state == ns2.network_state)) + eni_1 = net.render_interfaces(ns1.network_state) + eni_2 = net.render_interfaces(ns2.network_state) + print(eni_1) + print(eni_2) + print("eni_1 == eni_2 ?=> {}".format( + eni_1 == eni_2)) + + y = curtin_config.load_config(sys.argv[1]) + network_config = y.get('network') + test_parse(network_config) + test_dump_and_load(network_config) + test_output(network_config) diff --git a/cloudinit/net/udev.py b/cloudinit/net/udev.py new file mode 100644 index 00000000..6435ace0 --- /dev/null +++ b/cloudinit/net/udev.py @@ -0,0 +1,54 @@ +# Copyright (C) 2015 Canonical Ltd. +# +# Author: Ryan Harper +# +# Curtin is free software: you can redistribute it and/or modify it under +# the terms of the GNU Affero General Public License as published by the +# Free Software Foundation, either version 3 of the License, or (at your +# option) any later version. +# +# Curtin is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +# FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for +# more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Curtin. If not, see . + + +def compose_udev_equality(key, value): + """Return a udev comparison clause, like `ACTION=="add"`.""" + assert key == key.upper() + return '%s=="%s"' % (key, value) + + +def compose_udev_attr_equality(attribute, value): + """Return a udev attribute comparison clause, like `ATTR{type}=="1"`.""" + assert attribute == attribute.lower() + return 'ATTR{%s}=="%s"' % (attribute, value) + + +def compose_udev_setting(key, value): + """Return a udev assignment clause, like `NAME="eth0"`.""" + assert key == key.upper() + return '%s="%s"' % (key, value) + + +def generate_udev_rule(interface, mac): + """Return a udev rule to set the name of network interface with `mac`. + + The rule ends up as a single line looking something like: + + SUBSYSTEM=="net", ACTION=="add", DRIVERS=="?*", + ATTR{address}="ff:ee:dd:cc:bb:aa", NAME="eth0" + """ + rule = ', '.join([ + compose_udev_equality('SUBSYSTEM', 'net'), + compose_udev_equality('ACTION', 'add'), + compose_udev_equality('DRIVERS', '?*'), + compose_udev_attr_equality('address', mac), + compose_udev_setting('NAME', interface), + ]) + return '%s\n' % rule + +# vi: ts=4 expandtab syntax=python -- cgit v1.2.3 From 098d05b2f1253ba0808772851a1a8e85cb0108a9 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Thu, 10 Mar 2016 16:42:27 -0500 Subject: adjust net to fit with cloudinit at this point, this works: python -m cloudinit.net.network_state examples/network-all.yaml --- cloudinit/net/__init__.py | 11 ++++++----- cloudinit/net/network_state.py | 17 ++++++++++------- 2 files changed, 16 insertions(+), 12 deletions(-) diff --git a/cloudinit/net/__init__.py b/cloudinit/net/__init__.py index e5ca050e..3cf99604 100644 --- a/cloudinit/net/__init__.py +++ b/cloudinit/net/__init__.py @@ -21,12 +21,13 @@ import glob import os import re -from curtin.log import LOG -from curtin.udev import generate_udev_rule -import curtin.util as util -import curtin.config as config +from cloudinit import log as logging +from cloudinit import util +from .udev import generate_udev_rule from . import network_state +LOG = logging.getLogger(__name__) + SYS_CLASS_NET = "/sys/class/net/" NET_CONFIG_OPTIONS = [ @@ -272,7 +273,7 @@ def parse_net_config(path): """Parses a curtin network configuration file and return network state""" ns = None - net_config = config.load_config(path) + net_config = util.read_conf(path) if 'network' in net_config: ns = parse_net_config_data(net_config.get('network')) diff --git a/cloudinit/net/network_state.py b/cloudinit/net/network_state.py index 83303317..df04c526 100644 --- a/cloudinit/net/network_state.py +++ b/cloudinit/net/network_state.py @@ -15,8 +15,11 @@ # You should have received a copy of the GNU Affero General Public License # along with Curtin. If not, see . -from curtin.log import LOG -import curtin.config as curtin_config +from cloudinit import log as logging +from cloudinit import util +from cloudinit.util import yaml_dumps as dump_config + +LOG = logging.getLogger(__name__) NETWORK_STATE_VERSION = 1 NETWORK_STATE_REQUIRED_KEYS = { @@ -26,7 +29,7 @@ NETWORK_STATE_REQUIRED_KEYS = { def from_state_file(state_file): network_state = None - state = curtin_config.load_config(state_file) + state = util.read_conf(state_file) network_state = NetworkState() network_state.load(state) @@ -64,7 +67,7 @@ class NetworkState: 'config': self.config, 'network_state': self.network_state, } - return curtin_config.dump_config(state) + return dump_config(state) def load(self, state): if 'version' not in state: @@ -83,7 +86,7 @@ class NetworkState: self.command_handlers = self.get_command_handlers() def dump_network_state(self): - return curtin_config.dump_config(self.network_state) + return dump_config(self.network_state) def parse_config(self): # rebuild network state @@ -335,7 +338,7 @@ def cidr2mask(cidr): if __name__ == '__main__': import sys import random - from curtin import net + from cloudinit import net def load_config(nc): version = nc.get('version') @@ -393,7 +396,7 @@ if __name__ == '__main__': print("eni_1 == eni_2 ?=> {}".format( eni_1 == eni_2)) - y = curtin_config.load_config(sys.argv[1]) + y = util.read_conf(sys.argv[1]) network_config = y.get('network') test_parse(network_config) test_dump_and_load(network_config) -- cgit v1.2.3 From 781ded8127deefb49a8806e49bdb7bb6e4d4b245 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Thu, 10 Mar 2016 21:43:46 -0500 Subject: commit planned implementation of datasourcenocloud this adds the consumption of 'network-config' to the datasourcenocloud. There is an implementation of the network rendering taht is untested in distros/debian. --- cloudinit/distros/__init__.py | 11 ++++++++ cloudinit/distros/debian.py | 10 +++++++ cloudinit/sources/DataSourceNoCloud.py | 49 ++++++++++++++++++++++------------ 3 files changed, 53 insertions(+), 17 deletions(-) diff --git a/cloudinit/distros/__init__.py b/cloudinit/distros/__init__.py index a73acae5..461253a7 100644 --- a/cloudinit/distros/__init__.py +++ b/cloudinit/distros/__init__.py @@ -75,6 +75,9 @@ class Distro(object): # to write this blob out in a distro format raise NotImplementedError() + def _write_network_config(self, settings): + raise NotImplementedError() + def _find_tz_file(self, tz): tz_file = os.path.join(self.tz_zone_dir, str(tz)) if not os.path.isfile(tz_file): @@ -132,6 +135,14 @@ class Distro(object): return self._bring_up_interfaces(dev_names) return False + def apply_network_config(self, netconfig, bring_up=True): + # Write it out + dev_names = self._write_network_config(netconfig) + # Now try to bring them up + if bring_up: + return self._bring_up_interfaces(dev_names) + return False + @abc.abstractmethod def apply_locale(self, locale, out_fn=None): raise NotImplementedError() diff --git a/cloudinit/distros/debian.py b/cloudinit/distros/debian.py index db5890b1..89d8d28e 100644 --- a/cloudinit/distros/debian.py +++ b/cloudinit/distros/debian.py @@ -26,6 +26,8 @@ from cloudinit import distros from cloudinit import helpers from cloudinit import log as logging from cloudinit import util +from cloudinit import net +from cloudinit.net import network_state from cloudinit.distros.parsers.hostname import HostnameConf @@ -76,6 +78,14 @@ class Distro(distros.Distro): util.write_file(self.network_conf_fn, settings) return ['all'] + def _write_network_config(self, netconfig): + # TODO: THIS IS NOT TESTED + state = network_state.NetworkState() + state.load(netconfig) + state.parse_config() + net.render_network_state("/", state) + return ['all'] + def _bring_up_interfaces(self, device_names): use_all = False for d in device_names: diff --git a/cloudinit/sources/DataSourceNoCloud.py b/cloudinit/sources/DataSourceNoCloud.py index 4cad6877..e00210e7 100644 --- a/cloudinit/sources/DataSourceNoCloud.py +++ b/cloudinit/sources/DataSourceNoCloud.py @@ -50,21 +50,22 @@ class DataSourceNoCloud(sources.DataSource): } found = [] - mydata = {'meta-data': {}, 'user-data': "", 'vendor-data': ""} + mydata = {'meta-data': {}, 'user-data': "", 'vendor-data': "", + 'network-config': {}} try: # Parse the kernel command line, getting data passed in md = {} if parse_cmdline_data(self.cmdline_id, md): found.append("cmdline") - mydata['meta-data'].update(md) + mydata = _merge_new_seed({'meta-data': md}) except: util.logexc(LOG, "Unable to parse command line data") return False # Check to see if the seed dir has data. pp2d_kwargs = {'required': ['user-data', 'meta-data'], - 'optional': ['vendor-data']} + 'optional': ['vendor-data', 'network-config']} try: seeded = util.pathprefix2dict(self.seed_dir, **pp2d_kwargs) @@ -141,8 +142,7 @@ class DataSourceNoCloud(sources.DataSource): if len(found) == 0: return False - seeded_interfaces = None - + seeded_network = None # The special argument "seedfrom" indicates we should # attempt to seed the userdata / metadata from its value # its primarily value is in allowing the user to type less @@ -158,8 +158,9 @@ class DataSourceNoCloud(sources.DataSource): LOG.debug("Seed from %s not supported by %s", seedfrom, self) return False - if 'network-interfaces' in mydata['meta-data']: - seeded_interfaces = self.dsmode + if (mydata['meta-data'].get('network-interfaces') or + mydata.get('network-config')): + seeded_network = self.dsmode # This could throw errors, but the user told us to do it # so if errors are raised, let them raise @@ -176,15 +177,25 @@ class DataSourceNoCloud(sources.DataSource): mydata['meta-data'] = util.mergemanydict([mydata['meta-data'], defaults]) - # Update the network-interfaces if metadata had 'network-interfaces' - # entry and this is the local datasource, or 'seedfrom' was used - # and the source of the seed was self.dsmode - # ('local' for NoCloud, 'net' for NoCloudNet') - if ('network-interfaces' in mydata['meta-data'] and - (self.dsmode in ("local", seeded_interfaces))): - LOG.debug("Updating network interfaces from %s", self) - self.distro.apply_network( - mydata['meta-data']['network-interfaces']) + netdata = {'format': None, 'data': None} + if mydata['meta-data'].get('network-interfaces'): + netdata['format'] = 'interfaces' + netdata['data'] = mydata['meta-data']['network-interfaces'] + elif mydata.get('network-config'): + netdata['format'] = 'network-config' + netdata['data'] = mydata['network-config'] + + # if this is the local datasource or 'seedfrom' was used + # and the source of the seed was self.dsmode. + # Then see if there is network config to apply. + if self.dsmode in ("local", seeded_network): + if mydata['meta-data'].get('network-interfaces'): + LOG.debug("Updating network interfaces from %s", self) + self.distro.apply_network( + mydata['meta-data']['network-interfaces']) + elif mydata.get('network-config'): + LOG.debug("Updating network config from %s", self) + self.distro.apply_network_config(mydata['network-config']) if mydata['meta-data']['dsmode'] == self.dsmode: self.seed = ",".join(found) @@ -246,7 +257,11 @@ def _merge_new_seed(cur, seeded): ret = cur.copy() ret['meta-data'] = util.mergemanydict([cur['meta-data'], util.load_yaml(seeded['meta-data'])]) - ret['user-data'] = seeded['user-data'] + if seeded.get('network-config'): + ret['network-config'] = util.load_yaml(seeded['network-config']) + + if 'user-data' in seeded: + ret['user-data'] = seeded['user-data'] if 'vendor-data' in seeded: ret['vendor-data'] = seeded['vendor-data'] return ret -- cgit v1.2.3 From efd22ec256568b655c1dea1c61028b49b7fe04ab Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Fri, 11 Mar 2016 15:46:50 -0500 Subject: fix use of network state --- cloudinit/distros/debian.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/cloudinit/distros/debian.py b/cloudinit/distros/debian.py index 89d8d28e..24545fd4 100644 --- a/cloudinit/distros/debian.py +++ b/cloudinit/distros/debian.py @@ -44,6 +44,14 @@ APT_GET_WRAPPER = { } +def render_network_config(config, target="/"): + version = config['version'] + config = config['config'] + ns = network_state.NetworkState(version=version, config=config) + ns.parse_config() + net.render_network_state(target, ns.network_state) + + class Distro(distros.Distro): hostname_conf_fn = "/etc/hostname" locale_conf_fn = "/etc/default/locale" @@ -80,10 +88,7 @@ class Distro(distros.Distro): def _write_network_config(self, netconfig): # TODO: THIS IS NOT TESTED - state = network_state.NetworkState() - state.load(netconfig) - state.parse_config() - net.render_network_state("/", state) + render_network_config(netconfig) return ['all'] def _bring_up_interfaces(self, device_names): -- cgit v1.2.3 From 24a5e31f5ad96cde75315ed488b6d5a011533936 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Fri, 11 Mar 2016 16:07:49 -0500 Subject: minor changes use the helpers in cloudinit/net functional --- cloudinit/distros/debian.py | 14 +++----------- cloudinit/sources/DataSourceNoCloud.py | 3 ++- 2 files changed, 5 insertions(+), 12 deletions(-) diff --git a/cloudinit/distros/debian.py b/cloudinit/distros/debian.py index 24545fd4..36a844f1 100644 --- a/cloudinit/distros/debian.py +++ b/cloudinit/distros/debian.py @@ -44,14 +44,6 @@ APT_GET_WRAPPER = { } -def render_network_config(config, target="/"): - version = config['version'] - config = config['config'] - ns = network_state.NetworkState(version=version, config=config) - ns.parse_config() - net.render_network_state(target, ns.network_state) - - class Distro(distros.Distro): hostname_conf_fn = "/etc/hostname" locale_conf_fn = "/etc/default/locale" @@ -87,9 +79,9 @@ class Distro(distros.Distro): return ['all'] def _write_network_config(self, netconfig): - # TODO: THIS IS NOT TESTED - render_network_config(netconfig) - return ['all'] + ns = net.parse_net_config_data(netconfig) + net.render_network_state(network_state=ns, target="/") + return [] def _bring_up_interfaces(self, device_names): use_all = False diff --git a/cloudinit/sources/DataSourceNoCloud.py b/cloudinit/sources/DataSourceNoCloud.py index e00210e7..a3532463 100644 --- a/cloudinit/sources/DataSourceNoCloud.py +++ b/cloudinit/sources/DataSourceNoCloud.py @@ -195,7 +195,8 @@ class DataSourceNoCloud(sources.DataSource): mydata['meta-data']['network-interfaces']) elif mydata.get('network-config'): LOG.debug("Updating network config from %s", self) - self.distro.apply_network_config(mydata['network-config']) + self.distro.apply_network_config(mydata['network-config'], + bring_up=False) if mydata['meta-data']['dsmode'] == self.dsmode: self.seed = ",".join(found) -- cgit v1.2.3 From d0a706e0302e281179a7e274ca97d8b995e4570d Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Fri, 11 Mar 2016 16:10:47 -0500 Subject: fix tox --- cloudinit/distros/debian.py | 1 - 1 file changed, 1 deletion(-) diff --git a/cloudinit/distros/debian.py b/cloudinit/distros/debian.py index 36a844f1..909d6deb 100644 --- a/cloudinit/distros/debian.py +++ b/cloudinit/distros/debian.py @@ -27,7 +27,6 @@ from cloudinit import helpers from cloudinit import log as logging from cloudinit import util from cloudinit import net -from cloudinit.net import network_state from cloudinit.distros.parsers.hostname import HostnameConf -- cgit v1.2.3 From 24bd640d20383116d1285361d7e86eda9e9d20e8 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Mon, 14 Mar 2016 09:44:21 -0400 Subject: minor cleanup. long line and remove unused 'local' --- systemd/cloud-init-generator | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/systemd/cloud-init-generator b/systemd/cloud-init-generator index 8156db58..2d319695 100755 --- a/systemd/cloud-init-generator +++ b/systemd/cloud-init-generator @@ -35,9 +35,9 @@ etc_file() { read_proc_cmdline() { # return /proc/cmdline for non-container, and /proc/1/cmdline for container local ctname="systemd" - if [ -n "$CONTAINER" ] && ctname=$CONTAINER || systemd-detect-virt --container --quiet; then - local - if _RET=$(tr '\0' ' ' < /proc/1/cmdline) >/dev/null 2>&1; then + if [ -n "$CONTAINER" ] && ctname=$CONTAINER || + systemd-detect-virt --container --quiet; then + if { _RET=$(tr '\0' ' ' < /proc/1/cmdline); } 2>/dev/null; then _RET_MSG="container[$ctname]: pid 1 cmdline" return fi -- cgit v1.2.3 From f9180c9b0951420b052871bd7bb21082ce1e660e Mon Sep 17 00:00:00 2001 From: Wesley Wiedenmeier Date: Wed, 16 Mar 2016 13:39:47 -0500 Subject: In distros.debian call net.merge_from_cmdline_config to ensure that the results from any ip= parameters passed on the kernel cmdline are merged into network state --- cloudinit/distros/debian.py | 1 + 1 file changed, 1 insertion(+) diff --git a/cloudinit/distros/debian.py b/cloudinit/distros/debian.py index 909d6deb..c7a4ba07 100644 --- a/cloudinit/distros/debian.py +++ b/cloudinit/distros/debian.py @@ -79,6 +79,7 @@ class Distro(distros.Distro): def _write_network_config(self, netconfig): ns = net.parse_net_config_data(netconfig) + ns = net.merge_from_cmdline_config(ns) net.render_network_state(network_state=ns, target="/") return [] -- cgit v1.2.3 From 1dd9102afda920d486a144b3153d6c9951f45cf9 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Wed, 16 Mar 2016 21:06:28 -0400 Subject: fix regression when command line (ds=nocloud) is present parsing the command line parameters returned a dictionary but _merge_new_seed was expecting a string to be yaml loaded. Change is to make _merge_new_seed take either string or dict. --- cloudinit/sources/DataSourceNoCloud.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/cloudinit/sources/DataSourceNoCloud.py b/cloudinit/sources/DataSourceNoCloud.py index a3532463..64853385 100644 --- a/cloudinit/sources/DataSourceNoCloud.py +++ b/cloudinit/sources/DataSourceNoCloud.py @@ -58,7 +58,7 @@ class DataSourceNoCloud(sources.DataSource): md = {} if parse_cmdline_data(self.cmdline_id, md): found.append("cmdline") - mydata = _merge_new_seed({'meta-data': md}) + mydata = _merge_new_seed(mydata, {'meta-data': md}) except: util.logexc(LOG, "Unable to parse command line data") return False @@ -256,8 +256,12 @@ def parse_cmdline_data(ds_id, fill, cmdline=None): def _merge_new_seed(cur, seeded): ret = cur.copy() - ret['meta-data'] = util.mergemanydict([cur['meta-data'], - util.load_yaml(seeded['meta-data'])]) + + newmd = seeded.get('meta-data', {}) + if not isinstance(seeded['meta-data'], dict): + newmd = util.load_yaml(seeded['meta-data']) + ret['meta-data'] = util.mergemanydict([cur['meta-data'], newmd]) + if seeded.get('network-config'): ret['network-config'] = util.load_yaml(seeded['network-config']) -- cgit v1.2.3 From 16a44056eaa5cc36fd9f6b08fe3a6bb4700fe1e7 Mon Sep 17 00:00:00 2001 From: Wesley Wiedenmeier Date: Thu, 17 Mar 2016 21:54:57 -0500 Subject: Check for and merge in configuration caused by the 'ip' parameter on the kernel's cmdline during network configuration parsing. - Search for .conf files in /run with names starting with 'net', as these are created during early boot if the ip parameter is present - If any are present and valid they are merged with network configuration from the current data source - If the devices affected by the 'ip' parameter are already present in network configuration, then a subnet entry will be added to the device's configuration unless an identical entry is already present - If any of the devices affected are not present then a mostly blank configuration will be generated for the device and the appropriate subnet specified --- cloudinit/net/__init__.py | 56 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 56 insertions(+) diff --git a/cloudinit/net/__init__.py b/cloudinit/net/__init__.py index 3cf99604..e455040d 100644 --- a/cloudinit/net/__init__.py +++ b/cloudinit/net/__init__.py @@ -280,6 +280,62 @@ def parse_net_config(path): return ns +def load_key_value_pair_net_cfg(data_mapping): + """Process key value pairs from files written because of the ip parameter + on the kernel cmdline""" + subnets = [] + entry_ns = { + 'mtu': None, 'name': data_mapping['DEVICE'], 'type': 'physical', + 'mode': 'manual', 'inet': 'inet', 'gateway': None, 'address': None + } + + if data_mapping.get('PROTO') == 'dhcp': + if 'IPV4ADDR' in data_mapping: + subnets.append({'type': 'dhcp4'}) + if 'IPV6ADDR' in data_mapping: + subnets.append({'type': 'dhcp6'}) + entry_ns['subnets'] = subnets + + return entry_ns + + +def merge_from_cmdline_config(ns): + """If ip parameter passed on kernel cmdline then some initial network + configuration may have been done in initramfs. Files from the result of + this may have been written into /run. If any are present they should be + merged into network state""" + + if 'interfaces' not in ns: + ns['interfaces'] = {} + + for cfg_file in glob.glob('/run/net*.conf'): + with open(cfg_file, 'r') as fp: + data = [l.replace("'", "") for l in fp.readlines()] + try: + parsed = dict([l.strip().split('=') for l in data]) + except: + # if split did not work then this is likely not a netcfg file + continue + + dev_name = parsed.get('DEVICE') + if not dev_name: + # Not a net cfg file + continue + + loaded_ns = load_key_value_pair_net_cfg(parsed) + + if dev_name in ns['interfaces']: + if 'subnets' not in ns['interfaces'][dev_name]: + ns['interfaces'][dev_name]['subnets'] = [] + for newsubnet in loaded_ns['subnets']: + if newsubnet not in ns['interfaces'][dev_name]['subnets']: + ns['interfaces'][dev_name]['subnets'].append(newsubnet) + else: + ns['interfaces'][dev_name] = loaded_ns + + return ns + + def render_persistent_net(network_state): ''' Given state, emit udev rules to map mac to ifname -- cgit v1.2.3 From c03915a454cfa947c1d905ba5b55a5207ad85cda Mon Sep 17 00:00:00 2001 From: Wesley Wiedenmeier Date: Fri, 18 Mar 2016 02:30:59 -0500 Subject: Fully parse files at /run/net-dev.conf, loading parameters for address, broadcast, netmask, gateway and hostname if present --- cloudinit/net/__init__.py | 36 +++++++++++++++++++++++++++--------- 1 file changed, 27 insertions(+), 9 deletions(-) diff --git a/cloudinit/net/__init__.py b/cloudinit/net/__init__.py index e455040d..cce0773f 100644 --- a/cloudinit/net/__init__.py +++ b/cloudinit/net/__init__.py @@ -280,21 +280,39 @@ def parse_net_config(path): return ns -def load_key_value_pair_net_cfg(data_mapping): +def load_klibc_net_cfg(data_mapping): """Process key value pairs from files written because of the ip parameter on the kernel cmdline""" - subnets = [] entry_ns = { 'mtu': None, 'name': data_mapping['DEVICE'], 'type': 'physical', - 'mode': 'manual', 'inet': 'inet', 'gateway': None, 'address': None + 'mode': 'manual', 'inet': 'inet', 'gateway': None, 'address': None, + 'subnets': [] } if data_mapping.get('PROTO') == 'dhcp': - if 'IPV4ADDR' in data_mapping: - subnets.append({'type': 'dhcp4'}) - if 'IPV6ADDR' in data_mapping: - subnets.append({'type': 'dhcp6'}) - entry_ns['subnets'] = subnets + if data_mapping.get('IPV4ADDR'): + entry_ns['subnets'].append({'type': 'dhcp4'}) + if data_mapping.get('IPV6ADDR'): + entry_ns['subnets'].append({'type': 'dhcp6'}) + + if data_mapping.get('IPV4ADDR'): + entry_ns['address'] = data_mapping['IPV4ADDR'] + if data_mapping.get('IPV6ADDR'): + entry_ns['address'] = data_mapping['IPV6ADDR'] + if data_mapping.get('IPV4BROADCAST'): + entry_ns['broadcast'] = data_mapping['IPV4BROADCAST'] + if data_mapping.get('IPV6BROADCAST'): + entry_ns['broadcast'] = data_mapping['IPV6BROADCAST'] + if data_mapping.get('IPV4NETMASK'): + entry_ns['netmask'] = data_mapping['IPV4NETMASK'] + if data_mapping.get('IPV6NETMASK'): + entry_ns['netmask'] = data_mapping['IPV6NETMASK'] + if data_mapping.get('IPV4GATEWAY'): + entry_ns['gateway'] = data_mapping['IPV4GATEWAY'] + if data_mapping.get('IPV6GATEWAY'): + entry_ns['gateway'] = data_mapping['IPV6GATEWAY'] + if data_mapping.get('HOSTNAME'): + entry_ns['hostname'] = data_mapping['HOSTNAME'] return entry_ns @@ -322,7 +340,7 @@ def merge_from_cmdline_config(ns): # Not a net cfg file continue - loaded_ns = load_key_value_pair_net_cfg(parsed) + loaded_ns = load_klibc_net_cfg(parsed) if dev_name in ns['interfaces']: if 'subnets' not in ns['interfaces'][dev_name]: -- cgit v1.2.3 From b123a912448b74a1d51a50baed8d3d7edd4875d7 Mon Sep 17 00:00:00 2001 From: Wesley Wiedenmeier Date: Fri, 18 Mar 2016 02:40:13 -0500 Subject: If proto not specified, determine it using logic from: lp:cloud-initramfs-tools/dyn-netconf/scripts/init-bottom/cloud-initramfs-dyn-netconf --- cloudinit/net/__init__.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/cloudinit/net/__init__.py b/cloudinit/net/__init__.py index cce0773f..2bfaf149 100644 --- a/cloudinit/net/__init__.py +++ b/cloudinit/net/__init__.py @@ -289,6 +289,15 @@ def load_klibc_net_cfg(data_mapping): 'subnets': [] } + # ipconfig on precise does not write PROTO + # (lp:cloud-initramfs-tools/dyn-netconf/scripts/init-bottom/ + # cloud-initramfs-dyn-netconf) + if not data_mapping.get('PROTO'): + if data_mapping.get('filename'): + data_mapping['PROTO'] = 'dhcp' + else: + data_mapping['PROTO'] = 'static' + if data_mapping.get('PROTO') == 'dhcp': if data_mapping.get('IPV4ADDR'): entry_ns['subnets'].append({'type': 'dhcp4'}) -- cgit v1.2.3 From 266717afcc01c9ae2f9b4e8cfc2db08aad83de97 Mon Sep 17 00:00:00 2001 From: Wesley Wiedenmeier Date: Fri, 18 Mar 2016 12:17:15 -0500 Subject: Handle static ip= entries by appending a static subnet to the device --- cloudinit/net/__init__.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/cloudinit/net/__init__.py b/cloudinit/net/__init__.py index 2bfaf149..88d0061c 100644 --- a/cloudinit/net/__init__.py +++ b/cloudinit/net/__init__.py @@ -303,6 +303,9 @@ def load_klibc_net_cfg(data_mapping): entry_ns['subnets'].append({'type': 'dhcp4'}) if data_mapping.get('IPV6ADDR'): entry_ns['subnets'].append({'type': 'dhcp6'}) + elif data_mapping.get('PROTO') in ['static', 'none']: + entry_ns['subnets'].append( + {'type': 'static', 'address': data_mapping.get('IPV4ADDR')}) if data_mapping.get('IPV4ADDR'): entry_ns['address'] = data_mapping['IPV4ADDR'] -- cgit v1.2.3 From bb9cb8df25eaeca8740f5e1bcbc9cd2feafcba24 Mon Sep 17 00:00:00 2001 From: Wesley Wiedenmeier Date: Fri, 18 Mar 2016 14:41:44 -0500 Subject: Added comments --- cloudinit/net/__init__.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/cloudinit/net/__init__.py b/cloudinit/net/__init__.py index 88d0061c..e2dcaee7 100644 --- a/cloudinit/net/__init__.py +++ b/cloudinit/net/__init__.py @@ -282,7 +282,9 @@ def parse_net_config(path): def load_klibc_net_cfg(data_mapping): """Process key value pairs from files written because of the ip parameter - on the kernel cmdline""" + on the kernel cmdline, note that mode: manual is used because the + interface should already have been brought up by the kernel and + cloud-initramfs-tools""" entry_ns = { 'mtu': None, 'name': data_mapping['DEVICE'], 'type': 'physical', 'mode': 'manual', 'inet': 'inet', 'gateway': None, 'address': None, @@ -304,6 +306,8 @@ def load_klibc_net_cfg(data_mapping): if data_mapping.get('IPV6ADDR'): entry_ns['subnets'].append({'type': 'dhcp6'}) elif data_mapping.get('PROTO') in ['static', 'none']: + # It appears that specifying ipv6 static addrs does not work, so only + # check for ipv4 addr entry_ns['subnets'].append( {'type': 'static', 'address': data_mapping.get('IPV4ADDR')}) -- cgit v1.2.3 From 5916180c5ecb2ef74c1c993dcd32f95cb4581172 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Fri, 18 Mar 2016 19:55:53 -0400 Subject: add atomic_write_file and use it from atomic_write_json atomic_write_file just does less and easily utilized for the same purpose that atomic_write_json served. --- bin/cloud-init | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/bin/cloud-init b/bin/cloud-init index 7f665e7e..42166580 100755 --- a/bin/cloud-init +++ b/bin/cloud-init @@ -435,20 +435,24 @@ def main_single(name, args): return 0 -def atomic_write_json(path, data): +def atomic_write_file(path, content, mode='w'): tf = None try: tf = tempfile.NamedTemporaryFile(dir=os.path.dirname(path), - delete=False) - tf.write(util.encode_text(json.dumps(data, indent=1) + "\n")) + delete=False, mode=mode) + tf.write(content)u tf.close() os.rename(tf.name, path) except Exception as e: if tf is not None: - util.del_file(tf.name) + os.unlink(tf.name) raise e +def atomic_write_json(path, data): + return atomic_write_file(path, json.dumps(data, indent=1) + "\n") + + def status_wrapper(name, args, data_d=None, link_d=None): if data_d is None: data_d = os.path.normpath("/var/lib/cloud/data") -- cgit v1.2.3 From 72b56d0f59f321519f25b039937a24b0ce338295 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Fri, 18 Mar 2016 20:35:36 -0400 Subject: add this, its getting moved, but i wanted some of the content stored --- udev/79-cloud-init-net-setup-link.rules | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 udev/79-cloud-init-net-setup-link.rules diff --git a/udev/79-cloud-init-net-setup-link.rules b/udev/79-cloud-init-net-setup-link.rules new file mode 100644 index 00000000..03dba382 --- /dev/null +++ b/udev/79-cloud-init-net-setup-link.rules @@ -0,0 +1,18 @@ +# cloud-init rules to apply + +SUBSYSTEM!="net", GOTO="cloudinit_naming_end" + +IMPORT{builtin}="path_id" + +ACTION!="add", GOTO="cloudinit_naming_end" + +# net_setup_link provides us with systemd names for reference +IMPORT{builtin}="net_setup_link" +ATTR{address}!="", ENV{MAC_ADDRESS}="$attr{address}" +IMPORT{program}="/lib/udev/cloud-init-name-device" + +ENV{CLOUDINIT_NET_NAME}!="", NAME="$env{CLOUDINIT_NET_NAME}" + +LABEL="cloudinit_naming_end" + +# vi: ts=4 expandtab syntax=udevrules -- cgit v1.2.3 From 519c0936e3e80fc14225e500fbb61d0d12d28c35 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Fri, 18 Mar 2016 20:40:54 -0400 Subject: commit the systemd waiting mechanism Note, still broken as cloud-init local is not going to ever touch the CI_NET_READY file (/run/cloud-init/network-config-ready). So as this is , it will actually just block for 60 seconds and go on. --- setup.py | 3 +- systemd/cloud-init-generator | 3 ++ udev/79-cloud-init-net-setup-link.rules | 18 -------- udev/79-cloud-init-net-wait.rules | 10 ++++ udev/cloud-init-wait | 82 +++++++++++++++++++++++++++++++++ 5 files changed, 97 insertions(+), 19 deletions(-) delete mode 100644 udev/79-cloud-init-net-setup-link.rules create mode 100644 udev/79-cloud-init-net-wait.rules create mode 100755 udev/cloud-init-wait diff --git a/setup.py b/setup.py index 0b261dfe..f86727b2 100755 --- a/setup.py +++ b/setup.py @@ -183,7 +183,8 @@ else: [f for f in glob('doc/examples/*') if is_f(f)]), (USR + '/share/doc/cloud-init/examples/seed', [f for f in glob('doc/examples/seed/*') if is_f(f)]), - (LIB + '/udev/rules.d', ['udev/66-azure-ephemeral.rules']), + (LIB + '/udev/rules.d', [f for f in glob('udev/*.rules')]), + (LIB + '/udev', ['udev/cloud-init-wait']), ] # Use a subclass for install that handles # adding on the right init system configuration files diff --git a/systemd/cloud-init-generator b/systemd/cloud-init-generator index 2d319695..ae286d58 100755 --- a/systemd/cloud-init-generator +++ b/systemd/cloud-init-generator @@ -107,6 +107,9 @@ main() { "ln $CLOUD_SYSTEM_TARGET $link_path" fi fi + # this touches /run/cloud-init/enabled, which is read by + # udev/cloud-init-wait. If not present, it will exit quickly. + touch "$LOG_D/$ENABLE" elif [ "$result" = "$DISABLE" ]; then if [ -f "$link_path" ]; then if rm -f "$link_path"; then diff --git a/udev/79-cloud-init-net-setup-link.rules b/udev/79-cloud-init-net-setup-link.rules deleted file mode 100644 index 03dba382..00000000 --- a/udev/79-cloud-init-net-setup-link.rules +++ /dev/null @@ -1,18 +0,0 @@ -# cloud-init rules to apply - -SUBSYSTEM!="net", GOTO="cloudinit_naming_end" - -IMPORT{builtin}="path_id" - -ACTION!="add", GOTO="cloudinit_naming_end" - -# net_setup_link provides us with systemd names for reference -IMPORT{builtin}="net_setup_link" -ATTR{address}!="", ENV{MAC_ADDRESS}="$attr{address}" -IMPORT{program}="/lib/udev/cloud-init-name-device" - -ENV{CLOUDINIT_NET_NAME}!="", NAME="$env{CLOUDINIT_NET_NAME}" - -LABEL="cloudinit_naming_end" - -# vi: ts=4 expandtab syntax=udevrules diff --git a/udev/79-cloud-init-net-wait.rules b/udev/79-cloud-init-net-wait.rules new file mode 100644 index 00000000..8344222a --- /dev/null +++ b/udev/79-cloud-init-net-wait.rules @@ -0,0 +1,10 @@ +# cloud-init cold/hot-plug blocking mechanism +# this file blocks further processing of network events +# until cloud-init local has had a chance to read and apply network +SUBSYSTEM!="net", GOTO="cloudinit_naming_end" +ACTION!="add", GOTO="cloudinit_naming_end" + +IMPORT{program}="/lib/udev/cloud-init-wait" + +LABEL="cloudinit_naming_end" +# vi: ts=4 expandtab syntax=udevrules diff --git a/udev/cloud-init-wait b/udev/cloud-init-wait new file mode 100755 index 00000000..345333f9 --- /dev/null +++ b/udev/cloud-init-wait @@ -0,0 +1,82 @@ +#!/bin/sh + +CI_NET_READY="/run/cloud-init/network-config-ready" +LOG="/run/cloud-init/${0##*/}.log" +LOG_INIT=0 +DEBUG=0 + +find_name() { + local match="" name="" none="_UNSET" pound="#" + while read match name; do + [ "${match#${pound}}" = "$match" ] || continue + case "$match" in + ID_NET_NAME=${ID_NET_NAME:-$none}) _RET="$name"; return 0;; + ID_NET_NAME_PATH=${ID_NET_NAME_PATH:-$none}) _RET="$name"; return 0;; + MAC_ADDRESS=${MAC_ADDRESS:-$none}) _RET="$name"; return 0;; + INTERFACE=${INTERFACE:-$none}) _RET="$name"; return 0;; + esac + done + return 0 +} + +block_until_ready() { + local fname="$1" + local naplen="$2" max="$3" n=0 + while ! [ -f "$fname" ]; do + n=$(($n+1)) + [ "$n" -ge "$max" ] && return 1 + sleep $naplen + done +} + +log() { + [ -n "${LOG}" ] || return + [ "${DEBUG:-0}" = "0" ] && return + + if [ $LOG_INIT = 0 ]; then + if [ -d "${LOG%/*}" ] || mkdir -p "${LOG%/*}"; then + LOG_INIT=1 + else + echo "${0##*/}: WARN: log init to ${LOG%/*}" 1>&2 + return + fi + elif [ "$LOG_INIT" = "-1" ]; then + return + fi + local info="$$ $INTERFACE" + if [ "$DEBUG" -gt 1 ]; then + local up idle + read up idle < /proc/uptime + info="$$ $INTERFACE $up" + fi + echo "[$info]" "$@" >> "$LOG" +} + +main() { + local name="" readyfile="$CI_NET_READY" + local info="INTERFACE=${INTERFACE} ID_NET_NAME=${ID_NET_NAME}" + info="$info ID_NET_NAME_PATH=${ID_NET_NAME_PATH}" + info="$info MAC_ADDRESS=${MAC_ADDRESS}" + log "$info" + + ## Check to see if cloud-init.target is set. If cloud-init is + ## disabled we do not want to do anything. + if [ ! -f "/run/cloud-init/enabled" ]; then + log "cloud-init disabled" + return 0 + fi + + block_until_ready "$readyfile" .1 600 || + { log "failed waiting for ready on $INTERFACE"; return 1; } + + #find_name < "$CI_NET_RULES" && name="$_RET" || + # { log "failed to find match for $INTERFACE"; return 0; } + + log "net config ready" + #[ -z "$name" ] || echo "CLOUDINIT_NET_NAME=$name" +} + +main "$@" +exit + +# vi: ts=4 expandtab -- cgit v1.2.3 From db54b59b90c8db2fc4a637ae09d3f0df14e77acb Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Fri, 18 Mar 2016 20:42:49 -0400 Subject: remove the 'find_name' function that was here. I had left this in to commit it, it was my first pass at cloud-init doing the naming itself. That design was then replaced with the idea for cloud-init to instead write systemd.rules files. --- udev/cloud-init-wait | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/udev/cloud-init-wait b/udev/cloud-init-wait index 345333f9..f27309e3 100755 --- a/udev/cloud-init-wait +++ b/udev/cloud-init-wait @@ -5,20 +5,6 @@ LOG="/run/cloud-init/${0##*/}.log" LOG_INIT=0 DEBUG=0 -find_name() { - local match="" name="" none="_UNSET" pound="#" - while read match name; do - [ "${match#${pound}}" = "$match" ] || continue - case "$match" in - ID_NET_NAME=${ID_NET_NAME:-$none}) _RET="$name"; return 0;; - ID_NET_NAME_PATH=${ID_NET_NAME_PATH:-$none}) _RET="$name"; return 0;; - MAC_ADDRESS=${MAC_ADDRESS:-$none}) _RET="$name"; return 0;; - INTERFACE=${INTERFACE:-$none}) _RET="$name"; return 0;; - esac - done - return 0 -} - block_until_ready() { local fname="$1" local naplen="$2" max="$3" n=0 @@ -69,11 +55,7 @@ main() { block_until_ready "$readyfile" .1 600 || { log "failed waiting for ready on $INTERFACE"; return 1; } - #find_name < "$CI_NET_RULES" && name="$_RET" || - # { log "failed to find match for $INTERFACE"; return 0; } - log "net config ready" - #[ -z "$name" ] || echo "CLOUDINIT_NET_NAME=$name" } main "$@" -- cgit v1.2.3 From 1c8e0d93bb48338777e689e6303702bf84fed0d1 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Fri, 18 Mar 2016 20:47:06 -0400 Subject: cloud-init-local.service: touch file that cloud-init-wait will wait for this might work. And if it does means we could generally test this as the file that the cloud-init-wait will wait for will actually get created. --- systemd/cloud-init-local.service | 3 +++ 1 file changed, 3 insertions(+) diff --git a/systemd/cloud-init-local.service b/systemd/cloud-init-local.service index 475a2e11..f3a92e2f 100644 --- a/systemd/cloud-init-local.service +++ b/systemd/cloud-init-local.service @@ -10,6 +10,9 @@ Before=shutdown.target [Service] Type=oneshot ExecStart=/usr/bin/cloud-init init --local +## FIXME: remove this when cloud-initn local does it itself +## or otherwise better signals any blocking udev events +ExecStopPost=touch /run/cloud-init/network-config-ready RemainAfterExit=yes TimeoutSec=0 -- cgit v1.2.3 From 5d1d36f617f9ce342930a78183a13877b5a619cd Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Sun, 20 Mar 2016 06:26:11 -0400 Subject: fix syntax --- bin/cloud-init | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/cloud-init b/bin/cloud-init index 42166580..f101a713 100755 --- a/bin/cloud-init +++ b/bin/cloud-init @@ -440,7 +440,7 @@ def atomic_write_file(path, content, mode='w'): try: tf = tempfile.NamedTemporaryFile(dir=os.path.dirname(path), delete=False, mode=mode) - tf.write(content)u + tf.write(content) tf.close() os.rename(tf.name, path) except Exception as e: -- cgit v1.2.3 From 16751f75a51814e4873199eddec15040dd221561 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Sun, 20 Mar 2016 22:31:21 -0400 Subject: fix creation of network-config-ready and dont bother waiting on lo --- systemd/cloud-init-local.service | 4 +--- udev/cloud-init-wait | 4 ++++ 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/systemd/cloud-init-local.service b/systemd/cloud-init-local.service index f3a92e2f..dd737644 100644 --- a/systemd/cloud-init-local.service +++ b/systemd/cloud-init-local.service @@ -10,9 +10,7 @@ Before=shutdown.target [Service] Type=oneshot ExecStart=/usr/bin/cloud-init init --local -## FIXME: remove this when cloud-initn local does it itself -## or otherwise better signals any blocking udev events -ExecStopPost=touch /run/cloud-init/network-config-ready +ExecStart=/bin/touch /run/cloud-init/network-config-ready RemainAfterExit=yes TimeoutSec=0 diff --git a/udev/cloud-init-wait b/udev/cloud-init-wait index f27309e3..7d53dee4 100755 --- a/udev/cloud-init-wait +++ b/udev/cloud-init-wait @@ -52,6 +52,10 @@ main() { return 0 fi + if [ "${INTERFACE#lo}" != "$INTERFACE" ]; then + return 0 + fi + block_until_ready "$readyfile" .1 600 || { log "failed waiting for ready on $INTERFACE"; return 1; } -- cgit v1.2.3 From 7a22e352b2f87636554d9787f60cd3168f3d77bc Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Mon, 21 Mar 2016 08:59:55 -0400 Subject: cloud-init-local needs to want network-pre or it isnt guaranteed to start --- systemd/cloud-init-local.service | 1 + 1 file changed, 1 insertion(+) diff --git a/systemd/cloud-init-local.service b/systemd/cloud-init-local.service index dd737644..b19eeaee 100644 --- a/systemd/cloud-init-local.service +++ b/systemd/cloud-init-local.service @@ -2,6 +2,7 @@ Description=Initial cloud-init job (pre-networking) DefaultDependencies=no Wants=local-fs.target +Wants=network-pre.target After=local-fs.target Conflicts=shutdown.target Before=network-pre.target -- cgit v1.2.3 From bb58463474e334b8c8d1769101bd3afc48ebfef4 Mon Sep 17 00:00:00 2001 From: Wesley Wiedenmeier Date: Mon, 21 Mar 2016 20:42:26 -0500 Subject: Added net.find_fallback_network_device() to find an appropriate device to dhcp on in the event that no network configuration was provided to cloud-init - Devices in /sys/class/net aside from loopback devices are scanned - Each device is tested to determine if it has a carrier using /sys/class/net/DEV/carrier, devices which do are preferred as they are most likely connected to the outside world - Devices which do not have a carrier but which might still be connected due to being in a dormant or down state are used as fallbacks in case no devices are found which have a carrier - A network state dictionary is generated to be passed to render_network_state to write ENI - A systemd link file is generated that will rename the chosen device to eth0 --- cloudinit/net/__init__.py | 83 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 83 insertions(+) diff --git a/cloudinit/net/__init__.py b/cloudinit/net/__init__.py index 3cf99604..800ffe61 100644 --- a/cloudinit/net/__init__.py +++ b/cloudinit/net/__init__.py @@ -20,6 +20,8 @@ import errno import glob import os import re +import string +import textwrap from cloudinit import log as logging from cloudinit import util @@ -280,6 +282,87 @@ def parse_net_config(path): return ns +def find_fallback_network_device(): + """Determine which attached net dev is most likely to have a connection and + generate network state to run dhcp on that interface""" + ns = {'interfaces': {}, 'dns': {'search': [], 'nameservers': []}, + 'routes': []} + default_link_file = textwrap.dedent(""" + #cloud-init + [Match] + MACAddress={mac} + + [Link] + Name={name} + """) + + # get list of interfaces that could have connections + invalid_interfaces = set(['lo']) + potential_interfaces = set(os.listdir(SYS_CLASS_NET)) + potential_interfaces = potential_interfaces.difference(invalid_interfaces) + + # sort into interfaces with carrier, interfaces which could have carrier, + # and ignore interfaces that are definitely disconnected + connected = [] + possibly_connected = [] + for interface in potential_interfaces: + sysfs_carrier = os.path.join(SYS_CLASS_NET, interface, 'carrier') + carrier = int(util.load_file(sysfs_carrier).strip()) + if carrier: + connected.append(interface) + continue + # check if nic is dormant or down, as this may make a nick appear to + # not have a carrier even though it could acquire one when brought + # online by dhclient + sysfs_dormant = os.path.join(SYS_CLASS_NET, interface, 'dormant') + dormant = int(util.load_file(sysfs_dormant).strip()) + if dormant: + possibly_connected.append(interface) + continue + sysfs_operstate = os.path.join(SYS_CLASS_NET, interface, 'operstate') + operstate = util.load_file(sysfs_operstate).strip() + if operstate in ['dormant', 'down', 'lowerlayerdown', 'unknown']: + possibly_connected.append(interface) + continue + + # don't bother with interfaces that might not be connected if there are + # some that definitely are + if connected: + potential_interfaces = connected + else: + potential_interfaces = possibly_connected + + # if there are no interfaces, give up + if not potential_interfaces: + return + + # if eth0 exists use it above anything else, otherwise get the interface + # that looks 'first' + if 'eth0' in potential_interfaces: + name = 'eth0' + else: + name = potential_interfaces.sort( + key=lambda x: int(x.strip(string.ascii_letters)))[0] + + sysfs_mac = os.path.join(SYS_CLASS_NET, name, 'address') + mac = util.load_file(sysfs_mac).strip() + + # generate net config for interface, rename interface to eth0 for backwards + # compatibility, and attempt both dhcp4 and dhcp6 + ns['interfaces']['eth0'] = { + 'mac_address': mac, 'name': 'eth0', 'type': 'physical', + 'mode': 'manual', 'inet': 'inet', + 'subnets': [{'type': 'dhcp4'}, {'type': 'dhcp6'}] + } + + # insert params into link file + link_file = default_link_file.format(name=name, mac=mac) + + syslink_name = "/etc/systemd/network/50-cloud-init-{}.link".format(name) + + return (ns, link_file, syslink_name) + + def render_persistent_net(network_state): ''' Given state, emit udev rules to map mac to ifname -- cgit v1.2.3 From 9a146e83189ef3128a04c9e0c1d21c6181f554f1 Mon Sep 17 00:00:00 2001 From: Wesley Wiedenmeier Date: Mon, 21 Mar 2016 21:10:20 -0500 Subject: Added _write_network_fallback function to distros.debian and abstract to distros base, and apply_fallback_network to distros to call _write_network_fallback. Note that since _write_network_fallback is only implemented for debian and ubuntu a check is needed to ensure that it does not break behaviour for other distros. Added function to disable .cfg files to util, since it may be useful elsewhere --- cloudinit/distros/__init__.py | 12 ++++++++++++ cloudinit/distros/debian.py | 10 ++++++++++ cloudinit/util.py | 9 +++++++++ 3 files changed, 31 insertions(+) diff --git a/cloudinit/distros/__init__.py b/cloudinit/distros/__init__.py index 74b484a7..e32ddd57 100644 --- a/cloudinit/distros/__init__.py +++ b/cloudinit/distros/__init__.py @@ -78,6 +78,10 @@ class Distro(object): def _write_network_config(self, settings): raise NotImplementedError() + @abc.abstractmethod + def _write_network_fallback(self): + raise NotImplementedError() + def _find_tz_file(self, tz): tz_file = os.path.join(self.tz_zone_dir, str(tz)) if not os.path.isfile(tz_file): @@ -143,6 +147,14 @@ class Distro(object): return self._bring_up_interfaces(dev_names) return False + def apply_fallback_network(self, bring_up=True): + # Write it out + dev_names = self._write_network_fallback() + # Now try to bring them up + if bring_up: + return self._bring_up_interfaces(dev_names) + return False + @abc.abstractmethod def apply_locale(self, locale, out_fn=None): raise NotImplementedError() diff --git a/cloudinit/distros/debian.py b/cloudinit/distros/debian.py index 909d6deb..18d5d124 100644 --- a/cloudinit/distros/debian.py +++ b/cloudinit/distros/debian.py @@ -82,6 +82,16 @@ class Distro(distros.Distro): net.render_network_state(network_state=ns, target="/") return [] + def _write_network_fallback(self): + # old fallback configuration is obsolete, disable it + util.disable_cfg_file('/etc/network/interfaces.d/eth0.cfg') + (ns, link_file, syslink_name) = net.find_fallback_network_device() + if link_file is not None: + util.write_file(syslink_name, link_file) + if ns is not None: + net.render_network_stat(network_state=ns, target="/") + return [] + def _bring_up_interfaces(self, device_names): use_all = False for d in device_names: diff --git a/cloudinit/util.py b/cloudinit/util.py index 20916e53..fa3a6163 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -849,6 +849,15 @@ def read_seeded(base="", ext="", timeout=5, retries=10, file_retries=0): return (md, ud) +def disable_conf_file(conf): + # disable .cfg file by renaming it if it exists + if not os.path.exists(conf): + return None + target_path = os.path.join(conf, '.disabled') + rename(conf, target_path) + return target_path + + def read_conf_d(confd): # Get reverse sorted list (later trumps newer) confs = sorted(os.listdir(confd), reverse=True) -- cgit v1.2.3 From 4c3468985d93929df4e9486b2e68938806fbfa1b Mon Sep 17 00:00:00 2001 From: Wesley Wiedenmeier Date: Mon, 21 Mar 2016 23:41:47 -0500 Subject: Fix typo in disable_conf_file and mistake in call --- cloudinit/distros/debian.py | 2 +- cloudinit/util.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cloudinit/distros/debian.py b/cloudinit/distros/debian.py index 18d5d124..38d22d85 100644 --- a/cloudinit/distros/debian.py +++ b/cloudinit/distros/debian.py @@ -84,7 +84,7 @@ class Distro(distros.Distro): def _write_network_fallback(self): # old fallback configuration is obsolete, disable it - util.disable_cfg_file('/etc/network/interfaces.d/eth0.cfg') + util.disable_conf_file('/etc/network/interfaces.d/eth0.cfg') (ns, link_file, syslink_name) = net.find_fallback_network_device() if link_file is not None: util.write_file(syslink_name, link_file) diff --git a/cloudinit/util.py b/cloudinit/util.py index fa3a6163..58ab3c75 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -853,7 +853,7 @@ def disable_conf_file(conf): # disable .cfg file by renaming it if it exists if not os.path.exists(conf): return None - target_path = os.path.join(conf, '.disabled') + target_path = conf + '.disabled' rename(conf, target_path) return target_path -- cgit v1.2.3 From 2aacb06be37e7e8aa84d11ae8c566a26f9df27e4 Mon Sep 17 00:00:00 2001 From: Wesley Wiedenmeier Date: Tue, 22 Mar 2016 00:33:35 -0500 Subject: Wrap read calls to /sys/class/net/DEV/{carrier, dormant, operstate} in try/except blocks because there are sometimes read errors on the files and this should not cause a stacktrace --- cloudinit/net/__init__.py | 40 +++++++++++++++++++++++++--------------- 1 file changed, 25 insertions(+), 15 deletions(-) diff --git a/cloudinit/net/__init__.py b/cloudinit/net/__init__.py index 800ffe61..e5b45926 100644 --- a/cloudinit/net/__init__.py +++ b/cloudinit/net/__init__.py @@ -306,24 +306,34 @@ def find_fallback_network_device(): connected = [] possibly_connected = [] for interface in potential_interfaces: - sysfs_carrier = os.path.join(SYS_CLASS_NET, interface, 'carrier') - carrier = int(util.load_file(sysfs_carrier).strip()) - if carrier: - connected.append(interface) - continue + try: + sysfs_carrier = os.path.join(SYS_CLASS_NET, interface, 'carrier') + carrier = int(util.load_file(sysfs_carrier).strip()) + if carrier: + connected.append(interface) + continue + except OSError: + pass # check if nic is dormant or down, as this may make a nick appear to # not have a carrier even though it could acquire one when brought # online by dhclient - sysfs_dormant = os.path.join(SYS_CLASS_NET, interface, 'dormant') - dormant = int(util.load_file(sysfs_dormant).strip()) - if dormant: - possibly_connected.append(interface) - continue - sysfs_operstate = os.path.join(SYS_CLASS_NET, interface, 'operstate') - operstate = util.load_file(sysfs_operstate).strip() - if operstate in ['dormant', 'down', 'lowerlayerdown', 'unknown']: - possibly_connected.append(interface) - continue + try: + sysfs_dormant = os.path.join(SYS_CLASS_NET, interface, 'dormant') + dormant = int(util.load_file(sysfs_dormant).strip()) + if dormant: + possibly_connected.append(interface) + continue + except OSError: + pass + try: + sysfs_operstate = os.path.join(SYS_CLASS_NET, interface, + 'operstate') + operstate = util.load_file(sysfs_operstate).strip() + if operstate in ['dormant', 'down', 'lowerlayerdown', 'unknown']: + possibly_connected.append(interface) + continue + except OSError: + pass # don't bother with interfaces that might not be connected if there are # some that definitely are -- cgit v1.2.3 From 217d92372ca5a4e994f1e9bc9580363dbba59032 Mon Sep 17 00:00:00 2001 From: Wesley Wiedenmeier Date: Tue, 22 Mar 2016 00:43:31 -0500 Subject: Fix typo --- cloudinit/net/__init__.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/cloudinit/net/__init__.py b/cloudinit/net/__init__.py index e5b45926..4641e54d 100644 --- a/cloudinit/net/__init__.py +++ b/cloudinit/net/__init__.py @@ -351,8 +351,9 @@ def find_fallback_network_device(): if 'eth0' in potential_interfaces: name = 'eth0' else: - name = potential_interfaces.sort( - key=lambda x: int(x.strip(string.ascii_letters)))[0] + potential_interfaces.sort( + key=lambda x: int(x.strip(string.ascii_letters))) + name = potential_interfaces[0] sysfs_mac = os.path.join(SYS_CLASS_NET, name, 'address') mac = util.load_file(sysfs_mac).strip() -- cgit v1.2.3 From aab9331089abcf3f5074f3cf7659502ca0752114 Mon Sep 17 00:00:00 2001 From: Wesley Wiedenmeier Date: Tue, 22 Mar 2016 00:48:17 -0500 Subject: Typo fix --- cloudinit/distros/debian.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cloudinit/distros/debian.py b/cloudinit/distros/debian.py index 38d22d85..8e57f70e 100644 --- a/cloudinit/distros/debian.py +++ b/cloudinit/distros/debian.py @@ -89,7 +89,7 @@ class Distro(distros.Distro): if link_file is not None: util.write_file(syslink_name, link_file) if ns is not None: - net.render_network_stat(network_state=ns, target="/") + net.render_network_state(network_state=ns, target="/") return [] def _bring_up_interfaces(self, device_names): -- cgit v1.2.3 From 7e399773a95e21e4c825dab61847d6abcd2aa511 Mon Sep 17 00:00:00 2001 From: Wesley Wiedenmeier Date: Tue, 22 Mar 2016 01:17:12 -0500 Subject: For find_fallback_network_device, kwarg rename_to_default specifies whether or not to attempt renaming the network interface to the default interface. Default interface is controleld by net.DEFAULT_PRIMARY_INTERFACE and is currently set to eth0 for legacy reasons. By default cloud-init will not attempt to rename the device as this does not work in some situtations depending on the backing driver of the device. --- cloudinit/net/__init__.py | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/cloudinit/net/__init__.py b/cloudinit/net/__init__.py index 4641e54d..e2e50441 100644 --- a/cloudinit/net/__init__.py +++ b/cloudinit/net/__init__.py @@ -48,6 +48,8 @@ NET_CONFIG_BRIDGE_OPTIONS = [ "bridge_hello", "bridge_maxage", "bridge_maxwait", "bridge_stp", ] +DEFAULT_PRIMARY_INTERFACE = 'eth0' + def sys_dev_path(devname, path=""): return SYS_CLASS_NET + devname + "/" + path @@ -282,9 +284,10 @@ def parse_net_config(path): return ns -def find_fallback_network_device(): +def find_fallback_network_device(rename_to_default=False): """Determine which attached net dev is most likely to have a connection and generate network state to run dhcp on that interface""" + # by default use eth0 as primary interface ns = {'interfaces': {}, 'dns': {'search': [], 'nameservers': []}, 'routes': []} default_link_file = textwrap.dedent(""" @@ -348,8 +351,8 @@ def find_fallback_network_device(): # if eth0 exists use it above anything else, otherwise get the interface # that looks 'first' - if 'eth0' in potential_interfaces: - name = 'eth0' + if DEFAULT_PRIMARY_INTERFACE in potential_interfaces: + name = DEFAULT_PRIMARY_INTERFACE else: potential_interfaces.sort( key=lambda x: int(x.strip(string.ascii_letters))) @@ -358,18 +361,23 @@ def find_fallback_network_device(): sysfs_mac = os.path.join(SYS_CLASS_NET, name, 'address') mac = util.load_file(sysfs_mac).strip() + target_name = name + if rename_to_default: + target_name = DEFAULT_PRIMARY_INTERFACE + # generate net config for interface, rename interface to eth0 for backwards # compatibility, and attempt both dhcp4 and dhcp6 - ns['interfaces']['eth0'] = { - 'mac_address': mac, 'name': 'eth0', 'type': 'physical', + ns['interfaces'][target_name] = { + 'mac_address': mac, 'name': target_name, 'type': 'physical', 'mode': 'manual', 'inet': 'inet', 'subnets': [{'type': 'dhcp4'}, {'type': 'dhcp6'}] } # insert params into link file - link_file = default_link_file.format(name=name, mac=mac) + link_file = default_link_file.format(name=target_name, mac=mac) - syslink_name = "/etc/systemd/network/50-cloud-init-{}.link".format(name) + syslink_name = "/etc/systemd/network/50-cloud-init-{}.link".format( + target_name) return (ns, link_file, syslink_name) -- cgit v1.2.3 From 88bfbe22a2f1128f358501bc10f5a2cbd1f7facf Mon Sep 17 00:00:00 2001 From: Wesley Wiedenmeier Date: Tue, 22 Mar 2016 01:24:01 -0500 Subject: Basic code added to kick off network configuration and cause fallback network configuration to be run if there is no network configuration provided by the datasource. NOTE: the code added here will not behave correctly if a net datasource has network configuration. this code is temporary and should be reverted once support for network configuration for net datasources after retrieving config is in place. based on: http://paste.ubuntu.com/15443576/ With this in place cloud-init properly chooses a fallback interface, configures it and brings it online --- bin/cloud-init | 2 ++ cloudinit/sources/__init__.py | 4 ++++ cloudinit/stages.py | 11 +++++++++++ 3 files changed, 17 insertions(+) diff --git a/bin/cloud-init b/bin/cloud-init index f101a713..2dfa8ec7 100755 --- a/bin/cloud-init +++ b/bin/cloud-init @@ -251,6 +251,7 @@ def main_init(name, args): # Stage 5 try: init.fetch() + init.apply_networking() except sources.DataSourceNotFoundException: # In the case of 'cloud-init init' without '--local' it is a bit # more likely that the user would consider it failure if nothing was @@ -261,6 +262,7 @@ def main_init(name, args): else: util.logexc(LOG, ("No instance datasource found!" " Likely bad things to come!")) + init.apply_networking() if not args.force: if args.local: return (None, []) diff --git a/cloudinit/sources/__init__.py b/cloudinit/sources/__init__.py index d3cfa560..08058762 100644 --- a/cloudinit/sources/__init__.py +++ b/cloudinit/sources/__init__.py @@ -217,6 +217,10 @@ class DataSource(object): def get_package_mirror_info(self): return self.distro.get_package_mirror_info(data_source=self) + @property + def network_config(self): + return self.metadata.network_config + def normalize_pubkey_data(pubkey_data): keys = [] diff --git a/cloudinit/stages.py b/cloudinit/stages.py index dbcf3d55..d508897b 100644 --- a/cloudinit/stages.py +++ b/cloudinit/stages.py @@ -587,6 +587,17 @@ class Init(object): # Run the handlers self._do_handlers(user_data_msg, c_handlers_list, frequency) + def apply_networking(self): + """Attempt to apply network configuration, either using network + configuration from datasource or fallback configuration if that is + not available""" + if self.datasource and self.datasource.network_config: + ds_net_conf = self.datasource.network_config + res = self.distro.apply_network_config(ds_net_conf, bring_up=True) + else: + res = self.distro.apply_fallback_network(bring_up=True) + return res + class Modules(object): def __init__(self, init, cfg_files=None, reporter=None): -- cgit v1.2.3 From 3a7e3d198172f26b7b97707520d06fe5303fadbc Mon Sep 17 00:00:00 2001 From: Wesley Wiedenmeier Date: Tue, 22 Mar 2016 01:33:20 -0500 Subject: Got rid of blank lines in net.find_fallback_network_device --- cloudinit/net/__init__.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/cloudinit/net/__init__.py b/cloudinit/net/__init__.py index e2e50441..2596b4f5 100644 --- a/cloudinit/net/__init__.py +++ b/cloudinit/net/__init__.py @@ -303,7 +303,6 @@ def find_fallback_network_device(rename_to_default=False): invalid_interfaces = set(['lo']) potential_interfaces = set(os.listdir(SYS_CLASS_NET)) potential_interfaces = potential_interfaces.difference(invalid_interfaces) - # sort into interfaces with carrier, interfaces which could have carrier, # and ignore interfaces that are definitely disconnected connected = [] @@ -344,11 +343,9 @@ def find_fallback_network_device(rename_to_default=False): potential_interfaces = connected else: potential_interfaces = possibly_connected - # if there are no interfaces, give up if not potential_interfaces: return - # if eth0 exists use it above anything else, otherwise get the interface # that looks 'first' if DEFAULT_PRIMARY_INTERFACE in potential_interfaces: @@ -360,11 +357,9 @@ def find_fallback_network_device(rename_to_default=False): sysfs_mac = os.path.join(SYS_CLASS_NET, name, 'address') mac = util.load_file(sysfs_mac).strip() - target_name = name if rename_to_default: target_name = DEFAULT_PRIMARY_INTERFACE - # generate net config for interface, rename interface to eth0 for backwards # compatibility, and attempt both dhcp4 and dhcp6 ns['interfaces'][target_name] = { @@ -372,10 +367,8 @@ def find_fallback_network_device(rename_to_default=False): 'mode': 'manual', 'inet': 'inet', 'subnets': [{'type': 'dhcp4'}, {'type': 'dhcp6'}] } - # insert params into link file link_file = default_link_file.format(name=target_name, mac=mac) - syslink_name = "/etc/systemd/network/50-cloud-init-{}.link".format( target_name) -- cgit v1.2.3 From e66bcc8b2ea7648c15476cdd43d3753ee6c27ff1 Mon Sep 17 00:00:00 2001 From: Wesley Wiedenmeier Date: Tue, 22 Mar 2016 01:48:39 -0500 Subject: - Rename find_fallback_network_device to generate_fallback_config - Removed systemd .link file generation, as it is not needed right now - Changed return of generate_fallback_config to be just ns dict - In distros.debian don't attempt to write .link file --- cloudinit/distros/debian.py | 4 +--- cloudinit/net/__init__.py | 23 ++++------------------- 2 files changed, 5 insertions(+), 22 deletions(-) diff --git a/cloudinit/distros/debian.py b/cloudinit/distros/debian.py index 8e57f70e..0fa47274 100644 --- a/cloudinit/distros/debian.py +++ b/cloudinit/distros/debian.py @@ -85,9 +85,7 @@ class Distro(distros.Distro): def _write_network_fallback(self): # old fallback configuration is obsolete, disable it util.disable_conf_file('/etc/network/interfaces.d/eth0.cfg') - (ns, link_file, syslink_name) = net.find_fallback_network_device() - if link_file is not None: - util.write_file(syslink_name, link_file) + ns = net.generate_fallback_config() if ns is not None: net.render_network_state(network_state=ns, target="/") return [] diff --git a/cloudinit/net/__init__.py b/cloudinit/net/__init__.py index 2596b4f5..48b82a2c 100644 --- a/cloudinit/net/__init__.py +++ b/cloudinit/net/__init__.py @@ -21,7 +21,6 @@ import glob import os import re import string -import textwrap from cloudinit import log as logging from cloudinit import util @@ -284,20 +283,12 @@ def parse_net_config(path): return ns -def find_fallback_network_device(rename_to_default=False): +def generate_fallback_config(): """Determine which attached net dev is most likely to have a connection and generate network state to run dhcp on that interface""" # by default use eth0 as primary interface ns = {'interfaces': {}, 'dns': {'search': [], 'nameservers': []}, 'routes': []} - default_link_file = textwrap.dedent(""" - #cloud-init - [Match] - MACAddress={mac} - - [Link] - Name={name} - """) # get list of interfaces that could have connections invalid_interfaces = set(['lo']) @@ -358,21 +349,15 @@ def find_fallback_network_device(rename_to_default=False): sysfs_mac = os.path.join(SYS_CLASS_NET, name, 'address') mac = util.load_file(sysfs_mac).strip() target_name = name - if rename_to_default: - target_name = DEFAULT_PRIMARY_INTERFACE - # generate net config for interface, rename interface to eth0 for backwards - # compatibility, and attempt both dhcp4 and dhcp6 + + # generate net config for interface ns['interfaces'][target_name] = { 'mac_address': mac, 'name': target_name, 'type': 'physical', 'mode': 'manual', 'inet': 'inet', 'subnets': [{'type': 'dhcp4'}, {'type': 'dhcp6'}] } - # insert params into link file - link_file = default_link_file.format(name=target_name, mac=mac) - syslink_name = "/etc/systemd/network/50-cloud-init-{}.link".format( - target_name) - return (ns, link_file, syslink_name) + return ns def render_persistent_net(network_state): -- cgit v1.2.3 From 7eccb0f0f3693662b3f288e7a74cb5bd6d7814ea Mon Sep 17 00:00:00 2001 From: Wesley Wiedenmeier Date: Tue, 22 Mar 2016 02:02:22 -0500 Subject: In generate_fallback_config return full netconfig dict with 'config' and 'version' keys --- cloudinit/distros/debian.py | 5 +++-- cloudinit/net/__init__.py | 11 +++++++---- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/cloudinit/distros/debian.py b/cloudinit/distros/debian.py index 0fa47274..de8c4c6c 100644 --- a/cloudinit/distros/debian.py +++ b/cloudinit/distros/debian.py @@ -85,8 +85,9 @@ class Distro(distros.Distro): def _write_network_fallback(self): # old fallback configuration is obsolete, disable it util.disable_conf_file('/etc/network/interfaces.d/eth0.cfg') - ns = net.generate_fallback_config() - if ns is not None: + nconf = net.generate_fallback_config() + if nconf is not None: + ns = nconf['config'] net.render_network_state(network_state=ns, target="/") return [] diff --git a/cloudinit/net/__init__.py b/cloudinit/net/__init__.py index 48b82a2c..389c2afb 100644 --- a/cloudinit/net/__init__.py +++ b/cloudinit/net/__init__.py @@ -287,8 +287,11 @@ def generate_fallback_config(): """Determine which attached net dev is most likely to have a connection and generate network state to run dhcp on that interface""" # by default use eth0 as primary interface - ns = {'interfaces': {}, 'dns': {'search': [], 'nameservers': []}, - 'routes': []} + nconf = {'config': {'interfaces': {}, + 'dns': {'search': [], 'nameservers': []}, 'routes': [] + }, + 'version': 1 + } # get list of interfaces that could have connections invalid_interfaces = set(['lo']) @@ -351,13 +354,13 @@ def generate_fallback_config(): target_name = name # generate net config for interface - ns['interfaces'][target_name] = { + nconf['config']['interfaces'][target_name] = { 'mac_address': mac, 'name': target_name, 'type': 'physical', 'mode': 'manual', 'inet': 'inet', 'subnets': [{'type': 'dhcp4'}, {'type': 'dhcp6'}] } - return ns + return nconf def render_persistent_net(network_state): -- cgit v1.2.3 From 6ce134c1868478345471ba9166f1523f7d9bf19d Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Tue, 22 Mar 2016 03:02:31 -0400 Subject: move some of the pickle loading out of Init, into private methods I plan to re-use these methods later. They stand alone even if they dont end up getting used, though. --- cloudinit/stages.py | 65 ++++++++++++++++++++++++++++------------------------- 1 file changed, 35 insertions(+), 30 deletions(-) diff --git a/cloudinit/stages.py b/cloudinit/stages.py index edad6450..c230ec0d 100644 --- a/cloudinit/stages.py +++ b/cloudinit/stages.py @@ -193,40 +193,12 @@ class Init(object): # We try to restore from a current link and static path # by using the instance link, if purge_cache was called # the file wont exist. - pickled_fn = self.paths.get_ipath_cur('obj_pkl') - pickle_contents = None - try: - pickle_contents = util.load_file(pickled_fn, decode=False) - except Exception as e: - if os.path.isfile(pickled_fn): - LOG.warn("failed loading pickle in %s: %s" % (pickled_fn, e)) - pass - - # This is expected so just return nothing - # successfully loaded... - if not pickle_contents: - return None - try: - return pickle.loads(pickle_contents) - except Exception: - util.logexc(LOG, "Failed loading pickled blob from %s", pickled_fn) - return None + return _pkl_load(self.paths.get_ipath_cur('obj_pkl')) def _write_to_cache(self): if self.datasource is NULL_DATA_SOURCE: return False - pickled_fn = self.paths.get_ipath_cur("obj_pkl") - try: - pk_contents = pickle.dumps(self.datasource) - except Exception: - util.logexc(LOG, "Failed pickling datasource %s", self.datasource) - return False - try: - util.write_file(pickled_fn, pk_contents, omode="wb", mode=0o400) - except Exception: - util.logexc(LOG, "Failed pickling datasource to %s", pickled_fn) - return False - return True + return _pkl_store(self.datasource, self.paths.get_ipath_cur("obj_pkl")) def _get_datasources(self): # Any config provided??? @@ -796,3 +768,36 @@ def fetch_base_config(): base_cfgs.append(default_cfg) return util.mergemanydict(base_cfgs) + + +def _pkl_store(obj, fname): + try: + pk_contents = pickle.dumps(obj) + except Exception: + util.logexc(LOG, "Failed pickling datasource %s", obj) + return False + try: + util.write_file(fname, pk_contents, omode="wb", mode=0o400) + except Exception: + util.logexc(LOG, "Failed pickling datasource to %s", fname) + return False + return True + + +def _pkl_load(fname): + pickle_contents = None + try: + pickle_contents = util.load_file(fname, decode=False) + except Exception as e: + if os.path.isfile(fname): + LOG.warn("failed loading pickle in %s: %s" % (fname, e)) + pass + + # This is allowed so just return nothing successfully loaded... + if not pickle_contents: + return None + try: + return pickle.loads(pickle_contents) + except Exception: + util.logexc(LOG, "Failed loading pickled blob from %s", fname) + return None -- cgit v1.2.3 From 3a3f960d5cfee60766e7de9e1fced537cac72106 Mon Sep 17 00:00:00 2001 From: Wesley Wiedenmeier Date: Tue, 22 Mar 2016 02:20:06 -0500 Subject: In generate_fallback_config() fix function to sort potential interfaces to work on interfaces with characters between their numbers --- cloudinit/net/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cloudinit/net/__init__.py b/cloudinit/net/__init__.py index 389c2afb..36f07a02 100644 --- a/cloudinit/net/__init__.py +++ b/cloudinit/net/__init__.py @@ -346,7 +346,7 @@ def generate_fallback_config(): name = DEFAULT_PRIMARY_INTERFACE else: potential_interfaces.sort( - key=lambda x: int(x.strip(string.ascii_letters))) + key=lambda x: int(''.join(i for i in x if i in string.digits))) name = potential_interfaces[0] sysfs_mac = os.path.join(SYS_CLASS_NET, name, 'address') -- cgit v1.2.3 From 78c99ef3faecde46b3e460dffa1af69654e8bbff Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Tue, 22 Mar 2016 03:33:05 -0400 Subject: drop changes other than generate_fallback_config --- cloudinit/distros/__init__.py | 12 ------------ cloudinit/distros/debian.py | 9 --------- cloudinit/stages.py | 11 ----------- cloudinit/util.py | 9 --------- 4 files changed, 41 deletions(-) diff --git a/cloudinit/distros/__init__.py b/cloudinit/distros/__init__.py index e32ddd57..74b484a7 100644 --- a/cloudinit/distros/__init__.py +++ b/cloudinit/distros/__init__.py @@ -78,10 +78,6 @@ class Distro(object): def _write_network_config(self, settings): raise NotImplementedError() - @abc.abstractmethod - def _write_network_fallback(self): - raise NotImplementedError() - def _find_tz_file(self, tz): tz_file = os.path.join(self.tz_zone_dir, str(tz)) if not os.path.isfile(tz_file): @@ -147,14 +143,6 @@ class Distro(object): return self._bring_up_interfaces(dev_names) return False - def apply_fallback_network(self, bring_up=True): - # Write it out - dev_names = self._write_network_fallback() - # Now try to bring them up - if bring_up: - return self._bring_up_interfaces(dev_names) - return False - @abc.abstractmethod def apply_locale(self, locale, out_fn=None): raise NotImplementedError() diff --git a/cloudinit/distros/debian.py b/cloudinit/distros/debian.py index de8c4c6c..909d6deb 100644 --- a/cloudinit/distros/debian.py +++ b/cloudinit/distros/debian.py @@ -82,15 +82,6 @@ class Distro(distros.Distro): net.render_network_state(network_state=ns, target="/") return [] - def _write_network_fallback(self): - # old fallback configuration is obsolete, disable it - util.disable_conf_file('/etc/network/interfaces.d/eth0.cfg') - nconf = net.generate_fallback_config() - if nconf is not None: - ns = nconf['config'] - net.render_network_state(network_state=ns, target="/") - return [] - def _bring_up_interfaces(self, device_names): use_all = False for d in device_names: diff --git a/cloudinit/stages.py b/cloudinit/stages.py index 64da3b5b..c230ec0d 100644 --- a/cloudinit/stages.py +++ b/cloudinit/stages.py @@ -567,17 +567,6 @@ class Init(object): # Run the handlers self._do_handlers(user_data_msg, c_handlers_list, frequency) - def apply_networking(self): - """Attempt to apply network configuration, either using network - configuration from datasource or fallback configuration if that is - not available""" - if self.datasource and self.datasource.network_config: - ds_net_conf = self.datasource.network_config - res = self.distro.apply_network_config(ds_net_conf, bring_up=True) - else: - res = self.distro.apply_fallback_network(bring_up=True) - return res - class Modules(object): def __init__(self, init, cfg_files=None, reporter=None): diff --git a/cloudinit/util.py b/cloudinit/util.py index 58ab3c75..20916e53 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -849,15 +849,6 @@ def read_seeded(base="", ext="", timeout=5, retries=10, file_retries=0): return (md, ud) -def disable_conf_file(conf): - # disable .cfg file by renaming it if it exists - if not os.path.exists(conf): - return None - target_path = conf + '.disabled' - rename(conf, target_path) - return target_path - - def read_conf_d(confd): # Get reverse sorted list (later trumps newer) confs = sorted(os.listdir(confd), reverse=True) -- cgit v1.2.3 From 9c0a2abc8d2c0e390745ddb163f5eae07b20d61d Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Tue, 22 Mar 2016 03:50:28 -0400 Subject: add code to invoke networking config there is no data source that has a populated network_config() so at this point this doesn't do anything. --- bin/cloud-init | 4 ++++ cloudinit/distros/__init__.py | 2 +- cloudinit/net/__init__.py | 17 +++++++++++++++++ cloudinit/sources/__init__.py | 4 ++++ cloudinit/stages.py | 24 ++++++++++++++++++++++++ 5 files changed, 50 insertions(+), 1 deletion(-) diff --git a/bin/cloud-init b/bin/cloud-init index 63aa765b..8875d2f6 100755 --- a/bin/cloud-init +++ b/bin/cloud-init @@ -263,6 +263,10 @@ def main_init(name, args): return (None, []) else: return (None, ["No instance datasource found."]) + + if args.local: + init.apply_network_config() + # Stage 6 iid = init.instancify() LOG.debug("%s will now be targeting instance id: %s", name, iid) diff --git a/cloudinit/distros/__init__.py b/cloudinit/distros/__init__.py index 74b484a7..418421b9 100644 --- a/cloudinit/distros/__init__.py +++ b/cloudinit/distros/__init__.py @@ -135,7 +135,7 @@ class Distro(object): return self._bring_up_interfaces(dev_names) return False - def apply_network_config(self, netconfig, bring_up=True): + def apply_network_config(self, netconfig, bring_up=False): # Write it out dev_names = self._write_network_config(netconfig) # Now try to bring them up diff --git a/cloudinit/net/__init__.py b/cloudinit/net/__init__.py index 3cf99604..799cb97e 100644 --- a/cloudinit/net/__init__.py +++ b/cloudinit/net/__init__.py @@ -434,4 +434,21 @@ def render_network_state(target, network_state): with open(netrules, 'w+') as f: f.write(render_persistent_net(network_state)) + +def is_disabled_cfg(cfg): + if not cfg or not isinstance(cfg, dict): + return False + return cfg.get('config') == "disabled" + + +def generate_fallback_config(): + # FIXME: add implementation here + return None + + +def read_kernel_cmdline_config(): + # FIXME: add implementation here + return None + + # vi: ts=4 expandtab syntax=python diff --git a/cloudinit/sources/__init__.py b/cloudinit/sources/__init__.py index 28540a7b..c63464b2 100644 --- a/cloudinit/sources/__init__.py +++ b/cloudinit/sources/__init__.py @@ -221,6 +221,10 @@ class DataSource(object): # quickly (local check only) if self.instance_id is still return False + @property + def network_config(self): + return None + def normalize_pubkey_data(pubkey_data): keys = [] diff --git a/cloudinit/stages.py b/cloudinit/stages.py index c230ec0d..8e681e29 100644 --- a/cloudinit/stages.py +++ b/cloudinit/stages.py @@ -43,6 +43,7 @@ from cloudinit import distros from cloudinit import helpers from cloudinit import importer from cloudinit import log as logging +from cloudinit import net from cloudinit import sources from cloudinit import type_utils from cloudinit import util @@ -567,6 +568,29 @@ class Init(object): # Run the handlers self._do_handlers(user_data_msg, c_handlers_list, frequency) + def _find_networking_config(self): + cmdline_cfg = ('cmdline', net.read_kernel_cmdline_config()) + dscfg = ('ds', None) + if self.datasource and hasattr(self.datasource, 'network_config'): + dscfg = ('ds', self.datasource.network_config) + sys_cfg = ('system_cfg', self.cfg.get('network')) + + for loc, ncfg in (cmdline_cfg, dscfg, sys_cfg): + if net.is_disabled_cfg(ncfg): + LOG.debug("network config disabled by %s", loc) + return None + if ncfg: + return ncfg + return net.generate_fallback_config() + + def apply_network_config(self): + netcfg = self._find_networking_config() + if netcfg is None: + LOG.info("network config is disabled") + return + + return self.distro.apply_network_config(netcfg) + class Modules(object): def __init__(self, init, cfg_files=None, reporter=None): -- cgit v1.2.3 From ca00b0f1f8c8a40409328c595d44234bb61c24c4 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Tue, 22 Mar 2016 04:49:34 -0400 Subject: make NoCloud work for seeding network. Tested now with the generated fallback config in an lxc container. Had to change to return a config rather than a network state. Also this makes nocloud look in nocloud-net's seed dir. This way it will read the seed and clame the datasource but not do anything other than apply networking and the init_modules early. It is a change in behavior of the time that boothooks woudl run to do this. May need to change that back. --- cloudinit/net/__init__.py | 20 +++++--------------- cloudinit/sources/DataSourceNoCloud.py | 34 +++++++++++++++++++--------------- cloudinit/stages.py | 1 + 3 files changed, 25 insertions(+), 30 deletions(-) diff --git a/cloudinit/net/__init__.py b/cloudinit/net/__init__.py index b45153f4..63fad2fa 100644 --- a/cloudinit/net/__init__.py +++ b/cloudinit/net/__init__.py @@ -448,11 +448,7 @@ def generate_fallback_config(): """Determine which attached net dev is most likely to have a connection and generate network state to run dhcp on that interface""" # by default use eth0 as primary interface - nconf = {'config': {'interfaces': {}, - 'dns': {'search': [], 'nameservers': []}, 'routes': [] - }, - 'version': 1 - } + nconf = {'config': [], 'version': 1} # get list of interfaces that could have connections invalid_interfaces = set(['lo']) @@ -506,21 +502,15 @@ def generate_fallback_config(): if DEFAULT_PRIMARY_INTERFACE in potential_interfaces: name = DEFAULT_PRIMARY_INTERFACE else: - potential_interfaces.sort( - key=lambda x: int(''.join(i for i in x if i in string.digits))) - name = potential_interfaces[0] + name = sorted(potential_interfaces)[0] sysfs_mac = os.path.join(SYS_CLASS_NET, name, 'address') mac = util.load_file(sysfs_mac).strip() target_name = name - # generate net config for interface - nconf['config']['interfaces'][target_name] = { - 'mac_address': mac, 'name': target_name, 'type': 'physical', - 'mode': 'manual', 'inet': 'inet', - 'subnets': [{'type': 'dhcp4'}, {'type': 'dhcp6'}] - } - + nconf['config'].append( + {'type': 'physical', 'name': target_name, + 'mac_address': mac, 'subnets': [{'type': 'dhcp4'}]}) return nconf diff --git a/cloudinit/sources/DataSourceNoCloud.py b/cloudinit/sources/DataSourceNoCloud.py index 538df7d9..bd04a6fe 100644 --- a/cloudinit/sources/DataSourceNoCloud.py +++ b/cloudinit/sources/DataSourceNoCloud.py @@ -36,7 +36,9 @@ class DataSourceNoCloud(sources.DataSource): self.dsmode = 'local' self.seed = None self.cmdline_id = "ds=nocloud" - self.seed_dir = os.path.join(paths.seed_dir, 'nocloud') + self.seed_dirs = [os.path.join(paths.seed_dir, 'nocloud'), + os.path.join(paths.seed_dir, 'nocloud-net')] + self.seed_dir = None self.supported_seed_starts = ("/", "file://") def __str__(self): @@ -67,15 +69,15 @@ class DataSourceNoCloud(sources.DataSource): pp2d_kwargs = {'required': ['user-data', 'meta-data'], 'optional': ['vendor-data', 'network-config']} - try: - seeded = util.pathprefix2dict(self.seed_dir, **pp2d_kwargs) - found.append(self.seed_dir) - LOG.debug("Using seeded data from %s", self.seed_dir) - except ValueError as e: - pass - - if self.seed_dir in found: - mydata = _merge_new_seed(mydata, seeded) + for path in self.seed_dirs: + try: + seeded = util.pathprefix2dict(path, **pp2d_kwargs) + found.append(path) + LOG.debug("Using seeded data from %s", path) + mydata = _merge_new_seed(mydata, seeded) + break + except ValueError as e: + pass # If the datasource config had a 'seedfrom' entry, then that takes # precedence over a 'seedfrom' that was found in a filesystem @@ -188,21 +190,19 @@ class DataSourceNoCloud(sources.DataSource): # if this is the local datasource or 'seedfrom' was used # and the source of the seed was self.dsmode. # Then see if there is network config to apply. + # note this is obsolete network-interfaces style seeding. if self.dsmode in ("local", seeded_network): if mydata['meta-data'].get('network-interfaces'): LOG.debug("Updating network interfaces from %s", self) self.distro.apply_network( mydata['meta-data']['network-interfaces']) - elif mydata.get('network-config'): - LOG.debug("Updating network config from %s", self) - self.distro.apply_network_config(mydata['network-config'], - bring_up=False) if mydata['meta-data']['dsmode'] == self.dsmode: self.seed = ",".join(found) self.metadata = mydata['meta-data'] self.userdata_raw = mydata['user-data'] self.vendordata_raw = mydata['vendor-data'] + self._network_config = mydata['network-config'] return True LOG.debug("%s: not claiming datasource, dsmode=%s", self, @@ -222,6 +222,10 @@ class DataSourceNoCloud(sources.DataSource): return None return quick_id == current + @property + def network_config(self): + return self._network_config + def _quick_read_instance_id(cmdline_id, dirs=None): if dirs is None: @@ -312,7 +316,7 @@ class DataSourceNoCloudNet(DataSourceNoCloud): DataSourceNoCloud.__init__(self, sys_cfg, distro, paths) self.cmdline_id = "ds=nocloud-net" self.supported_seed_starts = ("http://", "https://", "ftp://") - self.seed_dir = os.path.join(paths.seed_dir, 'nocloud-net') + self.seed_dirs = [os.path.join(paths.seed_dir, 'nocloud-net')] self.dsmode = "net" diff --git a/cloudinit/stages.py b/cloudinit/stages.py index 8e681e29..73090025 100644 --- a/cloudinit/stages.py +++ b/cloudinit/stages.py @@ -589,6 +589,7 @@ class Init(object): LOG.info("network config is disabled") return + LOG.info("Applying configuration: %s", netcfg) return self.distro.apply_network_config(netcfg) -- cgit v1.2.3 From 4445b881380a39a56490d8a8f9e07bba4540ec62 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Tue, 22 Mar 2016 05:39:58 -0400 Subject: fix quick_read_instance_id in nocloud for seed_dirs change --- cloudinit/sources/DataSourceNoCloud.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cloudinit/sources/DataSourceNoCloud.py b/cloudinit/sources/DataSourceNoCloud.py index bd04a6fe..afd08935 100644 --- a/cloudinit/sources/DataSourceNoCloud.py +++ b/cloudinit/sources/DataSourceNoCloud.py @@ -217,7 +217,7 @@ class DataSourceNoCloud(sources.DataSource): return None quick_id = _quick_read_instance_id(cmdline_id=self.cmdline_id, - dirs=[self.seed_dir]) + dirs=self.seed_dirs) if not quick_id: return None return quick_id == current -- cgit v1.2.3 From 7f0871dc5b141ff4bf601b6d96021eba8a3bb0ec Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Tue, 22 Mar 2016 05:40:05 -0400 Subject: write to 50-cloud-init.cfg and write systemd.link rules. --- cloudinit/distros/debian.py | 8 +++-- cloudinit/net/__init__.py | 48 ++++++++++++++++++++------ tests/unittests/test_distros/test_netconfig.py | 5 +-- 3 files changed, 46 insertions(+), 15 deletions(-) diff --git a/cloudinit/distros/debian.py b/cloudinit/distros/debian.py index 909d6deb..b14fa3e2 100644 --- a/cloudinit/distros/debian.py +++ b/cloudinit/distros/debian.py @@ -46,7 +46,8 @@ APT_GET_WRAPPER = { class Distro(distros.Distro): hostname_conf_fn = "/etc/hostname" locale_conf_fn = "/etc/default/locale" - network_conf_fn = "/etc/network/interfaces" + network_conf_fn = "/etc/network/interfaces.d/50-cloud-init.cfg" + links_prefix = "/etc/systemd/network/50-cloud-init-" def __init__(self, name, cfg, paths): distros.Distro.__init__(self, name, cfg, paths) @@ -79,7 +80,10 @@ class Distro(distros.Distro): def _write_network_config(self, netconfig): ns = net.parse_net_config_data(netconfig) - net.render_network_state(network_state=ns, target="/") + net.render_network_state(target="/", network_state=ns, + eni=self.network_conf_fn, + links_prefix=self.links_prefix) + util.del_file("/etc/network/interfaces.d/eth0.cfg") return [] def _bring_up_interfaces(self, device_names): diff --git a/cloudinit/net/__init__.py b/cloudinit/net/__init__.py index 63fad2fa..ae7b1c04 100644 --- a/cloudinit/net/__init__.py +++ b/cloudinit/net/__init__.py @@ -20,7 +20,6 @@ import errno import glob import os import re -import string from cloudinit import log as logging from cloudinit import util @@ -30,6 +29,7 @@ from . import network_state LOG = logging.getLogger(__name__) SYS_CLASS_NET = "/sys/class/net/" +LINKS_FNAME_PREFIX = "etc/systemd/network/50-cloud-init-" NET_CONFIG_OPTIONS = [ "address", "netmask", "broadcast", "network", "metric", "gateway", @@ -423,19 +423,45 @@ def render_interfaces(network_state): return content -def render_network_state(target, network_state): - eni = 'etc/network/interfaces' - netrules = 'etc/udev/rules.d/70-persistent-net.rules' +def render_network_state(target, network_state, eni="etc/network/interfaces", + links_prefix=LINKS_FNAME_PREFIX, + netrules='etc/udev/rules.d/70-persistent-net.rules'): - eni = os.path.sep.join((target, eni,)) - util.ensure_dir(os.path.dirname(eni)) - with open(eni, 'w+') as f: + fpeni = os.path.sep.join((target, eni,)) + util.ensure_dir(os.path.dirname(fpeni)) + with open(fpeni, 'w+') as f: f.write(render_interfaces(network_state)) - netrules = os.path.sep.join((target, netrules,)) - util.ensure_dir(os.path.dirname(netrules)) - with open(netrules, 'w+') as f: - f.write(render_persistent_net(network_state)) + if netrules: + netrules = os.path.sep.join((target, netrules,)) + util.ensure_dir(os.path.dirname(netrules)) + with open(netrules, 'w+') as f: + f.write(render_persistent_net(network_state)) + + if links_prefix: + render_systemd_links(target, network_state, links_prefix) + + +def render_systemd_links(target, network_state, + links_prefix=LINKS_FNAME_PREFIX): + fp_prefix = os.path.sep.join((target, links_prefix)) + for f in glob.glob(fp_prefix + "*"): + os.unlink(f) + + interfaces = network_state.get('interfaces') + for iface in interfaces.values(): + if (iface['type'] == 'physical' and 'name' in iface and + 'mac_address' in iface): + fname = fp_prefix + iface['name'] + ".link" + with open(fname, "w") as fp: + fp.write("\n".join([ + "[Match]", + "MACAddress=" + iface['mac_address'], + "", + "[Link]", + "Name=" + iface['name'], + "" + ])) def is_disabled_cfg(cfg): diff --git a/tests/unittests/test_distros/test_netconfig.py b/tests/unittests/test_distros/test_netconfig.py index 6d30c5b8..2c2a424d 100644 --- a/tests/unittests/test_distros/test_netconfig.py +++ b/tests/unittests/test_distros/test_netconfig.py @@ -109,8 +109,9 @@ class TestNetCfgDistro(TestCase): ub_distro.apply_network(BASE_NET_CFG, False) self.assertEquals(len(write_bufs), 1) - self.assertIn('/etc/network/interfaces', write_bufs) - write_buf = write_bufs['/etc/network/interfaces'] + eni_name = '/etc/network/interfaces.d/50-cloud-init.cfg' + self.assertIn(eni_name, write_bufs) + write_buf = write_bufs[eni_name] self.assertEquals(str(write_buf).strip(), BASE_NET_CFG.strip()) self.assertEquals(write_buf.mode, 0o644) -- cgit v1.2.3 From ed55d6bac52c53b9473b9644ce50f61404bfd438 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Tue, 22 Mar 2016 20:02:29 -0400 Subject: better log message about applying networking. --- cloudinit/stages.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/cloudinit/stages.py b/cloudinit/stages.py index 73090025..8ebbe6a9 100644 --- a/cloudinit/stages.py +++ b/cloudinit/stages.py @@ -578,18 +578,18 @@ class Init(object): for loc, ncfg in (cmdline_cfg, dscfg, sys_cfg): if net.is_disabled_cfg(ncfg): LOG.debug("network config disabled by %s", loc) - return None + return (None, loc) if ncfg: - return ncfg - return net.generate_fallback_config() + return (ncfg, loc) + return (net.generate_fallback_config(), "fallback") def apply_network_config(self): - netcfg = self._find_networking_config() + netcfg, src = self._find_networking_config() if netcfg is None: - LOG.info("network config is disabled") + LOG.info("network config is disabled by %s", src) return - LOG.info("Applying configuration: %s", netcfg) + LOG.info("Applying network configuration from %s: %s", src, netcfg) return self.distro.apply_network_config(netcfg) -- cgit v1.2.3 From 5b3cad36be8981cd12cffdf5c5e539b522404000 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Wed, 23 Mar 2016 10:31:11 -0400 Subject: trust existing datasource in modules or single This fixes a bug where modules mode was not passing a 'existing' flag to fetch. fetch had existing default to 'check'. The DataSourceNoCloud when fed with data from a disk will return False to check() as it is not a guarantee'd hit. That caused fetch to go looking for a new datasource. That would have actually worked, but modules and single create the Init with deps=[]. So it went looking for Datasources that matched those deps, and only found DataSourceNone. I'm going to keep having modules and single specify deps=[] as that will prevent them from going to look for a DS and further making things worse. --- bin/cloud-init | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bin/cloud-init b/bin/cloud-init index 8875d2f6..341359e3 100755 --- a/bin/cloud-init +++ b/bin/cloud-init @@ -329,7 +329,7 @@ def main_modules(action_name, args): init.read_cfg(extract_fns(args)) # Stage 2 try: - init.fetch() + init.fetch(existing="trust") except sources.DataSourceNotFoundException: # There was no datasource found, theres nothing to do msg = ('Can not apply stage %s, no datasource found! Likely bad ' @@ -383,7 +383,7 @@ def main_single(name, args): init.read_cfg(extract_fns(args)) # Stage 2 try: - init.fetch() + init.fetch(existing="trust") except sources.DataSourceNotFoundException: # There was no datasource found, # that might be bad (or ok) depending on -- cgit v1.2.3 From 2b85dabb802766e0b3b1949d744c8860c0cb838a Mon Sep 17 00:00:00 2001 From: Ryan Harper Date: Wed, 23 Mar 2016 11:05:22 -0500 Subject: configdata: parse and convert openstack network_data json to network_config --- cloudinit/net/__init__.py | 34 +++-- cloudinit/net/network_state.py | 45 ++++++- cloudinit/sources/DataSourceConfigDrive.py | 137 +++++++++++++++++++++ cloudinit/sources/helpers/openstack.py | 34 ++++- .../unittests/test_datasource/test_configdrive.py | 50 +++++++- 5 files changed, 289 insertions(+), 11 deletions(-) diff --git a/cloudinit/net/__init__.py b/cloudinit/net/__init__.py index ae7b1c04..76cd4e8b 100644 --- a/cloudinit/net/__init__.py +++ b/cloudinit/net/__init__.py @@ -336,7 +336,7 @@ def iface_add_attrs(iface): 'index', 'subnets', ] - if iface['type'] not in ['bond', 'bridge']: + if iface['type'] not in ['bond', 'bridge', 'vlan']: ignore_map.append('mac_address') for key, value in iface.items(): @@ -348,19 +348,34 @@ def iface_add_attrs(iface): return content -def render_route(route): - content = "up route add" +def render_route(route, indent=""): + content = "" + up = indent + "post-up route add" + down = indent + "pre-down route del" + eol = " || true\n" mapping = { 'network': '-net', 'netmask': 'netmask', 'gateway': 'gw', 'metric': 'metric', } - for k in ['network', 'netmask', 'gateway', 'metric']: - if k in route: - content += " %s %s" % (mapping[k], route[k]) + if route['network'] == '0.0.0.0' and route['netmask'] == '0.0.0.0': + default_gw = " default gw %s" % route['gateway'] + content += up + default_gw + eol + content += down + default_gw + eol + elif route['network'] == '::' and route['netmask'] == 0: + # ipv6! + default_gw = " -A inet6 default gw %s" % route['gateway'] + content += up + default_gw + eol + content += down + default_gw + eol + else: + route_line = "" + for k in ['network', 'netmask', 'gateway', 'metric']: + if k in route: + route_line += " %s %s" % (mapping[k], route[k]) + content += up + route_line + eol + content += down + route_line + eol - content += '\n' return content @@ -384,6 +399,7 @@ def render_interfaces(network_state): if len(value): content += " dns-{} {}\n".format(dnskey, " ".join(value)) + content += "\n" for iface in sorted(interfaces.values(), key=lambda k: (order[k['type']], k['name'])): content += "auto {name}\n".format(**iface) @@ -409,6 +425,8 @@ def render_interfaces(network_state): content += iface_add_subnet(iface, subnet) content += iface_add_attrs(iface) + for route in subnet.get('routes', []): + content += render_route(route, indent=" ") content += "\n" else: content += "iface {name} {inet} {mode}\n".format(**iface) @@ -419,7 +437,7 @@ def render_interfaces(network_state): content += render_route(route) # global replacements until v2 format - content = content.replace('mac_address', 'hwaddress') + content = content.replace('mac_address', 'hwaddress ether') return content diff --git a/cloudinit/net/network_state.py b/cloudinit/net/network_state.py index df04c526..e32d2cdf 100644 --- a/cloudinit/net/network_state.py +++ b/cloudinit/net/network_state.py @@ -124,6 +124,17 @@ class NetworkState: iface = interfaces.get(command['name'], {}) for param, val in command.get('params', {}).items(): iface.update({param: val}) + + # convert subnet ipv6 netmask to cidr as needed + subnets = command.get('subnets') + if subnets: + for subnet in subnets: + if subnet['type'] == 'static': + if 'netmask' in subnet and ':' in subnet['address']: + subnet['netmask'] = mask2cidr(subnet['netmask']) + for route in subnet.get('routes', []): + if 'netmask' in route: + route['netmask'] = mask2cidr(route['netmask']) iface.update({ 'name': command.get('name'), 'type': command.get('type'), @@ -133,7 +144,7 @@ class NetworkState: 'mtu': command.get('mtu'), 'address': None, 'gateway': None, - 'subnets': command.get('subnets'), + 'subnets': subnets, }) self.network_state['interfaces'].update({command.get('name'): iface}) self.dump_network_state() @@ -144,6 +155,7 @@ class NetworkState: iface eth0.222 inet static address 10.10.10.1 netmask 255.255.255.0 + hwaddress ether BC:76:4E:06:96:B3 vlan-raw-device eth0 ''' required_keys = [ @@ -335,6 +347,37 @@ def cidr2mask(cidr): return ".".join([str(x) for x in mask]) +def ipv4mask2cidr(mask): + if '.' not in mask: + return mask + return sum([bin(int(x)).count('1') for x in mask.split('.')]) + + +def ipv6mask2cidr(mask): + if ':' not in mask: + return mask + + bitCount = [0, 0x8000, 0xc000, 0xe000, 0xf000, 0xf800, 0xfc00, 0xfe00, + 0xff00, 0xff80, 0xffc0, 0xffe0, 0xfff0, 0xfff8, 0xfffc, + 0xfffe, 0xffff] + cidr = 0 + for word in mask.split(':'): + if not word or int(word, 16) == 0: + break + cidr += bitCount.index(int(word, 16)) + + return cidr + + +def mask2cidr(mask): + if ':' in mask: + return ipv6mask2cidr(mask) + elif '.' in mask: + return ipv4mask2cidr(mask) + else: + return mask + + if __name__ == '__main__': import sys import random diff --git a/cloudinit/sources/DataSourceConfigDrive.py b/cloudinit/sources/DataSourceConfigDrive.py index 6fc9e05b..d84fab54 100644 --- a/cloudinit/sources/DataSourceConfigDrive.py +++ b/cloudinit/sources/DataSourceConfigDrive.py @@ -18,6 +18,7 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . +import copy import os from cloudinit import log as logging @@ -50,6 +51,8 @@ class DataSourceConfigDrive(openstack.SourceMixin, sources.DataSource): self.seed_dir = os.path.join(paths.seed_dir, 'config_drive') self.version = None self.ec2_metadata = None + self._network_config = None + self.network_json = None self.files = {} def __str__(self): @@ -144,12 +147,27 @@ class DataSourceConfigDrive(openstack.SourceMixin, sources.DataSource): LOG.warn("Invalid content in vendor-data: %s", e) self.vendordata_raw = None + nd = results.get('networkdata') + self.networkdata_pure = nd + try: + self.network_json = openstack.convert_networkdata_json(nd) + except ValueError as e: + LOG.warn("Invalid content in network-data: %s", e) + self.network_json = None + + if self.network_json: + self._network_config = convert_network_data(self.network_json) + return True def check_instance_id(self): # quickly (local check only) if self.instance_id is still valid return sources.instance_id_matches_system_uuid(self.get_instance_id()) + @property + def network_config(self): + return self._network_config + class DataSourceConfigDriveNet(DataSourceConfigDrive): def __init__(self, sys_cfg, distro, paths): @@ -287,3 +305,122 @@ datasources = [ # Return a list of data sources that match this set of dependencies def get_datasource_list(depends): return sources.list_from_depends(depends, datasources) + + +# Convert OpenStack ConfigDrive NetworkData json to network_config yaml +def convert_network_data(network_json=None): + """Return a dictionary of network_config by parsing provided + OpenStack ConfigDrive NetworkData json format + + OpenStack network_data.json provides a 3 element dictionary + - "links" (links are network devices, physical or virtual) + - "networks" (networks are ip network configurations for one or more + links) + - services (non-ip services, like dns) + + networks and links are combined via network items referencing specific + links via a 'link_id' which maps to a links 'id' field. + + To convert this format to network_config yaml, we first iterate over the + links and then walk the network list to determine if any of the networks + utilize the current link; if so we generate a subnet entry for the device + + We also need to map network_data.json fields to network_config fields. For + example, the network_data links 'id' field is equivalent to network_config + 'name' field for devices. We apply more of this mapping to the various + link types that we encounter. + + There are additional fields that are populated in the network_data.json + from OpenStack that are not relevant to network_config yaml, so we + enumerate a dictionary of valid keys for network_yaml and apply filtering + to drop these superflous keys from the network_config yaml. + """ + if network_json is None: + return None + + # dict of network_config key for filtering network_json + valid_keys = { + 'physical': [ + 'name', + 'type', + 'mac_address', + 'subnets', + 'params', + ], + 'subnet': [ + 'type', + 'address', + 'netmask', + 'broadcast', + 'metric', + 'gateway', + 'pointopoint', + 'mtu', + 'scope', + 'dns_nameservers', + 'dns_search', + 'routes', + ], + } + + links = network_json.get('links', []) + networks = network_json.get('networks', []) + services = network_json.get('services', []) + + config = [] + for link in links: + subnets = [] + cfg = {k: v for k, v in link.items() + if k in valid_keys['physical']} + cfg.update({'name': link['id']}) + for network in [net for net in networks + if net['link'] == link['id']]: + subnet = {k: v for k, v in network.items() + if k in valid_keys['subnet']} + if 'dhcp' in network['type']: + t = 'dhcp6' if network['type'].startswith('ipv6') else 'dhcp4' + subnet.update({ + 'type': t, + }) + else: + subnet.update({ + 'type': 'static', + 'address': network.get('ip_address'), + }) + subnets.append(subnet) + cfg.update({'subnets': subnets}) + if link['type'] in ['ethernet', 'vif', 'ovs']: + cfg.update({ + 'type': 'physical', + 'mac_address': link['ethernet_mac_address']}) + elif link['type'] in ['bond']: + params = {} + for k, v in link.items(): + if k == 'bond_links': + continue + elif k.startswith('bond'): + params.update({k: v}) + cfg.update({ + 'bond_interfaces': copy.deepcopy(link['bond_links']), + 'params': params, + }) + elif link['type'] in ['vlan']: + cfg.update({ + 'name': "%s.%s" % (link['vlan_link'], + link['vlan_id']), + 'vlan_link': link['vlan_link'], + 'vlan_id': link['vlan_id'], + 'mac_address': link['vlan_mac_address'], + }) + else: + raise ValueError( + 'Unknown network_data link type: %s' % link['type']) + + config.append(cfg) + + for service in services: + cfg = service + cfg.update({'type': 'nameserver'}) + config.append(cfg) + + return {'version': 1, 'config': config} diff --git a/cloudinit/sources/helpers/openstack.py b/cloudinit/sources/helpers/openstack.py index bd93d22f..eb50a7be 100644 --- a/cloudinit/sources/helpers/openstack.py +++ b/cloudinit/sources/helpers/openstack.py @@ -51,11 +51,13 @@ OS_LATEST = 'latest' OS_FOLSOM = '2012-08-10' OS_GRIZZLY = '2013-04-04' OS_HAVANA = '2013-10-17' +OS_KILO = '2015-10-15' # keep this in chronological order. new supported versions go at the end. OS_VERSIONS = ( OS_FOLSOM, OS_GRIZZLY, OS_HAVANA, + OS_KILO, ) @@ -229,6 +231,11 @@ class BaseReader(object): False, load_json_anytype, ) + files['networkdata'] = ( + self._path_join("openstack", version, 'network_data.json'), + False, + load_json_anytype, + ) return files results = { @@ -334,7 +341,7 @@ class ConfigDriveReader(BaseReader): path = self._path_join(self.base_path, 'openstack') found = [d for d in os.listdir(path) if os.path.isdir(os.path.join(path))] - self._versions = found + self._versions = sorted(found) return self._versions def _read_ec2_metadata(self): @@ -490,3 +497,28 @@ def convert_vendordata_json(data, recurse=True): recurse=False) raise ValueError("vendordata['cloud-init'] cannot be dict") raise ValueError("Unknown data type for vendordata: %s" % type(data)) + + +def convert_networkdata_json(data, recurse=True): + """ data: a loaded json *object* (strings, arrays, dicts). + return something suitable for cloudinit networkdata_raw. + + if data is: + None: return None + string: return string + list: return data + the list is then processed in UserDataProcessor + dict: return convert_networkdata_json(data.get('cloud-init')) + """ + if not data: + return None + if isinstance(data, six.string_types): + return data + if isinstance(data, list): + return copy.deepcopy(data) + if isinstance(data, dict): + if recurse is True: + return convert_networkdata_json(data.get('cloud-init'), + recurse=False) + raise ValueError("networkdata['cloud-init'] cannot be dict") + raise ValueError("Unknown data type for networkdata: %s" % type(data)) diff --git a/tests/unittests/test_datasource/test_configdrive.py b/tests/unittests/test_datasource/test_configdrive.py index bfd787d1..01f8c5ce 100644 --- a/tests/unittests/test_datasource/test_configdrive.py +++ b/tests/unittests/test_datasource/test_configdrive.py @@ -59,6 +59,34 @@ OSTACK_META = { CONTENT_0 = b'This is contents of /etc/foo.cfg\n' CONTENT_1 = b'# this is /etc/bar/bar.cfg\n' +NETWORK_DATA = { + 'services': [ + {'type': 'dns', 'address': '199.204.44.24'}, + {'type': 'dns', 'address': '199.204.47.54'} + ], + 'links': [ + {'vif_id': '2ecc7709-b3f7-4448-9580-e1ec32d75bbd', + 'ethernet_mac_address': 'fa:16:3e:69:b0:58', + 'type': 'ovs', 'mtu': None, 'id': 'tap2ecc7709-b3'}, + {'vif_id': '2f88d109-5b57-40e6-af32-2472df09dc33', + 'ethernet_mac_address': 'fa:16:3e:d4:57:ad', + 'type': 'ovs', 'mtu': None, 'id': 'tap2f88d109-5b'}, + {'vif_id': '1a5382f8-04c5-4d75-ab98-d666c1ef52cc', + 'ethernet_mac_address': 'fa:16:3e:05:30:fe', + 'type': 'ovs', 'mtu': None, 'id': 'tap1a5382f8-04'} + ], + 'networks': [ + {'link': 'tap2ecc7709-b3', 'type': 'ipv4_dhcp', + 'network_id': '6d6357ac-0f70-4afa-8bd7-c274cc4ea235', + 'id': 'network0'}, + {'link': 'tap2f88d109-5b', 'type': 'ipv4_dhcp', + 'network_id': 'd227a9b3-6960-4d94-8976-ee5788b44f54', + 'id': 'network1'}, + {'link': 'tap1a5382f8-04', 'type': 'ipv4_dhcp', + 'network_id': 'dab2ba57-cae2-4311-a5ed-010b263891f5', + 'id': 'network2'} + ] +} CFG_DRIVE_FILES_V2 = { 'ec2/2009-04-04/meta-data.json': json.dumps(EC2_META), @@ -70,7 +98,11 @@ CFG_DRIVE_FILES_V2 = { 'openstack/content/0000': CONTENT_0, 'openstack/content/0001': CONTENT_1, 'openstack/latest/meta_data.json': json.dumps(OSTACK_META), - 'openstack/latest/user_data': USER_DATA} + 'openstack/latest/user_data': USER_DATA, + 'openstack/latest/network_data.json': json.dumps(NETWORK_DATA), + 'openstack/2015-10-15/meta_data.json': json.dumps(OSTACK_META), + 'openstack/2015-10-15/user_data': USER_DATA, + 'openstack/2015-10-15/network_data.json': json.dumps(NETWORK_DATA)} class TestConfigDriveDataSource(TestCase): @@ -225,6 +257,7 @@ class TestConfigDriveDataSource(TestCase): self.assertEqual(USER_DATA, found['userdata']) self.assertEqual(expected_md, found['metadata']) + self.assertEqual(NETWORK_DATA, found['networkdata']) self.assertEqual(found['files']['/etc/foo.cfg'], CONTENT_0) self.assertEqual(found['files']['/etc/bar/bar.cfg'], CONTENT_1) @@ -321,6 +354,19 @@ class TestConfigDriveDataSource(TestCase): self.assertEqual(myds.get_public_ssh_keys(), [OSTACK_META['public_keys']['mykey']]) + def test_network_data_is_found(self): + """Verify that network_data is present in ds in config-drive-v2.""" + populate_dir(self.tmp, CFG_DRIVE_FILES_V2) + myds = cfg_ds_from_dir(self.tmp) + self.assertEqual(myds.network_json, NETWORK_DATA) + + def test_network_config_is_converted(self): + """Verify that network_data is converted and present on ds object.""" + populate_dir(self.tmp, CFG_DRIVE_FILES_V2) + myds = cfg_ds_from_dir(self.tmp) + network_config = ds.convert_network_data(NETWORK_DATA) + self.assertEqual(myds.network_config, network_config) + def cfg_ds_from_dir(seed_d): found = ds.read_config_drive(seed_d) @@ -339,6 +385,8 @@ def populate_ds_from_read_config(cfg_ds, source, results): cfg_ds.ec2_metadata = results.get('ec2-metadata') cfg_ds.userdata_raw = results.get('userdata') cfg_ds.version = results.get('version') + cfg_ds.network_json = results.get('networkdata') + cfg_ds._network_config = ds.convert_network_data(cfg_ds.network_json) def populate_dir(seed_dir, files): -- cgit v1.2.3 From 0f4811e307af08cb8f0dd552346f1148ea2f9c10 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Wed, 23 Mar 2016 13:52:14 -0400 Subject: add config_from_klibc_net_cfg and helper functions Wesley's loader returned network state, so that got me updating it, and i implemented as such. Then realized that actually ipconfig (klibc) has no support for ipv6. So even though i painfully generalized that, its pointless. next commit will drop it. --- cloudinit/net/__init__.py | 170 ++++++++++++++++++++++++---------------------- 1 file changed, 88 insertions(+), 82 deletions(-) diff --git a/cloudinit/net/__init__.py b/cloudinit/net/__init__.py index 0560fa45..3362d172 100644 --- a/cloudinit/net/__init__.py +++ b/cloudinit/net/__init__.py @@ -20,6 +20,7 @@ import errno import glob import os import re +import shlex from cloudinit import log as logging from cloudinit import util @@ -283,94 +284,99 @@ def parse_net_config(path): return ns -def load_klibc_net_cfg(data_mapping): - """Process key value pairs from files written because of the ip parameter - on the kernel cmdline, note that mode: manual is used because the - interface should already have been brought up by the kernel and - cloud-initramfs-tools""" - entry_ns = { - 'mtu': None, 'name': data_mapping['DEVICE'], 'type': 'physical', - 'mode': 'manual', 'inet': 'inet', 'gateway': None, 'address': None, - 'subnets': [] - } +def _load_shell_content(content, add_empty=False, empty_val=None): + """Given the content of a klibc created /run/net*.conf file, return + its data in dictionary form.""" + data = {} + for line in shlex.split(content): + key, value = line.split("=", maxsplit=1) + if not value: + value = empty_val + if add_empty or value: + data[key] = value - # ipconfig on precise does not write PROTO - # (lp:cloud-initramfs-tools/dyn-netconf/scripts/init-bottom/ - # cloud-initramfs-dyn-netconf) - if not data_mapping.get('PROTO'): - if data_mapping.get('filename'): - data_mapping['PROTO'] = 'dhcp' - else: - data_mapping['PROTO'] = 'static' - - if data_mapping.get('PROTO') == 'dhcp': - if data_mapping.get('IPV4ADDR'): - entry_ns['subnets'].append({'type': 'dhcp4'}) - if data_mapping.get('IPV6ADDR'): - entry_ns['subnets'].append({'type': 'dhcp6'}) - elif data_mapping.get('PROTO') in ['static', 'none']: - # It appears that specifying ipv6 static addrs does not work, so only - # check for ipv4 addr - entry_ns['subnets'].append( - {'type': 'static', 'address': data_mapping.get('IPV4ADDR')}) - - if data_mapping.get('IPV4ADDR'): - entry_ns['address'] = data_mapping['IPV4ADDR'] - if data_mapping.get('IPV6ADDR'): - entry_ns['address'] = data_mapping['IPV6ADDR'] - if data_mapping.get('IPV4BROADCAST'): - entry_ns['broadcast'] = data_mapping['IPV4BROADCAST'] - if data_mapping.get('IPV6BROADCAST'): - entry_ns['broadcast'] = data_mapping['IPV6BROADCAST'] - if data_mapping.get('IPV4NETMASK'): - entry_ns['netmask'] = data_mapping['IPV4NETMASK'] - if data_mapping.get('IPV6NETMASK'): - entry_ns['netmask'] = data_mapping['IPV6NETMASK'] - if data_mapping.get('IPV4GATEWAY'): - entry_ns['gateway'] = data_mapping['IPV4GATEWAY'] - if data_mapping.get('IPV6GATEWAY'): - entry_ns['gateway'] = data_mapping['IPV6GATEWAY'] - if data_mapping.get('HOSTNAME'): - entry_ns['hostname'] = data_mapping['HOSTNAME'] - - return entry_ns - - -def merge_from_cmdline_config(ns): - """If ip parameter passed on kernel cmdline then some initial network - configuration may have been done in initramfs. Files from the result of - this may have been written into /run. If any are present they should be - merged into network state""" - - if 'interfaces' not in ns: - ns['interfaces'] = {} - - for cfg_file in glob.glob('/run/net*.conf'): - with open(cfg_file, 'r') as fp: - data = [l.replace("'", "") for l in fp.readlines()] - try: - parsed = dict([l.strip().split('=') for l in data]) - except: - # if split did not work then this is likely not a netcfg file - continue + return data - dev_name = parsed.get('DEVICE') - if not dev_name: - # Not a net cfg file - continue - loaded_ns = load_klibc_net_cfg(parsed) +def _klibc_to_config_entry(content): + data = _load_shell_content(content) + try: + name = data['DEVICE'] + except KeyError: + raise ValueError("no 'DEVICE' entry in data") - if dev_name in ns['interfaces']: - if 'subnets' not in ns['interfaces'][dev_name]: - ns['interfaces'][dev_name]['subnets'] = [] - for newsubnet in loaded_ns['subnets']: - if newsubnet not in ns['interfaces'][dev_name]['subnets']: - ns['interfaces'][dev_name]['subnets'].append(newsubnet) + # ipconfig on precise does not write PROTO + proto = data.get('PROTO') + if not proto: + if data.get('filename'): + proto = 'dhcp' else: - ns['interfaces'][dev_name] = loaded_ns + proto = 'static' - return ns + if proto not in ('static', 'dhcp'): + raise ValueError("Unexpected value for PROTO: %s" % proto) + + iface = { + 'type': 'physical', + 'name': name, + 'subnets': [], + } + subnets = {} + + for v, pre in (('ipv4', 'IPV4'), ('ipv6', 'IPV6')): + # if no IPV4ADDR or IPV6ADDR, then go on. + if pre + "ADDR" not in data: + continue + subnet = {'type': proto} + + # these fields go right on the subnet + for key in ('NETMASK', 'BROADCAST', 'GATEWAY'): + if pre + key in data: + subnet[key.lower()] = data[pre + key] + + dns = [] + # handle IPV4DNS0 or IPV6DNS0 + for nskey in ('DNS0', 'DNS1'): + ns = data.get(pre + nskey) + # verify it has something other than 0.0.0.0 (or ipv6) + if ns and len(ns.strip(":.0")): + dns.append(data[pre + nskey]) + if dns: + subnet['dns_nameservers'] = dns + # add search to both ipv4 and ipv6, as it has no namespace + search = data.get('DOMAINSEARCH') + if search: + if ',' in search: + subnet['dns_search'] = search.split(",") + else: + subnet['dns_search'] = search.split() + + iface['subnets'].append(subnet) + + for subnet in subnets: + iface[subnet].append(subnet) + + return name, iface + + +def config_from_klibc_net_cfg(files=None): + if files is None: + files = glob.glob('/run/net*.conf') + + devs = {} + entries = [] + names = {} + for cfg_file in files: + name, entry = _klibc_to_config_entry(util.load_file(cfg_file)) + print("name: %s file: %s" % (name, cfg_file)) + if name in names: + raise ValueError( + "device '%s' defined multiple times: %s and %s" % ( + name, names[name], cfg_file + )) + names[name] = cfg_file + entries.append(entry) + return {'config': entries, 'version': 1} def render_persistent_net(network_state): -- cgit v1.2.3 From 3791d25694444fe49e026a575b556117a1ea99c3 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Wed, 23 Mar 2016 14:23:54 -0400 Subject: add sys_netdev_info helper, support reading macs in. --- cloudinit/net/__init__.py | 55 ++++++++++++++++++++++++++++------------------- 1 file changed, 33 insertions(+), 22 deletions(-) diff --git a/cloudinit/net/__init__.py b/cloudinit/net/__init__.py index 3362d172..a167c0a1 100644 --- a/cloudinit/net/__init__.py +++ b/cloudinit/net/__init__.py @@ -298,7 +298,10 @@ def _load_shell_content(content, add_empty=False, empty_val=None): return data -def _klibc_to_config_entry(content): +def _klibc_to_config_entry(content, mac_addrs=None): + if mac_addrs is None: + mac_addrs = {} + data = _load_shell_content(content) try: name = data['DEVICE'] @@ -321,14 +324,17 @@ def _klibc_to_config_entry(content): 'name': name, 'subnets': [], } - subnets = {} - for v, pre in (('ipv4', 'IPV4'), ('ipv6', 'IPV6')): + if name in mac_addrs: + iface['mac_address'] = mac_addrs[name] + + # originally believed there might be IPV6* values + for v, pre in (('ipv4', 'IPV4'),): # if no IPV4ADDR or IPV6ADDR, then go on. if pre + "ADDR" not in data: continue subnet = {'type': proto} - + # these fields go right on the subnet for key in ('NETMASK', 'BROADCAST', 'GATEWAY'): if pre + key in data: @@ -353,13 +359,10 @@ def _klibc_to_config_entry(content): iface['subnets'].append(subnet) - for subnet in subnets: - iface[subnet].append(subnet) - return name, iface -def config_from_klibc_net_cfg(files=None): +def config_from_klibc_net_cfg(files=None, mac_addrs=None): if files is None: files = glob.glob('/run/net*.conf') @@ -367,13 +370,13 @@ def config_from_klibc_net_cfg(files=None): entries = [] names = {} for cfg_file in files: - name, entry = _klibc_to_config_entry(util.load_file(cfg_file)) - print("name: %s file: %s" % (name, cfg_file)) + name, entry = _klibc_to_config_entry(util.load_file(cfg_file), + mac_addrs=mac_addrs) if name in names: raise ValueError( "device '%s' defined multiple times: %s and %s" % ( - name, names[name], cfg_file - )) + name, names[name], cfg_file)) + names[name] = cfg_file entries.append(entry) return {'config': entries, 'version': 1} @@ -566,6 +569,19 @@ def is_disabled_cfg(cfg): return cfg.get('config') == "disabled" +def sys_netdev_info(name, field): + if not os.path.exists(os.path.join(SYS_CLASS_NET, name)): + raise OSError("%s: interface does not exist in /sys" % name) + + fname = os.path.join(SYS_CLASS_NET, name, field) + if not os.path.exists(fname): + raise OSError("%s: %s does not exist in /sys" % (name, fname)) + data = util.load_file(fname) + if data[-1] == '\n': + data = data[:-1] + return data + + def generate_fallback_config(): """Determine which attached net dev is most likely to have a connection and generate network state to run dhcp on that interface""" @@ -574,7 +590,7 @@ def generate_fallback_config(): # get list of interfaces that could have connections invalid_interfaces = set(['lo']) - potential_interfaces = set(os.listdir(SYS_CLASS_NET)) + potential_interfaces = set(get_devicelist()) potential_interfaces = potential_interfaces.difference(invalid_interfaces) # sort into interfaces with carrier, interfaces which could have carrier, # and ignore interfaces that are definitely disconnected @@ -582,8 +598,7 @@ def generate_fallback_config(): possibly_connected = [] for interface in potential_interfaces: try: - sysfs_carrier = os.path.join(SYS_CLASS_NET, interface, 'carrier') - carrier = int(util.load_file(sysfs_carrier).strip()) + carrier = int(sys_netdev_info(interface, 'carrier')) if carrier: connected.append(interface) continue @@ -593,17 +608,14 @@ def generate_fallback_config(): # not have a carrier even though it could acquire one when brought # online by dhclient try: - sysfs_dormant = os.path.join(SYS_CLASS_NET, interface, 'dormant') - dormant = int(util.load_file(sysfs_dormant).strip()) + dormant = int(sys_netdev_info(interface, 'dormant')) if dormant: possibly_connected.append(interface) continue except OSError: pass try: - sysfs_operstate = os.path.join(SYS_CLASS_NET, interface, - 'operstate') - operstate = util.load_file(sysfs_operstate).strip() + operstate = sys_netdev_info(interface, 'operstate') if operstate in ['dormant', 'down', 'lowerlayerdown', 'unknown']: possibly_connected.append(interface) continue @@ -626,8 +638,7 @@ def generate_fallback_config(): else: name = sorted(potential_interfaces)[0] - sysfs_mac = os.path.join(SYS_CLASS_NET, name, 'address') - mac = util.load_file(sysfs_mac).strip() + mac = sys_netdev_info(name, 'address') target_name = name nconf['config'].append( -- cgit v1.2.3 From 5cf44d7847b085c5fc881c9eb39bcf6bc891e0d9 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Wed, 23 Mar 2016 14:29:22 -0400 Subject: add the implementation for read_kernel_cmdline_config --- cloudinit/net/__init__.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/cloudinit/net/__init__.py b/cloudinit/net/__init__.py index a167c0a1..9f5a7fd7 100644 --- a/cloudinit/net/__init__.py +++ b/cloudinit/net/__init__.py @@ -647,9 +647,11 @@ def generate_fallback_config(): return nconf -def read_kernel_cmdline_config(): - # FIXME: add implementation here - return None +def read_kernel_cmdline_config(files=None, mac_addrs=None): + if mac_addrs is None: + mac_addrs = {k: sys_netdev_info(k, 'address') + for k in get_devicelist()} + return config_from_klibc_net_cfg(files=files, mac_addrs=mac_addrs) # vi: ts=4 expandtab syntax=python -- cgit v1.2.3 From d8cfa30be45492140c04fb9c9ed69e83e9f041d9 Mon Sep 17 00:00:00 2001 From: Ryan Harper Date: Wed, 23 Mar 2016 13:32:23 -0500 Subject: unittest: fix bad json test with ConfigDrive Introduced a new path in configdrive, openstack/2015-10-15/, needed to add bogus data in that path as well to ensure config reader didn't find good data when testing for exception thrown. --- tests/unittests/test_datasource/test_configdrive.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/unittests/test_datasource/test_configdrive.py b/tests/unittests/test_datasource/test_configdrive.py index 01f8c5ce..89b15f54 100644 --- a/tests/unittests/test_datasource/test_configdrive.py +++ b/tests/unittests/test_datasource/test_configdrive.py @@ -283,6 +283,7 @@ class TestConfigDriveDataSource(TestCase): data = copy(CFG_DRIVE_FILES_V2) data["openstack/2012-08-10/meta_data.json"] = "non-json garbage {}" + data["openstack/2015-10-15/meta_data.json"] = "non-json garbage {}" data["openstack/latest/meta_data.json"] = "non-json garbage {}" populate_dir(self.tmp, data) -- cgit v1.2.3 From 6b79e2c6f9a7342163691be9e785cef1aa642541 Mon Sep 17 00:00:00 2001 From: Ryan Harper Date: Wed, 23 Mar 2016 14:17:10 -0500 Subject: fix openstack versions s/KILO/LIBERY drop networkdata read helper --- cloudinit/sources/DataSourceConfigDrive.py | 2 +- cloudinit/sources/helpers/openstack.py | 29 ++--------------------------- 2 files changed, 3 insertions(+), 28 deletions(-) diff --git a/cloudinit/sources/DataSourceConfigDrive.py b/cloudinit/sources/DataSourceConfigDrive.py index d84fab54..15dddefe 100644 --- a/cloudinit/sources/DataSourceConfigDrive.py +++ b/cloudinit/sources/DataSourceConfigDrive.py @@ -150,7 +150,7 @@ class DataSourceConfigDrive(openstack.SourceMixin, sources.DataSource): nd = results.get('networkdata') self.networkdata_pure = nd try: - self.network_json = openstack.convert_networkdata_json(nd) + self.network_json = util.load_json(nd) except ValueError as e: LOG.warn("Invalid content in network-data: %s", e) self.network_json = None diff --git a/cloudinit/sources/helpers/openstack.py b/cloudinit/sources/helpers/openstack.py index eb50a7be..1aa6bbae 100644 --- a/cloudinit/sources/helpers/openstack.py +++ b/cloudinit/sources/helpers/openstack.py @@ -51,13 +51,13 @@ OS_LATEST = 'latest' OS_FOLSOM = '2012-08-10' OS_GRIZZLY = '2013-04-04' OS_HAVANA = '2013-10-17' -OS_KILO = '2015-10-15' +OS_LIBERTY = '2015-10-15' # keep this in chronological order. new supported versions go at the end. OS_VERSIONS = ( OS_FOLSOM, OS_GRIZZLY, OS_HAVANA, - OS_KILO, + OS_LIBERTY, ) @@ -497,28 +497,3 @@ def convert_vendordata_json(data, recurse=True): recurse=False) raise ValueError("vendordata['cloud-init'] cannot be dict") raise ValueError("Unknown data type for vendordata: %s" % type(data)) - - -def convert_networkdata_json(data, recurse=True): - """ data: a loaded json *object* (strings, arrays, dicts). - return something suitable for cloudinit networkdata_raw. - - if data is: - None: return None - string: return string - list: return data - the list is then processed in UserDataProcessor - dict: return convert_networkdata_json(data.get('cloud-init')) - """ - if not data: - return None - if isinstance(data, six.string_types): - return data - if isinstance(data, list): - return copy.deepcopy(data) - if isinstance(data, dict): - if recurse is True: - return convert_networkdata_json(data.get('cloud-init'), - recurse=False) - raise ValueError("networkdata['cloud-init'] cannot be dict") - raise ValueError("Unknown data type for networkdata: %s" % type(data)) -- cgit v1.2.3 From 44f71ff4ccb72c89f6cdebc8a7b4e7a0d7029818 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Wed, 23 Mar 2016 15:32:51 -0400 Subject: add unit test --- tests/unittests/test_net.py | 94 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 94 insertions(+) create mode 100644 tests/unittests/test_net.py diff --git a/tests/unittests/test_net.py b/tests/unittests/test_net.py new file mode 100644 index 00000000..06a55643 --- /dev/null +++ b/tests/unittests/test_net.py @@ -0,0 +1,94 @@ +from cloudinit import util +from cloudinit import net +from .helpers import TestCase +import copy +import os + +DHCP_CONTENT_1 = """ +DEVICE='eth0' +PROTO='dhcp' +IPV4ADDR='192.168.122.89' +IPV4BROADCAST='192.168.122.255' +IPV4NETMASK='255.255.255.0' +IPV4GATEWAY='192.168.122.1' +IPV4DNS0='192.168.122.1' +IPV4DNS1='0.0.0.0' +HOSTNAME='foohost' +DNSDOMAIN='' +NISDOMAIN='' +ROOTSERVER='192.168.122.1' +ROOTPATH='' +filename='' +UPTIME='21' +DHCPLEASETIME='3600' +DOMAINSEARCH='foo.com' +""" + +DHCP_EXPECTED_1 = { + 'name': 'eth0', + 'type': 'physical', + 'subnets': [{'broadcast': '192.168.122.255', + 'gateway': '192.168.122.1', + 'dns_search': ['foo.com'], + 'type': 'dhcp', + 'netmask': '255.255.255.0', + 'dns_nameservers': ['192.168.122.1']}], +} + + +STATIC_CONTENT_1 = """ +DEVICE='eth1' +PROTO='static' +IPV4ADDR='10.0.0.2' +IPV4BROADCAST='10.0.0.255' +IPV4NETMASK='255.255.255.0' +IPV4GATEWAY='10.0.0.1' +IPV4DNS0='10.0.1.1' +IPV4DNS1='0.0.0.0' +HOSTNAME='foohost' +UPTIME='21' +DHCPLEASETIME='3600' +DOMAINSEARCH='foo.com' +""" + +STATIC_EXPECTED_1 = { + 'name': 'eth1', + 'type': 'physical', + 'subnets': [{'broadcast': '10.0.0.255', 'gateway': '10.0.0.1', + 'dns_search': ['foo.com'], 'type': 'static', + 'netmask': '255.255.255.0', + 'dns_nameservers': ['10.0.1.1']}], +} + +class TestNetConfigParsing(TestCase): + def test_klibc_convert_dhcp(self): + found = net._klibc_to_config_entry(DHCP_CONTENT_1) + self.assertEqual(found, ('eth0', DHCP_EXPECTED_1)) + + def test_klibc_convert_static(self): + found = net._klibc_to_config_entry(STATIC_CONTENT_1) + self.assertEqual(found, ('eth1', STATIC_EXPECTED_1)) + + def test_config_from_klibc_net_cfg(self): + files = [] + pairs = (('net-eth0.cfg', DHCP_CONTENT_1), + ('net-eth1.cfg', STATIC_CONTENT_1)) + + macs = {'eth1': 'b8:ae:ed:75:ff:2b', + 'eth0': 'b8:ae:ed:75:ff:2a'} + + dhcp = copy.deepcopy(DHCP_EXPECTED_1) + dhcp['mac_address'] = macs['eth0'] + + static = copy.deepcopy(STATIC_EXPECTED_1) + static['mac_address'] = macs['eth1'] + + expected = {'version': 1, 'config': [dhcp, static]} + with util.tempdir() as tmpd: + for fname, content in pairs: + fp = os.path.join(tmpd, fname) + files.append(fp) + util.write_file(fp, content) + + found = net.config_from_klibc_net_cfg(files=files, mac_addrs=macs) + self.assertEqual(found, expected) -- cgit v1.2.3 From 1471ddd3210a4ac5753330f97c12bf8960fedbf7 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Wed, 23 Mar 2016 15:37:11 -0400 Subject: fix tox issues --- cloudinit/net/__init__.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/cloudinit/net/__init__.py b/cloudinit/net/__init__.py index 9f5a7fd7..4f6b4dfc 100644 --- a/cloudinit/net/__init__.py +++ b/cloudinit/net/__init__.py @@ -289,7 +289,7 @@ def _load_shell_content(content, add_empty=False, empty_val=None): its data in dictionary form.""" data = {} for line in shlex.split(content): - key, value = line.split("=", maxsplit=1) + key, value = line.split("=", 1) if not value: value = empty_val if add_empty or value: @@ -366,7 +366,6 @@ def config_from_klibc_net_cfg(files=None, mac_addrs=None): if files is None: files = glob.glob('/run/net*.conf') - devs = {} entries = [] names = {} for cfg_file in files: -- cgit v1.2.3 From 51fa67a88dc0dc631c19770142b16c0b56c21384 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Wed, 23 Mar 2016 15:38:32 -0400 Subject: one more tox --- tests/unittests/test_net.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/unittests/test_net.py b/tests/unittests/test_net.py index 06a55643..11c0b1eb 100644 --- a/tests/unittests/test_net.py +++ b/tests/unittests/test_net.py @@ -60,6 +60,7 @@ STATIC_EXPECTED_1 = { 'dns_nameservers': ['10.0.1.1']}], } + class TestNetConfigParsing(TestCase): def test_klibc_convert_dhcp(self): found = net._klibc_to_config_entry(DHCP_CONTENT_1) -- cgit v1.2.3 From 740facb79c70cd8e3380e08acb4f5c5a285bb1ae Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Wed, 23 Mar 2016 15:53:37 -0400 Subject: support [untested] network-config= on kernel command line --- cloudinit/net/__init__.py | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/cloudinit/net/__init__.py b/cloudinit/net/__init__.py index 4f6b4dfc..3a208e43 100644 --- a/cloudinit/net/__init__.py +++ b/cloudinit/net/__init__.py @@ -16,6 +16,7 @@ # You should have received a copy of the GNU Affero General Public License # along with Curtin. If not, see . +import base64 import errno import glob import os @@ -646,10 +647,25 @@ def generate_fallback_config(): return nconf -def read_kernel_cmdline_config(files=None, mac_addrs=None): +def read_kernel_cmdline_config(files=None, mac_addrs=None, cmdline=None): + if cmdline is None: + cmdline = util.get_cmdline() + + if 'network-config=' in cmdline: + data64 = None + for tok in cmdline.split(): + if tok.startswith("network-config="): + data64 = tok.split("=", 1)[1] + if data64: + return util.load_yaml(base64.b64decode(data64)) + + if 'ip=' not in cmdline: + return None + if mac_addrs is None: mac_addrs = {k: sys_netdev_info(k, 'address') for k in get_devicelist()} + return config_from_klibc_net_cfg(files=files, mac_addrs=mac_addrs) -- cgit v1.2.3 From 32e81553907eaba84345252527f208d29151620f Mon Sep 17 00:00:00 2001 From: Ryan Harper Date: Wed, 23 Mar 2016 16:56:02 -0500 Subject: network_data: add link type 'phys', no need to reload json data --- cloudinit/sources/DataSourceConfigDrive.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/cloudinit/sources/DataSourceConfigDrive.py b/cloudinit/sources/DataSourceConfigDrive.py index 15dddefe..db813f6e 100644 --- a/cloudinit/sources/DataSourceConfigDrive.py +++ b/cloudinit/sources/DataSourceConfigDrive.py @@ -147,10 +147,8 @@ class DataSourceConfigDrive(openstack.SourceMixin, sources.DataSource): LOG.warn("Invalid content in vendor-data: %s", e) self.vendordata_raw = None - nd = results.get('networkdata') - self.networkdata_pure = nd try: - self.network_json = util.load_json(nd) + self.network_json = results.get('networkdata') except ValueError as e: LOG.warn("Invalid content in network-data: %s", e) self.network_json = None @@ -389,7 +387,7 @@ def convert_network_data(network_json=None): }) subnets.append(subnet) cfg.update({'subnets': subnets}) - if link['type'] in ['ethernet', 'vif', 'ovs']: + if link['type'] in ['ethernet', 'vif', 'ovs', 'phy']: cfg.update({ 'type': 'physical', 'mac_address': link['ethernet_mac_address']}) -- cgit v1.2.3 From f9cae62c2f70c582a6b12a98911dc82180881364 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Thu, 24 Mar 2016 11:19:41 -0400 Subject: add suport for base64 encoded gzipped text on command line add tests to show this functional. --- cloudinit/net/__init__.py | 34 +++++++++++++++++++++++++++++++++- tests/unittests/test_net.py | 32 ++++++++++++++++++++++++++++++++ 2 files changed, 65 insertions(+), 1 deletion(-) diff --git a/cloudinit/net/__init__.py b/cloudinit/net/__init__.py index 3a208e43..7d7f274d 100644 --- a/cloudinit/net/__init__.py +++ b/cloudinit/net/__init__.py @@ -19,6 +19,8 @@ import base64 import errno import glob +import gzip +import io import os import re import shlex @@ -647,6 +649,36 @@ def generate_fallback_config(): return nconf +def _decomp_gzip(blob, strict=True): + # decompress blob. raise exception if not compressed unless strict=False. + with io.BytesIO(blob) as iobuf: + gzfp = None + try: + gzfp = gzip.GzipFile(mode="rb", fileobj=iobuf) + return gzfp.read() + except IOError: + if strict: + raise + return blob + finally: + if gzfp: + gzfp.close() + + +def _b64dgz(b64str, gzipped="try"): + # decode a base64 string. If gzipped is true, transparently uncompresss + # if gzipped is 'try', then try gunzip, returning the original on fail. + try: + blob = base64.b64decode(b64str) + except TypeError: + raise ValueError("Invalid base64 text: %s" % b64str) + + if not gzipped: + return blob + + return _decomp_gzip(blob, strict=gzipped != "try") + + def read_kernel_cmdline_config(files=None, mac_addrs=None, cmdline=None): if cmdline is None: cmdline = util.get_cmdline() @@ -657,7 +689,7 @@ def read_kernel_cmdline_config(files=None, mac_addrs=None, cmdline=None): if tok.startswith("network-config="): data64 = tok.split("=", 1)[1] if data64: - return util.load_yaml(base64.b64decode(data64)) + return util.load_yaml(_b64dgz(data64)) if 'ip=' not in cmdline: return None diff --git a/tests/unittests/test_net.py b/tests/unittests/test_net.py index 11c0b1eb..16c44588 100644 --- a/tests/unittests/test_net.py +++ b/tests/unittests/test_net.py @@ -1,7 +1,12 @@ from cloudinit import util from cloudinit import net from .helpers import TestCase + +import base64 import copy +import io +import gzip +import json import os DHCP_CONTENT_1 = """ @@ -62,6 +67,11 @@ STATIC_EXPECTED_1 = { class TestNetConfigParsing(TestCase): + simple_cfg = { + 'config': [{"type": "physical", "name": "eth0", + "mac_address": "c0:d6:9f:2c:e8:80", + "subnets": [{"type": "dhcp4"}]}]} + def test_klibc_convert_dhcp(self): found = net._klibc_to_config_entry(DHCP_CONTENT_1) self.assertEqual(found, ('eth0', DHCP_EXPECTED_1)) @@ -93,3 +103,25 @@ class TestNetConfigParsing(TestCase): found = net.config_from_klibc_net_cfg(files=files, mac_addrs=macs) self.assertEqual(found, expected) + + def test_cmdline_with_b64(self): + data = base64.b64encode(json.dumps(self.simple_cfg).encode()) + encoded_text = data.decode() + cmdline = 'ro network-config=' + encoded_text + ' root=foo' + found = net.read_kernel_cmdline_config(cmdline=cmdline) + self.assertEqual(found, self.simple_cfg) + + def test_cmdline_with_b64_gz(self): + data = _gzip_data(json.dumps(self.simple_cfg).encode()) + encoded_text = base64.b64encode(data).decode() + cmdline = 'ro network-config=' + encoded_text + ' root=foo' + found = net.read_kernel_cmdline_config(cmdline=cmdline) + self.assertEqual(found, self.simple_cfg) + + +def _gzip_data(data): + with io.BytesIO() as iobuf: + gzfp = gzip.GzipFile(mode="wb", fileobj=iobuf) + gzfp.write(data) + gzfp.close() + return iobuf.getvalue() -- cgit v1.2.3 From 841a773fd36968419354507fa45f44afa6eb8470 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Thu, 24 Mar 2016 11:50:49 -0400 Subject: improve comment --- cloudinit/net/__init__.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/cloudinit/net/__init__.py b/cloudinit/net/__init__.py index 7d7f274d..e13ca470 100644 --- a/cloudinit/net/__init__.py +++ b/cloudinit/net/__init__.py @@ -288,8 +288,10 @@ def parse_net_config(path): def _load_shell_content(content, add_empty=False, empty_val=None): - """Given the content of a klibc created /run/net*.conf file, return - its data in dictionary form.""" + """Given shell like syntax (key=value\nkey2=value2\n) in content + return the data in dictionary form. If 'add_empty' is True + then add entries in to the returned dictionary for 'VAR=' + variables. Set their value to empty_val.""" data = {} for line in shlex.split(content): key, value = line.split("=", 1) -- cgit v1.2.3 From eb8b2f0e7b777b756a4965ea784ce1354b5c6396 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Thu, 24 Mar 2016 12:51:31 -0400 Subject: provide datasource.check_instance_id with access to system config Changing this interface to allow for easy change later. The thing that this will enable is: a.) maas datasource to look at the system config and see if it is configured with the same consumer_key b.) datasource config could allow setting a variable that it would look at. --- cloudinit/sources/DataSourceAzure.py | 2 +- cloudinit/sources/DataSourceNoCloud.py | 3 ++- cloudinit/sources/DataSourceOpenStack.py | 2 +- cloudinit/sources/__init__.py | 2 +- cloudinit/stages.py | 3 ++- 5 files changed, 7 insertions(+), 5 deletions(-) diff --git a/cloudinit/sources/DataSourceAzure.py b/cloudinit/sources/DataSourceAzure.py index 832b3063..698f4cac 100644 --- a/cloudinit/sources/DataSourceAzure.py +++ b/cloudinit/sources/DataSourceAzure.py @@ -254,7 +254,7 @@ class DataSourceAzureNet(sources.DataSource): def get_config_obj(self): return self.cfg - def check_instance_id(self): + def check_instance_id(self, sys_cfg): # quickly (local check only) if self.instance_id is still valid return sources.instance_id_matches_system_uuid(self.get_instance_id()) diff --git a/cloudinit/sources/DataSourceNoCloud.py b/cloudinit/sources/DataSourceNoCloud.py index afd08935..f786516b 100644 --- a/cloudinit/sources/DataSourceNoCloud.py +++ b/cloudinit/sources/DataSourceNoCloud.py @@ -209,13 +209,14 @@ class DataSourceNoCloud(sources.DataSource): mydata['meta-data']['dsmode']) return False - def check_instance_id(self): + def check_instance_id(self, sys_cfg): # quickly (local check only) if self.instance_id is still valid # we check kernel command line or files. current = self.get_instance_id() if not current: return None + LOG.info("Hi, I got some system config: %s", sys_cfg) quick_id = _quick_read_instance_id(cmdline_id=self.cmdline_id, dirs=self.seed_dirs) if not quick_id: diff --git a/cloudinit/sources/DataSourceOpenStack.py b/cloudinit/sources/DataSourceOpenStack.py index 79bb9d63..f7f4590b 100644 --- a/cloudinit/sources/DataSourceOpenStack.py +++ b/cloudinit/sources/DataSourceOpenStack.py @@ -150,7 +150,7 @@ class DataSourceOpenStack(openstack.SourceMixin, sources.DataSource): return True - def check_instance_id(self): + def check_instance_id(self, sys_cfg): # quickly (local check only) if self.instance_id is still valid return sources.instance_id_matches_system_uuid(self.get_instance_id()) diff --git a/cloudinit/sources/__init__.py b/cloudinit/sources/__init__.py index c63464b2..82cd3553 100644 --- a/cloudinit/sources/__init__.py +++ b/cloudinit/sources/__init__.py @@ -217,7 +217,7 @@ class DataSource(object): def get_package_mirror_info(self): return self.distro.get_package_mirror_info(data_source=self) - def check_instance_id(self): + def check_instance_id(self, sys_cfg): # quickly (local check only) if self.instance_id is still return False diff --git a/cloudinit/stages.py b/cloudinit/stages.py index 8ebbe6a9..5d6b0447 100644 --- a/cloudinit/stages.py +++ b/cloudinit/stages.py @@ -223,7 +223,8 @@ class Init(object): if ds and existing == "trust": myrep.description = "restored from cache: %s" % ds elif ds and existing == "check": - if hasattr(ds, 'check_instance_id') and ds.check_instance_id(): + if (hasattr(ds, 'check_instance_id') and + ds.check_instance_id(self.cfg)): myrep.description = "restored from checked cache: %s" % ds else: myrep.description = "cache invalid in datasource: %s" % ds -- cgit v1.2.3 From 5eedd9e6f15a49029e00aca83f863c89fdb6d198 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Thu, 24 Mar 2016 13:10:16 -0400 Subject: remove debug code --- cloudinit/sources/DataSourceNoCloud.py | 1 - 1 file changed, 1 deletion(-) diff --git a/cloudinit/sources/DataSourceNoCloud.py b/cloudinit/sources/DataSourceNoCloud.py index f786516b..802d515b 100644 --- a/cloudinit/sources/DataSourceNoCloud.py +++ b/cloudinit/sources/DataSourceNoCloud.py @@ -216,7 +216,6 @@ class DataSourceNoCloud(sources.DataSource): if not current: return None - LOG.info("Hi, I got some system config: %s", sys_cfg) quick_id = _quick_read_instance_id(cmdline_id=self.cmdline_id, dirs=self.seed_dirs) if not quick_id: -- cgit v1.2.3 From 9c0b3fc96fc33107dde8e89b02a63dbfb04e207c Mon Sep 17 00:00:00 2001 From: Ryan Harper Date: Thu, 24 Mar 2016 13:41:25 -0500 Subject: fix review comments net: add render_route comment to document why we added || true to route statements DataSourceConfigDrive: Only convert network_json to network_config when caller reads network_config attr. Cache the conversion. --- cloudinit/net/__init__.py | 14 ++++++++++++++ cloudinit/sources/DataSourceConfigDrive.py | 6 +++--- 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/cloudinit/net/__init__.py b/cloudinit/net/__init__.py index 76cd4e8b..2435055b 100644 --- a/cloudinit/net/__init__.py +++ b/cloudinit/net/__init__.py @@ -349,6 +349,20 @@ def iface_add_attrs(iface): def render_route(route, indent=""): + """ When rendering routes for an iface, in some cases applying a route + may result in the route command returning non-zero which produces + some confusing output for users manually using ifup/ifdown[1]. To + that end, we will optionally include an '|| true' postfix to each + route line allowing users to work with ifup/ifdown without using + --force option. + + We may at somepoint not want to emit this additional postfix, and + add a 'strict' flag to this function. When called with strict=True, + then we will not append the postfix. + + 1. http://askubuntu.com/questions/168033/ + how-to-set-static-routes-in-ubuntu-server + """ content = "" up = indent + "post-up route add" down = indent + "pre-down route del" diff --git a/cloudinit/sources/DataSourceConfigDrive.py b/cloudinit/sources/DataSourceConfigDrive.py index db813f6e..14676f97 100644 --- a/cloudinit/sources/DataSourceConfigDrive.py +++ b/cloudinit/sources/DataSourceConfigDrive.py @@ -153,9 +153,6 @@ class DataSourceConfigDrive(openstack.SourceMixin, sources.DataSource): LOG.warn("Invalid content in network-data: %s", e) self.network_json = None - if self.network_json: - self._network_config = convert_network_data(self.network_json) - return True def check_instance_id(self): @@ -164,6 +161,9 @@ class DataSourceConfigDrive(openstack.SourceMixin, sources.DataSource): @property def network_config(self): + if self._network_config is None: + if self.network_json is not None: + self._network_config = convert_network_data(self.network_json) return self._network_config -- cgit v1.2.3 From 3ad9929efcab614a6ffc170c75c1c6c81b57a2b8 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Thu, 24 Mar 2016 15:30:03 -0400 Subject: make get_cmdline read /proc/1/cmdline if inside a container This follows behavior of systemd/cloud-init-generator. This way you can feed a command line into lxc container. --- cloudinit/util.py | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/cloudinit/util.py b/cloudinit/util.py index 20916e53..0d21e11b 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -80,6 +80,8 @@ CONTAINER_TESTS = (['systemd-detect-virt', '--quiet', '--container'], ['running-in-container'], ['lxc-is-container']) +PROC_CMDLINE = None + def decode_binary(blob, encoding='utf-8'): # Converts a binary type into a text type using given encoding. @@ -1191,12 +1193,27 @@ def load_file(fname, read_cb=None, quiet=False, decode=True): def get_cmdline(): if 'DEBUG_PROC_CMDLINE' in os.environ: - cmdline = os.environ["DEBUG_PROC_CMDLINE"] + return os.environ["DEBUG_PROC_CMDLINE"] + + global PROC_CMDLINE + if PROC_CMDLINE is not None: + return PROC_CMDLINE + + if is_container(): + try: + contents = load_file("/proc/1/cmdline") + # replace nulls with space and drop trailing null + cmdline = contents.replace("\x00", " ")[:-1] + except Exception as e: + LOG.warn("failed reading /proc/1/cmdline: %s", e) + cmdline = "" else: try: cmdline = load_file("/proc/cmdline").strip() except: cmdline = "" + + PROC_CMDLINE = cmdline return cmdline @@ -1569,7 +1586,7 @@ def uptime(): try: if os.path.exists("/proc/uptime"): method = '/proc/uptime' - contents = load_file("/proc/uptime").strip() + contents = load_file("/proc/uptime") if contents: uptime_str = contents.split()[0] else: -- cgit v1.2.3 From 557650728d3c1b1c1bbd29f9292d43133c00cdd4 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Thu, 24 Mar 2016 15:46:52 -0400 Subject: add comments and improve error messages --- cloudinit/net/__init__.py | 20 +++++++++++++++++--- tests/unittests/test_net.py | 2 +- 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/cloudinit/net/__init__.py b/cloudinit/net/__init__.py index e13ca470..7af9b03a 100644 --- a/cloudinit/net/__init__.py +++ b/cloudinit/net/__init__.py @@ -304,6 +304,19 @@ def _load_shell_content(content, add_empty=False, empty_val=None): def _klibc_to_config_entry(content, mac_addrs=None): + """Convert a klibc writtent shell content file to a 'config' entry + When ip= is seen on the kernel command line in debian initramfs + and networking is brought up, ipconfig will populate + /run/net-.cfg. + + The files are shell style syntax, and examples are in the tests + provided here. There is no good documentation on this unfortunately. + + DEVICE= is expected/required and PROTO should indicate if + this is 'static' or 'dhcp'. + """ + + if mac_addrs is None: mac_addrs = {} @@ -575,11 +588,12 @@ def is_disabled_cfg(cfg): def sys_netdev_info(name, field): if not os.path.exists(os.path.join(SYS_CLASS_NET, name)): - raise OSError("%s: interface does not exist in /sys" % name) + raise OSError("%s: interface does not exist in %s" % + (name, SYS_CLASS_NET)) fname = os.path.join(SYS_CLASS_NET, name, field) if not os.path.exists(fname): - raise OSError("%s: %s does not exist in /sys" % (name, fname)) + raise OSError("%s: could not find sysfs entry: %s" % (name, fname)) data = util.load_file(fname) if data[-1] == '\n': data = data[:-1] @@ -647,7 +661,7 @@ def generate_fallback_config(): nconf['config'].append( {'type': 'physical', 'name': target_name, - 'mac_address': mac, 'subnets': [{'type': 'dhcp4'}]}) + 'mac_address': mac, 'subnets': [{'type': 'dhcp'}]}) return nconf diff --git a/tests/unittests/test_net.py b/tests/unittests/test_net.py index 16c44588..dfb31710 100644 --- a/tests/unittests/test_net.py +++ b/tests/unittests/test_net.py @@ -70,7 +70,7 @@ class TestNetConfigParsing(TestCase): simple_cfg = { 'config': [{"type": "physical", "name": "eth0", "mac_address": "c0:d6:9f:2c:e8:80", - "subnets": [{"type": "dhcp4"}]}]} + "subnets": [{"type": "dhcp"}]}]} def test_klibc_convert_dhcp(self): found = net._klibc_to_config_entry(DHCP_CONTENT_1) -- cgit v1.2.3 From e2536e758f22bbce9e1fc49a319738f2c3296930 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Thu, 24 Mar 2016 16:05:18 -0400 Subject: pep8 fixes from last comment merge --- cloudinit/net/__init__.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/cloudinit/net/__init__.py b/cloudinit/net/__init__.py index 66e0e9ee..57beb837 100644 --- a/cloudinit/net/__init__.py +++ b/cloudinit/net/__init__.py @@ -306,7 +306,7 @@ def _load_shell_content(content, add_empty=False, empty_val=None): def _klibc_to_config_entry(content, mac_addrs=None): """Convert a klibc writtent shell content file to a 'config' entry When ip= is seen on the kernel command line in debian initramfs - and networking is brought up, ipconfig will populate + and networking is brought up, ipconfig will populate /run/net-.cfg. The files are shell style syntax, and examples are in the tests @@ -315,8 +315,7 @@ def _klibc_to_config_entry(content, mac_addrs=None): DEVICE= is expected/required and PROTO should indicate if this is 'static' or 'dhcp'. """ - - + if mac_addrs is None: mac_addrs = {} -- cgit v1.2.3 From 6c49afad6134c5094c5e21784e76735faf510a29 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Thu, 24 Mar 2016 17:11:26 -0400 Subject: some final changes a.) do not write systemd link files if we do not have a mac address. the check is updated to check for value rather than just presense (ie, 'mac_address': None) b.) DataSourceNoCloudNet: search in the nocloud seed dir this is important because NoCloud if dsmode is Net will look only would pass by, expecting NoCloudNet to pick it up but NoCloudNet would not look in /var/lib/cloud/seed/nocloud and thus skip it. c.) support the disabling of network configuration via /var/lib/cloud/data/upgraded-network This is what the package upgrader is writing. --- cloudinit/net/__init__.py | 4 ++-- cloudinit/sources/DataSourceNoCloud.py | 1 - cloudinit/stages.py | 5 +++++ 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/cloudinit/net/__init__.py b/cloudinit/net/__init__.py index 57beb837..40929c6e 100644 --- a/cloudinit/net/__init__.py +++ b/cloudinit/net/__init__.py @@ -407,7 +407,7 @@ def render_persistent_net(network_state): for iface in interfaces.values(): # for physical interfaces write out a persist net udev rule if iface['type'] == 'physical' and \ - 'name' in iface and 'mac_address' in iface: + 'name' in iface and iface.get('mac_address'): content += generate_udev_rule(iface['name'], iface['mac_address']) @@ -598,7 +598,7 @@ def render_systemd_links(target, network_state, interfaces = network_state.get('interfaces') for iface in interfaces.values(): if (iface['type'] == 'physical' and 'name' in iface and - 'mac_address' in iface): + iface.get('mac_address')): fname = fp_prefix + iface['name'] + ".link" with open(fname, "w") as fp: fp.write("\n".join([ diff --git a/cloudinit/sources/DataSourceNoCloud.py b/cloudinit/sources/DataSourceNoCloud.py index 802d515b..c2fba4d2 100644 --- a/cloudinit/sources/DataSourceNoCloud.py +++ b/cloudinit/sources/DataSourceNoCloud.py @@ -316,7 +316,6 @@ class DataSourceNoCloudNet(DataSourceNoCloud): DataSourceNoCloud.__init__(self, sys_cfg, distro, paths) self.cmdline_id = "ds=nocloud-net" self.supported_seed_starts = ("http://", "https://", "ftp://") - self.seed_dirs = [os.path.join(paths.seed_dir, 'nocloud-net')] self.dsmode = "net" diff --git a/cloudinit/stages.py b/cloudinit/stages.py index 5d6b0447..143a4fc9 100644 --- a/cloudinit/stages.py +++ b/cloudinit/stages.py @@ -570,6 +570,11 @@ class Init(object): self._do_handlers(user_data_msg, c_handlers_list, frequency) def _find_networking_config(self): + disable_file = os.path.join( + self.paths.get_cpath('data'), 'upgraded-network') + if os.path.exists(disable_file): + return (None, disable_file) + cmdline_cfg = ('cmdline', net.read_kernel_cmdline_config()) dscfg = ('ds', None) if self.datasource and hasattr(self.datasource, 'network_config'): -- cgit v1.2.3 From 20cc8113dde9e6849e8a692aea64cf81a266406d Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Thu, 24 Mar 2016 17:29:35 -0400 Subject: pyflakes --- cloudinit/sources/DataSourceConfigDrive.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cloudinit/sources/DataSourceConfigDrive.py b/cloudinit/sources/DataSourceConfigDrive.py index 14676f97..3fa62ef3 100644 --- a/cloudinit/sources/DataSourceConfigDrive.py +++ b/cloudinit/sources/DataSourceConfigDrive.py @@ -382,8 +382,8 @@ def convert_network_data(network_json=None): }) else: subnet.update({ - 'type': 'static', - 'address': network.get('ip_address'), + 'type': 'static', + 'address': network.get('ip_address'), }) subnets.append(subnet) cfg.update({'subnets': subnets}) @@ -412,7 +412,7 @@ def convert_network_data(network_json=None): }) else: raise ValueError( - 'Unknown network_data link type: %s' % link['type']) + 'Unknown network_data link type: %s' % link['type']) config.append(cfg) -- cgit v1.2.3