diff options
Diffstat (limited to 'cloudinit/net')
| -rw-r--r-- | cloudinit/net/__init__.py | 693 | ||||
| -rw-r--r-- | cloudinit/net/cmdline.py | 203 | ||||
| -rw-r--r-- | cloudinit/net/eni.py | 460 | ||||
| -rw-r--r-- | cloudinit/net/network_state.py | 365 | ||||
| -rw-r--r-- | cloudinit/net/renderer.py | 48 | ||||
| -rw-r--r-- | cloudinit/net/sysconfig.py | 400 |
6 files changed, 1325 insertions, 844 deletions
diff --git a/cloudinit/net/__init__.py b/cloudinit/net/__init__.py index 0066561e..6959ad34 100644 --- a/cloudinit/net/__init__.py +++ b/cloudinit/net/__init__.py @@ -16,42 +16,15 @@ # You should have received a copy of the GNU Affero General Public License # along with Curtin. If not, see <http://www.gnu.org/licenses/>. -import base64 -import copy import errno -import glob -import gzip -import io +import logging import os import re -import shlex -from cloudinit import log as logging -from cloudinit.net import network_state -from cloudinit.net.udev import generate_udev_rule from cloudinit import util LOG = logging.getLogger(__name__) - SYS_CLASS_NET = "/sys/class/net/" -LINKS_FNAME_PREFIX = "etc/systemd/network/50-cloud-init-" - -NET_CONFIG_OPTIONS = [ - "address", "netmask", "broadcast", "network", "metric", "gateway", - "pointtopoint", "media", "mtu", "hostname", "leasehours", "leasetime", - "vendor", "client", "bootfile", "server", "hwaddr", "provider", "frame", - "netnum", "endpoint", "local", "ttl", -] - -NET_CONFIG_COMMANDS = [ - "pre-up", "up", "post-up", "down", "pre-down", "post-down", -] - -NET_CONFIG_BRIDGE_OPTIONS = [ - "bridge_ageing", "bridge_bridgeprio", "bridge_fd", "bridge_gcinit", - "bridge_hello", "bridge_maxage", "bridge_maxwait", "bridge_stp", -] - DEFAULT_PRIMARY_INTERFACE = 'eth0' @@ -61,23 +34,22 @@ def sys_dev_path(devname, path=""): def read_sys_net(devname, path, translate=None, enoent=None, keyerror=None): try: - contents = "" - with open(sys_dev_path(devname, path), "r") as fp: - contents = fp.read().strip() - if translate is None: - return contents - - try: - return translate.get(contents) - except KeyError: - LOG.debug("found unexpected value '%s' in '%s/%s'", contents, - devname, path) - if keyerror is not None: - return keyerror - raise - except OSError as e: - if e.errno == errno.ENOENT and enoent is not None: - return enoent + contents = util.load_file(sys_dev_path(devname, path)) + except (OSError, IOError) as e: + if getattr(e, 'errno', None) == errno.ENOENT: + if enoent is not None: + return enoent + raise + contents = contents.strip() + if translate is None: + return contents + try: + return translate.get(contents) + except KeyError: + LOG.debug("found unexpected value '%s' in '%s/%s'", contents, + devname, path) + if keyerror is not None: + return keyerror raise @@ -128,523 +100,7 @@ def get_devicelist(): class ParserError(Exception): - """Raised when parser has issue parsing the interfaces file.""" - - -def parse_deb_config_data(ifaces, contents, src_dir, src_path): - """Parses the file contents, placing result into ifaces. - - '_source_path' is added to every dictionary entry to define which file - the configration information came from. - - :param ifaces: interface dictionary - :param contents: contents of interfaces file - :param src_dir: directory interfaces file was located - :param src_path: file path the `contents` was read - """ - currif = None - for line in contents.splitlines(): - line = line.strip() - if line.startswith('#'): - continue - split = line.split(' ') - option = split[0] - if option == "source-directory": - parsed_src_dir = split[1] - if not parsed_src_dir.startswith("/"): - parsed_src_dir = os.path.join(src_dir, parsed_src_dir) - for expanded_path in glob.glob(parsed_src_dir): - dir_contents = os.listdir(expanded_path) - dir_contents = [ - os.path.join(expanded_path, path) - for path in dir_contents - if (os.path.isfile(os.path.join(expanded_path, path)) and - re.match("^[a-zA-Z0-9_-]+$", path) is not None) - ] - for entry in dir_contents: - with open(entry, "r") as fp: - src_data = fp.read().strip() - abs_entry = os.path.abspath(entry) - parse_deb_config_data( - ifaces, src_data, - os.path.dirname(abs_entry), abs_entry) - elif option == "source": - new_src_path = split[1] - if not new_src_path.startswith("/"): - new_src_path = os.path.join(src_dir, new_src_path) - for expanded_path in glob.glob(new_src_path): - with open(expanded_path, "r") as fp: - src_data = fp.read().strip() - abs_path = os.path.abspath(expanded_path) - parse_deb_config_data( - ifaces, src_data, - os.path.dirname(abs_path), abs_path) - elif option == "auto": - for iface in split[1:]: - if iface not in ifaces: - ifaces[iface] = { - # Include the source path this interface was found in. - "_source_path": src_path - } - ifaces[iface]['auto'] = True - elif option == "iface": - iface, family, method = split[1:4] - if iface not in ifaces: - ifaces[iface] = { - # Include the source path this interface was found in. - "_source_path": src_path - } - elif 'family' in ifaces[iface]: - raise ParserError( - "Interface %s can only be defined once. " - "Re-defined in '%s'." % (iface, src_path)) - ifaces[iface]['family'] = family - ifaces[iface]['method'] = method - currif = iface - elif option == "hwaddress": - if split[1] == "ether": - val = split[2] - else: - val = split[1] - ifaces[currif]['hwaddress'] = val - elif option in NET_CONFIG_OPTIONS: - ifaces[currif][option] = split[1] - elif option in NET_CONFIG_COMMANDS: - if option not in ifaces[currif]: - ifaces[currif][option] = [] - ifaces[currif][option].append(' '.join(split[1:])) - elif option.startswith('dns-'): - if 'dns' not in ifaces[currif]: - ifaces[currif]['dns'] = {} - if option == 'dns-search': - ifaces[currif]['dns']['search'] = [] - for domain in split[1:]: - ifaces[currif]['dns']['search'].append(domain) - elif option == 'dns-nameservers': - ifaces[currif]['dns']['nameservers'] = [] - for server in split[1:]: - ifaces[currif]['dns']['nameservers'].append(server) - elif option.startswith('bridge_'): - if 'bridge' not in ifaces[currif]: - ifaces[currif]['bridge'] = {} - if option in NET_CONFIG_BRIDGE_OPTIONS: - bridge_option = option.replace('bridge_', '', 1) - ifaces[currif]['bridge'][bridge_option] = split[1] - elif option == "bridge_ports": - ifaces[currif]['bridge']['ports'] = [] - for iface in split[1:]: - ifaces[currif]['bridge']['ports'].append(iface) - elif option == "bridge_hw" and split[1].lower() == "mac": - ifaces[currif]['bridge']['mac'] = split[2] - elif option == "bridge_pathcost": - if 'pathcost' not in ifaces[currif]['bridge']: - ifaces[currif]['bridge']['pathcost'] = {} - ifaces[currif]['bridge']['pathcost'][split[1]] = split[2] - elif option == "bridge_portprio": - if 'portprio' not in ifaces[currif]['bridge']: - ifaces[currif]['bridge']['portprio'] = {} - ifaces[currif]['bridge']['portprio'][split[1]] = split[2] - elif option.startswith('bond-'): - if 'bond' not in ifaces[currif]: - ifaces[currif]['bond'] = {} - bond_option = option.replace('bond-', '', 1) - ifaces[currif]['bond'][bond_option] = split[1] - for iface in ifaces.keys(): - if 'auto' not in ifaces[iface]: - ifaces[iface]['auto'] = False - - -def parse_deb_config(path): - """Parses a debian network configuration file.""" - ifaces = {} - with open(path, "r") as fp: - contents = fp.read().strip() - abs_path = os.path.abspath(path) - parse_deb_config_data( - ifaces, contents, - os.path.dirname(abs_path), abs_path) - return ifaces - - -def parse_net_config_data(net_config): - """Parses the config, returns NetworkState dictionary - - :param net_config: curtin network config dict - """ - state = None - if 'version' in net_config and 'config' in net_config: - ns = network_state.NetworkState(version=net_config.get('version'), - config=net_config.get('config')) - ns.parse_config() - state = ns.network_state - - return state - - -def parse_net_config(path): - """Parses a curtin network configuration file and - return network state""" - ns = None - net_config = util.read_conf(path) - if 'network' in net_config: - ns = parse_net_config_data(net_config.get('network')) - - return ns - - -def _load_shell_content(content, add_empty=False, empty_val=None): - """Given shell like syntax (key=value\nkey2=value2\n) in content - return the data in dictionary form. If 'add_empty' is True - then add entries in to the returned dictionary for 'VAR=' - variables. Set their value to empty_val.""" - data = {} - for line in shlex.split(content): - key, value = line.split("=", 1) - if not value: - value = empty_val - if add_empty or value: - data[key] = value - - return data - - -def _klibc_to_config_entry(content, mac_addrs=None): - """Convert a klibc writtent shell content file to a 'config' entry - When ip= is seen on the kernel command line in debian initramfs - and networking is brought up, ipconfig will populate - /run/net-<name>.cfg. - - The files are shell style syntax, and examples are in the tests - provided here. There is no good documentation on this unfortunately. - - DEVICE=<name> is expected/required and PROTO should indicate if - this is 'static' or 'dhcp'. - """ - - if mac_addrs is None: - mac_addrs = {} - - data = _load_shell_content(content) - try: - name = data['DEVICE'] - except KeyError: - raise ValueError("no 'DEVICE' entry in data") - - # ipconfig on precise does not write PROTO - proto = data.get('PROTO') - if not proto: - if data.get('filename'): - proto = 'dhcp' - else: - proto = 'static' - - if proto not in ('static', 'dhcp'): - raise ValueError("Unexpected value for PROTO: %s" % proto) - - iface = { - 'type': 'physical', - 'name': name, - 'subnets': [], - } - - if name in mac_addrs: - iface['mac_address'] = mac_addrs[name] - - # originally believed there might be IPV6* values - for v, pre in (('ipv4', 'IPV4'),): - # if no IPV4ADDR or IPV6ADDR, then go on. - if pre + "ADDR" not in data: - continue - subnet = {'type': proto, 'control': 'manual'} - - # these fields go right on the subnet - for key in ('NETMASK', 'BROADCAST', 'GATEWAY'): - if pre + key in data: - subnet[key.lower()] = data[pre + key] - - dns = [] - # handle IPV4DNS0 or IPV6DNS0 - for nskey in ('DNS0', 'DNS1'): - ns = data.get(pre + nskey) - # verify it has something other than 0.0.0.0 (or ipv6) - if ns and len(ns.strip(":.0")): - dns.append(data[pre + nskey]) - if dns: - subnet['dns_nameservers'] = dns - # add search to both ipv4 and ipv6, as it has no namespace - search = data.get('DOMAINSEARCH') - if search: - if ',' in search: - subnet['dns_search'] = search.split(",") - else: - subnet['dns_search'] = search.split() - - iface['subnets'].append(subnet) - - return name, iface - - -def config_from_klibc_net_cfg(files=None, mac_addrs=None): - if files is None: - files = glob.glob('/run/net*.conf') - - entries = [] - names = {} - for cfg_file in files: - name, entry = _klibc_to_config_entry(util.load_file(cfg_file), - mac_addrs=mac_addrs) - if name in names: - raise ValueError( - "device '%s' defined multiple times: %s and %s" % ( - name, names[name], cfg_file)) - - names[name] = cfg_file - entries.append(entry) - return {'config': entries, 'version': 1} - - -def render_persistent_net(network_state): - '''Given state, emit udev rules to map mac to ifname.''' - content = "" - interfaces = network_state.get('interfaces') - for iface in interfaces.values(): - # for physical interfaces write out a persist net udev rule - if iface['type'] == 'physical' and \ - 'name' in iface and iface.get('mac_address'): - content += generate_udev_rule(iface['name'], - iface['mac_address']) - - return content - - -# TODO: switch valid_map based on mode inet/inet6 -def iface_add_subnet(iface, subnet): - content = [] - valid_map = [ - 'address', - 'netmask', - 'broadcast', - 'metric', - 'gateway', - 'pointopoint', - 'mtu', - 'scope', - 'dns_search', - 'dns_nameservers', - ] - for key, value in subnet.items(): - if value and key in valid_map: - if type(value) == list: - value = " ".join(value) - if '_' in key: - key = key.replace('_', '-') - content.append(" {} {}".format(key, value)) - - return content - - -# TODO: switch to valid_map for attrs -def iface_add_attrs(iface): - content = [] - ignore_map = [ - 'control', - 'index', - 'inet', - 'mode', - 'name', - 'subnets', - 'type', - ] - if iface['type'] not in ['bond', 'bridge', 'vlan']: - ignore_map.append('mac_address') - - for key, value in iface.items(): - if value and key not in ignore_map: - if type(value) == list: - value = " ".join(value) - content.append(" {} {}".format(key, value)) - - return content - - -def render_route(route, indent=""): - """When rendering routes for an iface, in some cases applying a route - may result in the route command returning non-zero which produces - some confusing output for users manually using ifup/ifdown[1]. To - that end, we will optionally include an '|| true' postfix to each - route line allowing users to work with ifup/ifdown without using - --force option. - - We may at somepoint not want to emit this additional postfix, and - add a 'strict' flag to this function. When called with strict=True, - then we will not append the postfix. - - 1. http://askubuntu.com/questions/168033/ - how-to-set-static-routes-in-ubuntu-server - """ - content = [] - up = indent + "post-up route add" - down = indent + "pre-down route del" - eol = " || true" - mapping = { - 'network': '-net', - 'netmask': 'netmask', - 'gateway': 'gw', - 'metric': 'metric', - } - if route['network'] == '0.0.0.0' and route['netmask'] == '0.0.0.0': - default_gw = " default gw %s" % route['gateway'] - content.append(up + default_gw + eol) - content.append(down + default_gw + eol) - elif route['network'] == '::' and route['netmask'] == 0: - # ipv6! - default_gw = " -A inet6 default gw %s" % route['gateway'] - content.append(up + default_gw + eol) - content.append(down + default_gw + eol) - else: - route_line = "" - for k in ['network', 'netmask', 'gateway', 'metric']: - if k in route: - route_line += " %s %s" % (mapping[k], route[k]) - content.append(up + route_line + eol) - content.append(down + route_line + eol) - - return content - - -def iface_start_entry(iface, index): - fullname = iface['name'] - if index != 0: - fullname += ":%s" % index - - control = iface['control'] - if control == "auto": - cverb = "auto" - elif control in ("hotplug",): - cverb = "allow-" + control - else: - cverb = "# control-" + control - - subst = iface.copy() - subst.update({'fullname': fullname, 'cverb': cverb}) - - if 'inet' not in subst: - print("bug....iface: %s" % iface) - return ["{cverb} {fullname}".format(**subst), - "iface {fullname} {inet} {mode}".format(**subst)] - - -def _render_iface(iface): - lines = [] - subnets = iface.get('subnets', {}) - if subnets: - for index, subnet in zip(range(0, len(subnets)), subnets): - iface['index'] = index - iface['mode'] = subnet['type'] - iface['control'] = subnet.get('control', 'auto') - if iface['mode'].endswith('6'): - iface['inet'] += '6' - elif iface['mode'] == 'static' and ":" in subnet['address']: - iface['inet'] += '6' - if iface['mode'].startswith('dhcp'): - iface['mode'] = 'dhcp' - - lines.extend(iface_start_entry(iface, index)) - lines.extend(iface_add_subnet(iface, subnet)) - lines.extend(iface_add_attrs(iface)) - lines.append("") - else: - # ifenslave docs say to auto the slave devices - if 'bond-master' in iface: - lines.append("auto {name}".format(**iface)) - lines.append("iface {name} {inet} {mode}".format(**iface)) - lines.extend(iface_add_attrs(iface)) - return lines - - -def render_interfaces(network_state): - '''Given state, emit etc/network/interfaces content.''' - - content = "" - interfaces = network_state.get('interfaces') - ''' Apply a sort order to ensure that we write out - the physical interfaces first; this is critical for - bonding - ''' - order = { - 'physical': 0, - 'bond': 1, - 'bridge': 2, - 'vlan': 3, - } - - # handle 'lo' specifically as we need to insert the global dns entries - # there (as that is the only interface) that will be always up. - lo = {'name': 'lo', 'type': 'physical', 'inet': 'inet', - 'subnets': [{'type': 'loopback', 'control': 'auto'}]} - for iface in interfaces.values(): - if iface.get('name') == "lo": - lo = copy.deepcopy(iface) - for dnskey, value in network_state.get('dns', {}).items(): - if len(value): - lo['subnets'][0]["dns_" + dnskey] = value - - sections = [_render_iface(lo)] - - for iface in sorted(interfaces.values(), - key=lambda k: (order[k['type']], k['name'])): - if iface.get('name') == "lo": - continue - sections.append(_render_iface(iface)) - - for route in network_state.get('routes'): - sections.append(render_route(route)) - - content = ''.join(['\n'.join(s) + '\n\n' for s in sections]) - # global replacements until v2 format - content = content.replace('mac_address', 'hwaddress') - return content - - -def render_network_state(target, network_state, eni="etc/network/interfaces", - links_prefix=LINKS_FNAME_PREFIX, - netrules='etc/udev/rules.d/70-persistent-net.rules'): - - fpeni = os.path.sep.join((target, eni,)) - util.ensure_dir(os.path.dirname(fpeni)) - with open(fpeni, 'w+') as f: - f.write(render_interfaces(network_state)) - - if netrules: - netrules = os.path.sep.join((target, netrules,)) - util.ensure_dir(os.path.dirname(netrules)) - with open(netrules, 'w+') as f: - f.write(render_persistent_net(network_state)) - - if links_prefix: - render_systemd_links(target, network_state, links_prefix) - - -def render_systemd_links(target, network_state, - links_prefix=LINKS_FNAME_PREFIX): - fp_prefix = os.path.sep.join((target, links_prefix)) - for f in glob.glob(fp_prefix + "*"): - os.unlink(f) - - interfaces = network_state.get('interfaces') - for iface in interfaces.values(): - if (iface['type'] == 'physical' and 'name' in iface and - iface.get('mac_address')): - fname = fp_prefix + iface['name'] + ".link" - with open(fname, "w") as fp: - fp.write("\n".join([ - "[Match]", - "MACAddress=" + iface['mac_address'], - "", - "[Link]", - "Name=" + iface['name'], - "" - ])) + """Raised when a parser has issue parsing a file/content.""" def is_disabled_cfg(cfg): @@ -657,7 +113,6 @@ def sys_netdev_info(name, field): if not os.path.exists(os.path.join(SYS_CLASS_NET, name)): raise OSError("%s: interface does not exist in %s" % (name, SYS_CLASS_NET)) - fname = os.path.join(SYS_CLASS_NET, name, field) if not os.path.exists(fname): raise OSError("%s: could not find sysfs entry: %s" % (name, fname)) @@ -737,104 +192,6 @@ def generate_fallback_config(): return nconf -def _decomp_gzip(blob, strict=True): - # decompress blob. raise exception if not compressed unless strict=False. - with io.BytesIO(blob) as iobuf: - gzfp = None - try: - gzfp = gzip.GzipFile(mode="rb", fileobj=iobuf) - return gzfp.read() - except IOError: - if strict: - raise - return blob - finally: - if gzfp: - gzfp.close() - - -def _b64dgz(b64str, gzipped="try"): - # decode a base64 string. If gzipped is true, transparently uncompresss - # if gzipped is 'try', then try gunzip, returning the original on fail. - try: - blob = base64.b64decode(b64str) - except TypeError: - raise ValueError("Invalid base64 text: %s" % b64str) - - if not gzipped: - return blob - - return _decomp_gzip(blob, strict=gzipped != "try") - - -def read_kernel_cmdline_config(files=None, mac_addrs=None, cmdline=None): - if cmdline is None: - cmdline = util.get_cmdline() - - if 'network-config=' in cmdline: - data64 = None - for tok in cmdline.split(): - if tok.startswith("network-config="): - data64 = tok.split("=", 1)[1] - if data64: - return util.load_yaml(_b64dgz(data64)) - - if 'ip=' not in cmdline: - return None - - if mac_addrs is None: - mac_addrs = {k: sys_netdev_info(k, 'address') - for k in get_devicelist()} - - return config_from_klibc_net_cfg(files=files, mac_addrs=mac_addrs) - - -def convert_eni_data(eni_data): - # return a network config representation of what is in eni_data - ifaces = {} - parse_deb_config_data(ifaces, eni_data, src_dir=None, src_path=None) - return _ifaces_to_net_config_data(ifaces) - - -def _ifaces_to_net_config_data(ifaces): - """Return network config that represents the ifaces data provided. - ifaces = parse_deb_config("/etc/network/interfaces") - config = ifaces_to_net_config_data(ifaces) - state = parse_net_config_data(config).""" - devs = {} - for name, data in ifaces.items(): - # devname is 'eth0' for name='eth0:1' - devname = name.partition(":")[0] - if devname not in devs: - devs[devname] = {'type': 'physical', 'name': devname, - 'subnets': []} - # this isnt strictly correct, but some might specify - # hwaddress on a nic for matching / declaring name. - if 'hwaddress' in data: - devs[devname]['mac_address'] = data['hwaddress'] - subnet = {'_orig_eni_name': name, 'type': data['method']} - if data.get('auto'): - subnet['control'] = 'auto' - else: - subnet['control'] = 'manual' - - if data.get('method') == 'static': - subnet['address'] = data['address'] - - for copy_key in ('netmask', 'gateway', 'broadcast'): - if copy_key in data: - subnet[copy_key] = data[copy_key] - - if 'dns' in data: - for n in ('nameservers', 'search'): - if n in data['dns'] and data['dns'][n]: - subnet['dns_' + n] = data['dns'][n] - devs[devname]['subnets'].append(subnet) - - return {'version': 1, - 'config': [devs[d] for d in sorted(devs)]} - - def apply_network_config_names(netcfg, strict_present=True, strict_busy=True): """read the network config and rename devices accordingly. if strict_present is false, then do not raise exception if no devices @@ -850,7 +207,7 @@ def apply_network_config_names(netcfg, strict_present=True, strict_busy=True): continue renames.append([mac, name]) - return rename_interfaces(renames) + return _rename_interfaces(renames) def _get_current_rename_info(check_downable=True): @@ -878,8 +235,8 @@ def _get_current_rename_info(check_downable=True): return bymac -def rename_interfaces(renames, strict_present=True, strict_busy=True, - current_info=None): +def _rename_interfaces(renames, strict_present=True, strict_busy=True, + current_info=None): if current_info is None: current_info = _get_current_rename_info() @@ -990,7 +347,13 @@ def get_interface_mac(ifname): def get_interfaces_by_mac(devs=None): """Build a dictionary of tuples {mac: name}""" if devs is None: - devs = get_devicelist() + try: + devs = get_devicelist() + except OSError as e: + if e.errno == errno.ENOENT: + devs = [] + else: + raise ret = {} for name in devs: mac = get_interface_mac(name) diff --git a/cloudinit/net/cmdline.py b/cloudinit/net/cmdline.py new file mode 100644 index 00000000..822a020b --- /dev/null +++ b/cloudinit/net/cmdline.py @@ -0,0 +1,203 @@ +# Copyright (C) 2013-2014 Canonical Ltd. +# +# Author: Scott Moser <scott.moser@canonical.com> +# Author: Blake Rouse <blake.rouse@canonical.com> +# +# Curtin is free software: you can redistribute it and/or modify it under +# the terms of the GNU Affero General Public License as published by the +# Free Software Foundation, either version 3 of the License, or (at your +# option) any later version. +# +# Curtin is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +# FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for +# more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Curtin. If not, see <http://www.gnu.org/licenses/>. + +import base64 +import glob +import gzip +import io +import shlex +import sys + +import six + +from . import get_devicelist +from . import sys_netdev_info + +from cloudinit import util + +PY26 = sys.version_info[0:2] == (2, 6) + + +def _shlex_split(blob): + if PY26 and isinstance(blob, six.text_type): + # Older versions don't support unicode input + blob = blob.encode("utf8") + return shlex.split(blob) + + +def _load_shell_content(content, add_empty=False, empty_val=None): + """Given shell like syntax (key=value\nkey2=value2\n) in content + return the data in dictionary form. If 'add_empty' is True + then add entries in to the returned dictionary for 'VAR=' + variables. Set their value to empty_val.""" + data = {} + for line in _shlex_split(content): + key, value = line.split("=", 1) + if not value: + value = empty_val + if add_empty or value: + data[key] = value + + return data + + +def _klibc_to_config_entry(content, mac_addrs=None): + """Convert a klibc writtent shell content file to a 'config' entry + When ip= is seen on the kernel command line in debian initramfs + and networking is brought up, ipconfig will populate + /run/net-<name>.cfg. + + The files are shell style syntax, and examples are in the tests + provided here. There is no good documentation on this unfortunately. + + DEVICE=<name> is expected/required and PROTO should indicate if + this is 'static' or 'dhcp'. + """ + + if mac_addrs is None: + mac_addrs = {} + + data = _load_shell_content(content) + try: + name = data['DEVICE'] + except KeyError: + raise ValueError("no 'DEVICE' entry in data") + + # ipconfig on precise does not write PROTO + proto = data.get('PROTO') + if not proto: + if data.get('filename'): + proto = 'dhcp' + else: + proto = 'static' + + if proto not in ('static', 'dhcp'): + raise ValueError("Unexpected value for PROTO: %s" % proto) + + iface = { + 'type': 'physical', + 'name': name, + 'subnets': [], + } + + if name in mac_addrs: + iface['mac_address'] = mac_addrs[name] + + # originally believed there might be IPV6* values + for v, pre in (('ipv4', 'IPV4'),): + # if no IPV4ADDR or IPV6ADDR, then go on. + if pre + "ADDR" not in data: + continue + subnet = {'type': proto, 'control': 'manual'} + + # these fields go right on the subnet + for key in ('NETMASK', 'BROADCAST', 'GATEWAY'): + if pre + key in data: + subnet[key.lower()] = data[pre + key] + + dns = [] + # handle IPV4DNS0 or IPV6DNS0 + for nskey in ('DNS0', 'DNS1'): + ns = data.get(pre + nskey) + # verify it has something other than 0.0.0.0 (or ipv6) + if ns and len(ns.strip(":.0")): + dns.append(data[pre + nskey]) + if dns: + subnet['dns_nameservers'] = dns + # add search to both ipv4 and ipv6, as it has no namespace + search = data.get('DOMAINSEARCH') + if search: + if ',' in search: + subnet['dns_search'] = search.split(",") + else: + subnet['dns_search'] = search.split() + + iface['subnets'].append(subnet) + + return name, iface + + +def config_from_klibc_net_cfg(files=None, mac_addrs=None): + if files is None: + files = glob.glob('/run/net*.conf') + + entries = [] + names = {} + for cfg_file in files: + name, entry = _klibc_to_config_entry(util.load_file(cfg_file), + mac_addrs=mac_addrs) + if name in names: + raise ValueError( + "device '%s' defined multiple times: %s and %s" % ( + name, names[name], cfg_file)) + + names[name] = cfg_file + entries.append(entry) + return {'config': entries, 'version': 1} + + +def _decomp_gzip(blob, strict=True): + # decompress blob. raise exception if not compressed unless strict=False. + with io.BytesIO(blob) as iobuf: + gzfp = None + try: + gzfp = gzip.GzipFile(mode="rb", fileobj=iobuf) + return gzfp.read() + except IOError: + if strict: + raise + return blob + finally: + if gzfp: + gzfp.close() + + +def _b64dgz(b64str, gzipped="try"): + # decode a base64 string. If gzipped is true, transparently uncompresss + # if gzipped is 'try', then try gunzip, returning the original on fail. + try: + blob = base64.b64decode(b64str) + except TypeError: + raise ValueError("Invalid base64 text: %s" % b64str) + + if not gzipped: + return blob + + return _decomp_gzip(blob, strict=gzipped != "try") + + +def read_kernel_cmdline_config(files=None, mac_addrs=None, cmdline=None): + if cmdline is None: + cmdline = util.get_cmdline() + + if 'network-config=' in cmdline: + data64 = None + for tok in cmdline.split(): + if tok.startswith("network-config="): + data64 = tok.split("=", 1)[1] + if data64: + return util.load_yaml(_b64dgz(data64)) + + if 'ip=' not in cmdline: + return None + + if mac_addrs is None: + mac_addrs = dict((k, sys_netdev_info(k, 'address')) + for k in get_devicelist()) + + return config_from_klibc_net_cfg(files=files, mac_addrs=mac_addrs) diff --git a/cloudinit/net/eni.py b/cloudinit/net/eni.py new file mode 100644 index 00000000..5a91fcf2 --- /dev/null +++ b/cloudinit/net/eni.py @@ -0,0 +1,460 @@ +# vi: ts=4 expandtab +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. + +import copy +import glob +import os +import re + +from . import ParserError + +from . import renderer + +from cloudinit import util + + +NET_CONFIG_COMMANDS = [ + "pre-up", "up", "post-up", "down", "pre-down", "post-down", +] + +NET_CONFIG_BRIDGE_OPTIONS = [ + "bridge_ageing", "bridge_bridgeprio", "bridge_fd", "bridge_gcinit", + "bridge_hello", "bridge_maxage", "bridge_maxwait", "bridge_stp", +] + +NET_CONFIG_OPTIONS = [ + "address", "netmask", "broadcast", "network", "metric", "gateway", + "pointtopoint", "media", "mtu", "hostname", "leasehours", "leasetime", + "vendor", "client", "bootfile", "server", "hwaddr", "provider", "frame", + "netnum", "endpoint", "local", "ttl", +] + + +# TODO: switch valid_map based on mode inet/inet6 +def _iface_add_subnet(iface, subnet): + content = [] + valid_map = [ + 'address', + 'netmask', + 'broadcast', + 'metric', + 'gateway', + 'pointopoint', + 'mtu', + 'scope', + 'dns_search', + 'dns_nameservers', + ] + for key, value in subnet.items(): + if value and key in valid_map: + if type(value) == list: + value = " ".join(value) + if '_' in key: + key = key.replace('_', '-') + content.append(" {} {}".format(key, value)) + + return content + + +# TODO: switch to valid_map for attrs + +def _iface_add_attrs(iface): + content = [] + ignore_map = [ + 'control', + 'index', + 'inet', + 'mode', + 'name', + 'subnets', + 'type', + ] + if iface['type'] not in ['bond', 'bridge', 'vlan']: + ignore_map.append('mac_address') + + for key, value in iface.items(): + if value and key not in ignore_map: + if type(value) == list: + value = " ".join(value) + content.append(" {} {}".format(key, value)) + + return content + + +def _iface_start_entry(iface, index): + fullname = iface['name'] + if index != 0: + fullname += ":%s" % index + + control = iface['control'] + if control == "auto": + cverb = "auto" + elif control in ("hotplug",): + cverb = "allow-" + control + else: + cverb = "# control-" + control + + subst = iface.copy() + subst.update({'fullname': fullname, 'cverb': cverb}) + + return ["{cverb} {fullname}".format(**subst), + "iface {fullname} {inet} {mode}".format(**subst)] + + +def _parse_deb_config_data(ifaces, contents, src_dir, src_path): + """Parses the file contents, placing result into ifaces. + + '_source_path' is added to every dictionary entry to define which file + the configration information came from. + + :param ifaces: interface dictionary + :param contents: contents of interfaces file + :param src_dir: directory interfaces file was located + :param src_path: file path the `contents` was read + """ + currif = None + for line in contents.splitlines(): + line = line.strip() + if line.startswith('#'): + continue + split = line.split(' ') + option = split[0] + if option == "source-directory": + parsed_src_dir = split[1] + if not parsed_src_dir.startswith("/"): + parsed_src_dir = os.path.join(src_dir, parsed_src_dir) + for expanded_path in glob.glob(parsed_src_dir): + dir_contents = os.listdir(expanded_path) + dir_contents = [ + os.path.join(expanded_path, path) + for path in dir_contents + if (os.path.isfile(os.path.join(expanded_path, path)) and + re.match("^[a-zA-Z0-9_-]+$", path) is not None) + ] + for entry in dir_contents: + with open(entry, "r") as fp: + src_data = fp.read().strip() + abs_entry = os.path.abspath(entry) + _parse_deb_config_data( + ifaces, src_data, + os.path.dirname(abs_entry), abs_entry) + elif option == "source": + new_src_path = split[1] + if not new_src_path.startswith("/"): + new_src_path = os.path.join(src_dir, new_src_path) + for expanded_path in glob.glob(new_src_path): + with open(expanded_path, "r") as fp: + src_data = fp.read().strip() + abs_path = os.path.abspath(expanded_path) + _parse_deb_config_data( + ifaces, src_data, + os.path.dirname(abs_path), abs_path) + elif option == "auto": + for iface in split[1:]: + if iface not in ifaces: + ifaces[iface] = { + # Include the source path this interface was found in. + "_source_path": src_path + } + ifaces[iface]['auto'] = True + elif option == "iface": + iface, family, method = split[1:4] + if iface not in ifaces: + ifaces[iface] = { + # Include the source path this interface was found in. + "_source_path": src_path + } + elif 'family' in ifaces[iface]: + raise ParserError( + "Interface %s can only be defined once. " + "Re-defined in '%s'." % (iface, src_path)) + ifaces[iface]['family'] = family + ifaces[iface]['method'] = method + currif = iface + elif option == "hwaddress": + if split[1] == "ether": + val = split[2] + else: + val = split[1] + ifaces[currif]['hwaddress'] = val + elif option in NET_CONFIG_OPTIONS: + ifaces[currif][option] = split[1] + elif option in NET_CONFIG_COMMANDS: + if option not in ifaces[currif]: + ifaces[currif][option] = [] + ifaces[currif][option].append(' '.join(split[1:])) + elif option.startswith('dns-'): + if 'dns' not in ifaces[currif]: + ifaces[currif]['dns'] = {} + if option == 'dns-search': + ifaces[currif]['dns']['search'] = [] + for domain in split[1:]: + ifaces[currif]['dns']['search'].append(domain) + elif option == 'dns-nameservers': + ifaces[currif]['dns']['nameservers'] = [] + for server in split[1:]: + ifaces[currif]['dns']['nameservers'].append(server) + elif option.startswith('bridge_'): + if 'bridge' not in ifaces[currif]: + ifaces[currif]['bridge'] = {} + if option in NET_CONFIG_BRIDGE_OPTIONS: + bridge_option = option.replace('bridge_', '', 1) + ifaces[currif]['bridge'][bridge_option] = split[1] + elif option == "bridge_ports": + ifaces[currif]['bridge']['ports'] = [] + for iface in split[1:]: + ifaces[currif]['bridge']['ports'].append(iface) + elif option == "bridge_hw" and split[1].lower() == "mac": + ifaces[currif]['bridge']['mac'] = split[2] + elif option == "bridge_pathcost": + if 'pathcost' not in ifaces[currif]['bridge']: + ifaces[currif]['bridge']['pathcost'] = {} + ifaces[currif]['bridge']['pathcost'][split[1]] = split[2] + elif option == "bridge_portprio": + if 'portprio' not in ifaces[currif]['bridge']: + ifaces[currif]['bridge']['portprio'] = {} + ifaces[currif]['bridge']['portprio'][split[1]] = split[2] + elif option.startswith('bond-'): + if 'bond' not in ifaces[currif]: + ifaces[currif]['bond'] = {} + bond_option = option.replace('bond-', '', 1) + ifaces[currif]['bond'][bond_option] = split[1] + for iface in ifaces.keys(): + if 'auto' not in ifaces[iface]: + ifaces[iface]['auto'] = False + + +def parse_deb_config(path): + """Parses a debian network configuration file.""" + ifaces = {} + with open(path, "r") as fp: + contents = fp.read().strip() + abs_path = os.path.abspath(path) + _parse_deb_config_data( + ifaces, contents, + os.path.dirname(abs_path), abs_path) + return ifaces + + +def convert_eni_data(eni_data): + # return a network config representation of what is in eni_data + ifaces = {} + _parse_deb_config_data(ifaces, eni_data, src_dir=None, src_path=None) + return _ifaces_to_net_config_data(ifaces) + + +def _ifaces_to_net_config_data(ifaces): + """Return network config that represents the ifaces data provided. + ifaces = parse_deb_config("/etc/network/interfaces") + config = ifaces_to_net_config_data(ifaces) + state = parse_net_config_data(config).""" + devs = {} + for name, data in ifaces.items(): + # devname is 'eth0' for name='eth0:1' + devname = name.partition(":")[0] + if devname == "lo": + # currently provding 'lo' in network config results in duplicate + # entries. in rendered interfaces file. so skip it. + continue + if devname not in devs: + devs[devname] = {'type': 'physical', 'name': devname, + 'subnets': []} + # this isnt strictly correct, but some might specify + # hwaddress on a nic for matching / declaring name. + if 'hwaddress' in data: + devs[devname]['mac_address'] = data['hwaddress'] + subnet = {'_orig_eni_name': name, 'type': data['method']} + if data.get('auto'): + subnet['control'] = 'auto' + else: + subnet['control'] = 'manual' + + if data.get('method') == 'static': + subnet['address'] = data['address'] + + for copy_key in ('netmask', 'gateway', 'broadcast'): + if copy_key in data: + subnet[copy_key] = data[copy_key] + + if 'dns' in data: + for n in ('nameservers', 'search'): + if n in data['dns'] and data['dns'][n]: + subnet['dns_' + n] = data['dns'][n] + devs[devname]['subnets'].append(subnet) + + return {'version': 1, + 'config': [devs[d] for d in sorted(devs)]} + + +class Renderer(renderer.Renderer): + """Renders network information in a /etc/network/interfaces format.""" + + def __init__(self, config=None): + if not config: + config = {} + self.eni_path = config.get('eni_path', 'etc/network/interfaces') + self.links_path_prefix = config.get( + 'links_path_prefix', 'etc/systemd/network/50-cloud-init-') + self.netrules_path = config.get( + 'netrules_path', 'etc/udev/rules.d/70-persistent-net.rules') + + def _render_route(self, route, indent=""): + """When rendering routes for an iface, in some cases applying a route + may result in the route command returning non-zero which produces + some confusing output for users manually using ifup/ifdown[1]. To + that end, we will optionally include an '|| true' postfix to each + route line allowing users to work with ifup/ifdown without using + --force option. + + We may at somepoint not want to emit this additional postfix, and + add a 'strict' flag to this function. When called with strict=True, + then we will not append the postfix. + + 1. http://askubuntu.com/questions/168033/ + how-to-set-static-routes-in-ubuntu-server + """ + content = [] + up = indent + "post-up route add" + down = indent + "pre-down route del" + or_true = " || true" + mapping = { + 'network': '-net', + 'netmask': 'netmask', + 'gateway': 'gw', + 'metric': 'metric', + } + if route['network'] == '0.0.0.0' and route['netmask'] == '0.0.0.0': + default_gw = " default gw %s" % route['gateway'] + content.append(up + default_gw + or_true) + content.append(down + default_gw + or_true) + elif route['network'] == '::' and route['netmask'] == 0: + # ipv6! + default_gw = " -A inet6 default gw %s" % route['gateway'] + content.append(up + default_gw + or_true) + content.append(down + default_gw + or_true) + else: + route_line = "" + for k in ['network', 'netmask', 'gateway', 'metric']: + if k in route: + route_line += " %s %s" % (mapping[k], route[k]) + content.append(up + route_line + or_true) + content.append(down + route_line + or_true) + return content + + def _render_iface(self, iface): + lines = [] + subnets = iface.get('subnets', {}) + if subnets: + for index, subnet in zip(range(0, len(subnets)), subnets): + iface['index'] = index + iface['mode'] = subnet['type'] + iface['control'] = subnet.get('control', 'auto') + if iface['mode'].endswith('6'): + iface['inet'] += '6' + elif iface['mode'] == 'static' and ":" in subnet['address']: + iface['inet'] += '6' + if iface['mode'].startswith('dhcp'): + iface['mode'] = 'dhcp' + + lines.extend(_iface_start_entry(iface, index)) + lines.extend(_iface_add_subnet(iface, subnet)) + lines.extend(_iface_add_attrs(iface)) + lines.append("") + else: + # ifenslave docs say to auto the slave devices + if 'bond-master' in iface: + lines.append("auto {name}".format(**iface)) + lines.append("iface {name} {inet} {mode}".format(**iface)) + lines.extend(_iface_add_attrs(iface)) + return lines + + def _render_interfaces(self, network_state): + '''Given state, emit etc/network/interfaces content.''' + + content = "" + + # handle 'lo' specifically as we need to insert the global dns entries + # there (as that is the only interface) that will be always up. + lo = {'name': 'lo', 'type': 'physical', 'inet': 'inet', + 'subnets': [{'type': 'loopback', 'control': 'auto'}]} + for iface in network_state.iter_interfaces(): + if iface.get('name') == "lo": + lo = copy.deepcopy(iface) + + nameservers = network_state.dns_nameservers + if nameservers: + lo['subnets'][0]["dns_nameservers"] = (" ".join(nameservers)) + + searchdomains = network_state.dns_searchdomains + if searchdomains: + lo['subnets'][0]["dns_search"] = (" ".join(searchdomains)) + + ''' Apply a sort order to ensure that we write out + the physical interfaces first; this is critical for + bonding + ''' + order = { + 'physical': 0, + 'bond': 1, + 'bridge': 2, + 'vlan': 3, + } + + sections = [self._render_iface(lo)] + for iface in sorted(network_state.iter_interfaces(), + key=lambda k: (order[k['type']], k['name'])): + + if iface.get('name') == "lo": + continue + sections.append(self._render_iface(iface)) + + for route in network_state.iter_routes(): + sections.append(self._render_route(route)) + + # global replacements until v2 format + content = ''.join(['\n'.join(s) + '\n\n' for s in sections]) + return content + + def render_network_state(self, target, network_state): + fpeni = os.path.join(target, self.eni_path) + util.ensure_dir(os.path.dirname(fpeni)) + util.write_file(fpeni, self._render_interfaces(network_state)) + + if self.netrules_path: + netrules = os.path.join(target, self.netrules_path) + util.ensure_dir(os.path.dirname(netrules)) + util.write_file(netrules, + self._render_persistent_net(network_state)) + + if self.links_path_prefix: + self._render_systemd_links(target, network_state, + links_prefix=self.links_path_prefix) + + def _render_systemd_links(self, target, network_state, links_prefix): + fp_prefix = os.path.join(target, links_prefix) + for f in glob.glob(fp_prefix + "*"): + os.unlink(f) + for iface in network_state.iter_interfaces(): + if (iface['type'] == 'physical' and 'name' in iface and + iface.get('mac_address')): + fname = fp_prefix + iface['name'] + ".link" + content = "\n".join([ + "[Match]", + "MACAddress=" + iface['mac_address'], + "", + "[Link]", + "Name=" + iface['name'], + "" + ]) + util.write_file(fname, content) diff --git a/cloudinit/net/network_state.py b/cloudinit/net/network_state.py index 4c726ab4..8ca5106f 100644 --- a/cloudinit/net/network_state.py +++ b/cloudinit/net/network_state.py @@ -15,9 +15,13 @@ # You should have received a copy of the GNU Affero General Public License # along with Curtin. If not, see <http://www.gnu.org/licenses/>. -from cloudinit import log as logging +import copy +import functools +import logging + +import six + from cloudinit import util -from cloudinit.util import yaml_dumps as dump_config LOG = logging.getLogger(__name__) @@ -27,80 +31,198 @@ NETWORK_STATE_REQUIRED_KEYS = { } +def parse_net_config_data(net_config, skip_broken=True): + """Parses the config, returns NetworkState object + + :param net_config: curtin network config dict + """ + state = None + if 'version' in net_config and 'config' in net_config: + nsi = NetworkStateInterpreter(version=net_config.get('version'), + config=net_config.get('config')) + nsi.parse_config(skip_broken=skip_broken) + state = nsi.network_state + return state + + +def parse_net_config(path, skip_broken=True): + """Parses a curtin network configuration file and + return network state""" + ns = None + net_config = util.read_conf(path) + if 'network' in net_config: + ns = parse_net_config_data(net_config.get('network'), + skip_broken=skip_broken) + return ns + + def from_state_file(state_file): - network_state = None state = util.read_conf(state_file) - network_state = NetworkState() - network_state.load(state) + nsi = NetworkStateInterpreter() + nsi.load(state) + return nsi + + +def diff_keys(expected, actual): + missing = set(expected) + for key in actual: + missing.discard(key) + return missing + + +class InvalidCommand(Exception): + pass + + +def ensure_command_keys(required_keys): + + def wrapper(func): + + @functools.wraps(func) + def decorator(self, command, *args, **kwargs): + if required_keys: + missing_keys = diff_keys(required_keys, command) + if missing_keys: + raise InvalidCommand("Command missing %s of required" + " keys %s" % (missing_keys, + required_keys)) + return func(self, command, *args, **kwargs) + + return decorator + + return wrapper + + +class CommandHandlerMeta(type): + """Metaclass that dynamically creates a 'command_handlers' attribute. - return network_state + This will scan the to-be-created class for methods that start with + 'handle_' and on finding those will populate a class attribute mapping + so that those methods can be quickly located and called. + """ + def __new__(cls, name, parents, dct): + command_handlers = {} + for attr_name, attr in dct.items(): + if callable(attr) and attr_name.startswith('handle_'): + handles_what = attr_name[len('handle_'):] + if handles_what: + command_handlers[handles_what] = attr + dct['command_handlers'] = command_handlers + return super(CommandHandlerMeta, cls).__new__(cls, name, + parents, dct) class NetworkState(object): - def __init__(self, version=NETWORK_STATE_VERSION, config=None): - self.version = version - self.config = config - self.network_state = { - 'interfaces': {}, - 'routes': [], - 'dns': { - 'nameservers': [], - 'search': [], - } + + def __init__(self, network_state, version=NETWORK_STATE_VERSION): + self._network_state = copy.deepcopy(network_state) + self._version = version + + @property + def version(self): + return self._version + + def iter_routes(self, filter_func=None): + for route in self._network_state.get('routes', []): + if filter_func is not None: + if filter_func(route): + yield route + else: + yield route + + @property + def dns_nameservers(self): + try: + return self._network_state['dns']['nameservers'] + except KeyError: + return [] + + @property + def dns_searchdomains(self): + try: + return self._network_state['dns']['search'] + except KeyError: + return [] + + def iter_interfaces(self, filter_func=None): + ifaces = self._network_state.get('interfaces', {}) + for iface in six.itervalues(ifaces): + if filter_func is None: + yield iface + else: + if filter_func(iface): + yield iface + + +@six.add_metaclass(CommandHandlerMeta) +class NetworkStateInterpreter(object): + + initial_network_state = { + 'interfaces': {}, + 'routes': [], + 'dns': { + 'nameservers': [], + 'search': [], } - self.command_handlers = self.get_command_handlers() + } - def get_command_handlers(self): - METHOD_PREFIX = 'handle_' - methods = filter(lambda x: callable(getattr(self, x)) and - x.startswith(METHOD_PREFIX), dir(self)) - handlers = {} - for m in methods: - key = m.replace(METHOD_PREFIX, '') - handlers[key] = getattr(self, m) + def __init__(self, version=NETWORK_STATE_VERSION, config=None): + self._version = version + self._config = config + self._network_state = copy.deepcopy(self.initial_network_state) + self._parsed = False - return handlers + @property + def network_state(self): + return NetworkState(self._network_state, version=self._version) def dump(self): state = { - 'version': self.version, - 'config': self.config, - 'network_state': self.network_state, + 'version': self._version, + 'config': self._config, + 'network_state': self._network_state, } - return dump_config(state) + return util.yaml_dumps(state) def load(self, state): if 'version' not in state: LOG.error('Invalid state, missing version field') - raise Exception('Invalid state, missing version field') + raise ValueError('Invalid state, missing version field') required_keys = NETWORK_STATE_REQUIRED_KEYS[state['version']] - if not self.valid_command(state, required_keys): - msg = 'Invalid state, missing keys: {}'.format(required_keys) + missing_keys = diff_keys(required_keys, state) + if missing_keys: + msg = 'Invalid state, missing keys: %s' % (missing_keys) LOG.error(msg) - raise Exception(msg) + raise ValueError(msg) # v1 - direct attr mapping, except version for key in [k for k in required_keys if k not in ['version']]: setattr(self, key, state[key]) - self.command_handlers = self.get_command_handlers() def dump_network_state(self): - return dump_config(self.network_state) + return util.yaml_dumps(self._network_state) - def parse_config(self): + def parse_config(self, skip_broken=True): # rebuild network state - for command in self.config: - handler = self.command_handlers.get(command['type']) - handler(command) - - def valid_command(self, command, required_keys): - if not required_keys: - return False - - found_keys = [key for key in command.keys() if key in required_keys] - return len(found_keys) == len(required_keys) - + for command in self._config: + command_type = command['type'] + try: + handler = self.command_handlers[command_type] + except KeyError: + raise RuntimeError("No handler found for" + " command '%s'" % command_type) + try: + handler(self, command) + except InvalidCommand: + if not skip_broken: + raise + else: + LOG.warn("Skipping invalid command: %s", command, + exc_info=True) + LOG.debug(self.dump_network_state()) + + @ensure_command_keys(['name']) def handle_physical(self, command): ''' command = { @@ -112,15 +234,8 @@ class NetworkState(object): ] } ''' - required_keys = [ - 'name', - ] - if not self.valid_command(command, required_keys): - LOG.warn('Skipping Invalid command: {}'.format(command)) - LOG.debug(self.dump_network_state()) - return - interfaces = self.network_state.get('interfaces') + interfaces = self._network_state.get('interfaces', {}) iface = interfaces.get(command['name'], {}) for param, val in command.get('params', {}).items(): iface.update({param: val}) @@ -146,9 +261,10 @@ class NetworkState(object): 'gateway': None, 'subnets': subnets, }) - self.network_state['interfaces'].update({command.get('name'): iface}) + self._network_state['interfaces'].update({command.get('name'): iface}) self.dump_network_state() + @ensure_command_keys(['name', 'vlan_id', 'vlan_link']) def handle_vlan(self, command): ''' auto eth0.222 @@ -158,23 +274,14 @@ class NetworkState(object): hwaddress ether BC:76:4E:06:96:B3 vlan-raw-device eth0 ''' - required_keys = [ - 'name', - 'vlan_link', - 'vlan_id', - ] - if not self.valid_command(command, required_keys): - print('Skipping Invalid command: {}'.format(command)) - print(self.dump_network_state()) - return - - interfaces = self.network_state.get('interfaces') + interfaces = self._network_state.get('interfaces', {}) self.handle_physical(command) iface = interfaces.get(command.get('name'), {}) iface['vlan-raw-device'] = command.get('vlan_link') iface['vlan_id'] = command.get('vlan_id') interfaces.update({iface['name']: iface}) + @ensure_command_keys(['name', 'bond_interfaces', 'params']) def handle_bond(self, command): ''' #/etc/network/interfaces @@ -200,23 +307,14 @@ class NetworkState(object): bond-updelay 200 bond-lacp-rate 4 ''' - required_keys = [ - 'name', - 'bond_interfaces', - 'params', - ] - if not self.valid_command(command, required_keys): - print('Skipping Invalid command: {}'.format(command)) - print(self.dump_network_state()) - return self.handle_physical(command) - interfaces = self.network_state.get('interfaces') + interfaces = self._network_state.get('interfaces') iface = interfaces.get(command.get('name'), {}) for param, val in command.get('params').items(): iface.update({param: val}) iface.update({'bond-slaves': 'none'}) - self.network_state['interfaces'].update({iface['name']: iface}) + self._network_state['interfaces'].update({iface['name']: iface}) # handle bond slaves for ifname in command.get('bond_interfaces'): @@ -228,14 +326,15 @@ class NetworkState(object): # inject placeholder self.handle_physical(cmd) - interfaces = self.network_state.get('interfaces') + interfaces = self._network_state.get('interfaces', {}) bond_if = interfaces.get(ifname) bond_if['bond-master'] = command.get('name') # copy in bond config into slave for param, val in command.get('params').items(): bond_if.update({param: val}) - self.network_state['interfaces'].update({ifname: bond_if}) + self._network_state['interfaces'].update({ifname: bond_if}) + @ensure_command_keys(['name', 'bridge_interfaces', 'params']) def handle_bridge(self, command): ''' auto br0 @@ -263,19 +362,10 @@ class NetworkState(object): "bridge_waitport", ] ''' - required_keys = [ - 'name', - 'bridge_interfaces', - 'params', - ] - if not self.valid_command(command, required_keys): - print('Skipping Invalid command: {}'.format(command)) - print(self.dump_network_state()) - return # find one of the bridge port ifaces to get mac_addr # handle bridge_slaves - interfaces = self.network_state.get('interfaces') + interfaces = self._network_state.get('interfaces', {}) for ifname in command.get('bridge_interfaces'): if ifname in interfaces: continue @@ -286,7 +376,7 @@ class NetworkState(object): # inject placeholder self.handle_physical(cmd) - interfaces = self.network_state.get('interfaces') + interfaces = self._network_state.get('interfaces', {}) self.handle_physical(command) iface = interfaces.get(command.get('name'), {}) iface['bridge_ports'] = command['bridge_interfaces'] @@ -295,16 +385,9 @@ class NetworkState(object): interfaces.update({iface['name']: iface}) + @ensure_command_keys(['address']) def handle_nameserver(self, command): - required_keys = [ - 'address', - ] - if not self.valid_command(command, required_keys): - print('Skipping Invalid command: {}'.format(command)) - print(self.dump_network_state()) - return - - dns = self.network_state.get('dns') + dns = self._network_state.get('dns') if 'address' in command: addrs = command['address'] if not type(addrs) == list: @@ -318,16 +401,9 @@ class NetworkState(object): for path in paths: dns['search'].append(path) + @ensure_command_keys(['destination']) def handle_route(self, command): - required_keys = [ - 'destination', - ] - if not self.valid_command(command, required_keys): - print('Skipping Invalid command: {}'.format(command)) - print(self.dump_network_state()) - return - - routes = self.network_state.get('routes') + routes = self._network_state.get('routes', []) network, cidr = command['destination'].split("/") netmask = cidr2mask(int(cidr)) route = { @@ -376,72 +452,3 @@ def mask2cidr(mask): return ipv4mask2cidr(mask) else: return mask - - -if __name__ == '__main__': - import random - import sys - - from cloudinit import net - - def load_config(nc): - version = nc.get('version') - config = nc.get('config') - return (version, config) - - def test_parse(network_config): - (version, config) = load_config(network_config) - ns1 = NetworkState(version=version, config=config) - ns1.parse_config() - random.shuffle(config) - ns2 = NetworkState(version=version, config=config) - ns2.parse_config() - print("----NS1-----") - print(ns1.dump_network_state()) - print() - print("----NS2-----") - print(ns2.dump_network_state()) - print("NS1 == NS2 ?=> {}".format( - ns1.network_state == ns2.network_state)) - eni = net.render_interfaces(ns2.network_state) - print(eni) - udev_rules = net.render_persistent_net(ns2.network_state) - print(udev_rules) - - def test_dump_and_load(network_config): - print("Loading network_config into NetworkState") - (version, config) = load_config(network_config) - ns1 = NetworkState(version=version, config=config) - ns1.parse_config() - print("Dumping state to file") - ns1_dump = ns1.dump() - ns1_state = "/tmp/ns1.state" - with open(ns1_state, "w+") as f: - f.write(ns1_dump) - - print("Loading state from file") - ns2 = from_state_file(ns1_state) - print("NS1 == NS2 ?=> {}".format( - ns1.network_state == ns2.network_state)) - - def test_output(network_config): - (version, config) = load_config(network_config) - ns1 = NetworkState(version=version, config=config) - ns1.parse_config() - random.shuffle(config) - ns2 = NetworkState(version=version, config=config) - ns2.parse_config() - print("NS1 == NS2 ?=> {}".format( - ns1.network_state == ns2.network_state)) - eni_1 = net.render_interfaces(ns1.network_state) - eni_2 = net.render_interfaces(ns2.network_state) - print(eni_1) - print(eni_2) - print("eni_1 == eni_2 ?=> {}".format( - eni_1 == eni_2)) - - y = util.read_conf(sys.argv[1]) - network_config = y.get('network') - test_parse(network_config) - test_dump_and_load(network_config) - test_output(network_config) diff --git a/cloudinit/net/renderer.py b/cloudinit/net/renderer.py new file mode 100644 index 00000000..310cbe0d --- /dev/null +++ b/cloudinit/net/renderer.py @@ -0,0 +1,48 @@ +# Copyright (C) 2013-2014 Canonical Ltd. +# +# Author: Scott Moser <scott.moser@canonical.com> +# Author: Blake Rouse <blake.rouse@canonical.com> +# +# Curtin is free software: you can redistribute it and/or modify it under +# the terms of the GNU Affero General Public License as published by the +# Free Software Foundation, either version 3 of the License, or (at your +# option) any later version. +# +# Curtin is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +# FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for +# more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Curtin. If not, see <http://www.gnu.org/licenses/>. + +import six + +from .udev import generate_udev_rule + + +def filter_by_type(match_type): + return lambda iface: match_type == iface['type'] + + +def filter_by_name(match_name): + return lambda iface: match_name == iface['name'] + + +filter_by_physical = filter_by_type('physical') + + +class Renderer(object): + + @staticmethod + def _render_persistent_net(network_state): + """Given state, emit udev rules to map mac to ifname.""" + # TODO(harlowja): this seems shared between eni renderer and + # this, so move it to a shared location. + content = six.StringIO() + for iface in network_state.iter_interfaces(filter_by_physical): + # for physical interfaces write out a persist net udev rule + if 'name' in iface and iface.get('mac_address'): + content.write(generate_udev_rule(iface['name'], + iface['mac_address'])) + return content.getvalue() diff --git a/cloudinit/net/sysconfig.py b/cloudinit/net/sysconfig.py new file mode 100644 index 00000000..c53acf71 --- /dev/null +++ b/cloudinit/net/sysconfig.py @@ -0,0 +1,400 @@ +# vi: ts=4 expandtab +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. + +import os +import re + +import six + +from cloudinit.distros.parsers import resolv_conf +from cloudinit import util + +from . import renderer + + +def _make_header(sep='#'): + lines = [ + "Created by cloud-init on instance boot automatically, do not edit.", + "", + ] + for i in range(0, len(lines)): + if lines[i]: + lines[i] = sep + " " + lines[i] + else: + lines[i] = sep + return "\n".join(lines) + + +def _is_default_route(route): + if route['network'] == '::' and route['netmask'] == 0: + return True + if route['network'] == '0.0.0.0' and route['netmask'] == '0.0.0.0': + return True + return False + + +def _quote_value(value): + if re.search(r"\s", value): + # This doesn't handle complex cases... + if value.startswith('"') and value.endswith('"'): + return value + else: + return '"%s"' % value + else: + return value + + +class ConfigMap(object): + """Sysconfig like dictionary object.""" + + # Why does redhat prefer yes/no to true/false?? + _bool_map = { + True: 'yes', + False: 'no', + } + + def __init__(self): + self._conf = {} + + def __setitem__(self, key, value): + self._conf[key] = value + + def drop(self, key): + self._conf.pop(key, None) + + def __len__(self): + return len(self._conf) + + def to_string(self): + buf = six.StringIO() + buf.write(_make_header()) + if self._conf: + buf.write("\n") + for key in sorted(self._conf.keys()): + value = self._conf[key] + if isinstance(value, bool): + value = self._bool_map[value] + if not isinstance(value, six.string_types): + value = str(value) + buf.write("%s=%s\n" % (key, _quote_value(value))) + return buf.getvalue() + + +class Route(ConfigMap): + """Represents a route configuration.""" + + route_fn_tpl = '%(base)s/network-scripts/route-%(name)s' + + def __init__(self, route_name, base_sysconf_dir): + super(Route, self).__init__() + self.last_idx = 1 + self.has_set_default = False + self._route_name = route_name + self._base_sysconf_dir = base_sysconf_dir + + def copy(self): + r = Route(self._route_name, self._base_sysconf_dir) + r._conf = self._conf.copy() + r.last_idx = self.last_idx + r.has_set_default = self.has_set_default + return r + + @property + def path(self): + return self.route_fn_tpl % ({'base': self._base_sysconf_dir, + 'name': self._route_name}) + + +class NetInterface(ConfigMap): + """Represents a sysconfig/networking-script (and its config + children).""" + + iface_fn_tpl = '%(base)s/network-scripts/ifcfg-%(name)s' + + iface_types = { + 'ethernet': 'Ethernet', + 'bond': 'Bond', + 'bridge': 'Bridge', + } + + def __init__(self, iface_name, base_sysconf_dir, kind='ethernet'): + super(NetInterface, self).__init__() + self.children = [] + self.routes = Route(iface_name, base_sysconf_dir) + self._kind = kind + self._iface_name = iface_name + self._conf['DEVICE'] = iface_name + self._conf['TYPE'] = self.iface_types[kind] + self._base_sysconf_dir = base_sysconf_dir + + @property + def name(self): + return self._iface_name + + @name.setter + def name(self, iface_name): + self._iface_name = iface_name + self._conf['DEVICE'] = iface_name + + @property + def kind(self): + return self._kind + + @kind.setter + def kind(self, kind): + self._kind = kind + self._conf['TYPE'] = self.iface_types[kind] + + @property + def path(self): + return self.iface_fn_tpl % ({'base': self._base_sysconf_dir, + 'name': self.name}) + + def copy(self, copy_children=False, copy_routes=False): + c = NetInterface(self.name, self._base_sysconf_dir, kind=self._kind) + c._conf = self._conf.copy() + if copy_children: + c.children = list(self.children) + if copy_routes: + c.routes = self.routes.copy() + return c + + +class Renderer(renderer.Renderer): + """Renders network information in a /etc/sysconfig format.""" + + # See: https://access.redhat.com/documentation/en-US/\ + # Red_Hat_Enterprise_Linux/6/html/Deployment_Guide/\ + # s1-networkscripts-interfaces.html (or other docs for + # details about this) + + iface_defaults = tuple([ + ('ONBOOT', True), + ('USERCTL', False), + ('NM_CONTROLLED', False), + ('BOOTPROTO', 'none'), + ]) + + # If these keys exist, then there values will be used to form + # a BONDING_OPTS grouping; otherwise no grouping will be set. + bond_tpl_opts = tuple([ + ('bond_mode', "mode=%s"), + ('bond_xmit_hash_policy', "xmit_hash_policy=%s"), + ('bond_miimon', "miimon=%s"), + ]) + + bridge_opts_keys = tuple([ + ('bridge_stp', 'STP'), + ('bridge_ageing', 'AGEING'), + ('bridge_bridgeprio', 'PRIO'), + ]) + + def __init__(self, config=None): + if not config: + config = {} + self.sysconf_dir = config.get('sysconf_dir', 'etc/sysconfig/') + self.netrules_path = config.get( + 'netrules_path', 'etc/udev/rules.d/70-persistent-net.rules') + self.dns_path = config.get('dns_path', 'etc/resolv.conf') + + @classmethod + def _render_iface_shared(cls, iface, iface_cfg): + for k, v in cls.iface_defaults: + iface_cfg[k] = v + for (old_key, new_key) in [('mac_address', 'HWADDR'), ('mtu', 'MTU')]: + old_value = iface.get(old_key) + if old_value is not None: + iface_cfg[new_key] = old_value + + @classmethod + def _render_subnet(cls, iface_cfg, route_cfg, subnet): + subnet_type = subnet.get('type') + if subnet_type == 'dhcp6': + iface_cfg['DHCPV6C'] = True + iface_cfg['IPV6INIT'] = True + iface_cfg['BOOTPROTO'] = 'dhcp' + elif subnet_type in ['dhcp4', 'dhcp']: + iface_cfg['BOOTPROTO'] = 'dhcp' + elif subnet_type == 'static': + iface_cfg['BOOTPROTO'] = 'static' + if subnet.get('ipv6'): + iface_cfg['IPV6ADDR'] = subnet['address'] + iface_cfg['IPV6INIT'] = True + else: + iface_cfg['IPADDR'] = subnet['address'] + else: + raise ValueError("Unknown subnet type '%s' found" + " for interface '%s'" % (subnet_type, + iface_cfg.name)) + if 'netmask' in subnet: + iface_cfg['NETMASK'] = subnet['netmask'] + for route in subnet.get('routes', []): + if _is_default_route(route): + if route_cfg.has_set_default: + raise ValueError("Duplicate declaration of default" + " route found for interface '%s'" + % (iface_cfg.name)) + # NOTE(harlowja): ipv6 and ipv4 default gateways + gw_key = 'GATEWAY0' + nm_key = 'NETMASK0' + addr_key = 'ADDRESS0' + # The owning interface provides the default route. + # + # TODO(harlowja): add validation that no other iface has + # also provided the default route? + iface_cfg['DEFROUTE'] = True + if 'gateway' in route: + iface_cfg['GATEWAY'] = route['gateway'] + route_cfg.has_set_default = True + else: + gw_key = 'GATEWAY%s' % route_cfg.last_idx + nm_key = 'NETMASK%s' % route_cfg.last_idx + addr_key = 'ADDRESS%s' % route_cfg.last_idx + route_cfg.last_idx += 1 + for (old_key, new_key) in [('gateway', gw_key), + ('netmask', nm_key), + ('network', addr_key)]: + if old_key in route: + route_cfg[new_key] = route[old_key] + + @classmethod + def _render_bonding_opts(cls, iface_cfg, iface): + bond_opts = [] + for (bond_key, value_tpl) in cls.bond_tpl_opts: + # Seems like either dash or underscore is possible? + bond_keys = [bond_key, bond_key.replace("_", "-")] + for bond_key in bond_keys: + if bond_key in iface: + bond_value = iface[bond_key] + if isinstance(bond_value, (tuple, list)): + bond_value = " ".join(bond_value) + bond_opts.append(value_tpl % (bond_value)) + break + if bond_opts: + iface_cfg['BONDING_OPTS'] = " ".join(bond_opts) + + @classmethod + def _render_physical_interfaces(cls, network_state, iface_contents): + physical_filter = renderer.filter_by_physical + for iface in network_state.iter_interfaces(physical_filter): + iface_name = iface['name'] + iface_subnets = iface.get("subnets", []) + iface_cfg = iface_contents[iface_name] + route_cfg = iface_cfg.routes + if len(iface_subnets) == 1: + cls._render_subnet(iface_cfg, route_cfg, iface_subnets[0]) + elif len(iface_subnets) > 1: + for i, iface_subnet in enumerate(iface_subnets, + start=len(iface.children)): + iface_sub_cfg = iface_cfg.copy() + iface_sub_cfg.name = "%s:%s" % (iface_name, i) + iface.children.append(iface_sub_cfg) + cls._render_subnet(iface_sub_cfg, route_cfg, iface_subnet) + + @classmethod + def _render_bond_interfaces(cls, network_state, iface_contents): + bond_filter = renderer.filter_by_type('bond') + for iface in network_state.iter_interfaces(bond_filter): + iface_name = iface['name'] + iface_cfg = iface_contents[iface_name] + cls._render_bonding_opts(iface_cfg, iface) + iface_master_name = iface['bond-master'] + iface_cfg['MASTER'] = iface_master_name + iface_cfg['SLAVE'] = True + # Ensure that the master interface (and any of its children) + # are actually marked as being bond types... + master_cfg = iface_contents[iface_master_name] + master_cfgs = [master_cfg] + master_cfgs.extend(master_cfg.children) + for master_cfg in master_cfgs: + master_cfg['BONDING_MASTER'] = True + master_cfg.kind = 'bond' + + @staticmethod + def _render_vlan_interfaces(network_state, iface_contents): + vlan_filter = renderer.filter_by_type('vlan') + for iface in network_state.iter_interfaces(vlan_filter): + iface_name = iface['name'] + iface_cfg = iface_contents[iface_name] + iface_cfg['VLAN'] = True + iface_cfg['PHYSDEV'] = iface_name[:iface_name.rfind('.')] + + @staticmethod + def _render_dns(network_state, existing_dns_path=None): + content = resolv_conf.ResolvConf("") + if existing_dns_path and os.path.isfile(existing_dns_path): + content = resolv_conf.ResolvConf(util.load_file(existing_dns_path)) + for nameserver in network_state.dns_nameservers: + content.add_nameserver(nameserver) + for searchdomain in network_state.dns_searchdomains: + content.add_search_domain(searchdomain) + return "\n".join([_make_header(';'), str(content)]) + + @classmethod + def _render_bridge_interfaces(cls, network_state, iface_contents): + bridge_filter = renderer.filter_by_type('bridge') + for iface in network_state.iter_interfaces(bridge_filter): + iface_name = iface['name'] + iface_cfg = iface_contents[iface_name] + iface_cfg.kind = 'bridge' + for old_key, new_key in cls.bridge_opts_keys: + if old_key in iface: + iface_cfg[new_key] = iface[old_key] + # Is this the right key to get all the connected interfaces? + for bridged_iface_name in iface.get('bridge_ports', []): + # Ensure all bridged interfaces are correctly tagged + # as being bridged to this interface. + bridged_cfg = iface_contents[bridged_iface_name] + bridged_cfgs = [bridged_cfg] + bridged_cfgs.extend(bridged_cfg.children) + for bridge_cfg in bridged_cfgs: + bridge_cfg['BRIDGE'] = iface_name + + @classmethod + def _render_sysconfig(cls, base_sysconf_dir, network_state): + '''Given state, return /etc/sysconfig files + contents''' + iface_contents = {} + for iface in network_state.iter_interfaces(): + iface_name = iface['name'] + iface_cfg = NetInterface(iface_name, base_sysconf_dir) + cls._render_iface_shared(iface, iface_cfg) + iface_contents[iface_name] = iface_cfg + cls._render_physical_interfaces(network_state, iface_contents) + cls._render_bond_interfaces(network_state, iface_contents) + cls._render_vlan_interfaces(network_state, iface_contents) + cls._render_bridge_interfaces(network_state, iface_contents) + contents = {} + for iface_name, iface_cfg in iface_contents.items(): + if iface_cfg or iface_cfg.children: + contents[iface_cfg.path] = iface_cfg.to_string() + for iface_cfg in iface_cfg.children: + if iface_cfg: + contents[iface_cfg.path] = iface_cfg.to_string() + if iface_cfg.routes: + contents[iface_cfg.routes.path] = iface_cfg.routes.to_string() + return contents + + def render_network_state(self, target, network_state): + base_sysconf_dir = os.path.join(target, self.sysconf_dir) + for path, data in self._render_sysconfig(base_sysconf_dir, + network_state).items(): + util.write_file(path, data) + if self.dns_path: + dns_path = os.path.join(target, self.dns_path) + resolv_content = self._render_dns(network_state, + existing_dns_path=dns_path) + util.write_file(dns_path, resolv_content) + if self.netrules_path: + netrules_content = self._render_persistent_net(network_state) + netrules_path = os.path.join(target, self.netrules_path) + util.write_file(netrules_path, netrules_content) |