From 5beecdf88b630a397b3722ddb299e9a37ff02737 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Wed, 15 Mar 2017 14:33:45 -0400 Subject: net: add renderers for automatically selecting the renderer. Previously, the distro had hard coded which network renderer it would use. This adds support for just picking the right renderer based on what is available. Now, that can be set via a priority in system_info, but should generally work. That config looks like: system_info: network: renderers: ["eni", "sysconfig"] When no renderers are found, a specific RendererNotFoundError is raised. stages.py is modified to catch that and log it at error level. This path should not really be exercised, but could occur if for example an Ubuntu system did not have ifupdown, or a rhel system did not have sysconfig. In such a system previously we would have quietly rendered ENI configuration but that would have been ignored. This is one step better in that we at least log the error. --- cloudinit/util.py | 43 +++++++++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 14 deletions(-) (limited to 'cloudinit/util.py') diff --git a/cloudinit/util.py b/cloudinit/util.py index 7196a7ca..82f2f76b 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -2099,21 +2099,36 @@ def get_mount_info(path, log=LOG): return parse_mount(path) -def which(program): - # Return path of program for execution if found in path - def is_exe(fpath): - return os.path.isfile(fpath) and os.access(fpath, os.X_OK) - - _fpath, _ = os.path.split(program) - if _fpath: - if is_exe(program): +def is_exe(fpath): + # return boolean indicating if fpath exists and is executable. + return os.path.isfile(fpath) and os.access(fpath, os.X_OK) + + +def which(program, search=None, target=None): + target = target_path(target) + + if os.path.sep in program: + # if program had a '/' in it, then do not search PATH + # 'which' does consider cwd here. (cd / && which bin/ls) = bin/ls + # so effectively we set cwd to / (or target) + if is_exe(target_path(target, program)): return program - else: - for path in os.environ.get("PATH", "").split(os.pathsep): - path = path.strip('"') - exe_file = os.path.join(path, program) - if is_exe(exe_file): - return exe_file + + if search is None: + paths = [p.strip('"') for p in + os.environ.get("PATH", "").split(os.pathsep)] + if target == "/": + search = paths + else: + search = [p for p in paths if p.startswith("/")] + + # normalize path input + search = [os.path.abspath(p) for p in search] + + for path in search: + ppath = os.path.sep.join((path, program)) + if is_exe(target_path(target, ppath)): + return ppath return None -- cgit v1.2.3 From ef18b8ac4cf7e3dfd98830fbdb298380a192a0fc Mon Sep 17 00:00:00 2001 From: Ryan Harper Date: Sun, 19 Mar 2017 08:39:01 -0500 Subject: cloudinit.net: add network config v2 parsing and rendering Network configuration version 2 format is implemented in a package called netplan (nplan)[1] which allows consolidated network config for multiple network controllers. - Add a new netplan renderer - Update default policy, placing eni and sysconfig first This requires explicit policy to enable netplan over eni on systems which have both (Yakkety, Zesty, UC16) - Allow any network state (parsed from any format cloud-init supports) to render to v2 if system supports netplan. - Move eni's _subnet_is_ipv6 to common code for use by other renderers - Make sysconfig renderer always emit /etc/syconfig/network configuration - Update cloud-init.service systemd unit to also wait on systemd-networkd-wait-online.service 1. https://lists.ubuntu.com/archives/ubuntu-devel/2016-July/039464.html --- cloudinit/distros/debian.py | 16 +- cloudinit/net/eni.py | 15 +- cloudinit/net/netplan.py | 373 +++++++++++++++++++++++++ cloudinit/net/network_state.py | 312 ++++++++++++++++++++- cloudinit/net/renderers.py | 4 +- cloudinit/net/sysconfig.py | 20 +- cloudinit/util.py | 9 + systemd/cloud-init.service | 1 + tests/unittests/test_distros/test_netconfig.py | 351 ++++++++++++++++++++++- tests/unittests/test_net.py | 301 ++++++++++++++++++++ tools/net-convert.py | 84 ++++++ 11 files changed, 1450 insertions(+), 36 deletions(-) create mode 100644 cloudinit/net/netplan.py create mode 100755 tools/net-convert.py (limited to 'cloudinit/util.py') diff --git a/cloudinit/distros/debian.py b/cloudinit/distros/debian.py index 1101f02d..3f0f9d53 100644 --- a/cloudinit/distros/debian.py +++ b/cloudinit/distros/debian.py @@ -42,11 +42,16 @@ NETWORK_CONF_FN = "/etc/network/interfaces.d/50-cloud-init.cfg" class Distro(distros.Distro): hostname_conf_fn = "/etc/hostname" locale_conf_fn = "/etc/default/locale" + network_conf_fn = { + "eni": "/etc/network/interfaces.d/50-cloud-init.cfg", + "netplan": "/etc/netplan/50-cloud-init.yaml" + } renderer_configs = { - 'eni': { - 'eni_path': NETWORK_CONF_FN, - 'eni_header': ENI_HEADER, - } + "eni": {"eni_path": network_conf_fn["eni"], + "eni_header": ENI_HEADER}, + "netplan": {"netplan_path": network_conf_fn["netplan"], + "netplan_header": ENI_HEADER, + "postcmds": True} } def __init__(self, name, cfg, paths): @@ -75,7 +80,8 @@ class Distro(distros.Distro): self.package_command('install', pkgs=pkglist) def _write_network(self, settings): - util.write_file(NETWORK_CONF_FN, settings) + # this is a legacy method, it will always write eni + util.write_file(self.network_conf_fn["eni"], settings) return ['all'] def _write_network_config(self, netconfig): diff --git a/cloudinit/net/eni.py b/cloudinit/net/eni.py index f471e05f..9819d4f5 100644 --- a/cloudinit/net/eni.py +++ b/cloudinit/net/eni.py @@ -8,6 +8,7 @@ import re from . import ParserError from . import renderer +from .network_state import subnet_is_ipv6 from cloudinit import util @@ -111,16 +112,6 @@ def _iface_start_entry(iface, index, render_hwaddress=False): return lines -def _subnet_is_ipv6(subnet): - # 'static6' or 'dhcp6' - if subnet['type'].endswith('6'): - # This is a request for DHCPv6. - return True - elif subnet['type'] == 'static' and ":" in subnet['address']: - return True - return False - - def _parse_deb_config_data(ifaces, contents, src_dir, src_path): """Parses the file contents, placing result into ifaces. @@ -370,7 +361,7 @@ class Renderer(renderer.Renderer): iface['mode'] = subnet['type'] iface['control'] = subnet.get('control', 'auto') subnet_inet = 'inet' - if _subnet_is_ipv6(subnet): + if subnet_is_ipv6(subnet): subnet_inet += '6' iface['inet'] = subnet_inet if subnet['type'].startswith('dhcp'): @@ -486,7 +477,7 @@ class Renderer(renderer.Renderer): def network_state_to_eni(network_state, header=None, render_hwaddress=False): # render the provided network state, return a string of equivalent eni eni_path = 'etc/network/interfaces' - renderer = Renderer({ + renderer = Renderer(config={ 'eni_path': eni_path, 'eni_header': header, 'links_path_prefix': None, diff --git a/cloudinit/net/netplan.py b/cloudinit/net/netplan.py new file mode 100644 index 00000000..cd93b21c --- /dev/null +++ b/cloudinit/net/netplan.py @@ -0,0 +1,373 @@ +# This file is part of cloud-init. See LICENSE file ... + +import copy +import os + +from . import renderer +from .network_state import subnet_is_ipv6 + +from cloudinit import util +from cloudinit.net import SYS_CLASS_NET, get_devicelist + + +NET_CONFIG_TO_V2 = { + 'bond': {'bond-ad-select': 'ad-select', + 'bond-arp-interval': 'arp-interval', + 'bond-arp-ip-target': 'arp-ip-target', + 'bond-arp-validate': 'arp-validate', + 'bond-downdelay': 'down-delay', + 'bond-fail-over-mac': 'fail-over-mac-policy', + 'bond-lacp-rate': 'lacp-rate', + 'bond-miimon': 'mii-monitor-interval', + 'bond-min-links': 'min-links', + 'bond-mode': 'mode', + 'bond-num-grat-arp': 'gratuitious-arp', + 'bond-primary-reselect': 'primary-reselect-policy', + 'bond-updelay': 'up-delay', + 'bond-xmit_hash_policy': 'transmit_hash_policy'}, + 'bridge': {'bridge_ageing': 'ageing-time', + 'bridge_bridgeprio': 'priority', + 'bridge_fd': 'forward-delay', + 'bridge_gcint': None, + 'bridge_hello': 'hello-time', + 'bridge_maxage': 'max-age', + 'bridge_maxwait': None, + 'bridge_pathcost': 'path-cost', + 'bridge_portprio': None, + 'bridge_waitport': None}} + + +def _get_params_dict_by_match(config, match): + return dict((key, value) for (key, value) in config.items() + if key.startswith(match)) + + +def _extract_addresses(config, entry): + """This method parse a cloudinit.net.network_state dictionary (config) and + maps netstate keys/values into a dictionary (entry) to represent + netplan yaml. + + An example config dictionary might look like: + + {'mac_address': '52:54:00:12:34:00', + 'name': 'interface0', + 'subnets': [ + {'address': '192.168.1.2/24', + 'mtu': 1501, + 'type': 'static'}, + {'address': '2001:4800:78ff:1b:be76:4eff:fe06:1000", + 'mtu': 1480, + 'netmask': 64, + 'type': 'static'}], + 'type: physical' + } + + An entry dictionary looks like: + + {'set-name': 'interface0', + 'match': {'macaddress': '52:54:00:12:34:00'}, + 'mtu': 1501} + + After modification returns + + {'set-name': 'interface0', + 'match': {'macaddress': '52:54:00:12:34:00'}, + 'mtu': 1501, + 'address': ['192.168.1.2/24', '2001:4800:78ff:1b:be76:4eff:fe06:1000"], + 'mtu6': 1480} + + """ + + def _listify(obj, token=' '): + "Helper to convert strings to list of strings, handle single string" + if not obj or type(obj) not in [str]: + return obj + if token in obj: + return obj.split(token) + else: + return [obj, ] + + addresses = [] + routes = [] + nameservers = [] + searchdomains = [] + subnets = config.get('subnets', []) + if subnets is None: + subnets = [] + for subnet in subnets: + sn_type = subnet.get('type') + if sn_type.startswith('dhcp'): + if sn_type == 'dhcp': + sn_type += '4' + entry.update({sn_type: True}) + elif sn_type in ['static']: + addr = "%s" % subnet.get('address') + if 'netmask' in subnet: + addr += "/%s" % subnet.get('netmask') + if 'gateway' in subnet and subnet.get('gateway'): + gateway = subnet.get('gateway') + if ":" in gateway: + entry.update({'gateway6': gateway}) + else: + entry.update({'gateway4': gateway}) + if 'dns_nameservers' in subnet: + nameservers += _listify(subnet.get('dns_nameservers', [])) + if 'dns_search' in subnet: + searchdomains += _listify(subnet.get('dns_search', [])) + if 'mtu' in subnet: + mtukey = 'mtu' + if subnet_is_ipv6(subnet): + mtukey += '6' + entry.update({mtukey: subnet.get('mtu')}) + for route in subnet.get('routes', []): + to_net = "%s/%s" % (route.get('network'), + route.get('netmask')) + route = { + 'via': route.get('gateway'), + 'to': to_net, + } + if 'metric' in route: + route.update({'metric': route.get('metric', 100)}) + routes.append(route) + + addresses.append(addr) + + if len(addresses) > 0: + entry.update({'addresses': addresses}) + if len(routes) > 0: + entry.update({'routes': routes}) + if len(nameservers) > 0: + ns = {'addresses': nameservers} + entry.update({'nameservers': ns}) + if len(searchdomains) > 0: + ns = entry.get('nameservers', {}) + ns.update({'search': searchdomains}) + entry.update({'nameservers': ns}) + + +def _extract_bond_slaves_by_name(interfaces, entry, bond_master): + bond_slave_names = sorted([name for (name, cfg) in interfaces.items() + if cfg.get('bond-master', None) == bond_master]) + if len(bond_slave_names) > 0: + entry.update({'interfaces': bond_slave_names}) + + +class Renderer(renderer.Renderer): + """Renders network information in a /etc/netplan/network.yaml format.""" + + NETPLAN_GENERATE = ['netplan', 'generate'] + + def __init__(self, config=None): + if not config: + config = {} + self.netplan_path = config.get('netplan_path', + 'etc/netplan/50-cloud-init.yaml') + self.netplan_header = config.get('netplan_header', None) + self._postcmds = config.get('postcmds', False) + + def render_network_state(self, target, network_state): + # check network state for version + # if v2, then extract network_state.config + # else render_v2_from_state + fpnplan = os.path.join(target, self.netplan_path) + util.ensure_dir(os.path.dirname(fpnplan)) + header = self.netplan_header if self.netplan_header else "" + + # render from state + content = self._render_content(network_state) + + # ensure we poke udev to run net_setup_link + if not header.endswith("\n"): + header += "\n" + util.write_file(fpnplan, header + content) + + self._netplan_generate(run=self._postcmds) + self._net_setup_link(run=self._postcmds) + + def _netplan_generate(self, run=False): + if not run: + print("netplan postcmd disabled") + return + util.subp(self.NETPLAN_GENERATE, capture=True) + + def _net_setup_link(self, run=False): + """To ensure device link properties are applied, we poke + udev to re-evaluate networkd .link files and call + the setup_link udev builtin command + """ + if not run: + print("netsetup postcmd disabled") + return + setup_lnk = ['udevadm', 'test-builtin', 'net_setup_link'] + for cmd in [setup_lnk + [SYS_CLASS_NET + iface] + for iface in get_devicelist() if + os.path.islink(SYS_CLASS_NET + iface)]: + print(cmd) + util.subp(cmd, capture=True) + + def _render_content(self, network_state): + print('rendering v2 for victory!') + ethernets = {} + wifis = {} + bridges = {} + bonds = {} + vlans = {} + content = [] + + interfaces = network_state._network_state.get('interfaces', []) + + nameservers = network_state.dns_nameservers + searchdomains = network_state.dns_searchdomains + + for config in network_state.iter_interfaces(): + ifname = config.get('name') + # filter None entries up front so we can do simple if key in dict + ifcfg = dict((key, value) for (key, value) in config.items() + if value) + + if_type = ifcfg.get('type') + if if_type == 'physical': + # required_keys = ['name', 'mac_address'] + eth = { + 'set-name': ifname, + 'match': ifcfg.get('match', None), + } + if eth['match'] is None: + macaddr = ifcfg.get('mac_address', None) + if macaddr is not None: + eth['match'] = {'macaddress': macaddr.lower()} + else: + del eth['match'] + del eth['set-name'] + if 'mtu' in ifcfg: + eth['mtu'] = ifcfg.get('mtu') + + _extract_addresses(ifcfg, eth) + ethernets.update({ifname: eth}) + + elif if_type == 'bond': + # required_keys = ['name', 'bond_interfaces'] + bond = {} + bond_config = {} + # extract bond params and drop the bond_ prefix as it's + # redundent in v2 yaml format + v2_bond_map = NET_CONFIG_TO_V2.get('bond') + for match in ['bond_', 'bond-']: + bond_params = _get_params_dict_by_match(ifcfg, match) + for (param, value) in bond_params.items(): + newname = v2_bond_map.get(param) + if newname is None: + continue + bond_config.update({newname: value}) + + if len(bond_config) > 0: + bond.update({'parameters': bond_config}) + slave_interfaces = ifcfg.get('bond-slaves') + if slave_interfaces == 'none': + _extract_bond_slaves_by_name(interfaces, bond, ifname) + _extract_addresses(ifcfg, bond) + bonds.update({ifname: bond}) + + elif if_type == 'bridge': + # required_keys = ['name', 'bridge_ports'] + ports = sorted(copy.copy(ifcfg.get('bridge_ports'))) + bridge = { + 'interfaces': ports, + } + # extract bridge params and drop the bridge prefix as it's + # redundent in v2 yaml format + match_prefix = 'bridge_' + params = _get_params_dict_by_match(ifcfg, match_prefix) + br_config = {} + + # v2 yaml uses different names for the keys + # and at least one value format change + v2_bridge_map = NET_CONFIG_TO_V2.get('bridge') + for (param, value) in params.items(): + newname = v2_bridge_map.get(param) + if newname is None: + continue + br_config.update({newname: value}) + if newname == 'path-cost': + # -> : int() + newvalue = {} + for costval in value: + (port, cost) = costval.split() + newvalue[port] = int(cost) + br_config.update({newname: newvalue}) + if len(br_config) > 0: + bridge.update({'parameters': br_config}) + _extract_addresses(ifcfg, bridge) + bridges.update({ifname: bridge}) + + elif if_type == 'vlan': + # required_keys = ['name', 'vlan_id', 'vlan-raw-device'] + vlan = { + 'id': ifcfg.get('vlan_id'), + 'link': ifcfg.get('vlan-raw-device') + } + + _extract_addresses(ifcfg, vlan) + vlans.update({ifname: vlan}) + + # inject global nameserver values under each physical interface + if nameservers: + for _eth, cfg in ethernets.items(): + nscfg = cfg.get('nameservers', {}) + addresses = nscfg.get('addresses', []) + addresses += nameservers + nscfg.update({'addresses': addresses}) + cfg.update({'nameservers': nscfg}) + + if searchdomains: + for _eth, cfg in ethernets.items(): + nscfg = cfg.get('nameservers', {}) + search = nscfg.get('search', []) + search += searchdomains + nscfg.update({'search': search}) + cfg.update({'nameservers': nscfg}) + + # workaround yaml dictionary key sorting when dumping + def _render_section(name, section): + if section: + dump = util.yaml_dumps({name: section}, + explicit_start=False, + explicit_end=False) + txt = util.indent(dump, ' ' * 4) + return [txt] + return [] + + content.append("network:\n version: 2\n") + content += _render_section('ethernets', ethernets) + content += _render_section('wifis', wifis) + content += _render_section('bonds', bonds) + content += _render_section('bridges', bridges) + content += _render_section('vlans', vlans) + + return "".join(content) + + +def available(target=None): + expected = ['netplan'] + search = ['/usr/sbin', '/sbin'] + for p in expected: + if not util.which(p, search=search, target=target): + return False + return True + + +def network_state_to_netplan(network_state, header=None): + # render the provided network state, return a string of equivalent eni + netplan_path = 'etc/network/50-cloud-init.yaml' + renderer = Renderer({ + 'netplan_path': netplan_path, + 'netplan_header': header, + }) + if not header: + header = "" + if not header.endswith("\n"): + header += "\n" + contents = renderer._render_content(network_state) + return header + contents + +# vi: ts=4 expandtab diff --git a/cloudinit/net/network_state.py b/cloudinit/net/network_state.py index 90b2835a..701aaa4e 100644 --- a/cloudinit/net/network_state.py +++ b/cloudinit/net/network_state.py @@ -1,4 +1,4 @@ -# Copyright (C) 2013-2014 Canonical Ltd. +# Copyright (C) 2017 Canonical Ltd. # # Author: Ryan Harper # @@ -18,6 +18,10 @@ NETWORK_STATE_VERSION = 1 NETWORK_STATE_REQUIRED_KEYS = { 1: ['version', 'config', 'network_state'], } +NETWORK_V2_KEY_FILTER = [ + 'addresses', 'dhcp4', 'dhcp6', 'gateway4', 'gateway6', 'interfaces', + 'match', 'mtu', 'nameservers', 'renderer', 'set-name', 'wakeonlan' +] def parse_net_config_data(net_config, skip_broken=True): @@ -26,11 +30,18 @@ def parse_net_config_data(net_config, skip_broken=True): :param net_config: curtin network config dict """ state = None - if 'version' in net_config and 'config' in net_config: - nsi = NetworkStateInterpreter(version=net_config.get('version'), - config=net_config.get('config')) + version = net_config.get('version') + config = net_config.get('config') + if version == 2: + # v2 does not have explicit 'config' key so we + # pass the whole net-config as-is + config = net_config + + if version and config: + nsi = NetworkStateInterpreter(version=version, config=config) nsi.parse_config(skip_broken=skip_broken) - state = nsi.network_state + state = nsi.get_network_state() + return state @@ -106,6 +117,7 @@ class NetworkState(object): def __init__(self, network_state, version=NETWORK_STATE_VERSION): self._network_state = copy.deepcopy(network_state) self._version = version + self.use_ipv6 = network_state.get('use_ipv6', False) @property def version(self): @@ -152,7 +164,8 @@ class NetworkStateInterpreter(object): 'dns': { 'nameservers': [], 'search': [], - } + }, + 'use_ipv6': False, } def __init__(self, version=NETWORK_STATE_VERSION, config=None): @@ -165,6 +178,14 @@ class NetworkStateInterpreter(object): def network_state(self): return NetworkState(self._network_state, version=self._version) + @property + def use_ipv6(self): + return self._network_state.get('use_ipv6') + + @use_ipv6.setter + def use_ipv6(self, val): + self._network_state.update({'use_ipv6': val}) + def dump(self): state = { 'version': self._version, @@ -192,8 +213,22 @@ class NetworkStateInterpreter(object): def dump_network_state(self): return util.yaml_dumps(self._network_state) + def as_dict(self): + return {'version': self.version, 'config': self.config} + + def get_network_state(self): + ns = self.network_state + return ns + def parse_config(self, skip_broken=True): - # rebuild network state + if self._version == 1: + self.parse_config_v1(skip_broken=skip_broken) + self._parsed = True + elif self._version == 2: + self.parse_config_v2(skip_broken=skip_broken) + self._parsed = True + + def parse_config_v1(self, skip_broken=True): for command in self._config: command_type = command['type'] try: @@ -211,6 +246,26 @@ class NetworkStateInterpreter(object): exc_info=True) LOG.debug(self.dump_network_state()) + def parse_config_v2(self, skip_broken=True): + for command_type, command in self._config.items(): + if command_type == 'version': + continue + try: + handler = self.command_handlers[command_type] + except KeyError: + raise RuntimeError("No handler found for" + " command '%s'" % command_type) + try: + handler(self, command) + self._v2_common(command) + except InvalidCommand: + if not skip_broken: + raise + else: + LOG.warn("Skipping invalid command: %s", command, + exc_info=True) + LOG.debug(self.dump_network_state()) + @ensure_command_keys(['name']) def handle_loopback(self, command): return self.handle_physical(command) @@ -238,11 +293,16 @@ class NetworkStateInterpreter(object): if subnets: for subnet in subnets: if subnet['type'] == 'static': + if ':' in subnet['address']: + self.use_ipv6 = True if 'netmask' in subnet and ':' in subnet['address']: subnet['netmask'] = mask2cidr(subnet['netmask']) for route in subnet.get('routes', []): if 'netmask' in route: route['netmask'] = mask2cidr(route['netmask']) + elif subnet['type'].endswith('6'): + self.use_ipv6 = True + iface.update({ 'name': command.get('name'), 'type': command.get('type'), @@ -327,7 +387,7 @@ class NetworkStateInterpreter(object): bond_if.update({param: val}) self._network_state['interfaces'].update({ifname: bond_if}) - @ensure_command_keys(['name', 'bridge_interfaces', 'params']) + @ensure_command_keys(['name', 'bridge_interfaces']) def handle_bridge(self, command): ''' auto br0 @@ -373,7 +433,7 @@ class NetworkStateInterpreter(object): self.handle_physical(command) iface = interfaces.get(command.get('name'), {}) iface['bridge_ports'] = command['bridge_interfaces'] - for param, val in command.get('params').items(): + for param, val in command.get('params', {}).items(): iface.update({param: val}) interfaces.update({iface['name']: iface}) @@ -407,6 +467,240 @@ class NetworkStateInterpreter(object): } routes.append(route) + # V2 handlers + def handle_bonds(self, command): + ''' + v2_command = { + bond0: { + 'interfaces': ['interface0', 'interface1'], + 'miimon': 100, + 'mode': '802.3ad', + 'xmit_hash_policy': 'layer3+4'}, + bond1: { + 'bond-slaves': ['interface2', 'interface7'], + 'mode': 1 + } + } + + v1_command = { + 'type': 'bond' + 'name': 'bond0', + 'bond_interfaces': [interface0, interface1], + 'params': { + 'bond-mode': '802.3ad', + 'bond_miimon: 100, + 'bond_xmit_hash_policy': 'layer3+4', + } + } + + ''' + self._handle_bond_bridge(command, cmd_type='bond') + + def handle_bridges(self, command): + + ''' + v2_command = { + br0: { + 'interfaces': ['interface0', 'interface1'], + 'fd': 0, + 'stp': 'off', + 'maxwait': 0, + } + } + + v1_command = { + 'type': 'bridge' + 'name': 'br0', + 'bridge_interfaces': [interface0, interface1], + 'params': { + 'bridge_stp': 'off', + 'bridge_fd: 0, + 'bridge_maxwait': 0 + } + } + + ''' + self._handle_bond_bridge(command, cmd_type='bridge') + + def handle_ethernets(self, command): + ''' + ethernets: + eno1: + match: + macaddress: 00:11:22:33:44:55 + wakeonlan: true + dhcp4: true + dhcp6: false + addresses: + - 192.168.14.2/24 + - 2001:1::1/64 + gateway4: 192.168.14.1 + gateway6: 2001:1::2 + nameservers: + search: [foo.local, bar.local] + addresses: [8.8.8.8, 8.8.4.4] + lom: + match: + driver: ixgbe + set-name: lom1 + dhcp6: true + switchports: + match: + name: enp2* + mtu: 1280 + + command = { + 'type': 'physical', + 'mac_address': 'c0:d6:9f:2c:e8:80', + 'name': 'eth0', + 'subnets': [ + {'type': 'dhcp4'} + ] + } + ''' + for eth, cfg in command.items(): + phy_cmd = { + 'type': 'physical', + 'name': cfg.get('set-name', eth), + } + mac_address = cfg.get('match', {}).get('macaddress', None) + if not mac_address: + LOG.debug('NetworkState Version2: missing "macaddress" info ' + 'in config entry: %s: %s', eth, str(cfg)) + + for key in ['mtu', 'match', 'wakeonlan']: + if key in cfg: + phy_cmd.update({key: cfg.get(key)}) + + subnets = self._v2_to_v1_ipcfg(cfg) + if len(subnets) > 0: + phy_cmd.update({'subnets': subnets}) + + LOG.debug('v2(ethernets) -> v1(physical):\n%s', phy_cmd) + self.handle_physical(phy_cmd) + + def handle_vlans(self, command): + ''' + v2_vlans = { + 'eth0.123': { + 'id': 123, + 'link': 'eth0', + 'dhcp4': True, + } + } + + v1_command = { + 'type': 'vlan', + 'name': 'eth0.123', + 'vlan_link': 'eth0', + 'vlan_id': 123, + 'subnets': [{'type': 'dhcp4'}], + } + ''' + for vlan, cfg in command.items(): + vlan_cmd = { + 'type': 'vlan', + 'name': vlan, + 'vlan_id': cfg.get('id'), + 'vlan_link': cfg.get('link'), + } + subnets = self._v2_to_v1_ipcfg(cfg) + if len(subnets) > 0: + vlan_cmd.update({'subnets': subnets}) + LOG.debug('v2(vlans) -> v1(vlan):\n%s', vlan_cmd) + self.handle_vlan(vlan_cmd) + + def handle_wifis(self, command): + raise NotImplemented('NetworkState V2: Skipping wifi configuration') + + def _v2_common(self, cfg): + LOG.debug('v2_common: handling config:\n%s', cfg) + if 'nameservers' in cfg: + search = cfg.get('nameservers').get('search', []) + dns = cfg.get('nameservers').get('addresses', []) + name_cmd = {'type': 'nameserver'} + if len(search) > 0: + name_cmd.update({'search': search}) + if len(dns) > 0: + name_cmd.update({'addresses': dns}) + LOG.debug('v2(nameserver) -> v1(nameserver):\n%s', name_cmd) + self.handle_nameserver(name_cmd) + + def _handle_bond_bridge(self, command, cmd_type=None): + """Common handler for bond and bridge types""" + for item_name, item_cfg in command.items(): + item_params = dict((key, value) for (key, value) in + item_cfg.items() if key not in + NETWORK_V2_KEY_FILTER) + v1_cmd = { + 'type': cmd_type, + 'name': item_name, + cmd_type + '_interfaces': item_cfg.get('interfaces'), + 'params': item_params, + } + subnets = self._v2_to_v1_ipcfg(item_cfg) + if len(subnets) > 0: + v1_cmd.update({'subnets': subnets}) + + LOG.debug('v2(%ss) -> v1(%s):\n%s', cmd_type, cmd_type, v1_cmd) + self.handle_bridge(v1_cmd) + + def _v2_to_v1_ipcfg(self, cfg): + """Common ipconfig extraction from v2 to v1 subnets array.""" + + subnets = [] + if 'dhcp4' in cfg: + subnets.append({'type': 'dhcp4'}) + if 'dhcp6' in cfg: + self.use_ipv6 = True + subnets.append({'type': 'dhcp6'}) + + gateway4 = None + gateway6 = None + for address in cfg.get('addresses', []): + subnet = { + 'type': 'static', + 'address': address, + } + + routes = [] + for route in cfg.get('routes', []): + route_addr = route.get('to') + if "/" in route_addr: + route_addr, route_cidr = route_addr.split("/") + route_netmask = cidr2mask(route_cidr) + subnet_route = { + 'address': route_addr, + 'netmask': route_netmask, + 'gateway': route.get('via') + } + routes.append(subnet_route) + if len(routes) > 0: + subnet.update({'routes': routes}) + + if ":" in address: + if 'gateway6' in cfg and gateway6 is None: + gateway6 = cfg.get('gateway6') + subnet.update({'gateway': gateway6}) + else: + if 'gateway4' in cfg and gateway4 is None: + gateway4 = cfg.get('gateway4') + subnet.update({'gateway': gateway4}) + + subnets.append(subnet) + return subnets + + +def subnet_is_ipv6(subnet): + """Common helper for checking network_state subnets for ipv6.""" + # 'static6' or 'dhcp6' + if subnet['type'].endswith('6'): + # This is a request for DHCPv6. + return True + elif subnet['type'] == 'static' and ":" in subnet['address']: + return True + return False + def cidr2mask(cidr): mask = [0, 0, 0, 0] diff --git a/cloudinit/net/renderers.py b/cloudinit/net/renderers.py index 5ad84553..5117b4a5 100644 --- a/cloudinit/net/renderers.py +++ b/cloudinit/net/renderers.py @@ -1,15 +1,17 @@ # This file is part of cloud-init. See LICENSE file for license information. from . import eni +from . import netplan from . import RendererNotFoundError from . import sysconfig NAME_TO_RENDERER = { "eni": eni, + "netplan": netplan, "sysconfig": sysconfig, } -DEFAULT_PRIORITY = ["eni", "sysconfig"] +DEFAULT_PRIORITY = ["eni", "sysconfig", "netplan"] def search(priority=None, target=None, first=False): diff --git a/cloudinit/net/sysconfig.py b/cloudinit/net/sysconfig.py index 117b515c..504e4d02 100644 --- a/cloudinit/net/sysconfig.py +++ b/cloudinit/net/sysconfig.py @@ -9,6 +9,7 @@ from cloudinit.distros.parsers import resolv_conf from cloudinit import util from . import renderer +from .network_state import subnet_is_ipv6 def _make_header(sep='#'): @@ -194,7 +195,7 @@ class Renderer(renderer.Renderer): def __init__(self, config=None): if not config: config = {} - self.sysconf_dir = config.get('sysconf_dir', 'etc/sysconfig/') + self.sysconf_dir = config.get('sysconf_dir', 'etc/sysconfig') self.netrules_path = config.get( 'netrules_path', 'etc/udev/rules.d/70-persistent-net.rules') self.dns_path = config.get('dns_path', 'etc/resolv.conf') @@ -220,7 +221,7 @@ class Renderer(renderer.Renderer): iface_cfg['BOOTPROTO'] = 'dhcp' elif subnet_type == 'static': iface_cfg['BOOTPROTO'] = 'static' - if subnet.get('ipv6'): + if subnet_is_ipv6(subnet): iface_cfg['IPV6ADDR'] = subnet['address'] iface_cfg['IPV6INIT'] = True else: @@ -390,19 +391,28 @@ class Renderer(renderer.Renderer): return contents def render_network_state(self, network_state, target=None): + file_mode = 0o644 base_sysconf_dir = util.target_path(target, self.sysconf_dir) for path, data in self._render_sysconfig(base_sysconf_dir, network_state).items(): - util.write_file(path, data) + util.write_file(path, data, file_mode) if self.dns_path: dns_path = util.target_path(target, self.dns_path) resolv_content = self._render_dns(network_state, existing_dns_path=dns_path) - util.write_file(dns_path, resolv_content) + util.write_file(dns_path, resolv_content, file_mode) if self.netrules_path: netrules_content = self._render_persistent_net(network_state) netrules_path = util.target_path(target, self.netrules_path) - util.write_file(netrules_path, netrules_content) + util.write_file(netrules_path, netrules_content, file_mode) + + # always write /etc/sysconfig/network configuration + sysconfig_path = util.target_path(target, "etc/sysconfig/network") + netcfg = [_make_header(), 'NETWORKING=yes'] + if network_state.use_ipv6: + netcfg.append('NETWORKING_IPV6=yes') + netcfg.append('IPV6_AUTOCONF=no') + util.write_file(sysconfig_path, "\n".join(netcfg) + "\n", file_mode) def available(target=None): diff --git a/cloudinit/util.py b/cloudinit/util.py index 82f2f76b..33019579 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -2373,4 +2373,13 @@ def system_is_snappy(): return True return False + +def indent(text, prefix): + """replacement for indent from textwrap that is not available in 2.7.""" + lines = [] + for line in text.splitlines(True): + lines.append(prefix + line) + return ''.join(lines) + + # vi: ts=4 expandtab diff --git a/systemd/cloud-init.service b/systemd/cloud-init.service index fb3b918c..39acc20a 100644 --- a/systemd/cloud-init.service +++ b/systemd/cloud-init.service @@ -5,6 +5,7 @@ Wants=cloud-init-local.service Wants=sshd-keygen.service Wants=sshd.service After=cloud-init-local.service +After=systemd-networkd-wait-online.service After=networking.service Before=network-online.target Before=sshd-keygen.service diff --git a/tests/unittests/test_distros/test_netconfig.py b/tests/unittests/test_distros/test_netconfig.py index bde3bb50..b89b74ff 100644 --- a/tests/unittests/test_distros/test_netconfig.py +++ b/tests/unittests/test_distros/test_netconfig.py @@ -17,6 +17,7 @@ from ..helpers import TestCase from cloudinit import distros from cloudinit.distros.parsers.sys_conf import SysConf from cloudinit import helpers +from cloudinit.net import eni from cloudinit import settings from cloudinit import util @@ -28,10 +29,10 @@ iface lo inet loopback auto eth0 iface eth0 inet static address 192.168.1.5 - netmask 255.255.255.0 - network 192.168.0.0 broadcast 192.168.1.0 gateway 192.168.1.254 + netmask 255.255.255.0 + network 192.168.0.0 auto eth1 iface eth1 inet dhcp @@ -67,6 +68,100 @@ iface eth1 inet6 static gateway 2607:f0d0:1002:0011::1 ''' +V1_NET_CFG = {'config': [{'name': 'eth0', + + 'subnets': [{'address': '192.168.1.5', + 'broadcast': '192.168.1.0', + 'gateway': '192.168.1.254', + 'netmask': '255.255.255.0', + 'type': 'static'}], + 'type': 'physical'}, + {'name': 'eth1', + 'subnets': [{'control': 'auto', 'type': 'dhcp4'}], + 'type': 'physical'}], + 'version': 1} + +V1_NET_CFG_OUTPUT = """ +# This file is generated from information provided by +# the datasource. Changes to it will not persist across an instance. +# To disable cloud-init's network configuration capabilities, write a file +# /etc/cloud/cloud.cfg.d/99-disable-network-config.cfg with the following: +# network: {config: disabled} +auto lo +iface lo inet loopback + +auto eth0 +iface eth0 inet static + address 192.168.1.5 + broadcast 192.168.1.0 + gateway 192.168.1.254 + netmask 255.255.255.0 + +auto eth1 +iface eth1 inet dhcp +""" + +V1_NET_CFG_IPV6 = {'config': [{'name': 'eth0', + 'subnets': [{'address': + '2607:f0d0:1002:0011::2', + 'gateway': + '2607:f0d0:1002:0011::1', + 'netmask': '64', + 'type': 'static'}], + 'type': 'physical'}, + {'name': 'eth1', + 'subnets': [{'control': 'auto', + 'type': 'dhcp4'}], + 'type': 'physical'}], + 'version': 1} + + +V1_TO_V2_NET_CFG_OUTPUT = """ +# This file is generated from information provided by +# the datasource. Changes to it will not persist across an instance. +# To disable cloud-init's network configuration capabilities, write a file +# /etc/cloud/cloud.cfg.d/99-disable-network-config.cfg with the following: +# network: {config: disabled} +network: + version: 2 + ethernets: + eth0: + addresses: + - 192.168.1.5/255.255.255.0 + gateway4: 192.168.1.254 + eth1: + dhcp4: true +""" + +V2_NET_CFG = { + 'ethernets': { + 'eth7': { + 'addresses': ['192.168.1.5/255.255.255.0'], + 'gateway4': '192.168.1.254'}, + 'eth9': { + 'dhcp4': True} + }, + 'version': 2 +} + + +V2_TO_V2_NET_CFG_OUTPUT = """ +# This file is generated from information provided by +# the datasource. Changes to it will not persist across an instance. +# To disable cloud-init's network configuration capabilities, write a file +# /etc/cloud/cloud.cfg.d/99-disable-network-config.cfg with the following: +# network: {config: disabled} +network: + version: 2 + ethernets: + eth7: + addresses: + - 192.168.1.5/255.255.255.0 + gateway4: 192.168.1.254 + eth9: + dhcp4: true +""" + class WriteBuffer(object): def __init__(self): @@ -83,12 +178,14 @@ class WriteBuffer(object): class TestNetCfgDistro(TestCase): - def _get_distro(self, dname): + def _get_distro(self, dname, renderers=None): cls = distros.fetch(dname) cfg = settings.CFG_BUILTIN cfg['system_info']['distro'] = dname + if renderers: + cfg['system_info']['network'] = {'renderers': renderers} paths = helpers.Paths({}) - return cls(dname, cfg, paths) + return cls(dname, cfg.get('system_info'), paths) def test_simple_write_ub(self): ub_distro = self._get_distro('ubuntu') @@ -116,6 +213,107 @@ class TestNetCfgDistro(TestCase): self.assertEqual(str(write_buf).strip(), BASE_NET_CFG.strip()) self.assertEqual(write_buf.mode, 0o644) + def test_apply_network_config_eni_ub(self): + ub_distro = self._get_distro('ubuntu') + with ExitStack() as mocks: + write_bufs = {} + + def replace_write(filename, content, mode=0o644, omode="wb"): + buf = WriteBuffer() + buf.mode = mode + buf.omode = omode + buf.write(content) + write_bufs[filename] = buf + + # eni availability checks + mocks.enter_context( + mock.patch.object(util, 'which', return_value=True)) + mocks.enter_context( + mock.patch.object(eni, 'available', return_value=True)) + mocks.enter_context( + mock.patch.object(util, 'ensure_dir')) + mocks.enter_context( + mock.patch.object(util, 'write_file', replace_write)) + mocks.enter_context( + mock.patch.object(os.path, 'isfile', return_value=False)) + + ub_distro.apply_network_config(V1_NET_CFG, False) + + self.assertEqual(len(write_bufs), 2) + eni_name = '/etc/network/interfaces.d/50-cloud-init.cfg' + self.assertIn(eni_name, write_bufs) + write_buf = write_bufs[eni_name] + self.assertEqual(str(write_buf).strip(), V1_NET_CFG_OUTPUT.strip()) + self.assertEqual(write_buf.mode, 0o644) + + def test_apply_network_config_v1_to_netplan_ub(self): + renderers = ['netplan'] + ub_distro = self._get_distro('ubuntu', renderers=renderers) + with ExitStack() as mocks: + write_bufs = {} + + def replace_write(filename, content, mode=0o644, omode="wb"): + buf = WriteBuffer() + buf.mode = mode + buf.omode = omode + buf.write(content) + write_bufs[filename] = buf + + mocks.enter_context( + mock.patch.object(util, 'which', return_value=True)) + mocks.enter_context( + mock.patch.object(util, 'write_file', replace_write)) + mocks.enter_context( + mock.patch.object(util, 'ensure_dir')) + mocks.enter_context( + mock.patch.object(util, 'subp', return_value=(0, 0))) + mocks.enter_context( + mock.patch.object(os.path, 'isfile', return_value=False)) + + ub_distro.apply_network_config(V1_NET_CFG, False) + + self.assertEqual(len(write_bufs), 1) + netplan_name = '/etc/netplan/50-cloud-init.yaml' + self.assertIn(netplan_name, write_bufs) + write_buf = write_bufs[netplan_name] + self.assertEqual(str(write_buf).strip(), + V1_TO_V2_NET_CFG_OUTPUT.strip()) + self.assertEqual(write_buf.mode, 0o644) + + def test_apply_network_config_v2_passthrough_ub(self): + renderers = ['netplan'] + ub_distro = self._get_distro('ubuntu', renderers=renderers) + with ExitStack() as mocks: + write_bufs = {} + + def replace_write(filename, content, mode=0o644, omode="wb"): + buf = WriteBuffer() + buf.mode = mode + buf.omode = omode + buf.write(content) + write_bufs[filename] = buf + + mocks.enter_context( + mock.patch.object(util, 'which', return_value=True)) + mocks.enter_context( + mock.patch.object(util, 'write_file', replace_write)) + mocks.enter_context( + mock.patch.object(util, 'ensure_dir')) + mocks.enter_context( + mock.patch.object(util, 'subp', return_value=(0, 0))) + mocks.enter_context( + mock.patch.object(os.path, 'isfile', return_value=False)) + + ub_distro.apply_network_config(V2_NET_CFG, False) + + self.assertEqual(len(write_bufs), 1) + netplan_name = '/etc/netplan/50-cloud-init.yaml' + self.assertIn(netplan_name, write_bufs) + write_buf = write_bufs[netplan_name] + self.assertEqual(str(write_buf).strip(), + V2_TO_V2_NET_CFG_OUTPUT.strip()) + self.assertEqual(write_buf.mode, 0o644) + def assertCfgEquals(self, blob1, blob2): b1 = dict(SysConf(blob1.strip().splitlines())) b2 = dict(SysConf(blob2.strip().splitlines())) @@ -191,6 +389,79 @@ ONBOOT=yes expected_buf = ''' # Created by cloud-init v. 0.7 NETWORKING=yes +''' + self.assertCfgEquals(expected_buf, str(write_buf)) + self.assertEqual(write_buf.mode, 0o644) + + def test_apply_network_config_rh(self): + renderers = ['sysconfig'] + rh_distro = self._get_distro('rhel', renderers=renderers) + + write_bufs = {} + + def replace_write(filename, content, mode=0o644, omode="wb"): + buf = WriteBuffer() + buf.mode = mode + buf.omode = omode + buf.write(content) + write_bufs[filename] = buf + + with ExitStack() as mocks: + # sysconfig availability checks + mocks.enter_context( + mock.patch.object(util, 'which', return_value=True)) + mocks.enter_context( + mock.patch.object(util, 'write_file', replace_write)) + mocks.enter_context( + mock.patch.object(util, 'load_file', return_value='')) + mocks.enter_context( + mock.patch.object(os.path, 'isfile', return_value=True)) + + rh_distro.apply_network_config(V1_NET_CFG, False) + + self.assertEqual(len(write_bufs), 5) + + # eth0 + self.assertIn('/etc/sysconfig/network-scripts/ifcfg-eth0', + write_bufs) + write_buf = write_bufs['/etc/sysconfig/network-scripts/ifcfg-eth0'] + expected_buf = ''' +# Created by cloud-init on instance boot automatically, do not edit. +# +BOOTPROTO=static +DEVICE=eth0 +IPADDR=192.168.1.5 +NETMASK=255.255.255.0 +NM_CONTROLLED=no +ONBOOT=yes +TYPE=Ethernet +USERCTL=no +''' + self.assertCfgEquals(expected_buf, str(write_buf)) + self.assertEqual(write_buf.mode, 0o644) + + # eth1 + self.assertIn('/etc/sysconfig/network-scripts/ifcfg-eth1', + write_bufs) + write_buf = write_bufs['/etc/sysconfig/network-scripts/ifcfg-eth1'] + expected_buf = ''' +# Created by cloud-init on instance boot automatically, do not edit. +# +BOOTPROTO=dhcp +DEVICE=eth1 +NM_CONTROLLED=no +ONBOOT=yes +TYPE=Ethernet +USERCTL=no +''' + self.assertCfgEquals(expected_buf, str(write_buf)) + self.assertEqual(write_buf.mode, 0o644) + + self.assertIn('/etc/sysconfig/network', write_bufs) + write_buf = write_bufs['/etc/sysconfig/network'] + expected_buf = ''' +# Created by cloud-init v. 0.7 +NETWORKING=yes ''' self.assertCfgEquals(expected_buf, str(write_buf)) self.assertEqual(write_buf.mode, 0o644) @@ -270,6 +541,78 @@ IPV6_DEFAULTGW="2607:f0d0:1002:0011::1" NETWORKING=yes NETWORKING_IPV6=yes IPV6_AUTOCONF=no +''' + self.assertCfgEquals(expected_buf, str(write_buf)) + self.assertEqual(write_buf.mode, 0o644) + + def test_apply_network_config_ipv6_rh(self): + renderers = ['sysconfig'] + rh_distro = self._get_distro('rhel', renderers=renderers) + + write_bufs = {} + + def replace_write(filename, content, mode=0o644, omode="wb"): + buf = WriteBuffer() + buf.mode = mode + buf.omode = omode + buf.write(content) + write_bufs[filename] = buf + + with ExitStack() as mocks: + mocks.enter_context( + mock.patch.object(util, 'which', return_value=True)) + mocks.enter_context( + mock.patch.object(util, 'write_file', replace_write)) + mocks.enter_context( + mock.patch.object(util, 'load_file', return_value='')) + mocks.enter_context( + mock.patch.object(os.path, 'isfile', return_value=True)) + + rh_distro.apply_network_config(V1_NET_CFG_IPV6, False) + + self.assertEqual(len(write_bufs), 5) + + self.assertIn('/etc/sysconfig/network-scripts/ifcfg-eth0', + write_bufs) + write_buf = write_bufs['/etc/sysconfig/network-scripts/ifcfg-eth0'] + expected_buf = ''' +# Created by cloud-init on instance boot automatically, do not edit. +# +BOOTPROTO=static +DEVICE=eth0 +IPV6ADDR=2607:f0d0:1002:0011::2 +IPV6INIT=yes +NETMASK=64 +NM_CONTROLLED=no +ONBOOT=yes +TYPE=Ethernet +USERCTL=no +''' + self.assertCfgEquals(expected_buf, str(write_buf)) + self.assertEqual(write_buf.mode, 0o644) + self.assertIn('/etc/sysconfig/network-scripts/ifcfg-eth1', + write_bufs) + write_buf = write_bufs['/etc/sysconfig/network-scripts/ifcfg-eth1'] + expected_buf = ''' +# Created by cloud-init on instance boot automatically, do not edit. +# +BOOTPROTO=dhcp +DEVICE=eth1 +NM_CONTROLLED=no +ONBOOT=yes +TYPE=Ethernet +USERCTL=no +''' + self.assertCfgEquals(expected_buf, str(write_buf)) + self.assertEqual(write_buf.mode, 0o644) + + self.assertIn('/etc/sysconfig/network', write_bufs) + write_buf = write_bufs['/etc/sysconfig/network'] + expected_buf = ''' +# Created by cloud-init v. 0.7 +NETWORKING=yes +NETWORKING_IPV6=yes +IPV6_AUTOCONF=no ''' self.assertCfgEquals(expected_buf, str(write_buf)) self.assertEqual(write_buf.mode, 0o644) diff --git a/tests/unittests/test_net.py b/tests/unittests/test_net.py index 902204a0..4f07d804 100644 --- a/tests/unittests/test_net.py +++ b/tests/unittests/test_net.py @@ -3,6 +3,7 @@ from cloudinit import net from cloudinit.net import cmdline from cloudinit.net import eni +from cloudinit.net import netplan from cloudinit.net import network_state from cloudinit.net import renderers from cloudinit.net import sysconfig @@ -408,6 +409,41 @@ NETWORK_CONFIGS = { post-up route add default gw 65.61.151.37 || true pre-down route del default gw 65.61.151.37 || true """).rstrip(' '), + 'expected_netplan': textwrap.dedent(""" + network: + version: 2 + ethernets: + eth1: + match: + macaddress: cf:d6:af:48:e8:80 + nameservers: + addresses: + - 1.2.3.4 + - 5.6.7.8 + search: + - wark.maas + set-name: eth1 + eth99: + addresses: + - 192.168.21.3/24 + dhcp4: true + match: + macaddress: c0:d6:9f:2c:e8:80 + nameservers: + addresses: + - 8.8.8.8 + - 8.8.4.4 + - 1.2.3.4 + - 5.6.7.8 + search: + - barley.maas + - sach.maas + - wark.maas + routes: + - to: 0.0.0.0/0.0.0.0 + via: 65.61.151.37 + set-name: eth99 + """).rstrip(' '), 'yaml': textwrap.dedent(""" version: 1 config: @@ -450,6 +486,14 @@ NETWORK_CONFIGS = { # control-alias iface0 iface iface0 inet6 dhcp """).rstrip(' '), + 'expected_netplan': textwrap.dedent(""" + network: + version: 2 + ethernets: + iface0: + dhcp4: true + dhcp6: true + """).rstrip(' '), 'yaml': textwrap.dedent("""\ version: 1 config: @@ -524,6 +568,126 @@ iface eth0.101 inet static post-up route add -net 10.0.0.0 netmask 255.0.0.0 gw 11.0.0.1 metric 3 || true pre-down route del -net 10.0.0.0 netmask 255.0.0.0 gw 11.0.0.1 metric 3 || true """), + 'expected_netplan': textwrap.dedent(""" + network: + version: 2 + ethernets: + eth0: + match: + macaddress: c0:d6:9f:2c:e8:80 + nameservers: + addresses: + - 8.8.8.8 + - 4.4.4.4 + - 8.8.4.4 + search: + - barley.maas + - wark.maas + - foobar.maas + set-name: eth0 + eth1: + match: + macaddress: aa:d6:9f:2c:e8:80 + nameservers: + addresses: + - 8.8.8.8 + - 4.4.4.4 + - 8.8.4.4 + search: + - barley.maas + - wark.maas + - foobar.maas + set-name: eth1 + eth2: + match: + macaddress: c0:bb:9f:2c:e8:80 + nameservers: + addresses: + - 8.8.8.8 + - 4.4.4.4 + - 8.8.4.4 + search: + - barley.maas + - wark.maas + - foobar.maas + set-name: eth2 + eth3: + match: + macaddress: 66:bb:9f:2c:e8:80 + nameservers: + addresses: + - 8.8.8.8 + - 4.4.4.4 + - 8.8.4.4 + search: + - barley.maas + - wark.maas + - foobar.maas + set-name: eth3 + eth4: + match: + macaddress: 98:bb:9f:2c:e8:80 + nameservers: + addresses: + - 8.8.8.8 + - 4.4.4.4 + - 8.8.4.4 + search: + - barley.maas + - wark.maas + - foobar.maas + set-name: eth4 + eth5: + dhcp4: true + match: + macaddress: 98:bb:9f:2c:e8:8a + nameservers: + addresses: + - 8.8.8.8 + - 4.4.4.4 + - 8.8.4.4 + search: + - barley.maas + - wark.maas + - foobar.maas + set-name: eth5 + bonds: + bond0: + dhcp6: true + interfaces: + - eth1 + - eth2 + parameters: + mode: active-backup + bridges: + br0: + addresses: + - 192.168.14.2/24 + - 2001:1::1/64 + interfaces: + - eth3 + - eth4 + vlans: + bond0.200: + dhcp4: true + id: 200 + link: bond0 + eth0.101: + addresses: + - 192.168.0.2/24 + - 192.168.2.10/24 + gateway4: 192.168.0.1 + id: 101 + link: eth0 + nameservers: + addresses: + - 192.168.0.10 + - 10.23.23.134 + search: + - barley.maas + - sacchromyces.maas + - brettanomyces.maas + """).rstrip(' '), 'yaml': textwrap.dedent(""" version: 1 config: @@ -808,6 +972,99 @@ iface eth0 inet dhcp expected, dir2dict(tmp_dir)['/etc/network/interfaces']) +class TestNetplanNetRendering(CiTestCase): + + @mock.patch("cloudinit.net.sys_dev_path") + @mock.patch("cloudinit.net.read_sys_net") + @mock.patch("cloudinit.net.get_devicelist") + def test_default_generation(self, mock_get_devicelist, + mock_read_sys_net, + mock_sys_dev_path): + tmp_dir = self.tmp_dir() + _setup_test(tmp_dir, mock_get_devicelist, + mock_read_sys_net, mock_sys_dev_path) + + network_cfg = net.generate_fallback_config() + ns = network_state.parse_net_config_data(network_cfg, + skip_broken=False) + + render_dir = os.path.join(tmp_dir, "render") + os.makedirs(render_dir) + + render_target = 'netplan.yaml' + renderer = netplan.Renderer( + {'netplan_path': render_target, 'postcmds': False}) + renderer.render_network_state(render_dir, ns) + + self.assertTrue(os.path.exists(os.path.join(render_dir, + render_target))) + with open(os.path.join(render_dir, render_target)) as fh: + contents = fh.read() + print(contents) + + expected = """ +network: + version: 2 + ethernets: + eth1000: + dhcp4: true + match: + macaddress: 07-1c-c6-75-a4-be + set-name: eth1000 +""" + self.assertEqual(expected.lstrip(), contents.lstrip()) + + +class TestNetplanPostcommands(CiTestCase): + mycfg = { + 'config': [{"type": "physical", "name": "eth0", + "mac_address": "c0:d6:9f:2c:e8:80", + "subnets": [{"type": "dhcp"}]}], + 'version': 1} + + @mock.patch.object(netplan.Renderer, '_netplan_generate') + @mock.patch.object(netplan.Renderer, '_net_setup_link') + def test_netplan_render_calls_postcmds(self, mock_netplan_generate, + mock_net_setup_link): + tmp_dir = self.tmp_dir() + ns = network_state.parse_net_config_data(self.mycfg, + skip_broken=False) + + render_dir = os.path.join(tmp_dir, "render") + os.makedirs(render_dir) + + render_target = 'netplan.yaml' + renderer = netplan.Renderer( + {'netplan_path': render_target, 'postcmds': True}) + renderer.render_network_state(render_dir, ns) + + mock_netplan_generate.assert_called_with(run=True) + mock_net_setup_link.assert_called_with(run=True) + + @mock.patch.object(netplan, "get_devicelist") + @mock.patch('cloudinit.util.subp') + def test_netplan_postcmds(self, mock_subp, mock_devlist): + mock_devlist.side_effect = [['lo']] + tmp_dir = self.tmp_dir() + ns = network_state.parse_net_config_data(self.mycfg, + skip_broken=False) + + render_dir = os.path.join(tmp_dir, "render") + os.makedirs(render_dir) + + render_target = 'netplan.yaml' + renderer = netplan.Renderer( + {'netplan_path': render_target, 'postcmds': True}) + renderer.render_network_state(render_dir, ns) + + expected = [ + mock.call(['netplan', 'generate'], capture=True), + mock.call(['udevadm', 'test-builtin', 'net_setup_link', + '/sys/class/net/lo'], capture=True), + ] + mock_subp.assert_has_calls(expected) + + class TestEniNetworkStateToEni(CiTestCase): mycfg = { 'config': [{"type": "physical", "name": "eth0", @@ -953,6 +1210,50 @@ class TestCmdlineReadKernelConfig(CiTestCase): self.assertEqual(found['config'], expected) +class TestNetplanRoundTrip(CiTestCase): + def _render_and_read(self, network_config=None, state=None, + netplan_path=None, dir=None): + if dir is None: + dir = self.tmp_dir() + + if network_config: + ns = network_state.parse_net_config_data(network_config) + elif state: + ns = state + else: + raise ValueError("Expected data or state, got neither") + + if netplan_path is None: + netplan_path = 'etc/netplan/50-cloud-init.yaml' + + renderer = netplan.Renderer( + config={'netplan_path': netplan_path}) + + renderer.render_network_state(dir, ns) + return dir2dict(dir) + + def testsimple_render_small_netplan(self): + entry = NETWORK_CONFIGS['small'] + files = self._render_and_read(network_config=yaml.load(entry['yaml'])) + self.assertEqual( + entry['expected_netplan'].splitlines(), + files['/etc/netplan/50-cloud-init.yaml'].splitlines()) + + def testsimple_render_v4_and_v6(self): + entry = NETWORK_CONFIGS['v4_and_v6'] + files = self._render_and_read(network_config=yaml.load(entry['yaml'])) + self.assertEqual( + entry['expected_netplan'].splitlines(), + files['/etc/netplan/50-cloud-init.yaml'].splitlines()) + + def testsimple_render_all(self): + entry = NETWORK_CONFIGS['all'] + files = self._render_and_read(network_config=yaml.load(entry['yaml'])) + self.assertEqual( + entry['expected_netplan'].splitlines(), + files['/etc/netplan/50-cloud-init.yaml'].splitlines()) + + class TestEniRoundTrip(CiTestCase): def _render_and_read(self, network_config=None, state=None, eni_path=None, links_prefix=None, netrules_path=None, dir=None): diff --git a/tools/net-convert.py b/tools/net-convert.py new file mode 100755 index 00000000..870da639 --- /dev/null +++ b/tools/net-convert.py @@ -0,0 +1,84 @@ +#!/usr/bin/python3 +# This file is part of cloud-init. See LICENSE file for license information. + +import argparse +import json +import os +import yaml + +from cloudinit.sources.helpers import openstack + +from cloudinit.net import eni +from cloudinit.net import network_state +from cloudinit.net import netplan +from cloudinit.net import sysconfig + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--network-data", "-p", type=open, + metavar="PATH", required=True) + parser.add_argument("--kind", "-k", + choices=['eni', 'network_data.json', 'yaml'], + required=True) + parser.add_argument("-d", "--directory", + metavar="PATH", + help="directory to place output in", + required=True) + parser.add_argument("-m", "--mac", + metavar="name,mac", + action='append', + help="interface name to mac mapping") + parser.add_argument("--output-kind", "-ok", + choices=['eni', 'netplan', 'sysconfig'], + required=True) + args = parser.parse_args() + + if not os.path.isdir(args.directory): + os.makedirs(args.directory) + + if args.mac: + known_macs = {} + for item in args.mac: + iface_name, iface_mac = item.split(",", 1) + known_macs[iface_mac] = iface_name + else: + known_macs = None + + net_data = args.network_data.read() + if args.kind == "eni": + pre_ns = eni.convert_eni_data(net_data) + ns = network_state.parse_net_config_data(pre_ns) + elif args.kind == "yaml": + pre_ns = yaml.load(net_data) + if 'network' in pre_ns: + pre_ns = pre_ns.get('network') + print("Input YAML") + print(yaml.dump(pre_ns, default_flow_style=False, indent=4)) + ns = network_state.parse_net_config_data(pre_ns) + else: + pre_ns = openstack.convert_net_json( + json.loads(net_data), known_macs=known_macs) + ns = network_state.parse_net_config_data(pre_ns) + + if not ns: + raise RuntimeError("No valid network_state object created from" + "input data") + + print("\nInternal State") + print(yaml.dump(ns, default_flow_style=False, indent=4)) + if args.output_kind == "eni": + r_cls = eni.Renderer + elif args.output_kind == "netplan": + r_cls = netplan.Renderer + else: + r_cls = sysconfig.Renderer + + r = r_cls() + r.render_network_state(ns, target=args.directory) + + +if __name__ == '__main__': + main() + +# vi: ts=4 expandtab -- cgit v1.2.3 From 0e2030ca7fe783ead06100c748f4714895461799 Mon Sep 17 00:00:00 2001 From: Steve Langasek Date: Tue, 28 Mar 2017 21:47:50 -0700 Subject: support resizing partition and rootfs on system booted without initramfs. When booted without an initramfs, the root device will be /dev/root, not a named device. There is partial support for this when resizing filesystems, but not for growing partitions, without which it doesn't do much good. Move the /dev/root resolution code to util.py and use it from cc_growpart.py. Also, booting without an initramfs only works with a root= argument that's either a kernel device name (which is unstable) or a partition UUID. Handle the case of root=PARTUUID=value, not just LABEL and UUID. LP: #1677376 --- cloudinit/config/cc_growpart.py | 11 ++++++++++- cloudinit/config/cc_resizefs.py | 21 +-------------------- cloudinit/util.py | 29 +++++++++++++++++++++++++++++ 3 files changed, 40 insertions(+), 21 deletions(-) (limited to 'cloudinit/util.py') diff --git a/cloudinit/config/cc_growpart.py b/cloudinit/config/cc_growpart.py index 832bb3fd..089693e8 100644 --- a/cloudinit/config/cc_growpart.py +++ b/cloudinit/config/cc_growpart.py @@ -247,7 +247,16 @@ def devent2dev(devent): result = util.get_mount_info(devent) if not result: raise ValueError("Could not determine device of '%s' % dev_ent") - return result[0] + dev = result[0] + + container = util.is_container() + + # Ensure the path is a block device. + if (dev == "/dev/root" and not os.path.exists(dev) and not container): + dev = util.rootdev_from_cmdline(util.get_cmdline()) + if dev is None: + raise ValueError("Unable to find device '/dev/root'") + return dev def resize_devices(resizer, devices): diff --git a/cloudinit/config/cc_resizefs.py b/cloudinit/config/cc_resizefs.py index e028abf4..60e3ab53 100644 --- a/cloudinit/config/cc_resizefs.py +++ b/cloudinit/config/cc_resizefs.py @@ -71,25 +71,6 @@ RESIZE_FS_PREFIXES_CMDS = [ NOBLOCK = "noblock" -def rootdev_from_cmdline(cmdline): - found = None - for tok in cmdline.split(): - if tok.startswith("root="): - found = tok[5:] - break - if found is None: - return None - - if found.startswith("/dev/"): - return found - if found.startswith("LABEL="): - return "/dev/disk/by-label/" + found[len("LABEL="):] - if found.startswith("UUID="): - return "/dev/disk/by-uuid/" + found[len("UUID="):] - - return "/dev/" + found - - def handle(name, cfg, _cloud, log, args): if len(args) != 0: resize_root = args[0] @@ -121,7 +102,7 @@ def handle(name, cfg, _cloud, log, args): # Ensure the path is a block device. if (devpth == "/dev/root" and not os.path.exists(devpth) and not container): - devpth = rootdev_from_cmdline(util.get_cmdline()) + devpth = util.rootdev_from_cmdline(util.get_cmdline()) if devpth is None: log.warn("Unable to find device '/dev/root'") return diff --git a/cloudinit/util.py b/cloudinit/util.py index 33019579..17abdf81 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -2382,4 +2382,33 @@ def indent(text, prefix): return ''.join(lines) +def rootdev_from_cmdline(cmdline): + found = None + for tok in cmdline.split(): + if tok.startswith("root="): + found = tok[5:] + break + if found is None: + return None + + if found.startswith("/dev/"): + return found + if found.startswith("LABEL="): + return "/dev/disk/by-label/" + found[len("LABEL="):] + if found.startswith("UUID="): + return "/dev/disk/by-uuid/" + found[len("UUID="):] + if found.startswith("PARTUUID="): + disks_path = "/dev/disk/by-partuuid/" + found[len("PARTUUID="):] + if os.path.exists(disks_path): + return disks_path + results = find_devs_with(found) + if results: + return results[0] + # we know this doesn't exist, but for consistency return the path as + # it /would/ exist + return disks_path + + return "/dev/" + found + + # vi: ts=4 expandtab -- cgit v1.2.3