cloudinit.net: add network config v2 parsing and rendering

Network configuration version 2 format is implemented in a package
called netplan (nplan)[1] which allows consolidated network config
for multiple network controllers.

- Add a new netplan renderer
- Update default policy, placing eni and sysconfig first
  This requires explicit policy to enable netplan over eni
  on systems which have both (Yakkety, Zesty, UC16)
- Allow any network state (parsed from any format cloud-init supports) to
  render to v2 if system supports netplan.
- Move eni's _subnet_is_ipv6 to common code for use by other renderers
- Make sysconfig renderer always emit /etc/syconfig/network configuration
- Update cloud-init.service systemd unit to also wait on
  systemd-networkd-wait-online.service

1. https://lists.ubuntu.com/archives/ubuntu-devel/2016-July/039464.html

1 files changed, 373 insertions, 0 deletions

diff --git a/cloudinit/net/netplan.py b/cloudinit/net/netplan.py
new file mode 100644
index 00000000..cd93b21c
--- /dev/null
+++ b/cloudinit/net/netplan.py
@@ -0,0 +1,373 @@
+# This file is part of cloud-init.  See LICENSE file ...
+
+import copy
+import os
+
+from . import renderer
+from .network_state import subnet_is_ipv6
+
+from cloudinit import util
+from cloudinit.net import SYS_CLASS_NET, get_devicelist
+
+
+NET_CONFIG_TO_V2 = {
+    'bond': {'bond-ad-select': 'ad-select',
+             'bond-arp-interval': 'arp-interval',
+             'bond-arp-ip-target': 'arp-ip-target',
+             'bond-arp-validate': 'arp-validate',
+             'bond-downdelay': 'down-delay',
+             'bond-fail-over-mac': 'fail-over-mac-policy',
+             'bond-lacp-rate': 'lacp-rate',
+             'bond-miimon': 'mii-monitor-interval',
+             'bond-min-links': 'min-links',
+             'bond-mode': 'mode',
+             'bond-num-grat-arp': 'gratuitious-arp',
+             'bond-primary-reselect': 'primary-reselect-policy',
+             'bond-updelay': 'up-delay',
+             'bond-xmit_hash_policy': 'transmit_hash_policy'},
+    'bridge': {'bridge_ageing': 'ageing-time',
+               'bridge_bridgeprio': 'priority',
+               'bridge_fd': 'forward-delay',
+               'bridge_gcint': None,
+               'bridge_hello': 'hello-time',
+               'bridge_maxage': 'max-age',
+               'bridge_maxwait': None,
+               'bridge_pathcost': 'path-cost',
+               'bridge_portprio': None,
+               'bridge_waitport': None}}
+
+
+def _get_params_dict_by_match(config, match):
+    return dict((key, value) for (key, value) in config.items()
+                if key.startswith(match))
+
+
+def _extract_addresses(config, entry):
+    """This method parse a cloudinit.net.network_state dictionary (config) and
+       maps netstate keys/values into a dictionary (entry) to represent
+       netplan yaml.
+
+    An example config dictionary might look like:
+
+    {'mac_address': '52:54:00:12:34:00',
+     'name': 'interface0',
+     'subnets': [
+        {'address': '192.168.1.2/24',
+         'mtu': 1501,
+         'type': 'static'},
+        {'address': '2001:4800:78ff:1b:be76:4eff:fe06:1000",
+         'mtu': 1480,
+         'netmask': 64,
+         'type': 'static'}],
+      'type: physical'
+    }
+
+    An entry dictionary looks like:
+
+    {'set-name': 'interface0',
+     'match': {'macaddress': '52:54:00:12:34:00'},
+     'mtu': 1501}
+
+    After modification returns
+
+    {'set-name': 'interface0',
+     'match': {'macaddress': '52:54:00:12:34:00'},
+     'mtu': 1501,
+     'address': ['192.168.1.2/24', '2001:4800:78ff:1b:be76:4eff:fe06:1000"],
+     'mtu6': 1480}
+
+    """
+
+    def _listify(obj, token=' '):
+        "Helper to convert strings to list of strings, handle single string"
+        if not obj or type(obj) not in [str]:
+            return obj
+        if token in obj:
+            return obj.split(token)
+        else:
+            return [obj, ]
+
+    addresses = []
+    routes = []
+    nameservers = []
+    searchdomains = []
+    subnets = config.get('subnets', [])
+    if subnets is None:
+        subnets = []
+    for subnet in subnets:
+        sn_type = subnet.get('type')
+        if sn_type.startswith('dhcp'):
+            if sn_type == 'dhcp':
+                sn_type += '4'
+            entry.update({sn_type: True})
+        elif sn_type in ['static']:
+            addr = "%s" % subnet.get('address')
+            if 'netmask' in subnet:
+                addr += "/%s" % subnet.get('netmask')
+            if 'gateway' in subnet and subnet.get('gateway'):
+                gateway = subnet.get('gateway')
+                if ":" in gateway:
+                    entry.update({'gateway6': gateway})
+                else:
+                    entry.update({'gateway4': gateway})
+            if 'dns_nameservers' in subnet:
+                nameservers += _listify(subnet.get('dns_nameservers', []))
+            if 'dns_search' in subnet:
+                searchdomains += _listify(subnet.get('dns_search', []))
+            if 'mtu' in subnet:
+                mtukey = 'mtu'
+                if subnet_is_ipv6(subnet):
+                    mtukey += '6'
+                entry.update({mtukey: subnet.get('mtu')})
+            for route in subnet.get('routes', []):
+                to_net = "%s/%s" % (route.get('network'),
+                                    route.get('netmask'))
+                route = {
+                    'via': route.get('gateway'),
+                    'to': to_net,
+                }
+                if 'metric' in route:
+                    route.update({'metric': route.get('metric', 100)})
+                routes.append(route)
+
+            addresses.append(addr)
+
+    if len(addresses) > 0:
+        entry.update({'addresses': addresses})
+    if len(routes) > 0:
+        entry.update({'routes': routes})
+    if len(nameservers) > 0:
+        ns = {'addresses': nameservers}
+        entry.update({'nameservers': ns})
+    if len(searchdomains) > 0:
+        ns = entry.get('nameservers', {})
+        ns.update({'search': searchdomains})
+        entry.update({'nameservers': ns})
+
+
+def _extract_bond_slaves_by_name(interfaces, entry, bond_master):
+    bond_slave_names = sorted([name for (name, cfg) in interfaces.items()
+                               if cfg.get('bond-master', None) == bond_master])
+    if len(bond_slave_names) > 0:
+        entry.update({'interfaces': bond_slave_names})
+
+
+class Renderer(renderer.Renderer):
+    """Renders network information in a /etc/netplan/network.yaml format."""
+
+    NETPLAN_GENERATE = ['netplan', 'generate']
+
+    def __init__(self, config=None):
+        if not config:
+            config = {}
+        self.netplan_path = config.get('netplan_path',
+                                       'etc/netplan/50-cloud-init.yaml')
+        self.netplan_header = config.get('netplan_header', None)
+        self._postcmds = config.get('postcmds', False)
+
+    def render_network_state(self, target, network_state):
+        # check network state for version
+        # if v2, then extract network_state.config
+        # else render_v2_from_state
+        fpnplan = os.path.join(target, self.netplan_path)
+        util.ensure_dir(os.path.dirname(fpnplan))
+        header = self.netplan_header if self.netplan_header else ""
+
+        # render from state
+        content = self._render_content(network_state)
+
+        # ensure we poke udev to run net_setup_link
+        if not header.endswith("\n"):
+            header += "\n"
+        util.write_file(fpnplan, header + content)
+
+        self._netplan_generate(run=self._postcmds)
+        self._net_setup_link(run=self._postcmds)
+
+    def _netplan_generate(self, run=False):
+        if not run:
+            print("netplan postcmd disabled")
+            return
+        util.subp(self.NETPLAN_GENERATE, capture=True)
+
+    def _net_setup_link(self, run=False):
+        """To ensure device link properties are applied, we poke
+           udev to re-evaluate networkd .link files and call
+           the setup_link udev builtin command
+        """
+        if not run:
+            print("netsetup postcmd disabled")
+            return
+        setup_lnk = ['udevadm', 'test-builtin', 'net_setup_link']
+        for cmd in [setup_lnk + [SYS_CLASS_NET + iface]
+                    for iface in get_devicelist() if
+                    os.path.islink(SYS_CLASS_NET + iface)]:
+            print(cmd)
+            util.subp(cmd, capture=True)
+
+    def _render_content(self, network_state):
+        print('rendering v2 for victory!')
+        ethernets = {}
+        wifis = {}
+        bridges = {}
+        bonds = {}
+        vlans = {}
+        content = []
+
+        interfaces = network_state._network_state.get('interfaces', [])
+
+        nameservers = network_state.dns_nameservers
+        searchdomains = network_state.dns_searchdomains
+
+        for config in network_state.iter_interfaces():
+            ifname = config.get('name')
+            # filter None entries up front so we can do simple if key in dict
+            ifcfg = dict((key, value) for (key, value) in config.items()
+                         if value)
+
+            if_type = ifcfg.get('type')
+            if if_type == 'physical':
+                # required_keys = ['name', 'mac_address']
+                eth = {
+                    'set-name': ifname,
+                    'match': ifcfg.get('match', None),
+                }
+                if eth['match'] is None:
+                    macaddr = ifcfg.get('mac_address', None)
+                    if macaddr is not None:
+                        eth['match'] = {'macaddress': macaddr.lower()}
+                    else:
+                        del eth['match']
+                        del eth['set-name']
+                if 'mtu' in ifcfg:
+                    eth['mtu'] = ifcfg.get('mtu')
+
+                _extract_addresses(ifcfg, eth)
+                ethernets.update({ifname: eth})
+
+            elif if_type == 'bond':
+                # required_keys = ['name', 'bond_interfaces']
+                bond = {}
+                bond_config = {}
+                # extract bond params and drop the bond_ prefix as it's
+                # redundent in v2 yaml format
+                v2_bond_map = NET_CONFIG_TO_V2.get('bond')
+                for match in ['bond_', 'bond-']:
+                    bond_params = _get_params_dict_by_match(ifcfg, match)
+                    for (param, value) in bond_params.items():
+                        newname = v2_bond_map.get(param)
+                        if newname is None:
+                            continue
+                        bond_config.update({newname: value})
+
+                if len(bond_config) > 0:
+                    bond.update({'parameters': bond_config})
+                slave_interfaces = ifcfg.get('bond-slaves')
+                if slave_interfaces == 'none':
+                    _extract_bond_slaves_by_name(interfaces, bond, ifname)
+                _extract_addresses(ifcfg, bond)
+                bonds.update({ifname: bond})
+
+            elif if_type == 'bridge':
+                # required_keys = ['name', 'bridge_ports']
+                ports = sorted(copy.copy(ifcfg.get('bridge_ports')))
+                bridge = {
+                    'interfaces': ports,
+                }
+                # extract bridge params and drop the bridge prefix as it's
+                # redundent in v2 yaml format
+                match_prefix = 'bridge_'
+                params = _get_params_dict_by_match(ifcfg, match_prefix)
+                br_config = {}
+
+                # v2 yaml uses different names for the keys
+                # and at least one value format change
+                v2_bridge_map = NET_CONFIG_TO_V2.get('bridge')
+                for (param, value) in params.items():
+                    newname = v2_bridge_map.get(param)
+                    if newname is None:
+                        continue
+                    br_config.update({newname: value})
+                    if newname == 'path-cost':
+                        # <interface> <cost> -> <interface>: int(<cost>)
+                        newvalue = {}
+                        for costval in value:
+                            (port, cost) = costval.split()
+                            newvalue[port] = int(cost)
+                        br_config.update({newname: newvalue})
+                if len(br_config) > 0:
+                    bridge.update({'parameters': br_config})
+                _extract_addresses(ifcfg, bridge)
+                bridges.update({ifname: bridge})
+
+            elif if_type == 'vlan':
+                # required_keys = ['name', 'vlan_id', 'vlan-raw-device']
+                vlan = {
+                    'id': ifcfg.get('vlan_id'),
+                    'link': ifcfg.get('vlan-raw-device')
+                }
+
+                _extract_addresses(ifcfg, vlan)
+                vlans.update({ifname: vlan})
+
+        # inject global nameserver values under each physical interface
+        if nameservers:
+            for _eth, cfg in ethernets.items():
+                nscfg = cfg.get('nameservers', {})
+                addresses = nscfg.get('addresses', [])
+                addresses += nameservers
+                nscfg.update({'addresses': addresses})
+                cfg.update({'nameservers': nscfg})
+
+        if searchdomains:
+            for _eth, cfg in ethernets.items():
+                nscfg = cfg.get('nameservers', {})
+                search = nscfg.get('search', [])
+                search += searchdomains
+                nscfg.update({'search': search})
+                cfg.update({'nameservers': nscfg})
+
+        # workaround yaml dictionary key sorting when dumping
+        def _render_section(name, section):
+            if section:
+                dump = util.yaml_dumps({name: section},
+                                       explicit_start=False,
+                                       explicit_end=False)
+                txt = util.indent(dump, ' ' * 4)
+                return [txt]
+            return []
+
+        content.append("network:\n    version: 2\n")
+        content += _render_section('ethernets', ethernets)
+        content += _render_section('wifis', wifis)
+        content += _render_section('bonds', bonds)
+        content += _render_section('bridges', bridges)
+        content += _render_section('vlans', vlans)
+
+        return "".join(content)
+
+
+def available(target=None):
+    expected = ['netplan']
+    search = ['/usr/sbin', '/sbin']
+    for p in expected:
+        if not util.which(p, search=search, target=target):
+            return False
+    return True
+
+
+def network_state_to_netplan(network_state, header=None):
+    # render the provided network state, return a string of equivalent eni
+    netplan_path = 'etc/network/50-cloud-init.yaml'
+    renderer = Renderer({
+        'netplan_path': netplan_path,
+        'netplan_header': header,
+    })
+    if not header:
+        header = ""
+    if not header.endswith("\n"):
+        header += "\n"
+    contents = renderer._render_content(network_state)
+    return header + contents
+
+# vi: ts=4 expandtab