net: normalize data in network_state object

The network_state object's network and route keys would have different
information depending upon how the network_state object was populated.

This change cleans that up. Now:
  * address will always contain an IP address.
  * prefix will always include an integer value that is the
    network_prefix for the address.
  * netmask will be present only if the address is ipv4, and its
    value will always correlate to the 'prefix'.

4 files changed, 211 insertions, 74 deletions

diff --git a/cloudinit/net/eni.py b/cloudinit/net/eni.py
index 20e19f5b..98ce01e4 100644
--- a/cloudinit/net/eni.py
+++ b/cloudinit/net/eni.py
@@ -46,6 +46,10 @@ def _iface_add_subnet(iface, subnet):
         'dns_nameservers',
     ]
     for key, value in subnet.items():
+        if key == 'netmask':
+            continue
+        if key == 'address':
+            value = "%s/%s" % (subnet['address'], subnet['prefix'])
         if value and key in valid_map:
             if type(value) == list:
                 value = " ".join(value)
diff --git a/cloudinit/net/netplan.py b/cloudinit/net/netplan.py
index a715f3b0..67543305 100644
--- a/cloudinit/net/netplan.py
+++ b/cloudinit/net/netplan.py
@@ -4,7 +4,7 @@ import copy
 import os
 
 from . import renderer
-from .network_state import mask2cidr, subnet_is_ipv6
+from .network_state import subnet_is_ipv6
 
 from cloudinit import log as logging
 from cloudinit import util
@@ -118,10 +118,9 @@ def _extract_addresses(config, entry):
                 sn_type += '4'
             entry.update({sn_type: True})
         elif sn_type in ['static']:
-            addr = '%s' % subnet.get('address')
-            netmask = subnet.get('netmask')
-            if netmask and '/' not in addr:
-                addr += '/%s' % mask2cidr(netmask)
+            addr = "%s" % subnet.get('address')
+            if 'prefix' in subnet:
+                addr += "/%d" % subnet.get('prefix')
             if 'gateway' in subnet and subnet.get('gateway'):
                 gateway = subnet.get('gateway')
                 if ":" in gateway:
@@ -138,9 +137,8 @@ def _extract_addresses(config, entry):
                     mtukey += '6'
                 entry.update({mtukey: subnet.get('mtu')})
             for route in subnet.get('routes', []):
-                network = route.get('network')
-                netmask = route.get('netmask')
-                to_net = '%s/%s' % (network, mask2cidr(netmask))
+                to_net = "%s/%s" % (route.get('network'),
+                                    route.get('prefix'))
                 route = {
                     'via': route.get('gateway'),
                     'to': to_net,
diff --git a/cloudinit/net/network_state.py b/cloudinit/net/network_state.py
index 9e9c05a0..87a7222d 100644
--- a/cloudinit/net/network_state.py
+++ b/cloudinit/net/network_state.py
@@ -289,19 +289,15 @@ class NetworkStateInterpreter(object):
             iface.update({param: val})
 
         # convert subnet ipv6 netmask to cidr as needed
-        subnets = command.get('subnets')
-        if subnets:
+        subnets = _normalize_subnets(command.get('subnets'))
+
+        # automatically set 'use_ipv6' if any addresses are ipv6
+        if not self.use_ipv6:
             for subnet in subnets:
-                if subnet['type'] == 'static':
-                    if ':' in subnet['address']:
-                        self.use_ipv6 = True
-                    if 'netmask' in subnet and ':' in subnet['address']:
-                        subnet['netmask'] = mask2cidr(subnet['netmask'])
-                        for route in subnet.get('routes', []):
-                            if 'netmask' in route:
-                                route['netmask'] = mask2cidr(route['netmask'])
-                elif subnet['type'].endswith('6'):
+                if (subnet.get('type').endswith('6') or
+                        is_ipv6_addr(subnet.get('address'))):
                     self.use_ipv6 = True
+                    break
 
         iface.update({
             'name': command.get('name'),
@@ -456,16 +452,7 @@ class NetworkStateInterpreter(object):
 
     @ensure_command_keys(['destination'])
     def handle_route(self, command):
-        routes = self._network_state.get('routes', [])
-        network, cidr = command['destination'].split("/")
-        netmask = cidr2mask(int(cidr))
-        route = {
-            'network': network,
-            'netmask': netmask,
-            'gateway': command.get('gateway'),
-            'metric': command.get('metric'),
-        }
-        routes.append(route)
+        self._network_state['routes'].append(_normalize_route(command))
 
     # V2 handlers
     def handle_bonds(self, command):
@@ -666,18 +653,9 @@ class NetworkStateInterpreter(object):
 
             routes = []
             for route in cfg.get('routes', []):
-                route_addr = route.get('to')
-                if "/" in route_addr:
-                    route_addr, route_cidr = route_addr.split("/")
-                route_netmask = cidr2mask(route_cidr)
-                subnet_route = {
-                    'address': route_addr,
-                    'netmask': route_netmask,
-                    'gateway': route.get('via')
-                }
-                routes.append(subnet_route)
-            if len(routes) > 0:
-                subnet.update({'routes': routes})
+                routes.append(_normalize_route(
+                    {'address': route.get('to'), 'gateway': route.get('via')}))
+            subnet['routes'] = routes
 
             if ":" in address:
                 if 'gateway6' in cfg and gateway6 is None:
@@ -692,53 +670,219 @@ class NetworkStateInterpreter(object):
         return subnets
 
 
+def _normalize_subnet(subnet):
+    # Prune all keys with None values.
+    subnet = copy.deepcopy(subnet)
+    normal_subnet = dict((k, v) for k, v in subnet.items() if v)
+
+    if subnet.get('type') in ('static', 'static6'):
+        normal_subnet.update(
+            _normalize_net_keys(normal_subnet, address_keys=('address',)))
+    normal_subnet['routes'] = [_normalize_route(r)
+                               for r in subnet.get('routes', [])]
+    return normal_subnet
+
+
+def _normalize_net_keys(network, address_keys=()):
+    """Normalize dictionary network keys returning prefix and address keys.
+
+    @param network: A dict of network-related definition containing prefix,
+        netmask and address_keys.
+    @param address_keys: A tuple of keys to search for representing the address
+        or cidr. The first address_key discovered will be used for
+        normalization.
+
+    @returns: A dict containing normalized prefix and matching addr_key.
+    """
+    net = dict((k, v) for k, v in network.items() if v)
+    addr_key = None
+    for key in address_keys:
+        if net.get(key):
+            addr_key = key
+            break
+    if not addr_key:
+        message = (
+            'No config network address keys [%s] found in %s' %
+            (','.join(address_keys), network))
+        LOG.error(message)
+        raise ValueError(message)
+
+    addr = net.get(addr_key)
+    ipv6 = is_ipv6_addr(addr)
+    netmask = net.get('netmask')
+    if "/" in addr:
+        addr_part, _, maybe_prefix = addr.partition("/")
+        net[addr_key] = addr_part
+        try:
+            prefix = int(maybe_prefix)
+        except ValueError:
+            # this supports input of <address>/255.255.255.0
+            prefix = mask_to_net_prefix(maybe_prefix)
+    elif netmask:
+        prefix = mask_to_net_prefix(netmask)
+    elif 'prefix' in net:
+        prefix = int(prefix)
+    else:
+        prefix = 64 if ipv6 else 24
+
+    if 'prefix' in net and str(net['prefix']) != str(prefix):
+        LOG.warning("Overwriting existing 'prefix' with '%s' in "
+                    "network info: %s", prefix, net)
+    net['prefix'] = prefix
+
+    if ipv6:
+        # TODO: we could/maybe should add this back with the very uncommon
+        # 'netmask' for ipv6.  We need a 'net_prefix_to_ipv6_mask' for that.
+        if 'netmask' in net:
+            del net['netmask']
+    else:
+        net['netmask'] = net_prefix_to_ipv4_mask(net['prefix'])
+
+    return net
+
+
+def _normalize_route(route):
+    """normalize a route.
+    return a dictionary with only:
+       'type': 'route' (only present if it was present in input)
+       'network': the network portion of the route as a string.
+       'prefix': the network prefix for address as an integer.
+       'metric': integer metric (only if present in input).
+       'netmask': netmask (string) equivalent to prefix iff network is ipv4.
+       """
+    # Prune None-value keys.  Specifically allow 0 (a valid metric).
+    normal_route = dict((k, v) for k, v in route.items()
+                        if v not in ("", None))
+    if 'destination' in normal_route:
+        normal_route['network'] = normal_route['destination']
+        del normal_route['destination']
+
+    normal_route.update(
+        _normalize_net_keys(
+            normal_route, address_keys=('network', 'destination')))
+
+    metric = normal_route.get('metric')
+    if metric:
+        try:
+            normal_route['metric'] = int(metric)
+        except ValueError:
+            raise TypeError(
+                'Route config metric {} is not an integer'.format(metric))
+    return normal_route
+
+
+def _normalize_subnets(subnets):
+    if not subnets:
+        subnets = []
+    return [_normalize_subnet(s) for s in subnets]
+
+
+def is_ipv6_addr(address):
+    if not address:
+        return False
+    return ":" in str(address)
+
+
 def subnet_is_ipv6(subnet):
     """Common helper for checking network_state subnets for ipv6."""
     # 'static6' or 'dhcp6'
     if subnet['type'].endswith('6'):
         # This is a request for DHCPv6.
         return True
-    elif subnet['type'] == 'static' and ":" in subnet['address']:
+    elif subnet['type'] == 'static' and is_ipv6_addr(subnet.get('address')):
         return True
     return False
 
 
-def cidr2mask(cidr):
+def net_prefix_to_ipv4_mask(prefix):
+    """Convert a network prefix to an ipv4 netmask.
+
+    This is the inverse of ipv4_mask_to_net_prefix.
+        24 -> "255.255.255.0"
+    Also supports input as a string."""
+
     mask = [0, 0, 0, 0]
-    for i in list(range(0, cidr)):
+    for i in list(range(0, int(prefix))):
         idx = int(i / 8)
         mask[idx] = mask[idx] + (1 << (7 - i % 8))
     return ".".join([str(x) for x in mask])
 
 
-def ipv4mask2cidr(mask):
-    if '.' not in mask:
+def ipv4_mask_to_net_prefix(mask):
+    """Convert an ipv4 netmask into a network prefix length.
+
+    If the input is already an integer or a string representation of
+    an integer, then int(mask) will be returned.
+       "255.255.255.0" => 24
+       str(24)         => 24
+       "24"            => 24
+    """
+    if isinstance(mask, int):
         return mask
-    return sum([bin(int(x)).count('1') for x in mask.split('.')])
+    if isinstance(mask, six.string_types):
+        try:
+            return int(mask)
+        except ValueError:
+            pass
+    else:
+        raise TypeError("mask '%s' is not a string or int")
 
+    if '.' not in mask:
+        raise ValueError("netmask '%s' does not contain a '.'" % mask)
 
-def ipv6mask2cidr(mask):
-    if ':' not in mask:
+    toks = mask.split(".")
+    if len(toks) != 4:
+        raise ValueError("netmask '%s' had only %d parts" % (mask, len(toks)))
+
+    return sum([bin(int(x)).count('1') for x in toks])
+
+
+def ipv6_mask_to_net_prefix(mask):
+    """Convert an ipv6 netmask (very uncommon) or prefix (64) to prefix.
+
+    If 'mask' is an integer or string representation of one then
+    int(mask) will be returned.
+    """
+
+    if isinstance(mask, int):
         return mask
+    if isinstance(mask, six.string_types):
+        try:
+            return int(mask)
+        except ValueError:
+            pass
+    else:
+        raise TypeError("mask '%s' is not a string or int")
+
+    if ':' not in mask:
+        raise ValueError("mask '%s' does not have a ':'")
 
     bitCount = [0, 0x8000, 0xc000, 0xe000, 0xf000, 0xf800, 0xfc00, 0xfe00,
                 0xff00, 0xff80, 0xffc0, 0xffe0, 0xfff0, 0xfff8, 0xfffc,
                 0xfffe, 0xffff]
-    cidr = 0
+    prefix = 0
     for word in mask.split(':'):
         if not word or int(word, 16) == 0:
             break
-        cidr += bitCount.index(int(word, 16))
+        prefix += bitCount.index(int(word, 16))
+
+    return prefix
 
-    return cidr
 
+def mask_to_net_prefix(mask):
+    """Return the network prefix for the netmask provided.
 
-def mask2cidr(mask):
-    if ':' in str(mask):
-        return ipv6mask2cidr(mask)
-    elif '.' in str(mask):
-        return ipv4mask2cidr(mask)
+    Supports ipv4 or ipv6 netmasks."""
+    try:
+        # if 'mask' is a prefix that is an integer.
+        # then just return it.
+        return int(mask)
+    except ValueError:
+        pass
+    if is_ipv6_addr(mask):
+        return ipv6_mask_to_net_prefix(mask)
     else:
-        return mask
+        return ipv4_mask_to_net_prefix(mask)
+
 
 # vi: ts=4 expandtab
diff --git a/cloudinit/net/sysconfig.py b/cloudinit/net/sysconfig.py
index f7d45482..5d9b3d10 100644
--- a/cloudinit/net/sysconfig.py
+++ b/cloudinit/net/sysconfig.py
@@ -9,7 +9,7 @@ from cloudinit.distros.parsers import resolv_conf
 from cloudinit import util
 
 from . import renderer
-from .network_state import subnet_is_ipv6
+from .network_state import subnet_is_ipv6, net_prefix_to_ipv4_mask
 
 
 def _make_header(sep='#'):
@@ -26,11 +26,8 @@ def _make_header(sep='#'):
 
 
 def _is_default_route(route):
-    if route['network'] == '::' and route['netmask'] == 0:
-        return True
-    if route['network'] == '0.0.0.0' and route['netmask'] == '0.0.0.0':
-        return True
-    return False
+    default_nets = ('::', '0.0.0.0')
+    return route['prefix'] == 0 and route['network'] in default_nets
 
 
 def _quote_value(value):
@@ -323,16 +320,10 @@ class Renderer(renderer.Renderer):
                             " " + ipv6_cidr)
                 else:
                     ipv4_index = ipv4_index + 1
-                    if ipv4_index == 0:
-                        iface_cfg['IPADDR'] = subnet['address']
-                        if 'netmask' in subnet:
-                            iface_cfg['NETMASK'] = subnet['netmask']
-                    else:
-                        iface_cfg['IPADDR' + str(ipv4_index)] = \
-                            subnet['address']
-                        if 'netmask' in subnet:
-                            iface_cfg['NETMASK' + str(ipv4_index)] = \
-                                subnet['netmask']
+                    suff = "" if ipv4_index == 0 else str(ipv4_index)
+                    iface_cfg['IPADDR' + suff] = subnet['address']
+                    iface_cfg['NETMASK' + suff] = \
+                        net_prefix_to_ipv4_mask(subnet['prefix'])
 
     @classmethod
     def _render_subnet_routes(cls, iface_cfg, route_cfg, subnets):