ConfigDrive: convert OpenStack network_data.json to network_config.yaml

OpenStack clouds may provide network_data.json information via the
MetadataService in ConfigDrive.

Teach ConfigDrive to read, store and convert the data into network_config
yaml format. Making this available allows cloud-init to read network
config from OpenStack and use the distro.apply_network_config() method to
render the network_config from OpenStack into a distro network
configuration file.

5 files changed, 277 insertions, 11 deletions

diff --git a/cloudinit/net/__init__.py b/cloudinit/net/__init__.py
index ae7b1c04..2435055b 100644
--- a/cloudinit/net/__init__.py
+++ b/cloudinit/net/__init__.py
@@ -336,7 +336,7 @@ def iface_add_attrs(iface):
         'index',
         'subnets',
     ]
-    if iface['type'] not in ['bond', 'bridge']:
+    if iface['type'] not in ['bond', 'bridge', 'vlan']:
         ignore_map.append('mac_address')
 
     for key, value in iface.items():
@@ -348,19 +348,48 @@ def iface_add_attrs(iface):
     return content
 
 
-def render_route(route):
-    content = "up route add"
+def render_route(route, indent=""):
+    """ When rendering routes for an iface, in some cases applying a route
+    may result in the route command returning non-zero which produces
+    some confusing output for users manually using ifup/ifdown[1].  To
+    that end, we will optionally include an '|| true' postfix to each
+    route line allowing users to work with ifup/ifdown without using
+    --force option.
+
+    We may at somepoint not want to emit this additional postfix, and
+    add a 'strict' flag to this function.  When called with strict=True,
+    then we will not append the postfix.
+
+    1. http://askubuntu.com/questions/168033/
+             how-to-set-static-routes-in-ubuntu-server
+    """
+    content = ""
+    up = indent + "post-up route add"
+    down = indent + "pre-down route del"
+    eol = " || true\n"
     mapping = {
         'network': '-net',
         'netmask': 'netmask',
         'gateway': 'gw',
         'metric': 'metric',
     }
-    for k in ['network', 'netmask', 'gateway', 'metric']:
-        if k in route:
-            content += " %s %s" % (mapping[k], route[k])
+    if route['network'] == '0.0.0.0' and route['netmask'] == '0.0.0.0':
+        default_gw = " default gw %s" % route['gateway']
+        content += up + default_gw + eol
+        content += down + default_gw + eol
+    elif route['network'] == '::' and route['netmask'] == 0:
+        # ipv6!
+        default_gw = " -A inet6 default gw %s" % route['gateway']
+        content += up + default_gw + eol
+        content += down + default_gw + eol
+    else:
+        route_line = ""
+        for k in ['network', 'netmask', 'gateway', 'metric']:
+            if k in route:
+                route_line += " %s %s" % (mapping[k], route[k])
+        content += up + route_line + eol
+        content += down + route_line + eol
 
-    content += '\n'
     return content
 
 
@@ -384,6 +413,7 @@ def render_interfaces(network_state):
         if len(value):
             content += "    dns-{} {}\n".format(dnskey, " ".join(value))
 
+    content += "\n"
     for iface in sorted(interfaces.values(),
                         key=lambda k: (order[k['type']], k['name'])):
         content += "auto {name}\n".format(**iface)
@@ -409,6 +439,8 @@ def render_interfaces(network_state):
 
                 content += iface_add_subnet(iface, subnet)
                 content += iface_add_attrs(iface)
+                for route in subnet.get('routes', []):
+                    content += render_route(route, indent="    ")
                 content += "\n"
         else:
             content += "iface {name} {inet} {mode}\n".format(**iface)
@@ -419,7 +451,7 @@ def render_interfaces(network_state):
         content += render_route(route)
 
     # global replacements until v2 format
-    content = content.replace('mac_address', 'hwaddress')
+    content = content.replace('mac_address', 'hwaddress ether')
     return content
 
 
diff --git a/cloudinit/net/network_state.py b/cloudinit/net/network_state.py
index df04c526..e32d2cdf 100644
--- a/cloudinit/net/network_state.py
+++ b/cloudinit/net/network_state.py
@@ -124,6 +124,17 @@ class NetworkState:
         iface = interfaces.get(command['name'], {})
         for param, val in command.get('params', {}).items():
             iface.update({param: val})
+
+        # convert subnet ipv6 netmask to cidr as needed
+        subnets = command.get('subnets')
+        if subnets:
+            for subnet in subnets:
+                if subnet['type'] == 'static':
+                    if 'netmask' in subnet and ':' in subnet['address']:
+                        subnet['netmask'] = mask2cidr(subnet['netmask'])
+                        for route in subnet.get('routes', []):
+                            if 'netmask' in route:
+                                route['netmask'] = mask2cidr(route['netmask'])
         iface.update({
             'name': command.get('name'),
             'type': command.get('type'),
@@ -133,7 +144,7 @@ class NetworkState:
             'mtu': command.get('mtu'),
             'address': None,
             'gateway': None,
-            'subnets': command.get('subnets'),
+            'subnets': subnets,
         })
         self.network_state['interfaces'].update({command.get('name'): iface})
         self.dump_network_state()
@@ -144,6 +155,7 @@ class NetworkState:
             iface eth0.222 inet static
                     address 10.10.10.1
                     netmask 255.255.255.0
+                    hwaddress ether BC:76:4E:06:96:B3
                     vlan-raw-device eth0
         '''
         required_keys = [
@@ -335,6 +347,37 @@ def cidr2mask(cidr):
     return ".".join([str(x) for x in mask])
 
 
+def ipv4mask2cidr(mask):
+    if '.' not in mask:
+        return mask
+    return sum([bin(int(x)).count('1') for x in mask.split('.')])
+
+
+def ipv6mask2cidr(mask):
+    if ':' not in mask:
+        return mask
+
+    bitCount = [0, 0x8000, 0xc000, 0xe000, 0xf000, 0xf800, 0xfc00, 0xfe00,
+                0xff00, 0xff80, 0xffc0, 0xffe0, 0xfff0, 0xfff8, 0xfffc,
+                0xfffe, 0xffff]
+    cidr = 0
+    for word in mask.split(':'):
+        if not word or int(word, 16) == 0:
+            break
+        cidr += bitCount.index(int(word, 16))
+
+    return cidr
+
+
+def mask2cidr(mask):
+    if ':' in mask:
+        return ipv6mask2cidr(mask)
+    elif '.' in mask:
+        return ipv4mask2cidr(mask)
+    else:
+        return mask
+
+
 if __name__ == '__main__':
     import sys
     import random
diff --git a/cloudinit/sources/DataSourceConfigDrive.py b/cloudinit/sources/DataSourceConfigDrive.py
index 6fc9e05b..14676f97 100644
--- a/cloudinit/sources/DataSourceConfigDrive.py
+++ b/cloudinit/sources/DataSourceConfigDrive.py
@@ -18,6 +18,7 @@
 #    You should have received a copy of the GNU General Public License
 #    along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
+import copy
 import os
 
 from cloudinit import log as logging
@@ -50,6 +51,8 @@ class DataSourceConfigDrive(openstack.SourceMixin, sources.DataSource):
         self.seed_dir = os.path.join(paths.seed_dir, 'config_drive')
         self.version = None
         self.ec2_metadata = None
+        self._network_config = None
+        self.network_json = None
         self.files = {}
 
     def __str__(self):
@@ -144,12 +147,25 @@ class DataSourceConfigDrive(openstack.SourceMixin, sources.DataSource):
             LOG.warn("Invalid content in vendor-data: %s", e)
             self.vendordata_raw = None
 
+        try:
+            self.network_json = results.get('networkdata')
+        except ValueError as e:
+            LOG.warn("Invalid content in network-data: %s", e)
+            self.network_json = None
+
         return True
 
     def check_instance_id(self):
         # quickly (local check only) if self.instance_id is still valid
         return sources.instance_id_matches_system_uuid(self.get_instance_id())
 
+    @property
+    def network_config(self):
+        if self._network_config is None:
+            if self.network_json is not None:
+                self._network_config = convert_network_data(self.network_json)
+        return self._network_config
+
 
 class DataSourceConfigDriveNet(DataSourceConfigDrive):
     def __init__(self, sys_cfg, distro, paths):
@@ -287,3 +303,122 @@ datasources = [
 # Return a list of data sources that match this set of dependencies
 def get_datasource_list(depends):
     return sources.list_from_depends(depends, datasources)
+
+
+# Convert OpenStack ConfigDrive NetworkData json to network_config yaml
+def convert_network_data(network_json=None):
+    """Return a dictionary of network_config by parsing provided
+       OpenStack ConfigDrive NetworkData json format
+
+    OpenStack network_data.json provides a 3 element dictionary
+      - "links" (links are network devices, physical or virtual)
+      - "networks" (networks are ip network configurations for one or more
+                    links)
+      -  services (non-ip services, like dns)
+
+    networks and links are combined via network items referencing specific
+    links via a 'link_id' which maps to a links 'id' field.
+
+    To convert this format to network_config yaml, we first iterate over the
+    links and then walk the network list to determine if any of the networks
+    utilize the current link; if so we generate a subnet entry for the device
+
+    We also need to map network_data.json fields to network_config fields. For
+    example, the network_data links 'id' field is equivalent to network_config
+    'name' field for devices.  We apply more of this mapping to the various
+    link types that we encounter.
+
+    There are additional fields that are populated in the network_data.json
+    from OpenStack that are not relevant to network_config yaml, so we
+    enumerate a dictionary of valid keys for network_yaml and apply filtering
+    to drop these superflous keys from the network_config yaml.
+    """
+    if network_json is None:
+        return None
+
+    # dict of network_config key for filtering network_json
+    valid_keys = {
+        'physical': [
+            'name',
+            'type',
+            'mac_address',
+            'subnets',
+            'params',
+        ],
+        'subnet': [
+            'type',
+            'address',
+            'netmask',
+            'broadcast',
+            'metric',
+            'gateway',
+            'pointopoint',
+            'mtu',
+            'scope',
+            'dns_nameservers',
+            'dns_search',
+            'routes',
+        ],
+    }
+
+    links = network_json.get('links', [])
+    networks = network_json.get('networks', [])
+    services = network_json.get('services', [])
+
+    config = []
+    for link in links:
+        subnets = []
+        cfg = {k: v for k, v in link.items()
+               if k in valid_keys['physical']}
+        cfg.update({'name': link['id']})
+        for network in [net for net in networks
+                        if net['link'] == link['id']]:
+            subnet = {k: v for k, v in network.items()
+                      if k in valid_keys['subnet']}
+            if 'dhcp' in network['type']:
+                t = 'dhcp6' if network['type'].startswith('ipv6') else 'dhcp4'
+                subnet.update({
+                    'type': t,
+                })
+            else:
+                subnet.update({
+                   'type': 'static',
+                   'address': network.get('ip_address'),
+                })
+            subnets.append(subnet)
+        cfg.update({'subnets': subnets})
+        if link['type'] in ['ethernet', 'vif', 'ovs', 'phy']:
+            cfg.update({
+                'type': 'physical',
+                'mac_address': link['ethernet_mac_address']})
+        elif link['type'] in ['bond']:
+            params = {}
+            for k, v in link.items():
+                if k == 'bond_links':
+                    continue
+                elif k.startswith('bond'):
+                    params.update({k: v})
+            cfg.update({
+                'bond_interfaces': copy.deepcopy(link['bond_links']),
+                'params': params,
+            })
+        elif link['type'] in ['vlan']:
+            cfg.update({
+                'name': "%s.%s" % (link['vlan_link'],
+                                   link['vlan_id']),
+                'vlan_link': link['vlan_link'],
+                'vlan_id': link['vlan_id'],
+                'mac_address': link['vlan_mac_address'],
+            })
+        else:
+            raise ValueError(
+                    'Unknown network_data link type: %s' % link['type'])
+
+        config.append(cfg)
+
+    for service in services:
+        cfg = service
+        cfg.update({'type': 'nameserver'})
+        config.append(cfg)
+
+    return {'version': 1, 'config': config}
diff --git a/cloudinit/sources/helpers/openstack.py b/cloudinit/sources/helpers/openstack.py
index bd93d22f..1aa6bbae 100644
--- a/cloudinit/sources/helpers/openstack.py
+++ b/cloudinit/sources/helpers/openstack.py
@@ -51,11 +51,13 @@ OS_LATEST = 'latest'
 OS_FOLSOM = '2012-08-10'
 OS_GRIZZLY = '2013-04-04'
 OS_HAVANA = '2013-10-17'
+OS_LIBERTY = '2015-10-15'
 # keep this in chronological order. new supported versions go at the end.
 OS_VERSIONS = (
     OS_FOLSOM,
     OS_GRIZZLY,
     OS_HAVANA,
+    OS_LIBERTY,
 )
 
 
@@ -229,6 +231,11 @@ class BaseReader(object):
                 False,
                 load_json_anytype,
             )
+            files['networkdata'] = (
+                self._path_join("openstack", version, 'network_data.json'),
+                False,
+                load_json_anytype,
+            )
             return files
 
         results = {
@@ -334,7 +341,7 @@ class ConfigDriveReader(BaseReader):
             path = self._path_join(self.base_path, 'openstack')
             found = [d for d in os.listdir(path)
                      if os.path.isdir(os.path.join(path))]
-            self._versions = found
+            self._versions = sorted(found)
         return self._versions
 
     def _read_ec2_metadata(self):
diff --git a/tests/unittests/test_datasource/test_configdrive.py b/tests/unittests/test_datasource/test_configdrive.py
index bfd787d1..89b15f54 100644
--- a/tests/unittests/test_datasource/test_configdrive.py
+++ b/tests/unittests/test_datasource/test_configdrive.py
@@ -59,6 +59,34 @@ OSTACK_META = {
 
 CONTENT_0 = b'This is contents of /etc/foo.cfg\n'
 CONTENT_1 = b'# this is /etc/bar/bar.cfg\n'
+NETWORK_DATA = {
+    'services': [
+        {'type': 'dns', 'address': '199.204.44.24'},
+        {'type': 'dns', 'address': '199.204.47.54'}
+    ],
+    'links': [
+        {'vif_id': '2ecc7709-b3f7-4448-9580-e1ec32d75bbd',
+         'ethernet_mac_address': 'fa:16:3e:69:b0:58',
+         'type': 'ovs', 'mtu': None, 'id': 'tap2ecc7709-b3'},
+        {'vif_id': '2f88d109-5b57-40e6-af32-2472df09dc33',
+         'ethernet_mac_address': 'fa:16:3e:d4:57:ad',
+         'type': 'ovs', 'mtu': None, 'id': 'tap2f88d109-5b'},
+        {'vif_id': '1a5382f8-04c5-4d75-ab98-d666c1ef52cc',
+         'ethernet_mac_address': 'fa:16:3e:05:30:fe',
+         'type': 'ovs', 'mtu': None, 'id': 'tap1a5382f8-04'}
+    ],
+    'networks': [
+        {'link': 'tap2ecc7709-b3', 'type': 'ipv4_dhcp',
+         'network_id': '6d6357ac-0f70-4afa-8bd7-c274cc4ea235',
+         'id': 'network0'},
+        {'link': 'tap2f88d109-5b', 'type': 'ipv4_dhcp',
+         'network_id': 'd227a9b3-6960-4d94-8976-ee5788b44f54',
+         'id': 'network1'},
+        {'link': 'tap1a5382f8-04', 'type': 'ipv4_dhcp',
+         'network_id': 'dab2ba57-cae2-4311-a5ed-010b263891f5',
+         'id': 'network2'}
+    ]
+}
 
 CFG_DRIVE_FILES_V2 = {
     'ec2/2009-04-04/meta-data.json': json.dumps(EC2_META),
@@ -70,7 +98,11 @@ CFG_DRIVE_FILES_V2 = {
     'openstack/content/0000': CONTENT_0,
     'openstack/content/0001': CONTENT_1,
     'openstack/latest/meta_data.json': json.dumps(OSTACK_META),
-    'openstack/latest/user_data': USER_DATA}
+    'openstack/latest/user_data': USER_DATA,
+    'openstack/latest/network_data.json': json.dumps(NETWORK_DATA),
+    'openstack/2015-10-15/meta_data.json': json.dumps(OSTACK_META),
+    'openstack/2015-10-15/user_data': USER_DATA,
+    'openstack/2015-10-15/network_data.json': json.dumps(NETWORK_DATA)}
 
 
 class TestConfigDriveDataSource(TestCase):
@@ -225,6 +257,7 @@ class TestConfigDriveDataSource(TestCase):
 
         self.assertEqual(USER_DATA, found['userdata'])
         self.assertEqual(expected_md, found['metadata'])
+        self.assertEqual(NETWORK_DATA, found['networkdata'])
         self.assertEqual(found['files']['/etc/foo.cfg'], CONTENT_0)
         self.assertEqual(found['files']['/etc/bar/bar.cfg'], CONTENT_1)
 
@@ -250,6 +283,7 @@ class TestConfigDriveDataSource(TestCase):
         data = copy(CFG_DRIVE_FILES_V2)
 
         data["openstack/2012-08-10/meta_data.json"] = "non-json garbage {}"
+        data["openstack/2015-10-15/meta_data.json"] = "non-json garbage {}"
         data["openstack/latest/meta_data.json"] = "non-json garbage {}"
 
         populate_dir(self.tmp, data)
@@ -321,6 +355,19 @@ class TestConfigDriveDataSource(TestCase):
         self.assertEqual(myds.get_public_ssh_keys(),
                          [OSTACK_META['public_keys']['mykey']])
 
+    def test_network_data_is_found(self):
+        """Verify that network_data is present in ds in config-drive-v2."""
+        populate_dir(self.tmp, CFG_DRIVE_FILES_V2)
+        myds = cfg_ds_from_dir(self.tmp)
+        self.assertEqual(myds.network_json, NETWORK_DATA)
+
+    def test_network_config_is_converted(self):
+        """Verify that network_data is converted and present on ds object."""
+        populate_dir(self.tmp, CFG_DRIVE_FILES_V2)
+        myds = cfg_ds_from_dir(self.tmp)
+        network_config = ds.convert_network_data(NETWORK_DATA)
+        self.assertEqual(myds.network_config, network_config)
+
 
 def cfg_ds_from_dir(seed_d):
     found = ds.read_config_drive(seed_d)
@@ -339,6 +386,8 @@ def populate_ds_from_read_config(cfg_ds, source, results):
     cfg_ds.ec2_metadata = results.get('ec2-metadata')
     cfg_ds.userdata_raw = results.get('userdata')
     cfg_ds.version = results.get('version')
+    cfg_ds.network_json = results.get('networkdata')
+    cfg_ds._network_config = ds.convert_network_data(cfg_ds.network_json)
 
 
 def populate_dir(seed_dir, files):