OpenNebula: Update network to return v2 config rather than ENI.

OpenNebulaNetwork.gen_conf() was previously returning ENI format.
This is updated to return netplan/v2 config.
The changes here also adds support for IPv6 configuration distributed
from OpenNebula and fixes some issues about nameserver information.

3 files changed, 261 insertions, 119 deletions

diff --git a/cloudinit/net/network_state.py b/cloudinit/net/network_state.py
index 1dd7ded7..6d63e5c5 100644
--- a/cloudinit/net/network_state.py
+++ b/cloudinit/net/network_state.py
@@ -708,6 +708,7 @@ class NetworkStateInterpreter(object):
 
         gateway4 = None
         gateway6 = None
+        nameservers = {}
         for address in cfg.get('addresses', []):
             subnet = {
                 'type': 'static',
@@ -723,6 +724,15 @@ class NetworkStateInterpreter(object):
                     gateway4 = cfg.get('gateway4')
                     subnet.update({'gateway': gateway4})
 
+            if 'nameservers' in cfg and not nameservers:
+                addresses = cfg.get('nameservers').get('addresses')
+                if addresses:
+                    nameservers['dns_nameservers'] = addresses
+                search = cfg.get('nameservers').get('search')
+                if search:
+                    nameservers['dns_search'] = search
+                subnet.update(nameservers)
+
             subnets.append(subnet)
 
         routes = []
diff --git a/cloudinit/sources/DataSourceOpenNebula.py b/cloudinit/sources/DataSourceOpenNebula.py
index 9450835e..02cb98f7 100644
--- a/cloudinit/sources/DataSourceOpenNebula.py
+++ b/cloudinit/sources/DataSourceOpenNebula.py
@@ -20,7 +20,6 @@ import string
 
 from cloudinit import log as logging
 from cloudinit import net
-from cloudinit.net import eni
 from cloudinit import sources
 from cloudinit import util
 
@@ -91,15 +90,15 @@ class DataSourceOpenNebula(sources.DataSource):
             return False
 
         self.seed = seed
-        self.network_eni = results.get('network-interfaces')
+        self.network = results.get('network-interfaces')
         self.metadata = md
         self.userdata_raw = results.get('userdata')
         return True
 
     @property
     def network_config(self):
-        if self.network_eni is not None:
-            return eni.convert_eni_data(self.network_eni)
+        if self.network is not None:
+            return self.network
         else:
             return None
 
@@ -143,18 +142,42 @@ class OpenNebulaNetwork(object):
     def mac2network(self, mac):
         return self.mac2ip(mac).rpartition(".")[0] + ".0"
 
-    def get_dns(self, dev):
-        return self.get_field(dev, "dns", "").split()
+    def get_nameservers(self, dev):
+        nameservers = {}
+        dns = self.get_field(dev, "dns", "").split()
+        dns.extend(self.context.get('DNS', "").split())
+        if dns:
+            nameservers['addresses'] = dns
+        search_domain = self.get_field(dev, "search_domain", "").split()
+        if search_domain:
+            nameservers['search'] = search_domain
+        return nameservers
 
-    def get_domain(self, dev):
-        return self.get_field(dev, "domain")
+    def get_mtu(self, dev):
+        return self.get_field(dev, "mtu")
 
     def get_ip(self, dev, mac):
         return self.get_field(dev, "ip", self.mac2ip(mac))
 
+    def get_ip6(self, dev):
+        addresses6 = []
+        ip6 = self.get_field(dev, "ip6")
+        if ip6:
+            addresses6.append(ip6)
+        ip6_ula = self.get_field(dev, "ip6_ula")
+        if ip6_ula:
+            addresses6.append(ip6_ula)
+        return addresses6
+
+    def get_ip6_prefix(self, dev):
+        return self.get_field(dev, "ip6_prefix_length", "64")
+
     def get_gateway(self, dev):
         return self.get_field(dev, "gateway")
 
+    def get_gateway6(self, dev):
+        return self.get_field(dev, "gateway6")
+
     def get_mask(self, dev):
         return self.get_field(dev, "mask", "255.255.255.0")
 
@@ -171,10 +194,11 @@ class OpenNebulaNetwork(object):
         return default if val in (None, "") else val
 
     def gen_conf(self):
-        global_dns = self.context.get('DNS', "").split()
-
-        conf = ['auto lo', 'iface lo inet loopback', '']
+        netconf = {}
+        netconf['version'] = 2
+        netconf['ethernets'] = {}
 
+        ethernets = {}
         for mac, dev in self.ifaces.items():
             mac = mac.lower()
 
@@ -182,29 +206,49 @@ class OpenNebulaNetwork(object):
             # dev stores the current system name.
             c_dev = self.context_devname.get(mac, dev)
 
-            conf.append('auto ' + dev)
-            conf.append('iface ' + dev + ' inet static')
-            conf.append('  #hwaddress %s' % mac)
-            conf.append('  address ' + self.get_ip(c_dev, mac))
-            conf.append('  network ' + self.get_network(c_dev, mac))
-            conf.append('  netmask ' + self.get_mask(c_dev))
+            devconf = {}
+
+            # Set MAC address
+            devconf['match'] = {'macaddress': mac}
 
+            # Set IPv4 address
+            devconf['addresses'] = []
+            mask = self.get_mask(c_dev)
+            prefix = str(net.mask_to_net_prefix(mask))
+            devconf['addresses'].append(
+                self.get_ip(c_dev, mac) + '/' + prefix)
+
+            # Set IPv6 Global and ULA address
+            addresses6 = self.get_ip6(c_dev)
+            if addresses6:
+                prefix6 = self.get_ip6_prefix(c_dev)
+                devconf['addresses'].extend(
+                    [i + '/' + prefix6 for i in addresses6])
+
+            # Set IPv4 default gateway
             gateway = self.get_gateway(c_dev)
             if gateway:
-                conf.append('  gateway ' + gateway)
+                devconf['gateway4'] = gateway
+
+            # Set IPv6 default gateway
+            gateway6 = self.get_gateway6(c_dev)
+            if gateway:
+                devconf['gateway6'] = gateway6
 
-            domain = self.get_domain(c_dev)
-            if domain:
-                conf.append('  dns-search ' + domain)
+            # Set DNS servers and search domains
+            nameservers = self.get_nameservers(c_dev)
+            if nameservers:
+                devconf['nameservers'] = nameservers
 
-            # add global DNS servers to all interfaces
-            dns = self.get_dns(c_dev)
-            if global_dns or dns:
-                conf.append('  dns-nameservers ' + ' '.join(global_dns + dns))
+            # Set MTU size
+            mtu = self.get_mtu(c_dev)
+            if mtu:
+                devconf['mtu'] = mtu
 
-            conf.append('')
+            ethernets[dev] = devconf
 
-        return "\n".join(conf)
+        netconf['ethernets'] = ethernets
+        return(netconf)
 
 
 def find_candidate_devs():
@@ -390,10 +434,10 @@ def read_context_disk_dir(source_dir, asuser=None):
             except TypeError:
                 LOG.warning("Failed base64 decoding of userdata")
 
-    # generate static /etc/network/interfaces
+    # generate Network Configuration v2
     # only if there are any required context variables
-    # http://opennebula.org/documentation:rel3.8:cong#network_configuration
-    ipaddr_keys = [k for k in context if re.match(r'^ETH\d+_IP$', k)]
+    # http://docs.opennebula.org/5.4/operation/references/template.html#context-section
+    ipaddr_keys = [k for k in context if re.match(r'^ETH\d+_IP.*$', k)]
     if ipaddr_keys:
         onet = OpenNebulaNetwork(context)
         results['network-interfaces'] = onet.gen_conf()
diff --git a/tests/unittests/test_datasource/test_opennebula.py b/tests/unittests/test_datasource/test_opennebula.py
index 5c3ba012..ab42f344 100644
--- a/tests/unittests/test_datasource/test_opennebula.py
+++ b/tests/unittests/test_datasource/test_opennebula.py
@@ -4,7 +4,6 @@ from cloudinit import helpers
 from cloudinit.sources import DataSourceOpenNebula as ds
 from cloudinit import util
 from cloudinit.tests.helpers import mock, populate_dir, CiTestCase
-from textwrap import dedent
 
 import os
 import pwd
@@ -33,6 +32,11 @@ HOSTNAME = 'foo.example.com'
 PUBLIC_IP = '10.0.0.3'
 MACADDR = '02:00:0a:12:01:01'
 IP_BY_MACADDR = '10.18.1.1'
+IP4_PREFIX = '24'
+IP6_GLOBAL = '2001:db8:1:0:400:c0ff:fea8:1ba'
+IP6_ULA = 'fd01:dead:beaf:0:400:c0ff:fea8:1ba'
+IP6_GW = '2001:db8:1::ffff'
+IP6_PREFIX = '48'
 
 DS_PATH = "cloudinit.sources.DataSourceOpenNebula"
 
@@ -221,7 +225,9 @@ class TestOpenNebulaDataSource(CiTestCase):
             results = ds.read_context_disk_dir(self.seed_dir)
 
             self.assertTrue('network-interfaces' in results)
-            self.assertTrue(IP_BY_MACADDR in results['network-interfaces'])
+            self.assertTrue(
+                IP_BY_MACADDR + '/' + IP4_PREFIX in
+                results['network-interfaces']['ethernets'][dev]['addresses'])
 
             # ETH0_IP and ETH0_MAC
             populate_context_dir(
@@ -229,7 +235,9 @@ class TestOpenNebulaDataSource(CiTestCase):
             results = ds.read_context_disk_dir(self.seed_dir)
 
             self.assertTrue('network-interfaces' in results)
-            self.assertTrue(IP_BY_MACADDR in results['network-interfaces'])
+            self.assertTrue(
+                IP_BY_MACADDR + '/' + IP4_PREFIX in
+                results['network-interfaces']['ethernets'][dev]['addresses'])
 
             # ETH0_IP with empty string and ETH0_MAC
             # in the case of using Virtual Network contains
@@ -239,55 +247,91 @@ class TestOpenNebulaDataSource(CiTestCase):
             results = ds.read_context_disk_dir(self.seed_dir)
 
             self.assertTrue('network-interfaces' in results)
-            self.assertTrue(IP_BY_MACADDR in results['network-interfaces'])
+            self.assertTrue(
+                IP_BY_MACADDR + '/' + IP4_PREFIX in
+                results['network-interfaces']['ethernets'][dev]['addresses'])
 
-            # ETH0_NETWORK
+            # ETH0_MASK
             populate_context_dir(
                 self.seed_dir, {
                     'ETH0_IP': IP_BY_MACADDR,
                     'ETH0_MAC': MACADDR,
-                    'ETH0_NETWORK': '10.18.0.0'
+                    'ETH0_MASK': '255.255.0.0'
                 })
             results = ds.read_context_disk_dir(self.seed_dir)
 
             self.assertTrue('network-interfaces' in results)
-            self.assertTrue('10.18.0.0' in results['network-interfaces'])
+            self.assertTrue(
+                IP_BY_MACADDR + '/16' in
+                results['network-interfaces']['ethernets'][dev]['addresses'])
 
-            # ETH0_NETWORK with empty string
+            # ETH0_MASK with empty string
             populate_context_dir(
                 self.seed_dir, {
                     'ETH0_IP': IP_BY_MACADDR,
                     'ETH0_MAC': MACADDR,
-                    'ETH0_NETWORK': ''
+                    'ETH0_MASK': ''
                 })
             results = ds.read_context_disk_dir(self.seed_dir)
 
             self.assertTrue('network-interfaces' in results)
-            self.assertTrue('10.18.1.0' in results['network-interfaces'])
+            self.assertTrue(
+                IP_BY_MACADDR + '/' + IP4_PREFIX in
+                results['network-interfaces']['ethernets'][dev]['addresses'])
 
-            # ETH0_MASK
+            # ETH0_IP6
             populate_context_dir(
                 self.seed_dir, {
-                    'ETH0_IP': IP_BY_MACADDR,
+                    'ETH0_IP6': IP6_GLOBAL,
                     'ETH0_MAC': MACADDR,
-                    'ETH0_MASK': '255.255.0.0'
                 })
             results = ds.read_context_disk_dir(self.seed_dir)
 
             self.assertTrue('network-interfaces' in results)
-            self.assertTrue('255.255.0.0' in results['network-interfaces'])
+            self.assertTrue(
+                IP6_GLOBAL + '/64' in
+                results['network-interfaces']['ethernets'][dev]['addresses'])
 
-            # ETH0_MASK with empty string
+            # ETH0_IP6_ULA
             populate_context_dir(
                 self.seed_dir, {
-                    'ETH0_IP': IP_BY_MACADDR,
+                    'ETH0_IP6_ULA': IP6_ULA,
+                    'ETH0_MAC': MACADDR,
+                })
+            results = ds.read_context_disk_dir(self.seed_dir)
+
+            self.assertTrue('network-interfaces' in results)
+            self.assertTrue(
+                IP6_ULA + '/64' in
+                results['network-interfaces']['ethernets'][dev]['addresses'])
+
+            # ETH0_IP6 and ETH0_IP6_PREFIX_LENGTH
+            populate_context_dir(
+                self.seed_dir, {
+                    'ETH0_IP6': IP6_GLOBAL,
+                    'ETH0_IP6_PREFIX_LENGTH': IP6_PREFIX,
+                    'ETH0_MAC': MACADDR,
+                })
+            results = ds.read_context_disk_dir(self.seed_dir)
+
+            self.assertTrue('network-interfaces' in results)
+            self.assertTrue(
+                IP6_GLOBAL + '/' + IP6_PREFIX in
+                results['network-interfaces']['ethernets'][dev]['addresses'])
+
+            # ETH0_IP6 and ETH0_IP6_PREFIX_LENGTH with empty string
+            populate_context_dir(
+                self.seed_dir, {
+                    'ETH0_IP6': IP6_GLOBAL,
+                    'ETH0_IP6_PREFIX_LENGTH': '',
                     'ETH0_MAC': MACADDR,
-                    'ETH0_MASK': ''
                 })
             results = ds.read_context_disk_dir(self.seed_dir)
 
             self.assertTrue('network-interfaces' in results)
-            self.assertTrue('255.255.255.0' in results['network-interfaces'])
+            self.assertTrue(
+                IP6_GLOBAL + '/64' in
+                results['network-interfaces']['ethernets'][dev]['addresses'])
 
     def test_find_candidates(self):
         def my_devs_with(criteria):
@@ -310,108 +354,152 @@ class TestOpenNebulaNetwork(unittest.TestCase):
 
     system_nics = ('eth0', 'ens3')
 
-    def test_lo(self):
-        net = ds.OpenNebulaNetwork(context={}, system_nics_by_mac={})
-        self.assertEqual(net.gen_conf(), u'''\
-auto lo
-iface lo inet loopback
-''')
-
     @mock.patch(DS_PATH + ".get_physical_nics_by_mac")
     def test_eth0(self, m_get_phys_by_mac):
         for nic in self.system_nics:
             m_get_phys_by_mac.return_value = {MACADDR: nic}
             net = ds.OpenNebulaNetwork({})
-            self.assertEqual(net.gen_conf(), dedent("""\
-                auto lo
-                iface lo inet loopback
-
-                auto {dev}
-                iface {dev} inet static
-                  #hwaddress {macaddr}
-                  address 10.18.1.1
-                  network 10.18.1.0
-                  netmask 255.255.255.0
-                """.format(dev=nic, macaddr=MACADDR)))
+            expected = {
+                'version': 2,
+                'ethernets': {
+                    nic: {
+                        'match': {'macaddress': MACADDR},
+                        'addresses': [IP_BY_MACADDR + '/' + IP4_PREFIX]}}}
+
+            self.assertEqual(net.gen_conf(), expected)
 
     def test_eth0_override(self):
+        self.maxDiff = None
         context = {
             'DNS': '1.2.3.8',
-            'ETH0_IP': '10.18.1.1',
-            'ETH0_NETWORK': '10.18.0.0',
-            'ETH0_MASK': '255.255.0.0',
+            'ETH0_DNS': '1.2.3.6 1.2.3.7',
             'ETH0_GATEWAY': '1.2.3.5',
-            'ETH0_DOMAIN': 'example.com',
+            'ETH0_GATEWAY6': '',
+            'ETH0_IP': IP_BY_MACADDR,
+            'ETH0_IP6': '',
+            'ETH0_IP6_PREFIX_LENGTH': '',
+            'ETH0_IP6_ULA': '',
+            'ETH0_MAC': '02:00:0a:12:01:01',
+            'ETH0_MASK': '255.255.0.0',
+            'ETH0_MTU': '',
+            'ETH0_NETWORK': '10.18.0.0',
+            'ETH0_SEARCH_DOMAIN': '',
+        }
+        for nic in self.system_nics:
+            net = ds.OpenNebulaNetwork(context,
+                                       system_nics_by_mac={MACADDR: nic})
+            expected = {
+                'version': 2,
+                'ethernets': {
+                    nic: {
+                        'match': {'macaddress': MACADDR},
+                        'addresses': [IP_BY_MACADDR + '/16'],
+                        'gateway4': '1.2.3.5',
+                        'gateway6': None,
+                        'nameservers': {
+                            'addresses': ['1.2.3.6', '1.2.3.7', '1.2.3.8']}}}}
+
+            self.assertEqual(expected, net.gen_conf())
+
+    def test_eth0_v4v6_override(self):
+        self.maxDiff = None
+        context = {
+            'DNS': '1.2.3.8',
             'ETH0_DNS': '1.2.3.6 1.2.3.7',
-            'ETH0_MAC': '02:00:0a:12:01:01'
+            'ETH0_GATEWAY': '1.2.3.5',
+            'ETH0_GATEWAY6': IP6_GW,
+            'ETH0_IP': IP_BY_MACADDR,
+            'ETH0_IP6': IP6_GLOBAL,
+            'ETH0_IP6_PREFIX_LENGTH': IP6_PREFIX,
+            'ETH0_IP6_ULA': IP6_ULA,
+            'ETH0_MAC': '02:00:0a:12:01:01',
+            'ETH0_MASK': '255.255.0.0',
+            'ETH0_MTU': '1280',
+            'ETH0_NETWORK': '10.18.0.0',
+            'ETH0_SEARCH_DOMAIN': 'example.com example.org',
         }
         for nic in self.system_nics:
-            expected = dedent("""\
-                auto lo
-                iface lo inet loopback
-
-                auto {dev}
-                iface {dev} inet static
-                  #hwaddress {macaddr}
-                  address 10.18.1.1
-                  network 10.18.0.0
-                  netmask 255.255.0.0
-                  gateway 1.2.3.5
-                  dns-search example.com
-                  dns-nameservers 1.2.3.8 1.2.3.6 1.2.3.7
-                  """).format(dev=nic, macaddr=MACADDR)
             net = ds.OpenNebulaNetwork(context,
                                        system_nics_by_mac={MACADDR: nic})
+
+            expected = {
+                'version': 2,
+                'ethernets': {
+                    nic: {
+                        'match': {'macaddress': MACADDR},
+                        'addresses': [
+                            IP_BY_MACADDR + '/16',
+                            IP6_GLOBAL + '/' + IP6_PREFIX,
+                            IP6_ULA + '/' + IP6_PREFIX],
+                        'gateway4': '1.2.3.5',
+                        'gateway6': IP6_GW,
+                        'nameservers': {
+                            'addresses': ['1.2.3.6', '1.2.3.7', '1.2.3.8'],
+                            'search': ['example.com', 'example.org']},
+                        'mtu': '1280'}}}
+
             self.assertEqual(expected, net.gen_conf())
 
     def test_multiple_nics(self):
         """Test rendering multiple nics with names that differ from context."""
+        self.maxDiff = None
         MAC_1 = "02:00:0a:12:01:01"
         MAC_2 = "02:00:0a:12:01:02"
         context = {
             'DNS': '1.2.3.8',
-            'ETH0_IP': '10.18.1.1',
-            'ETH0_NETWORK': '10.18.0.0',
-            'ETH0_MASK': '255.255.0.0',
-            'ETH0_GATEWAY': '1.2.3.5',
-            'ETH0_DOMAIN': 'example.com',
             'ETH0_DNS': '1.2.3.6 1.2.3.7',
+            'ETH0_GATEWAY': '1.2.3.5',
+            'ETH0_GATEWAY6': IP6_GW,
+            'ETH0_IP': '10.18.1.1',
+            'ETH0_IP6': IP6_GLOBAL,
+            'ETH0_IP6_PREFIX_LENGTH': '',
+            'ETH0_IP6_ULA': IP6_ULA,
             'ETH0_MAC': MAC_2,
-            'ETH3_IP': '10.3.1.3',
-            'ETH3_NETWORK': '10.3.0.0',
-            'ETH3_MASK': '255.255.0.0',
-            'ETH3_GATEWAY': '10.3.0.1',
-            'ETH3_DOMAIN': 'third.example.com',
+            'ETH0_MASK': '255.255.0.0',
+            'ETH0_MTU': '1280',
+            'ETH0_NETWORK': '10.18.0.0',
+            'ETH0_SEARCH_DOMAIN': 'example.com',
             'ETH3_DNS': '10.3.1.2',
+            'ETH3_GATEWAY': '10.3.0.1',
+            'ETH3_GATEWAY6': '',
+            'ETH3_IP': '10.3.1.3',
+            'ETH3_IP6': '',
+            'ETH3_IP6_PREFIX_LENGTH': '',
+            'ETH3_IP6_ULA': '',
             'ETH3_MAC': MAC_1,
+            'ETH3_MASK': '255.255.0.0',
+            'ETH3_MTU': '',
+            'ETH3_NETWORK': '10.3.0.0',
+            'ETH3_SEARCH_DOMAIN': 'third.example.com third.example.org',
         }
         net = ds.OpenNebulaNetwork(
             context, system_nics_by_mac={MAC_1: 'enp0s25', MAC_2: 'enp1s2'})
 
-        expected = dedent("""\
-            auto lo
-            iface lo inet loopback
-
-            auto enp0s25
-            iface enp0s25 inet static
-              #hwaddress 02:00:0a:12:01:01
-              address 10.3.1.3
-              network 10.3.0.0
-              netmask 255.255.0.0
-              gateway 10.3.0.1
-              dns-search third.example.com
-              dns-nameservers 1.2.3.8 10.3.1.2
-
-            auto enp1s2
-            iface enp1s2 inet static
-              #hwaddress 02:00:0a:12:01:02
-              address 10.18.1.1
-              network 10.18.0.0
-              netmask 255.255.0.0
-              gateway 1.2.3.5
-              dns-search example.com
-              dns-nameservers 1.2.3.8 1.2.3.6 1.2.3.7
-            """)
+        expected = {
+            'version': 2,
+            'ethernets': {
+                'enp1s2': {
+                    'match': {'macaddress': MAC_2},
+                    'addresses': [
+                        '10.18.1.1/16',
+                        IP6_GLOBAL + '/64',
+                        IP6_ULA + '/64'],
+                    'gateway4': '1.2.3.5',
+                    'gateway6': IP6_GW,
+                    'nameservers': {
+                        'addresses': ['1.2.3.6', '1.2.3.7', '1.2.3.8'],
+                        'search': ['example.com']},
+                    'mtu': '1280'},
+                'enp0s25': {
+                    'match': {'macaddress': MAC_1},
+                    'addresses': ['10.3.1.3/16'],
+                    'gateway4': '10.3.0.1',
+                    'gateway6': None,
+                    'nameservers': {
+                        'addresses': ['10.3.1.2', '1.2.3.8'],
+                        'search': [
+                            'third.example.com',
+                            'third.example.org']}}}}
 
         self.assertEqual(expected, net.gen_conf())