Merge branch 'master' into ubuntu/xenial

2 files changed, 140 insertions, 47 deletions

diff --git a/cloudinit/sources/helpers/azure.py b/cloudinit/sources/helpers/azure.py
index 63ccf10e..689ed4cc 100644
--- a/cloudinit/sources/helpers/azure.py
+++ b/cloudinit/sources/helpers/azure.py
@@ -1,3 +1,4 @@
+import json
 import logging
 import os
 import re
@@ -6,6 +7,7 @@ import struct
 import tempfile
 import time
 
+from cloudinit import stages
 from contextlib import contextmanager
 from xml.etree import ElementTree
 
@@ -187,19 +189,33 @@ class WALinuxAgentShim(object):
         '  </Container>',
         '</Health>'])
 
-    def __init__(self):
-        LOG.debug('WALinuxAgentShim instantiated...')
-        self.endpoint = self.find_endpoint()
+    def __init__(self, fallback_lease_file=None):
+        LOG.debug('WALinuxAgentShim instantiated, fallback_lease_file=%s',
+                  fallback_lease_file)
+        self.dhcpoptions = None
+        self._endpoint = None
         self.openssl_manager = None
         self.values = {}
+        self.lease_file = fallback_lease_file
 
     def clean_up(self):
         if self.openssl_manager is not None:
             self.openssl_manager.clean_up()
 
     @staticmethod
-    def get_ip_from_lease_value(lease_value):
-        unescaped_value = lease_value.replace('\\', '')
+    def _get_hooks_dir():
+        _paths = stages.Init()
+        return os.path.join(_paths.paths.get_runpath(), "dhclient.hooks")
+
+    @property
+    def endpoint(self):
+        if self._endpoint is None:
+            self._endpoint = self.find_endpoint(self.lease_file)
+        return self._endpoint
+
+    @staticmethod
+    def get_ip_from_lease_value(fallback_lease_value):
+        unescaped_value = fallback_lease_value.replace('\\', '')
         if len(unescaped_value) > 4:
             hex_string = ''
             for hex_pair in unescaped_value.split(':'):
@@ -213,15 +229,75 @@ class WALinuxAgentShim(object):
         return socket.inet_ntoa(packed_bytes)
 
     @staticmethod
-    def find_endpoint():
-        LOG.debug('Finding Azure endpoint...')
-        content = util.load_file('/var/lib/dhcp/dhclient.eth0.leases')
-        value = None
+    def _get_value_from_leases_file(fallback_lease_file):
+        leases = []
+        content = util.load_file(fallback_lease_file)
+        LOG.debug("content is {}".format(content))
         for line in content.splitlines():
             if 'unknown-245' in line:
-                value = line.strip(' ').split(' ', 2)[-1].strip(';\n"')
+                # Example line from Ubuntu
+                # option unknown-245 a8:3f:81:10;
+                leases.append(line.strip(' ').split(' ', 2)[-1].strip(';\n"'))
+        # Return the "most recent" one in the list
+        if len(leases) < 1:
+            return None
+        else:
+            return leases[-1]
+
+    @staticmethod
+    def _load_dhclient_json():
+        dhcp_options = {}
+        hooks_dir = WALinuxAgentShim._get_hooks_dir()
+        if not os.path.exists(hooks_dir):
+            LOG.debug("%s not found.", hooks_dir)
+            return None
+        hook_files = [os.path.join(hooks_dir, x)
+                      for x in os.listdir(hooks_dir)]
+        for hook_file in hook_files:
+            try:
+                name = os.path.basename(hook_file).replace('.json', '')
+                dhcp_options[name] = json.loads(util.load_file((hook_file)))
+            except ValueError:
+                raise ValueError("%s is not valid JSON data", hook_file)
+        return dhcp_options
+
+    @staticmethod
+    def _get_value_from_dhcpoptions(dhcp_options):
+        if dhcp_options is None:
+            return None
+        # the MS endpoint server is given to us as DHPC option 245
+        _value = None
+        for interface in dhcp_options:
+            _value = dhcp_options[interface].get('unknown_245', None)
+            if _value is not None:
+                LOG.debug("Endpoint server found in dhclient options")
+                break
+        return _value
+
+    @staticmethod
+    def find_endpoint(fallback_lease_file=None):
+        LOG.debug('Finding Azure endpoint...')
+        value = None
+        # Option-245 stored in /run/cloud-init/dhclient.hooks/<ifc>.json
+        # a dhclient exit hook that calls cloud-init-dhclient-hook
+        dhcp_options = WALinuxAgentShim._load_dhclient_json()
+        value = WALinuxAgentShim._get_value_from_dhcpoptions(dhcp_options)
         if value is None:
-            raise ValueError('No endpoint found in DHCP config.')
+            # Fallback and check the leases file if unsuccessful
+            LOG.debug("Unable to find endpoint in dhclient logs. "
+                      " Falling back to check lease files")
+            if fallback_lease_file is None:
+                LOG.warn("No fallback lease file was specified.")
+                value = None
+            else:
+                LOG.debug("Looking for endpoint in lease file %s",
+                          fallback_lease_file)
+                value = WALinuxAgentShim._get_value_from_leases_file(
+                    fallback_lease_file)
+
+        if value is None:
+            raise ValueError('No endpoint found.')
+
         endpoint_ip_address = WALinuxAgentShim.get_ip_from_lease_value(value)
         LOG.debug('Azure endpoint found at %s', endpoint_ip_address)
         return endpoint_ip_address
@@ -271,8 +347,8 @@ class WALinuxAgentShim(object):
         LOG.info('Reported ready to Azure fabric.')
 
 
-def get_metadata_from_fabric():
-    shim = WALinuxAgentShim()
+def get_metadata_from_fabric(fallback_lease_file=None):
+    shim = WALinuxAgentShim(fallback_lease_file=fallback_lease_file)
     try:
         return shim.register_with_azure_and_fetch_data()
     finally:
diff --git a/cloudinit/sources/helpers/openstack.py b/cloudinit/sources/helpers/openstack.py
index 2e7a1d47..a5a2a1d6 100644
--- a/cloudinit/sources/helpers/openstack.py
+++ b/cloudinit/sources/helpers/openstack.py
@@ -539,6 +539,10 @@ def convert_net_json(network_json=None, known_macs=None):
     networks = network_json.get('networks', [])
     services = network_json.get('services', [])
 
+    link_updates = []
+    link_id_info = {}
+    bond_name_fmt = "bond%d"
+    bond_number = 0
     config = []
     for link in links:
         subnets = []
@@ -551,6 +555,13 @@ def convert_net_json(network_json=None, known_macs=None):
         if 'name' in link:
             cfg['name'] = link['name']
 
+        if link.get('ethernet_mac_address'):
+            link_id_info[link['id']] = link.get('ethernet_mac_address')
+
+        curinfo = {'name': cfg.get('name'),
+                   'mac': link.get('ethernet_mac_address'),
+                   'id': link['id'], 'type': link['type']}
+
         for network in [n for n in networks
                         if n['link'] == link['id']]:
             subnet = dict((k, v) for k, v in network.items()
@@ -571,7 +582,7 @@ def convert_net_json(network_json=None, known_macs=None):
                 subnet['ipv6'] = True
             subnets.append(subnet)
         cfg.update({'subnets': subnets})
-        if link['type'] in ['ethernet', 'vif', 'ovs', 'phy', 'bridge']:
+        if link['type'] in ['ethernet', 'vif', 'ovs', 'phy', 'bridge', 'tap']:
             cfg.update({
                 'type': 'physical',
                 'mac_address': link['ethernet_mac_address']})
@@ -582,31 +593,56 @@ def convert_net_json(network_json=None, known_macs=None):
                     continue
                 elif k.startswith('bond'):
                     params.update({k: v})
-            cfg.update({
-                'bond_interfaces': copy.deepcopy(link['bond_links']),
-                'params': params,
-            })
+
+            # openstack does not provide a name for the bond.
+            # they do provide an 'id', but that is possibly non-sensical.
+            # so we just create our own name.
+            link_name = bond_name_fmt % bond_number
+            bond_number += 1
+
+            # bond_links reference links by their id, but we need to add
+            # to the network config by their nic name.
+            # store that in bond_links_needed, and update these later.
+            link_updates.append(
+                (cfg, 'bond_interfaces', '%s',
+                 copy.deepcopy(link['bond_links']))
+            )
+            cfg.update({'params': params, 'name': link_name})
+
+            curinfo['name'] = link_name
         elif link['type'] in ['vlan']:
+            name = "%s.%s" % (link['vlan_link'], link['vlan_id'])
             cfg.update({
-                'name': "%s.%s" % (link['vlan_link'],
-                                   link['vlan_id']),
-                'vlan_link': link['vlan_link'],
+                'name': name,
                 'vlan_id': link['vlan_id'],
                 'mac_address': link['vlan_mac_address'],
             })
+            link_updates.append((cfg, 'vlan_link', '%s', link['vlan_link']))
+            link_updates.append((cfg, 'name', "%%s.%s" % link['vlan_id'],
+                                 link['vlan_link']))
+            curinfo.update({'mac': link['vlan_mac_address'],
+                            'name': name})
         else:
             raise ValueError(
                 'Unknown network_data link type: %s' % link['type'])
 
         config.append(cfg)
+        link_id_info[curinfo['id']] = curinfo
 
     need_names = [d for d in config
                   if d.get('type') == 'physical' and 'name' not in d]
 
-    if need_names:
+    if need_names or link_updates:
         if known_macs is None:
             known_macs = net.get_interfaces_by_mac()
 
+        # go through and fill out the link_id_info with names
+        for link_id, info in link_id_info.items():
+            if info.get('name'):
+                continue
+            if info.get('mac') in known_macs:
+                info['name'] = known_macs[info['mac']]
+
         for d in need_names:
             mac = d.get('mac_address')
             if not mac:
@@ -615,34 +651,15 @@ def convert_net_json(network_json=None, known_macs=None):
                 raise ValueError("Unable to find a system nic for %s" % d)
             d['name'] = known_macs[mac]
 
+        for cfg, key, fmt, target in link_updates:
+            if isinstance(target, (list, tuple)):
+                cfg[key] = [fmt % link_id_info[l]['name'] for l in target]
+            else:
+                cfg[key] = fmt % link_id_info[target]['name']
+
     for service in services:
         cfg = service
         cfg.update({'type': 'nameserver'})
         config.append(cfg)
 
     return {'version': 1, 'config': config}
-
-
-def convert_vendordata_json(data, recurse=True):
-    """data: a loaded json *object* (strings, arrays, dicts).
-    return something suitable for cloudinit vendordata_raw.
-
-    if data is:
-       None: return None
-       string: return string
-       list: return data
-             the list is then processed in UserDataProcessor
-       dict: return convert_vendordata_json(data.get('cloud-init'))
-    """
-    if not data:
-        return None
-    if isinstance(data, six.string_types):
-        return data
-    if isinstance(data, list):
-        return copy.deepcopy(data)
-    if isinstance(data, dict):
-        if recurse is True:
-            return convert_vendordata_json(data.get('cloud-init'),
-                                           recurse=False)
-        raise ValueError("vendordata['cloud-init'] cannot be dict")
-    raise ValueError("Unknown data type for vendordata: %s" % type(data))