Rebase against master

4 files changed, 271 insertions, 101 deletions

diff --git a/cloudinit/net/__init__.py b/cloudinit/net/__init__.py
index f8df58f0..f5668fff 100644
--- a/cloudinit/net/__init__.py
+++ b/cloudinit/net/__init__.py
@@ -19,77 +19,14 @@
 import errno
 import logging
 import os
+import re
 
-from .import compat
-
-import yaml
+from cloudinit import util
 
 LOG = logging.getLogger(__name__)
 SYS_CLASS_NET = "/sys/class/net/"
-LINKS_FNAME_PREFIX = "etc/systemd/network/50-cloud-init-"
-
-NET_CONFIG_OPTIONS = [
-    "address", "netmask", "broadcast", "network", "metric", "gateway",
-    "pointtopoint", "media", "mtu", "hostname", "leasehours", "leasetime",
-    "vendor", "client", "bootfile", "server", "hwaddr", "provider", "frame",
-    "netnum", "endpoint", "local", "ttl",
-]
-
-NET_CONFIG_COMMANDS = [
-    "pre-up", "up", "post-up", "down", "pre-down", "post-down",
-]
-
-NET_CONFIG_BRIDGE_OPTIONS = [
-    "bridge_ageing", "bridge_bridgeprio", "bridge_fd", "bridge_gcinit",
-    "bridge_hello", "bridge_maxage", "bridge_maxwait", "bridge_stp",
-]
 DEFAULT_PRIMARY_INTERFACE = 'eth0'
-
-# NOTE(harlowja): some of these are similar to what is in cloudinit main
-# source or utils tree/module but the reason that is done is so that this
-# whole module can be easily extracted and placed into other
-# code-bases (curtin for example).
-
-
-def write_file(path, content):
-    base_path = os.path.dirname(path)
-    if not os.path.isdir(base_path):
-        os.makedirs(base_path)
-    with open(path, "wb+") as fh:
-        if isinstance(content, compat.text_type):
-            content = content.encode("utf8")
-        fh.write(content)
-
-
-def read_file(path, decode='utf8', enoent=None):
-    try:
-        with open(path, "rb") as fh:
-            contents = fh.read()
-    except OSError as e:
-        if e.errno == errno.ENOENT and enoent is not None:
-            return enoent
-        raise
-    if decode:
-        return contents.decode(decode)
-    return contents
-
-
-def dump_yaml(obj):
-    return yaml.safe_dump(obj,
-                          line_break="\n",
-                          indent=4,
-                          explicit_start=True,
-                          explicit_end=True,
-                          default_flow_style=False)
-
-
-def read_yaml_file(path):
-    val = yaml.safe_load(read_file(path))
-    if not isinstance(val, dict):
-        gotten_type_name = type(val).__name__
-        raise TypeError("Expected dict to be loaded from %s, got"
-                        " '%s' instead" % (path, gotten_type_name))
-    return val
+LINKS_FNAME_PREFIX = "etc/systemd/network/50-cloud-init-"
 
 
 def sys_dev_path(devname, path=""):
@@ -97,7 +34,13 @@ def sys_dev_path(devname, path=""):
 
 
 def read_sys_net(devname, path, translate=None, enoent=None, keyerror=None):
-    contents = read_file(sys_dev_path(devname, path), enoent=enoent)
+    try:
+        contents = util.load_file(sys_dev_path(devname, path))
+    except (OSError, IOError) as e:
+        if getattr(e, 'errno', None) == errno.ENOENT:
+            if enoent is not None:
+                return enoent
+        raise
     contents = contents.strip()
     if translate is None:
         return contents
@@ -158,7 +101,7 @@ def get_devicelist():
 
 
 class ParserError(Exception):
-    """Raised when parser has issue parsing the interfaces file."""
+    """Raised when a parser has issue parsing a file/content."""
 
 
 def is_disabled_cfg(cfg):
@@ -171,11 +114,10 @@ def sys_netdev_info(name, field):
     if not os.path.exists(os.path.join(SYS_CLASS_NET, name)):
         raise OSError("%s: interface does not exist in %s" %
                       (name, SYS_CLASS_NET))
-
     fname = os.path.join(SYS_CLASS_NET, name, field)
     if not os.path.exists(fname):
         raise OSError("%s: could not find sysfs entry: %s" % (name, fname))
-    data = read_file(fname)
+    data = util.load_file(fname)
     if data[-1] == '\n':
         data = data[:-1]
     return data
@@ -251,4 +193,174 @@ def generate_fallback_config():
     return nconf
 
 
+def apply_network_config_names(netcfg, strict_present=True, strict_busy=True):
+    """read the network config and rename devices accordingly.
+    if strict_present is false, then do not raise exception if no devices
+    match.  if strict_busy is false, then do not raise exception if the
+    device cannot be renamed because it is currently configured."""
+    renames = []
+    for ent in netcfg.get('config', {}):
+        if ent.get('type') != 'physical':
+            continue
+        mac = ent.get('mac_address')
+        name = ent.get('name')
+        if not mac:
+            continue
+        renames.append([mac, name])
+
+    return _rename_interfaces(renames)
+
+
+def _get_current_rename_info(check_downable=True):
+    """Collect information necessary for rename_interfaces."""
+    names = get_devicelist()
+    bymac = {}
+    for n in names:
+        bymac[get_interface_mac(n)] = {
+            'name': n, 'up': is_up(n), 'downable': None}
+
+    if check_downable:
+        nmatch = re.compile(r"[0-9]+:\s+(\w+)[@:]")
+        ipv6, _err = util.subp(['ip', '-6', 'addr', 'show', 'permanent',
+                                'scope', 'global'], capture=True)
+        ipv4, _err = util.subp(['ip', '-4', 'addr', 'show'], capture=True)
+
+        nics_with_addresses = set()
+        for bytes_out in (ipv6, ipv4):
+            nics_with_addresses.update(nmatch.findall(bytes_out))
+
+        for d in bymac.values():
+            d['downable'] = (d['up'] is False or
+                             d['name'] not in nics_with_addresses)
+
+    return bymac
+
+
+def _rename_interfaces(renames, strict_present=True, strict_busy=True,
+                       current_info=None):
+    if current_info is None:
+        current_info = _get_current_rename_info()
+
+    cur_bymac = {}
+    for mac, data in current_info.items():
+        cur = data.copy()
+        cur['mac'] = mac
+        cur_bymac[mac] = cur
+
+    def update_byname(bymac):
+        return {data['name']: data for data in bymac.values()}
+
+    def rename(cur, new):
+        util.subp(["ip", "link", "set", cur, "name", new], capture=True)
+
+    def down(name):
+        util.subp(["ip", "link", "set", name, "down"], capture=True)
+
+    def up(name):
+        util.subp(["ip", "link", "set", name, "up"], capture=True)
+
+    ops = []
+    errors = []
+    ups = []
+    cur_byname = update_byname(cur_bymac)
+    tmpname_fmt = "cirename%d"
+    tmpi = -1
+
+    for mac, new_name in renames:
+        cur = cur_bymac.get(mac, {})
+        cur_name = cur.get('name')
+        cur_ops = []
+        if cur_name == new_name:
+            # nothing to do
+            continue
+
+        if not cur_name:
+            if strict_present:
+                errors.append(
+                    "[nic not present] Cannot rename mac=%s to %s"
+                    ", not available." % (mac, new_name))
+            continue
+
+        if cur['up']:
+            msg = "[busy] Error renaming mac=%s from %s to %s"
+            if not cur['downable']:
+                if strict_busy:
+                    errors.append(msg % (mac, cur_name, new_name))
+                continue
+            cur['up'] = False
+            cur_ops.append(("down", mac, new_name, (cur_name,)))
+            ups.append(("up", mac, new_name, (new_name,)))
+
+        if new_name in cur_byname:
+            target = cur_byname[new_name]
+            if target['up']:
+                msg = "[busy-target] Error renaming mac=%s from %s to %s."
+                if not target['downable']:
+                    if strict_busy:
+                        errors.append(msg % (mac, cur_name, new_name))
+                    continue
+                else:
+                    cur_ops.append(("down", mac, new_name, (new_name,)))
+
+            tmp_name = None
+            while tmp_name is None or tmp_name in cur_byname:
+                tmpi += 1
+                tmp_name = tmpname_fmt % tmpi
+
+            cur_ops.append(("rename", mac, new_name, (new_name, tmp_name)))
+            target['name'] = tmp_name
+            cur_byname = update_byname(cur_bymac)
+            if target['up']:
+                ups.append(("up", mac, new_name, (tmp_name,)))
+
+        cur_ops.append(("rename", mac, new_name, (cur['name'], new_name)))
+        cur['name'] = new_name
+        cur_byname = update_byname(cur_bymac)
+        ops += cur_ops
+
+    opmap = {'rename': rename, 'down': down, 'up': up}
+
+    if len(ops) + len(ups) == 0:
+        if len(errors):
+            LOG.debug("unable to do any work for renaming of %s", renames)
+        else:
+            LOG.debug("no work necessary for renaming of %s", renames)
+    else:
+        LOG.debug("achieving renaming of %s with ops %s", renames, ops + ups)
+
+        for op, mac, new_name, params in ops + ups:
+            try:
+                opmap.get(op)(*params)
+            except Exception as e:
+                errors.append(
+                    "[unknown] Error performing %s%s for %s, %s: %s" %
+                    (op, params, mac, new_name, e))
+
+    if len(errors):
+        raise Exception('\n'.join(errors))
+
+
+def get_interface_mac(ifname):
+    """Returns the string value of an interface's MAC Address"""
+    return read_sys_net(ifname, "address", enoent=False)
+
+
+def get_interfaces_by_mac(devs=None):
+    """Build a dictionary of tuples {mac: name}"""
+    if devs is None:
+        try:
+            devs = get_devicelist()
+        except OSError as e:
+            if e.errno == errno.ENOENT:
+                devs = []
+            else:
+                raise
+    ret = {}
+    for name in devs:
+        mac = get_interface_mac(name)
+        # some devices may not have a mac (tun0)
+        if mac:
+            ret[mac] = name
+    return ret
+
 # vi: ts=4 expandtab syntax=python
diff --git a/cloudinit/net/cmdline.py b/cloudinit/net/cmdline.py
index 41cba893..39523be2 100644
--- a/cloudinit/net/cmdline.py
+++ b/cloudinit/net/cmdline.py
@@ -43,17 +43,13 @@ def _load_shell_content(content, add_empty=False, empty_val=None):
        then add entries in to the returned dictionary for 'VAR='
        variables.  Set their value to empty_val."""
     data = {}
-    for line in _shlex_split(content):
-        try:
-            key, value = line.split("=", 1)
-        except ValueError:
-            # Unsplittable line, skip it...
-            pass
-        else:
-            if not value:
-                value = empty_val
-            if add_empty or value:
-                data[key] = value
+    for line in shlex.split(content):
+        key, value = line.split("=", 1)
+        if not value:
+            value = empty_val
+        if add_empty or value:
+            data[key] = value
+
     return data
 
 
diff --git a/cloudinit/net/eni.py b/cloudinit/net/eni.py
index f82c7f54..a695f5ed 100644
--- a/cloudinit/net/eni.py
+++ b/cloudinit/net/eni.py
@@ -18,10 +18,11 @@ import re
 
 from . import LINKS_FNAME_PREFIX
 from . import ParserError
-from . import write_file
 
 from .udev import generate_udev_rule
 
+from cloudinit import util
+
 
 NET_CONFIG_COMMANDS = [
     "pre-up", "up", "post-up", "down", "pre-down", "post-down",
@@ -67,6 +68,7 @@ def _iface_add_subnet(iface, subnet):
 
 
 # TODO: switch to valid_map for attrs
+
 def _iface_add_attrs(iface):
     content = ""
     ignore_map = [
@@ -181,7 +183,11 @@ def _parse_deb_config_data(ifaces, contents, src_dir, src_path):
             ifaces[iface]['method'] = method
             currif = iface
         elif option == "hwaddress":
-            ifaces[currif]['hwaddress'] = split[1]
+            if split[1] == "ether":
+                val = split[2]
+            else:
+                val = split[1]
+            ifaces[currif]['hwaddress'] = val
         elif option in NET_CONFIG_OPTIONS:
             ifaces[currif][option] = split[1]
         elif option in NET_CONFIG_COMMANDS:
@@ -229,7 +235,7 @@ def _parse_deb_config_data(ifaces, contents, src_dir, src_path):
             ifaces[iface]['auto'] = False
 
 
-def _parse_deb_config(path):
+def parse_deb_config(path):
     """Parses a debian network configuration file."""
     ifaces = {}
     with open(path, "r") as fp:
@@ -241,6 +247,56 @@ def _parse_deb_config(path):
     return ifaces
 
 
+def convert_eni_data(eni_data):
+    # return a network config representation of what is in eni_data
+    ifaces = {}
+    _parse_deb_config_data(ifaces, eni_data, src_dir=None, src_path=None)
+    return _ifaces_to_net_config_data(ifaces)
+
+
+def _ifaces_to_net_config_data(ifaces):
+    """Return network config that represents the ifaces data provided.
+    ifaces = parse_deb_config("/etc/network/interfaces")
+    config = ifaces_to_net_config_data(ifaces)
+    state = parse_net_config_data(config)."""
+    devs = {}
+    for name, data in ifaces.items():
+        # devname is 'eth0' for name='eth0:1'
+        devname = name.partition(":")[0]
+        if devname == "lo":
+            # currently provding 'lo' in network config results in duplicate
+            # entries. in rendered interfaces file. so skip it.
+            continue
+        if devname not in devs:
+            devs[devname] = {'type': 'physical', 'name': devname,
+                             'subnets': []}
+            # this isnt strictly correct, but some might specify
+            # hwaddress on a nic for matching / declaring name.
+            if 'hwaddress' in data:
+                devs[devname]['mac_address'] = data['hwaddress']
+        subnet = {'_orig_eni_name': name, 'type': data['method']}
+        if data.get('auto'):
+            subnet['control'] = 'auto'
+        else:
+            subnet['control'] = 'manual'
+
+        if data.get('method') == 'static':
+            subnet['address'] = data['address']
+
+        for copy_key in ('netmask', 'gateway', 'broadcast'):
+            if copy_key in data:
+                subnet[copy_key] = data[copy_key]
+
+        if 'dns' in data:
+            for n in ('nameservers', 'search'):
+                if n in data['dns'] and data['dns'][n]:
+                    subnet['dns_' + n] = data['dns'][n]
+        devs[devname]['subnets'].append(subnet)
+
+    return {'version': 1,
+            'config': [devs[d] for d in sorted(devs)]}
+
+
 class Renderer(object):
     """Renders network information in a /etc/network/interfaces format."""
 
@@ -298,11 +354,10 @@ class Renderer(object):
                     route_line += " %s %s" % (mapping[k], route[k])
             content += up + route_line + eol
             content += down + route_line + eol
-
         return content
 
     def _render_interfaces(self, network_state):
-        '''Given state, emit etc/network/interfaces content'''
+        '''Given state, emit etc/network/interfaces content.'''
 
         content = ""
         interfaces = network_state.get('interfaces')
@@ -345,6 +400,8 @@ class Renderer(object):
                     content += _iface_start_entry(iface, index)
                     content += _iface_add_subnet(iface, subnet)
                     content += _iface_add_attrs(iface)
+                    for route in subnet.get('routes', []):
+                        content += self._render_route(route, indent="    ")
             else:
                 # ifenslave docs say to auto the slave devices
                 if 'bond-master' in iface:
@@ -360,19 +417,24 @@ class Renderer(object):
         return content
 
     def render_network_state(
-            self, target, network_state,
-            eni="etc/network/interfaces", links_prefix=LINKS_FNAME_PREFIX,
-            netrules='etc/udev/rules.d/70-persistent-net.rules'):
+        self, target, network_state, eni="etc/network/interfaces",
+            links_prefix=LINKS_FNAME_PREFIX,
+            netrules='etc/udev/rules.d/70-persistent-net.rules',
+            writer=None):
 
-        fpeni = os.path.join(target, eni)
-        write_file(fpeni, self._render_interfaces(network_state))
+        fpeni = os.path.sep.join((target, eni,))
+        util.ensure_dir(os.path.dirname(fpeni))
+        util.write_file(fpeni, self._render_interfaces(network_state))
 
         if netrules:
-            netrules = os.path.join(target, netrules)
-            write_file(netrules, self._render_persistent_net(network_state))
+            netrules = os.path.sep.join((target, netrules,))
+            util.ensure_dir(os.path.dirname(netrules))
+            util.write_file(netrules,
+                            self._render_persistent_net(network_state))
 
         if links_prefix:
-            self._render_systemd_links(target, network_state, links_prefix)
+            self._render_systemd_links(target, network_state,
+                                       links_prefix=links_prefix)
 
     def _render_systemd_links(self, target, network_state,
                               links_prefix=LINKS_FNAME_PREFIX):
@@ -392,4 +454,4 @@ class Renderer(object):
                     "Name=" + iface['name'],
                     ""
                 ])
-                write_file(fname, content)
+                util.write_file(fname, content)
diff --git a/cloudinit/net/network_state.py b/cloudinit/net/network_state.py
index 32c48229..1e82e75d 100644
--- a/cloudinit/net/network_state.py
+++ b/cloudinit/net/network_state.py
@@ -20,8 +20,8 @@ import functools
 import logging
 
 from . import compat
-from . import dump_yaml
-from . import read_yaml_file
+
+from cloudinit import util
 
 LOG = logging.getLogger(__name__)
 
@@ -49,7 +49,7 @@ def parse_net_config(path, skip_broken=True):
     """Parses a curtin network configuration file and
        return network state"""
     ns = None
-    net_config = read_yaml_file(path)
+    net_config = util.read_conf(path)
     if 'network' in net_config:
         ns = parse_net_config_data(net_config.get('network'),
                                    skip_broken=skip_broken)
@@ -58,7 +58,7 @@ def parse_net_config(path, skip_broken=True):
 
 def from_state_file(state_file):
     network_state = None
-    state = read_yaml_file(state_file)
+    state = util.read_conf(state_file)
     network_state = NetworkState()
     network_state.load(state)
     return network_state
@@ -136,7 +136,7 @@ class NetworkState(object):
             'config': self.config,
             'network_state': self.network_state,
         }
-        return dump_yaml(state)
+        return util.yaml_dumps(state)
 
     def load(self, state):
         if 'version' not in state:
@@ -155,7 +155,7 @@ class NetworkState(object):
             setattr(self, key, state[key])
 
     def dump_network_state(self):
-        return dump_yaml(self.network_state)
+        return util.yaml_dumps(self.network_state)
 
     def parse_config(self, skip_broken=True):
         # rebuild network state