Adjust net module to be more isolated

This allows it to be used outside of cloudinit
more easily in the future.

7 files changed, 129 insertions, 87 deletions

diff --git a/cloudinit/net/__init__.py b/cloudinit/net/__init__.py
index 0202cbd8..07e7307e 100644
--- a/cloudinit/net/__init__.py
+++ b/cloudinit/net/__init__.py
@@ -17,44 +17,81 @@
 #   along with Curtin.  If not, see <http://www.gnu.org/licenses/>.
 
 import errno
+import logging
 import os
 
-
-from cloudinit import log as logging
-from cloudinit.net import network_state
-from cloudinit import util
-
+import six
+import yaml
 
 LOG = logging.getLogger(__name__)
 SYS_CLASS_NET = "/sys/class/net/"
 LINKS_FNAME_PREFIX = "etc/systemd/network/50-cloud-init-"
 DEFAULT_PRIMARY_INTERFACE = 'eth0'
 
+# NOTE(harlowja): some of these are similar to what is in cloudinit main
+# source or utils tree/module but the reason that is done is so that this
+# whole module can be easily extracted and placed into other
+# code-bases (curtin for example).
 
-def sys_dev_path(devname, path=""):
-    return SYS_CLASS_NET + devname + "/" + path
+def write_file(path, content):
+    """Simple writing a file helper."""
+    base_path = os.path.dirname(path)
+    if not os.path.isdir(base_path):
+        os.makedirs(base_path)
+    with open(path, "wb+") as fh:
+        if isinstance(content, six.text_type):
+            content = content.encode("utf8")
+        fh.write(content)
 
 
-def read_sys_net(devname, path, translate=None, enoent=None, keyerror=None):
+def read_file(path, decode='utf8', enoent=None):
     try:
-        contents = ""
-        with open(sys_dev_path(devname, path), "r") as fp:
-            contents = fp.read().strip()
-        if translate is None:
-            return contents
-
-        try:
-            return translate.get(contents)
-        except KeyError:
-            LOG.debug("found unexpected value '%s' in '%s/%s'", contents,
-                      devname, path)
-            if keyerror is not None:
-                return keyerror
-            raise
+        with open(path, "rb") as fh:
+            contents = fh.load()
     except OSError as e:
         if e.errno == errno.ENOENT and enoent is not None:
             return enoent
         raise
+    if decode:
+        return contents.decode(decode)
+    return contents
+
+
+def dump_yaml(obj):
+    return yaml.safe_dump(obj,
+                          line_break="\n",
+                          indent=4,
+                          explicit_start=True,
+                          explicit_end=True,
+                          default_flow_style=False)
+
+
+def read_yaml_file(path):
+    val = yaml.safe_load(read_file(path))
+    if not isinstance(val, dict):
+        gotten_type_name = type(val).__name__
+        raise TypeError("Expected dict to be loaded from %s, got"
+                        " '%s' instead" % (path, gotten_type_name))
+    return val
+
+
+def sys_dev_path(devname, path=""):
+    return SYS_CLASS_NET + devname + "/" + path
+
+
+def read_sys_net(devname, path, translate=None, enoent=None, keyerror=None):
+    contents = read_file(sys_dev_path(devname, path), enoent=enoent)
+    contents = contents.strip()
+    if translate is None:
+        return contents
+    try:
+        return translate.get(contents)
+    except KeyError:
+        LOG.debug("found unexpected value '%s' in '%s/%s'", contents,
+                  devname, path)
+        if keyerror is not None:
+            return keyerror
+        raise
 
 
 def is_up(devname):
@@ -107,31 +144,6 @@ class ParserError(Exception):
     """Raised when parser has issue parsing the interfaces file."""
 
 
-def parse_net_config_data(net_config, skip_broken=True):
-    """Parses the config, returns NetworkState object
-
-    :param net_config: curtin network config dict
-    """
-    state = None
-    if 'version' in net_config and 'config' in net_config:
-        ns = network_state.NetworkState(version=net_config.get('version'),
-                                        config=net_config.get('config'))
-        ns.parse_config(skip_broken=skip_broken)
-        state = ns.network_state
-    return state
-
-
-def parse_net_config(path, skip_broken=True):
-    """Parses a curtin network configuration file and
-       return network state"""
-    ns = None
-    net_config = util.read_conf(path)
-    if 'network' in net_config:
-        ns = parse_net_config_data(net_config.get('network'),
-                                   skip_broken=skip_broken)
-    return ns
-
-
 def is_disabled_cfg(cfg):
     if not cfg or not isinstance(cfg, dict):
         return False
@@ -146,7 +158,7 @@ def sys_netdev_info(name, field):
     fname = os.path.join(SYS_CLASS_NET, name, field)
     if not os.path.exists(fname):
         raise OSError("%s: could not find sysfs entry: %s" % (name, fname))
-    data = util.load_file(fname)
+    data = read_file(fname)
     if data[-1] == '\n':
         data = data[:-1]
     return data
diff --git a/cloudinit/net/cmdline.py b/cloudinit/net/cmdline.py
index f5712533..b85d4b0a 100644
--- a/cloudinit/net/cmdline.py
+++ b/cloudinit/net/cmdline.py
@@ -20,12 +20,25 @@ import base64
 import glob
 import gzip
 import io
+import shlex
+import sys
+
+import six
 
 from cloudinit.net import get_devicelist
 from cloudinit.net import sys_netdev_info
 
 from cloudinit import util
 
+PY26 = sys.version_info[0:2] == (2, 6)
+
+
+def _shlex_split(blob):
+    if PY26 and isinstance(blob, six.text_type):
+        # Older versions don't support unicode input
+        blob = blob.encode("utf8")
+    return shlex.split(blob)
+
 
 def _load_shell_content(content, add_empty=False, empty_val=None):
     """Given shell like syntax (key=value\nkey2=value2\n) in content
@@ -33,7 +46,7 @@ def _load_shell_content(content, add_empty=False, empty_val=None):
        then add entries in to the returned dictionary for 'VAR='
        variables.  Set their value to empty_val."""
     data = {}
-    for line in util.shlex_split(content):
+    for line in _shlex_split(content):
         try:
             key, value = line.split("=", 1)
         except ValueError:
diff --git a/cloudinit/net/eni.py b/cloudinit/net/eni.py
index b1bdac24..adb31c22 100644
--- a/cloudinit/net/eni.py
+++ b/cloudinit/net/eni.py
@@ -16,10 +16,11 @@ import glob
 import os
 import re
 
+from cloudinit import net
+
 from cloudinit.net import LINKS_FNAME_PREFIX
 from cloudinit.net import ParserError
 from cloudinit.net.udev import generate_udev_rule
-from cloudinit import util
 
 
 NET_CONFIG_COMMANDS = [
@@ -363,16 +364,13 @@ class Renderer(object):
         links_prefix=LINKS_FNAME_PREFIX,
         netrules='etc/udev/rules.d/70-persistent-net.rules'):
 
-        fpeni = os.path.sep.join((target, eni,))
-        util.ensure_dir(os.path.dirname(fpeni))
-        with open(fpeni, 'w+') as f:
-            f.write(self._render_interfaces(network_state))
+        fpeni = os.path.join(target, eni)
+        net.write_file(fpeni, self._render_interfaces(network_state))
 
         if netrules:
-            netrules = os.path.sep.join((target, netrules,))
-            util.ensure_dir(os.path.dirname(netrules))
-            with open(netrules, 'w+') as f:
-                f.write(self._render_persistent_net(network_state))
+            netrules = os.path.join(target, netrules)
+            net.write_file(netrules,
+                           self._render_persistent_net(network_state))
 
         if links_prefix:
             self._render_systemd_links(target, network_state, links_prefix)
@@ -382,18 +380,17 @@ class Renderer(object):
         fp_prefix = os.path.sep.join((target, links_prefix))
         for f in glob.glob(fp_prefix + "*"):
             os.unlink(f)
-
         interfaces = network_state.get('interfaces')
         for iface in interfaces.values():
             if (iface['type'] == 'physical' and 'name' in iface and
                     iface.get('mac_address')):
                 fname = fp_prefix + iface['name'] + ".link"
-                with open(fname, "w") as fp:
-                    fp.write("\n".join([
-                        "[Match]",
-                        "MACAddress=" + iface['mac_address'],
-                        "",
-                        "[Link]",
-                        "Name=" + iface['name'],
-                        ""
-                    ]))
+                content = "\n".join([
+                    "[Match]",
+                    "MACAddress=" + iface['mac_address'],
+                    "",
+                    "[Link]",
+                    "Name=" + iface['name'],
+                    ""
+                ])
+                net.write_file(fname, content)
diff --git a/cloudinit/net/network_state.py b/cloudinit/net/network_state.py
index 2feffa71..c5aeadb5 100644
--- a/cloudinit/net/network_state.py
+++ b/cloudinit/net/network_state.py
@@ -16,12 +16,11 @@
 #   along with Curtin.  If not, see <http://www.gnu.org/licenses/>.
 
 import copy
+import logging
 
 import six
 
-from cloudinit import log as logging
-from cloudinit import util
-from cloudinit.util import yaml_dumps as dump_config
+from cloudinit import net
 
 LOG = logging.getLogger(__name__)
 
@@ -31,9 +30,34 @@ NETWORK_STATE_REQUIRED_KEYS = {
 }
 
 
+def parse_net_config_data(net_config, skip_broken=True):
+    """Parses the config, returns NetworkState object
+
+    :param net_config: curtin network config dict
+    """
+    state = None
+    if 'version' in net_config and 'config' in net_config:
+        ns = NetworkState(version=net_config.get('version'),
+                          config=net_config.get('config'))
+        ns.parse_config(skip_broken=skip_broken)
+        state = ns.network_state
+    return state
+
+
+def parse_net_config(path, skip_broken=True):
+    """Parses a curtin network configuration file and
+       return network state"""
+    ns = None
+    net_config = net.read_yaml_file(path)
+    if 'network' in net_config:
+        ns = parse_net_config_data(net_config.get('network'),
+                                   skip_broken=skip_broken)
+    return ns
+
+
 def from_state_file(state_file):
     network_state = None
-    state = util.read_conf(state_file)
+    state = net.read_yaml_file(state_file)
     network_state = NetworkState()
     network_state.load(state)
     return network_state
@@ -111,7 +135,7 @@ class NetworkState(object):
             'config': self.config,
             'network_state': self.network_state,
         }
-        return dump_config(state)
+        return net.dump_yaml(state)
 
     def load(self, state):
         if 'version' not in state:
@@ -121,7 +145,7 @@ class NetworkState(object):
         required_keys = NETWORK_STATE_REQUIRED_KEYS[state['version']]
         missing_keys = diff_keys(required_keys, state)
         if missing_keys:
-            msg = 'Invalid state, missing keys: %s'.format(missing_keys)
+            msg = 'Invalid state, missing keys: %s' % (missing_keys)
             LOG.error(msg)
             raise ValueError(msg)
 
@@ -130,7 +154,7 @@ class NetworkState(object):
             setattr(self, key, state[key])
 
     def dump_network_state(self):
-        return dump_config(self.network_state)
+        return net.dump_yaml(self.network_state)
 
     def parse_config(self, skip_broken=True):
         # rebuild network state
diff --git a/cloudinit/sources/helpers/openstack.py b/cloudinit/sources/helpers/openstack.py
index b2acc648..f85fd864 100644
--- a/cloudinit/sources/helpers/openstack.py
+++ b/cloudinit/sources/helpers/openstack.py
@@ -551,6 +551,10 @@ def convert_net_json(network_json):
                     'type': 'static',
                     'address': network.get('ip_address'),
                 })
+                if network['type'] == 'ipv6':
+                    subnet['ipv6'] = True
+                else:
+                    subnet['ipv4'] = True
             subnets.append(subnet)
         cfg.update({'subnets': subnets})
         if link['type'] in ['ethernet', 'vif', 'ovs', 'phy']:
diff --git a/cloudinit/util.py b/cloudinit/util.py
index 7562b97a..2bec476e 100644
--- a/cloudinit/util.py
+++ b/cloudinit/util.py
@@ -37,7 +37,6 @@ import pwd
 import random
 import re
 import shutil
-import shlex
 import socket
 import stat
 import string
@@ -82,7 +81,6 @@ CONTAINER_TESTS = (['systemd-detect-virt', '--quiet', '--container'],
                    ['lxc-is-container'])
 
 PROC_CMDLINE = None
-PY26 = sys.version_info[0:2] == (2, 6)
 
 
 def decode_binary(blob, encoding='utf-8'):
@@ -1202,13 +1200,6 @@ def load_file(fname, read_cb=None, quiet=False, decode=True):
         return contents
 
 
-def shlex_split(blob):
-    if PY26 and isinstance(blob, six.text_type):
-        # Older versions don't support unicode input
-        blob = blob.encode("utf8")
-    return shlex.split(blob)
-
-
 def get_cmdline():
     if 'DEBUG_PROC_CMDLINE' in os.environ:
         return os.environ["DEBUG_PROC_CMDLINE"]
diff --git a/tests/unittests/test_net.py b/tests/unittests/test_net.py
index ed2c6d0f..75c433f6 100644
--- a/tests/unittests/test_net.py
+++ b/tests/unittests/test_net.py
@@ -2,6 +2,7 @@ from cloudinit import util
 from cloudinit import net
 from cloudinit.net import cmdline
 from cloudinit.net import eni
+from cloudinit.net import network_state
 from .helpers import TestCase
 from .helpers import mock
 
@@ -112,14 +113,14 @@ class TestEniNetRendering(TestCase):
         mock_sys_dev_path.side_effect = sys_dev_path
 
         network_cfg = net.generate_fallback_config()
-        network_state = net.parse_net_config_data(network_cfg,
-                                                  skip_broken=False)
+        ns = network_state.parse_net_config_data(network_cfg,
+                                                 skip_broken=False)
 
         render_dir = os.path.join(tmp_dir, "render")
         os.makedirs(render_dir)
 
         renderer = eni.Renderer()
-        renderer.render_network_state(render_dir, network_state,
+        renderer.render_network_state(render_dir, ns,
                                       eni="interfaces",
                                       links_prefix=None,
                                       netrules=None)