From 880d9fc2f9c62abf19b1506595aa81e5417dea45 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Thu, 19 May 2016 14:13:07 -0700 Subject: Adjust net module to be more isolated This allows it to be used outside of cloudinit more easily in the future. --- cloudinit/net/__init__.py | 108 ++++++++++++++++++--------------- cloudinit/net/cmdline.py | 15 ++++- cloudinit/net/eni.py | 35 +++++------ cloudinit/net/network_state.py | 38 +++++++++--- cloudinit/sources/helpers/openstack.py | 4 ++ cloudinit/util.py | 9 --- tests/unittests/test_net.py | 7 ++- 7 files changed, 129 insertions(+), 87 deletions(-) diff --git a/cloudinit/net/__init__.py b/cloudinit/net/__init__.py index 0202cbd8..07e7307e 100644 --- a/cloudinit/net/__init__.py +++ b/cloudinit/net/__init__.py @@ -17,44 +17,81 @@ # along with Curtin. If not, see . import errno +import logging import os - -from cloudinit import log as logging -from cloudinit.net import network_state -from cloudinit import util - +import six +import yaml LOG = logging.getLogger(__name__) SYS_CLASS_NET = "/sys/class/net/" LINKS_FNAME_PREFIX = "etc/systemd/network/50-cloud-init-" DEFAULT_PRIMARY_INTERFACE = 'eth0' +# NOTE(harlowja): some of these are similar to what is in cloudinit main +# source or utils tree/module but the reason that is done is so that this +# whole module can be easily extracted and placed into other +# code-bases (curtin for example). -def sys_dev_path(devname, path=""): - return SYS_CLASS_NET + devname + "/" + path +def write_file(path, content): + """Simple writing a file helper.""" + base_path = os.path.dirname(path) + if not os.path.isdir(base_path): + os.makedirs(base_path) + with open(path, "wb+") as fh: + if isinstance(content, six.text_type): + content = content.encode("utf8") + fh.write(content) -def read_sys_net(devname, path, translate=None, enoent=None, keyerror=None): +def read_file(path, decode='utf8', enoent=None): try: - contents = "" - with open(sys_dev_path(devname, path), "r") as fp: - contents = fp.read().strip() - if translate is None: - return contents - - try: - return translate.get(contents) - except KeyError: - LOG.debug("found unexpected value '%s' in '%s/%s'", contents, - devname, path) - if keyerror is not None: - return keyerror - raise + with open(path, "rb") as fh: + contents = fh.load() except OSError as e: if e.errno == errno.ENOENT and enoent is not None: return enoent raise + if decode: + return contents.decode(decode) + return contents + + +def dump_yaml(obj): + return yaml.safe_dump(obj, + line_break="\n", + indent=4, + explicit_start=True, + explicit_end=True, + default_flow_style=False) + + +def read_yaml_file(path): + val = yaml.safe_load(read_file(path)) + if not isinstance(val, dict): + gotten_type_name = type(val).__name__ + raise TypeError("Expected dict to be loaded from %s, got" + " '%s' instead" % (path, gotten_type_name)) + return val + + +def sys_dev_path(devname, path=""): + return SYS_CLASS_NET + devname + "/" + path + + +def read_sys_net(devname, path, translate=None, enoent=None, keyerror=None): + contents = read_file(sys_dev_path(devname, path), enoent=enoent) + contents = contents.strip() + if translate is None: + return contents + try: + return translate.get(contents) + except KeyError: + LOG.debug("found unexpected value '%s' in '%s/%s'", contents, + devname, path) + if keyerror is not None: + return keyerror + raise def is_up(devname): @@ -107,31 +144,6 @@ class ParserError(Exception): """Raised when parser has issue parsing the interfaces file.""" -def parse_net_config_data(net_config, skip_broken=True): - """Parses the config, returns NetworkState object - - :param net_config: curtin network config dict - """ - state = None - if 'version' in net_config and 'config' in net_config: - ns = network_state.NetworkState(version=net_config.get('version'), - config=net_config.get('config')) - ns.parse_config(skip_broken=skip_broken) - state = ns.network_state - return state - - -def parse_net_config(path, skip_broken=True): - """Parses a curtin network configuration file and - return network state""" - ns = None - net_config = util.read_conf(path) - if 'network' in net_config: - ns = parse_net_config_data(net_config.get('network'), - skip_broken=skip_broken) - return ns - - def is_disabled_cfg(cfg): if not cfg or not isinstance(cfg, dict): return False @@ -146,7 +158,7 @@ def sys_netdev_info(name, field): fname = os.path.join(SYS_CLASS_NET, name, field) if not os.path.exists(fname): raise OSError("%s: could not find sysfs entry: %s" % (name, fname)) - data = util.load_file(fname) + data = read_file(fname) if data[-1] == '\n': data = data[:-1] return data diff --git a/cloudinit/net/cmdline.py b/cloudinit/net/cmdline.py index f5712533..b85d4b0a 100644 --- a/cloudinit/net/cmdline.py +++ b/cloudinit/net/cmdline.py @@ -20,12 +20,25 @@ import base64 import glob import gzip import io +import shlex +import sys + +import six from cloudinit.net import get_devicelist from cloudinit.net import sys_netdev_info from cloudinit import util +PY26 = sys.version_info[0:2] == (2, 6) + + +def _shlex_split(blob): + if PY26 and isinstance(blob, six.text_type): + # Older versions don't support unicode input + blob = blob.encode("utf8") + return shlex.split(blob) + def _load_shell_content(content, add_empty=False, empty_val=None): """Given shell like syntax (key=value\nkey2=value2\n) in content @@ -33,7 +46,7 @@ def _load_shell_content(content, add_empty=False, empty_val=None): then add entries in to the returned dictionary for 'VAR=' variables. Set their value to empty_val.""" data = {} - for line in util.shlex_split(content): + for line in _shlex_split(content): try: key, value = line.split("=", 1) except ValueError: diff --git a/cloudinit/net/eni.py b/cloudinit/net/eni.py index b1bdac24..adb31c22 100644 --- a/cloudinit/net/eni.py +++ b/cloudinit/net/eni.py @@ -16,10 +16,11 @@ import glob import os import re +from cloudinit import net + from cloudinit.net import LINKS_FNAME_PREFIX from cloudinit.net import ParserError from cloudinit.net.udev import generate_udev_rule -from cloudinit import util NET_CONFIG_COMMANDS = [ @@ -363,16 +364,13 @@ class Renderer(object): links_prefix=LINKS_FNAME_PREFIX, netrules='etc/udev/rules.d/70-persistent-net.rules'): - fpeni = os.path.sep.join((target, eni,)) - util.ensure_dir(os.path.dirname(fpeni)) - with open(fpeni, 'w+') as f: - f.write(self._render_interfaces(network_state)) + fpeni = os.path.join(target, eni) + net.write_file(fpeni, self._render_interfaces(network_state)) if netrules: - netrules = os.path.sep.join((target, netrules,)) - util.ensure_dir(os.path.dirname(netrules)) - with open(netrules, 'w+') as f: - f.write(self._render_persistent_net(network_state)) + netrules = os.path.join(target, netrules) + net.write_file(netrules, + self._render_persistent_net(network_state)) if links_prefix: self._render_systemd_links(target, network_state, links_prefix) @@ -382,18 +380,17 @@ class Renderer(object): fp_prefix = os.path.sep.join((target, links_prefix)) for f in glob.glob(fp_prefix + "*"): os.unlink(f) - interfaces = network_state.get('interfaces') for iface in interfaces.values(): if (iface['type'] == 'physical' and 'name' in iface and iface.get('mac_address')): fname = fp_prefix + iface['name'] + ".link" - with open(fname, "w") as fp: - fp.write("\n".join([ - "[Match]", - "MACAddress=" + iface['mac_address'], - "", - "[Link]", - "Name=" + iface['name'], - "" - ])) + content = "\n".join([ + "[Match]", + "MACAddress=" + iface['mac_address'], + "", + "[Link]", + "Name=" + iface['name'], + "" + ]) + net.write_file(fname, content) diff --git a/cloudinit/net/network_state.py b/cloudinit/net/network_state.py index 2feffa71..c5aeadb5 100644 --- a/cloudinit/net/network_state.py +++ b/cloudinit/net/network_state.py @@ -16,12 +16,11 @@ # along with Curtin. If not, see . import copy +import logging import six -from cloudinit import log as logging -from cloudinit import util -from cloudinit.util import yaml_dumps as dump_config +from cloudinit import net LOG = logging.getLogger(__name__) @@ -31,9 +30,34 @@ NETWORK_STATE_REQUIRED_KEYS = { } +def parse_net_config_data(net_config, skip_broken=True): + """Parses the config, returns NetworkState object + + :param net_config: curtin network config dict + """ + state = None + if 'version' in net_config and 'config' in net_config: + ns = NetworkState(version=net_config.get('version'), + config=net_config.get('config')) + ns.parse_config(skip_broken=skip_broken) + state = ns.network_state + return state + + +def parse_net_config(path, skip_broken=True): + """Parses a curtin network configuration file and + return network state""" + ns = None + net_config = net.read_yaml_file(path) + if 'network' in net_config: + ns = parse_net_config_data(net_config.get('network'), + skip_broken=skip_broken) + return ns + + def from_state_file(state_file): network_state = None - state = util.read_conf(state_file) + state = net.read_yaml_file(state_file) network_state = NetworkState() network_state.load(state) return network_state @@ -111,7 +135,7 @@ class NetworkState(object): 'config': self.config, 'network_state': self.network_state, } - return dump_config(state) + return net.dump_yaml(state) def load(self, state): if 'version' not in state: @@ -121,7 +145,7 @@ class NetworkState(object): required_keys = NETWORK_STATE_REQUIRED_KEYS[state['version']] missing_keys = diff_keys(required_keys, state) if missing_keys: - msg = 'Invalid state, missing keys: %s'.format(missing_keys) + msg = 'Invalid state, missing keys: %s' % (missing_keys) LOG.error(msg) raise ValueError(msg) @@ -130,7 +154,7 @@ class NetworkState(object): setattr(self, key, state[key]) def dump_network_state(self): - return dump_config(self.network_state) + return net.dump_yaml(self.network_state) def parse_config(self, skip_broken=True): # rebuild network state diff --git a/cloudinit/sources/helpers/openstack.py b/cloudinit/sources/helpers/openstack.py index b2acc648..f85fd864 100644 --- a/cloudinit/sources/helpers/openstack.py +++ b/cloudinit/sources/helpers/openstack.py @@ -551,6 +551,10 @@ def convert_net_json(network_json): 'type': 'static', 'address': network.get('ip_address'), }) + if network['type'] == 'ipv6': + subnet['ipv6'] = True + else: + subnet['ipv4'] = True subnets.append(subnet) cfg.update({'subnets': subnets}) if link['type'] in ['ethernet', 'vif', 'ovs', 'phy']: diff --git a/cloudinit/util.py b/cloudinit/util.py index 7562b97a..2bec476e 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -37,7 +37,6 @@ import pwd import random import re import shutil -import shlex import socket import stat import string @@ -82,7 +81,6 @@ CONTAINER_TESTS = (['systemd-detect-virt', '--quiet', '--container'], ['lxc-is-container']) PROC_CMDLINE = None -PY26 = sys.version_info[0:2] == (2, 6) def decode_binary(blob, encoding='utf-8'): @@ -1202,13 +1200,6 @@ def load_file(fname, read_cb=None, quiet=False, decode=True): return contents -def shlex_split(blob): - if PY26 and isinstance(blob, six.text_type): - # Older versions don't support unicode input - blob = blob.encode("utf8") - return shlex.split(blob) - - def get_cmdline(): if 'DEBUG_PROC_CMDLINE' in os.environ: return os.environ["DEBUG_PROC_CMDLINE"] diff --git a/tests/unittests/test_net.py b/tests/unittests/test_net.py index ed2c6d0f..75c433f6 100644 --- a/tests/unittests/test_net.py +++ b/tests/unittests/test_net.py @@ -2,6 +2,7 @@ from cloudinit import util from cloudinit import net from cloudinit.net import cmdline from cloudinit.net import eni +from cloudinit.net import network_state from .helpers import TestCase from .helpers import mock @@ -112,14 +113,14 @@ class TestEniNetRendering(TestCase): mock_sys_dev_path.side_effect = sys_dev_path network_cfg = net.generate_fallback_config() - network_state = net.parse_net_config_data(network_cfg, - skip_broken=False) + ns = network_state.parse_net_config_data(network_cfg, + skip_broken=False) render_dir = os.path.join(tmp_dir, "render") os.makedirs(render_dir) renderer = eni.Renderer() - renderer.render_network_state(render_dir, network_state, + renderer.render_network_state(render_dir, ns, eni="interfaces", links_prefix=None, netrules=None) -- cgit v1.2.3