summaryrefslogtreecommitdiff
path: root/cloudinit
diff options
context:
space:
mode:
Diffstat (limited to 'cloudinit')
-rw-r--r--cloudinit/cs_utils.py3
-rw-r--r--cloudinit/distros/debian.py10
-rw-r--r--cloudinit/net/__init__.py638
-rw-r--r--cloudinit/net/cmdline.py207
-rw-r--r--cloudinit/net/eni.py396
-rw-r--r--cloudinit/net/network_state.py284
-rw-r--r--cloudinit/serial.py50
-rw-r--r--cloudinit/sources/DataSourceAzure.py2
-rw-r--r--cloudinit/sources/DataSourceConfigDrive.py127
-rw-r--r--cloudinit/sources/DataSourceSmartOS.py4
-rw-r--r--cloudinit/sources/helpers/openstack.py125
-rw-r--r--cloudinit/stages.py3
-rw-r--r--cloudinit/util.py16
13 files changed, 990 insertions, 875 deletions
diff --git a/cloudinit/cs_utils.py b/cloudinit/cs_utils.py
index 83ac1a0e..412431f2 100644
--- a/cloudinit/cs_utils.py
+++ b/cloudinit/cs_utils.py
@@ -33,7 +33,8 @@ API Docs: http://cloudsigma-docs.readthedocs.org/en/latest/server_context.html
import json
import platform
-import serial
+from cloudinit import serial
+
# these high timeouts are necessary as read may read a lot of data.
READ_TIMEOUT = 60
diff --git a/cloudinit/distros/debian.py b/cloudinit/distros/debian.py
index 32bef1cd..e71aaa97 100644
--- a/cloudinit/distros/debian.py
+++ b/cloudinit/distros/debian.py
@@ -26,6 +26,7 @@ from cloudinit import distros
from cloudinit import helpers
from cloudinit import log as logging
from cloudinit import net
+from cloudinit.net import eni
from cloudinit import util
from cloudinit.distros.parsers.hostname import HostnameConf
@@ -56,6 +57,7 @@ class Distro(distros.Distro):
# should only happen say once per instance...)
self._runner = helpers.Runners(paths)
self.osfamily = 'debian'
+ self._net_renderer = eni.Renderer()
def apply_locale(self, locale, out_fn=None):
if not out_fn:
@@ -80,10 +82,10 @@ class Distro(distros.Distro):
def _write_network_config(self, netconfig):
ns = net.parse_net_config_data(netconfig)
- net.render_network_state(target="/", network_state=ns,
- eni=self.network_conf_fn,
- links_prefix=self.links_prefix,
- netrules=None)
+ self._net_renderer.render_network_state(
+ target="/", network_state=ns,
+ eni=self.network_conf_fn, links_prefix=self.links_prefix,
+ netrules=None)
_maybe_remove_legacy_eth0()
return []
diff --git a/cloudinit/net/__init__.py b/cloudinit/net/__init__.py
index 91e36aca..ad44911b 100644
--- a/cloudinit/net/__init__.py
+++ b/cloudinit/net/__init__.py
@@ -16,22 +16,14 @@
# You should have received a copy of the GNU Affero General Public License
# along with Curtin. If not, see <http://www.gnu.org/licenses/>.
-import base64
import errno
-import glob
-import gzip
-import io
+import logging
import os
-import re
-import shlex
-from cloudinit import log as logging
-from cloudinit.net import network_state
-from cloudinit.net.udev import generate_udev_rule
-from cloudinit import util
+import six
+import yaml
LOG = logging.getLogger(__name__)
-
SYS_CLASS_NET = "/sys/class/net/"
LINKS_FNAME_PREFIX = "etc/systemd/network/50-cloud-init-"
@@ -50,34 +42,72 @@ NET_CONFIG_BRIDGE_OPTIONS = [
"bridge_ageing", "bridge_bridgeprio", "bridge_fd", "bridge_gcinit",
"bridge_hello", "bridge_maxage", "bridge_maxwait", "bridge_stp",
]
-
DEFAULT_PRIMARY_INTERFACE = 'eth0'
+# NOTE(harlowja): some of these are similar to what is in cloudinit main
+# source or utils tree/module but the reason that is done is so that this
+# whole module can be easily extracted and placed into other
+# code-bases (curtin for example).
-def sys_dev_path(devname, path=""):
- return SYS_CLASS_NET + devname + "/" + path
+def write_file(path, content):
+ """Simple writing a file helper."""
+ base_path = os.path.dirname(path)
+ if not os.path.isdir(base_path):
+ os.makedirs(base_path)
+ with open(path, "wb+") as fh:
+ if isinstance(content, six.text_type):
+ content = content.encode("utf8")
+ fh.write(content)
-def read_sys_net(devname, path, translate=None, enoent=None, keyerror=None):
+def read_file(path, decode='utf8', enoent=None):
try:
- contents = ""
- with open(sys_dev_path(devname, path), "r") as fp:
- contents = fp.read().strip()
- if translate is None:
- return contents
-
- try:
- return translate.get(contents)
- except KeyError:
- LOG.debug("found unexpected value '%s' in '%s/%s'", contents,
- devname, path)
- if keyerror is not None:
- return keyerror
- raise
+ with open(path, "rb") as fh:
+ contents = fh.load()
except OSError as e:
if e.errno == errno.ENOENT and enoent is not None:
return enoent
raise
+ if decode:
+ return contents.decode(decode)
+ return contents
+
+
+def dump_yaml(obj):
+ return yaml.safe_dump(obj,
+ line_break="\n",
+ indent=4,
+ explicit_start=True,
+ explicit_end=True,
+ default_flow_style=False)
+
+
+def read_yaml_file(path):
+ val = yaml.safe_load(read_file(path))
+ if not isinstance(val, dict):
+ gotten_type_name = type(val).__name__
+ raise TypeError("Expected dict to be loaded from %s, got"
+ " '%s' instead" % (path, gotten_type_name))
+ return val
+
+
+def sys_dev_path(devname, path=""):
+ return SYS_CLASS_NET + devname + "/" + path
+
+
+def read_sys_net(devname, path, translate=None, enoent=None, keyerror=None):
+ contents = read_file(sys_dev_path(devname, path), enoent=enoent)
+ contents = contents.strip()
+ if translate is None:
+ return contents
+ try:
+ return translate.get(contents)
+ except KeyError:
+ LOG.debug("found unexpected value '%s' in '%s/%s'", contents,
+ devname, path)
+ if keyerror is not None:
+ return keyerror
+ raise
def is_up(devname):
@@ -130,502 +160,6 @@ class ParserError(Exception):
"""Raised when parser has issue parsing the interfaces file."""
-def parse_deb_config_data(ifaces, contents, src_dir, src_path):
- """Parses the file contents, placing result into ifaces.
-
- '_source_path' is added to every dictionary entry to define which file
- the configration information came from.
-
- :param ifaces: interface dictionary
- :param contents: contents of interfaces file
- :param src_dir: directory interfaces file was located
- :param src_path: file path the `contents` was read
- """
- currif = None
- for line in contents.splitlines():
- line = line.strip()
- if line.startswith('#'):
- continue
- split = line.split(' ')
- option = split[0]
- if option == "source-directory":
- parsed_src_dir = split[1]
- if not parsed_src_dir.startswith("/"):
- parsed_src_dir = os.path.join(src_dir, parsed_src_dir)
- for expanded_path in glob.glob(parsed_src_dir):
- dir_contents = os.listdir(expanded_path)
- dir_contents = [
- os.path.join(expanded_path, path)
- for path in dir_contents
- if (os.path.isfile(os.path.join(expanded_path, path)) and
- re.match("^[a-zA-Z0-9_-]+$", path) is not None)
- ]
- for entry in dir_contents:
- with open(entry, "r") as fp:
- src_data = fp.read().strip()
- abs_entry = os.path.abspath(entry)
- parse_deb_config_data(
- ifaces, src_data,
- os.path.dirname(abs_entry), abs_entry)
- elif option == "source":
- new_src_path = split[1]
- if not new_src_path.startswith("/"):
- new_src_path = os.path.join(src_dir, new_src_path)
- for expanded_path in glob.glob(new_src_path):
- with open(expanded_path, "r") as fp:
- src_data = fp.read().strip()
- abs_path = os.path.abspath(expanded_path)
- parse_deb_config_data(
- ifaces, src_data,
- os.path.dirname(abs_path), abs_path)
- elif option == "auto":
- for iface in split[1:]:
- if iface not in ifaces:
- ifaces[iface] = {
- # Include the source path this interface was found in.
- "_source_path": src_path
- }
- ifaces[iface]['auto'] = True
- elif option == "iface":
- iface, family, method = split[1:4]
- if iface not in ifaces:
- ifaces[iface] = {
- # Include the source path this interface was found in.
- "_source_path": src_path
- }
- elif 'family' in ifaces[iface]:
- raise ParserError(
- "Interface %s can only be defined once. "
- "Re-defined in '%s'." % (iface, src_path))
- ifaces[iface]['family'] = family
- ifaces[iface]['method'] = method
- currif = iface
- elif option == "hwaddress":
- ifaces[currif]['hwaddress'] = split[1]
- elif option in NET_CONFIG_OPTIONS:
- ifaces[currif][option] = split[1]
- elif option in NET_CONFIG_COMMANDS:
- if option not in ifaces[currif]:
- ifaces[currif][option] = []
- ifaces[currif][option].append(' '.join(split[1:]))
- elif option.startswith('dns-'):
- if 'dns' not in ifaces[currif]:
- ifaces[currif]['dns'] = {}
- if option == 'dns-search':
- ifaces[currif]['dns']['search'] = []
- for domain in split[1:]:
- ifaces[currif]['dns']['search'].append(domain)
- elif option == 'dns-nameservers':
- ifaces[currif]['dns']['nameservers'] = []
- for server in split[1:]:
- ifaces[currif]['dns']['nameservers'].append(server)
- elif option.startswith('bridge_'):
- if 'bridge' not in ifaces[currif]:
- ifaces[currif]['bridge'] = {}
- if option in NET_CONFIG_BRIDGE_OPTIONS:
- bridge_option = option.replace('bridge_', '', 1)
- ifaces[currif]['bridge'][bridge_option] = split[1]
- elif option == "bridge_ports":
- ifaces[currif]['bridge']['ports'] = []
- for iface in split[1:]:
- ifaces[currif]['bridge']['ports'].append(iface)
- elif option == "bridge_hw" and split[1].lower() == "mac":
- ifaces[currif]['bridge']['mac'] = split[2]
- elif option == "bridge_pathcost":
- if 'pathcost' not in ifaces[currif]['bridge']:
- ifaces[currif]['bridge']['pathcost'] = {}
- ifaces[currif]['bridge']['pathcost'][split[1]] = split[2]
- elif option == "bridge_portprio":
- if 'portprio' not in ifaces[currif]['bridge']:
- ifaces[currif]['bridge']['portprio'] = {}
- ifaces[currif]['bridge']['portprio'][split[1]] = split[2]
- elif option.startswith('bond-'):
- if 'bond' not in ifaces[currif]:
- ifaces[currif]['bond'] = {}
- bond_option = option.replace('bond-', '', 1)
- ifaces[currif]['bond'][bond_option] = split[1]
- for iface in ifaces.keys():
- if 'auto' not in ifaces[iface]:
- ifaces[iface]['auto'] = False
-
-
-def parse_deb_config(path):
- """Parses a debian network configuration file."""
- ifaces = {}
- with open(path, "r") as fp:
- contents = fp.read().strip()
- abs_path = os.path.abspath(path)
- parse_deb_config_data(
- ifaces, contents,
- os.path.dirname(abs_path), abs_path)
- return ifaces
-
-
-def parse_net_config_data(net_config):
- """Parses the config, returns NetworkState dictionary
-
- :param net_config: curtin network config dict
- """
- state = None
- if 'version' in net_config and 'config' in net_config:
- ns = network_state.NetworkState(version=net_config.get('version'),
- config=net_config.get('config'))
- ns.parse_config()
- state = ns.network_state
-
- return state
-
-
-def parse_net_config(path):
- """Parses a curtin network configuration file and
- return network state"""
- ns = None
- net_config = util.read_conf(path)
- if 'network' in net_config:
- ns = parse_net_config_data(net_config.get('network'))
-
- return ns
-
-
-def _load_shell_content(content, add_empty=False, empty_val=None):
- """Given shell like syntax (key=value\nkey2=value2\n) in content
- return the data in dictionary form. If 'add_empty' is True
- then add entries in to the returned dictionary for 'VAR='
- variables. Set their value to empty_val."""
- data = {}
- for line in shlex.split(content):
- key, value = line.split("=", 1)
- if not value:
- value = empty_val
- if add_empty or value:
- data[key] = value
-
- return data
-
-
-def _klibc_to_config_entry(content, mac_addrs=None):
- """Convert a klibc writtent shell content file to a 'config' entry
- When ip= is seen on the kernel command line in debian initramfs
- and networking is brought up, ipconfig will populate
- /run/net-<name>.cfg.
-
- The files are shell style syntax, and examples are in the tests
- provided here. There is no good documentation on this unfortunately.
-
- DEVICE=<name> is expected/required and PROTO should indicate if
- this is 'static' or 'dhcp'.
- """
-
- if mac_addrs is None:
- mac_addrs = {}
-
- data = _load_shell_content(content)
- try:
- name = data['DEVICE']
- except KeyError:
- raise ValueError("no 'DEVICE' entry in data")
-
- # ipconfig on precise does not write PROTO
- proto = data.get('PROTO')
- if not proto:
- if data.get('filename'):
- proto = 'dhcp'
- else:
- proto = 'static'
-
- if proto not in ('static', 'dhcp'):
- raise ValueError("Unexpected value for PROTO: %s" % proto)
-
- iface = {
- 'type': 'physical',
- 'name': name,
- 'subnets': [],
- }
-
- if name in mac_addrs:
- iface['mac_address'] = mac_addrs[name]
-
- # originally believed there might be IPV6* values
- for v, pre in (('ipv4', 'IPV4'),):
- # if no IPV4ADDR or IPV6ADDR, then go on.
- if pre + "ADDR" not in data:
- continue
- subnet = {'type': proto, 'control': 'manual'}
-
- # these fields go right on the subnet
- for key in ('NETMASK', 'BROADCAST', 'GATEWAY'):
- if pre + key in data:
- subnet[key.lower()] = data[pre + key]
-
- dns = []
- # handle IPV4DNS0 or IPV6DNS0
- for nskey in ('DNS0', 'DNS1'):
- ns = data.get(pre + nskey)
- # verify it has something other than 0.0.0.0 (or ipv6)
- if ns and len(ns.strip(":.0")):
- dns.append(data[pre + nskey])
- if dns:
- subnet['dns_nameservers'] = dns
- # add search to both ipv4 and ipv6, as it has no namespace
- search = data.get('DOMAINSEARCH')
- if search:
- if ',' in search:
- subnet['dns_search'] = search.split(",")
- else:
- subnet['dns_search'] = search.split()
-
- iface['subnets'].append(subnet)
-
- return name, iface
-
-
-def config_from_klibc_net_cfg(files=None, mac_addrs=None):
- if files is None:
- files = glob.glob('/run/net*.conf')
-
- entries = []
- names = {}
- for cfg_file in files:
- name, entry = _klibc_to_config_entry(util.load_file(cfg_file),
- mac_addrs=mac_addrs)
- if name in names:
- raise ValueError(
- "device '%s' defined multiple times: %s and %s" % (
- name, names[name], cfg_file))
-
- names[name] = cfg_file
- entries.append(entry)
- return {'config': entries, 'version': 1}
-
-
-def render_persistent_net(network_state):
- '''Given state, emit udev rules to map mac to ifname.'''
- content = ""
- interfaces = network_state.get('interfaces')
- for iface in interfaces.values():
- # for physical interfaces write out a persist net udev rule
- if iface['type'] == 'physical' and \
- 'name' in iface and iface.get('mac_address'):
- content += generate_udev_rule(iface['name'],
- iface['mac_address'])
-
- return content
-
-
-# TODO: switch valid_map based on mode inet/inet6
-def iface_add_subnet(iface, subnet):
- content = ""
- valid_map = [
- 'address',
- 'netmask',
- 'broadcast',
- 'metric',
- 'gateway',
- 'pointopoint',
- 'mtu',
- 'scope',
- 'dns_search',
- 'dns_nameservers',
- ]
- for key, value in subnet.items():
- if value and key in valid_map:
- if type(value) == list:
- value = " ".join(value)
- if '_' in key:
- key = key.replace('_', '-')
- content += " {} {}\n".format(key, value)
-
- return content
-
-
-# TODO: switch to valid_map for attrs
-def iface_add_attrs(iface):
- content = ""
- ignore_map = [
- 'control',
- 'index',
- 'inet',
- 'mode',
- 'name',
- 'subnets',
- 'type',
- ]
- if iface['type'] not in ['bond', 'bridge', 'vlan']:
- ignore_map.append('mac_address')
-
- for key, value in iface.items():
- if value and key not in ignore_map:
- if type(value) == list:
- value = " ".join(value)
- content += " {} {}\n".format(key, value)
-
- return content
-
-
-def render_route(route, indent=""):
- """When rendering routes for an iface, in some cases applying a route
- may result in the route command returning non-zero which produces
- some confusing output for users manually using ifup/ifdown[1]. To
- that end, we will optionally include an '|| true' postfix to each
- route line allowing users to work with ifup/ifdown without using
- --force option.
-
- We may at somepoint not want to emit this additional postfix, and
- add a 'strict' flag to this function. When called with strict=True,
- then we will not append the postfix.
-
- 1. http://askubuntu.com/questions/168033/
- how-to-set-static-routes-in-ubuntu-server
- """
- content = ""
- up = indent + "post-up route add"
- down = indent + "pre-down route del"
- eol = " || true\n"
- mapping = {
- 'network': '-net',
- 'netmask': 'netmask',
- 'gateway': 'gw',
- 'metric': 'metric',
- }
- if route['network'] == '0.0.0.0' and route['netmask'] == '0.0.0.0':
- default_gw = " default gw %s" % route['gateway']
- content += up + default_gw + eol
- content += down + default_gw + eol
- elif route['network'] == '::' and route['netmask'] == 0:
- # ipv6!
- default_gw = " -A inet6 default gw %s" % route['gateway']
- content += up + default_gw + eol
- content += down + default_gw + eol
- else:
- route_line = ""
- for k in ['network', 'netmask', 'gateway', 'metric']:
- if k in route:
- route_line += " %s %s" % (mapping[k], route[k])
- content += up + route_line + eol
- content += down + route_line + eol
-
- return content
-
-
-def iface_start_entry(iface, index):
- fullname = iface['name']
- if index != 0:
- fullname += ":%s" % index
-
- control = iface['control']
- if control == "auto":
- cverb = "auto"
- elif control in ("hotplug",):
- cverb = "allow-" + control
- else:
- cverb = "# control-" + control
-
- subst = iface.copy()
- subst.update({'fullname': fullname, 'cverb': cverb})
-
- return ("{cverb} {fullname}\n"
- "iface {fullname} {inet} {mode}\n").format(**subst)
-
-
-def render_interfaces(network_state):
- '''Given state, emit etc/network/interfaces content.'''
-
- content = ""
- interfaces = network_state.get('interfaces')
- ''' Apply a sort order to ensure that we write out
- the physical interfaces first; this is critical for
- bonding
- '''
- order = {
- 'physical': 0,
- 'bond': 1,
- 'bridge': 2,
- 'vlan': 3,
- }
- content += "auto lo\niface lo inet loopback\n"
- for dnskey, value in network_state.get('dns', {}).items():
- if len(value):
- content += " dns-{} {}\n".format(dnskey, " ".join(value))
-
- for iface in sorted(interfaces.values(),
- key=lambda k: (order[k['type']], k['name'])):
-
- if content[-2:] != "\n\n":
- content += "\n"
- subnets = iface.get('subnets', {})
- if subnets:
- for index, subnet in zip(range(0, len(subnets)), subnets):
- if content[-2:] != "\n\n":
- content += "\n"
- iface['index'] = index
- iface['mode'] = subnet['type']
- iface['control'] = subnet.get('control', 'auto')
- if iface['mode'].endswith('6'):
- iface['inet'] += '6'
- elif iface['mode'] == 'static' and ":" in subnet['address']:
- iface['inet'] += '6'
- if iface['mode'].startswith('dhcp'):
- iface['mode'] = 'dhcp'
-
- content += iface_start_entry(iface, index)
- content += iface_add_subnet(iface, subnet)
- content += iface_add_attrs(iface)
- else:
- # ifenslave docs say to auto the slave devices
- if 'bond-master' in iface:
- content += "auto {name}\n".format(**iface)
- content += "iface {name} {inet} {mode}\n".format(**iface)
- content += iface_add_attrs(iface)
-
- for route in network_state.get('routes'):
- content += render_route(route)
-
- # global replacements until v2 format
- content = content.replace('mac_address', 'hwaddress')
- return content
-
-
-def render_network_state(target, network_state, eni="etc/network/interfaces",
- links_prefix=LINKS_FNAME_PREFIX,
- netrules='etc/udev/rules.d/70-persistent-net.rules'):
-
- fpeni = os.path.sep.join((target, eni,))
- util.ensure_dir(os.path.dirname(fpeni))
- with open(fpeni, 'w+') as f:
- f.write(render_interfaces(network_state))
-
- if netrules:
- netrules = os.path.sep.join((target, netrules,))
- util.ensure_dir(os.path.dirname(netrules))
- with open(netrules, 'w+') as f:
- f.write(render_persistent_net(network_state))
-
- if links_prefix:
- render_systemd_links(target, network_state, links_prefix)
-
-
-def render_systemd_links(target, network_state,
- links_prefix=LINKS_FNAME_PREFIX):
- fp_prefix = os.path.sep.join((target, links_prefix))
- for f in glob.glob(fp_prefix + "*"):
- os.unlink(f)
-
- interfaces = network_state.get('interfaces')
- for iface in interfaces.values():
- if (iface['type'] == 'physical' and 'name' in iface and
- iface.get('mac_address')):
- fname = fp_prefix + iface['name'] + ".link"
- with open(fname, "w") as fp:
- fp.write("\n".join([
- "[Match]",
- "MACAddress=" + iface['mac_address'],
- "",
- "[Link]",
- "Name=" + iface['name'],
- ""
- ]))
-
-
def is_disabled_cfg(cfg):
if not cfg or not isinstance(cfg, dict):
return False
@@ -640,7 +174,7 @@ def sys_netdev_info(name, field):
fname = os.path.join(SYS_CLASS_NET, name, field)
if not os.path.exists(fname):
raise OSError("%s: could not find sysfs entry: %s" % (name, fname))
- data = util.load_file(fname)
+ data = read_file(fname)
if data[-1] == '\n':
data = data[:-1]
return data
@@ -716,56 +250,4 @@ def generate_fallback_config():
return nconf
-def _decomp_gzip(blob, strict=True):
- # decompress blob. raise exception if not compressed unless strict=False.
- with io.BytesIO(blob) as iobuf:
- gzfp = None
- try:
- gzfp = gzip.GzipFile(mode="rb", fileobj=iobuf)
- return gzfp.read()
- except IOError:
- if strict:
- raise
- return blob
- finally:
- if gzfp:
- gzfp.close()
-
-
-def _b64dgz(b64str, gzipped="try"):
- # decode a base64 string. If gzipped is true, transparently uncompresss
- # if gzipped is 'try', then try gunzip, returning the original on fail.
- try:
- blob = base64.b64decode(b64str)
- except TypeError:
- raise ValueError("Invalid base64 text: %s" % b64str)
-
- if not gzipped:
- return blob
-
- return _decomp_gzip(blob, strict=gzipped != "try")
-
-
-def read_kernel_cmdline_config(files=None, mac_addrs=None, cmdline=None):
- if cmdline is None:
- cmdline = util.get_cmdline()
-
- if 'network-config=' in cmdline:
- data64 = None
- for tok in cmdline.split():
- if tok.startswith("network-config="):
- data64 = tok.split("=", 1)[1]
- if data64:
- return util.load_yaml(_b64dgz(data64))
-
- if 'ip=' not in cmdline:
- return None
-
- if mac_addrs is None:
- mac_addrs = {k: sys_netdev_info(k, 'address')
- for k in get_devicelist()}
-
- return config_from_klibc_net_cfg(files=files, mac_addrs=mac_addrs)
-
-
# vi: ts=4 expandtab syntax=python
diff --git a/cloudinit/net/cmdline.py b/cloudinit/net/cmdline.py
new file mode 100644
index 00000000..b85d4b0a
--- /dev/null
+++ b/cloudinit/net/cmdline.py
@@ -0,0 +1,207 @@
+# Copyright (C) 2013-2014 Canonical Ltd.
+#
+# Author: Scott Moser <scott.moser@canonical.com>
+# Author: Blake Rouse <blake.rouse@canonical.com>
+#
+# Curtin is free software: you can redistribute it and/or modify it under
+# the terms of the GNU Affero General Public License as published by the
+# Free Software Foundation, either version 3 of the License, or (at your
+# option) any later version.
+#
+# Curtin is distributed in the hope that it will be useful, but WITHOUT ANY
+# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for
+# more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with Curtin. If not, see <http://www.gnu.org/licenses/>.
+
+import base64
+import glob
+import gzip
+import io
+import shlex
+import sys
+
+import six
+
+from cloudinit.net import get_devicelist
+from cloudinit.net import sys_netdev_info
+
+from cloudinit import util
+
+PY26 = sys.version_info[0:2] == (2, 6)
+
+
+def _shlex_split(blob):
+ if PY26 and isinstance(blob, six.text_type):
+ # Older versions don't support unicode input
+ blob = blob.encode("utf8")
+ return shlex.split(blob)
+
+
+def _load_shell_content(content, add_empty=False, empty_val=None):
+ """Given shell like syntax (key=value\nkey2=value2\n) in content
+ return the data in dictionary form. If 'add_empty' is True
+ then add entries in to the returned dictionary for 'VAR='
+ variables. Set their value to empty_val."""
+ data = {}
+ for line in _shlex_split(content):
+ try:
+ key, value = line.split("=", 1)
+ except ValueError:
+ # Unsplittable line, skip it...
+ pass
+ else:
+ if not value:
+ value = empty_val
+ if add_empty or value:
+ data[key] = value
+ return data
+
+
+def _klibc_to_config_entry(content, mac_addrs=None):
+ """Convert a klibc writtent shell content file to a 'config' entry
+ When ip= is seen on the kernel command line in debian initramfs
+ and networking is brought up, ipconfig will populate
+ /run/net-<name>.cfg.
+
+ The files are shell style syntax, and examples are in the tests
+ provided here. There is no good documentation on this unfortunately.
+
+ DEVICE=<name> is expected/required and PROTO should indicate if
+ this is 'static' or 'dhcp'.
+ """
+
+ if mac_addrs is None:
+ mac_addrs = {}
+
+ data = _load_shell_content(content)
+ try:
+ name = data['DEVICE']
+ except KeyError:
+ raise ValueError("no 'DEVICE' entry in data")
+
+ # ipconfig on precise does not write PROTO
+ proto = data.get('PROTO')
+ if not proto:
+ if data.get('filename'):
+ proto = 'dhcp'
+ else:
+ proto = 'static'
+
+ if proto not in ('static', 'dhcp'):
+ raise ValueError("Unexpected value for PROTO: %s" % proto)
+
+ iface = {
+ 'type': 'physical',
+ 'name': name,
+ 'subnets': [],
+ }
+
+ if name in mac_addrs:
+ iface['mac_address'] = mac_addrs[name]
+
+ # originally believed there might be IPV6* values
+ for v, pre in (('ipv4', 'IPV4'),):
+ # if no IPV4ADDR or IPV6ADDR, then go on.
+ if pre + "ADDR" not in data:
+ continue
+ subnet = {'type': proto, 'control': 'manual'}
+
+ # these fields go right on the subnet
+ for key in ('NETMASK', 'BROADCAST', 'GATEWAY'):
+ if pre + key in data:
+ subnet[key.lower()] = data[pre + key]
+
+ dns = []
+ # handle IPV4DNS0 or IPV6DNS0
+ for nskey in ('DNS0', 'DNS1'):
+ ns = data.get(pre + nskey)
+ # verify it has something other than 0.0.0.0 (or ipv6)
+ if ns and len(ns.strip(":.0")):
+ dns.append(data[pre + nskey])
+ if dns:
+ subnet['dns_nameservers'] = dns
+ # add search to both ipv4 and ipv6, as it has no namespace
+ search = data.get('DOMAINSEARCH')
+ if search:
+ if ',' in search:
+ subnet['dns_search'] = search.split(",")
+ else:
+ subnet['dns_search'] = search.split()
+
+ iface['subnets'].append(subnet)
+
+ return name, iface
+
+
+def config_from_klibc_net_cfg(files=None, mac_addrs=None):
+ if files is None:
+ files = glob.glob('/run/net*.conf')
+
+ entries = []
+ names = {}
+ for cfg_file in files:
+ name, entry = _klibc_to_config_entry(util.load_file(cfg_file),
+ mac_addrs=mac_addrs)
+ if name in names:
+ raise ValueError(
+ "device '%s' defined multiple times: %s and %s" % (
+ name, names[name], cfg_file))
+
+ names[name] = cfg_file
+ entries.append(entry)
+ return {'config': entries, 'version': 1}
+
+
+def _decomp_gzip(blob, strict=True):
+ # decompress blob. raise exception if not compressed unless strict=False.
+ with io.BytesIO(blob) as iobuf:
+ gzfp = None
+ try:
+ gzfp = gzip.GzipFile(mode="rb", fileobj=iobuf)
+ return gzfp.read()
+ except IOError:
+ if strict:
+ raise
+ return blob
+ finally:
+ if gzfp:
+ gzfp.close()
+
+
+def _b64dgz(b64str, gzipped="try"):
+ # decode a base64 string. If gzipped is true, transparently uncompresss
+ # if gzipped is 'try', then try gunzip, returning the original on fail.
+ try:
+ blob = base64.b64decode(b64str)
+ except TypeError:
+ raise ValueError("Invalid base64 text: %s" % b64str)
+
+ if not gzipped:
+ return blob
+
+ return _decomp_gzip(blob, strict=gzipped != "try")
+
+
+def read_kernel_cmdline_config(files=None, mac_addrs=None, cmdline=None):
+ if cmdline is None:
+ cmdline = util.get_cmdline()
+
+ if 'network-config=' in cmdline:
+ data64 = None
+ for tok in cmdline.split():
+ if tok.startswith("network-config="):
+ data64 = tok.split("=", 1)[1]
+ if data64:
+ return util.load_yaml(_b64dgz(data64))
+
+ if 'ip=' not in cmdline:
+ return None
+
+ if mac_addrs is None:
+ mac_addrs = dict((k, sys_netdev_info(k, 'address'))
+ for k in get_devicelist())
+
+ return config_from_klibc_net_cfg(files=files, mac_addrs=mac_addrs)
diff --git a/cloudinit/net/eni.py b/cloudinit/net/eni.py
new file mode 100644
index 00000000..adb31c22
--- /dev/null
+++ b/cloudinit/net/eni.py
@@ -0,0 +1,396 @@
+# vi: ts=4 expandtab
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 3, as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+import glob
+import os
+import re
+
+from cloudinit import net
+
+from cloudinit.net import LINKS_FNAME_PREFIX
+from cloudinit.net import ParserError
+from cloudinit.net.udev import generate_udev_rule
+
+
+NET_CONFIG_COMMANDS = [
+ "pre-up", "up", "post-up", "down", "pre-down", "post-down",
+]
+
+NET_CONFIG_BRIDGE_OPTIONS = [
+ "bridge_ageing", "bridge_bridgeprio", "bridge_fd", "bridge_gcinit",
+ "bridge_hello", "bridge_maxage", "bridge_maxwait", "bridge_stp",
+]
+
+NET_CONFIG_OPTIONS = [
+ "address", "netmask", "broadcast", "network", "metric", "gateway",
+ "pointtopoint", "media", "mtu", "hostname", "leasehours", "leasetime",
+ "vendor", "client", "bootfile", "server", "hwaddr", "provider", "frame",
+ "netnum", "endpoint", "local", "ttl",
+]
+
+
+# TODO: switch valid_map based on mode inet/inet6
+def _iface_add_subnet(iface, subnet):
+ content = ""
+ valid_map = [
+ 'address',
+ 'netmask',
+ 'broadcast',
+ 'metric',
+ 'gateway',
+ 'pointopoint',
+ 'mtu',
+ 'scope',
+ 'dns_search',
+ 'dns_nameservers',
+ ]
+ for key, value in subnet.items():
+ if value and key in valid_map:
+ if type(value) == list:
+ value = " ".join(value)
+ if '_' in key:
+ key = key.replace('_', '-')
+ content += " {} {}\n".format(key, value)
+
+ return content
+
+
+# TODO: switch to valid_map for attrs
+def _iface_add_attrs(iface):
+ content = ""
+ ignore_map = [
+ 'control',
+ 'index',
+ 'inet',
+ 'mode',
+ 'name',
+ 'subnets',
+ 'type',
+ ]
+ if iface['type'] not in ['bond', 'bridge', 'vlan']:
+ ignore_map.append('mac_address')
+
+ for key, value in iface.items():
+ if value and key not in ignore_map:
+ if type(value) == list:
+ value = " ".join(value)
+ content += " {} {}\n".format(key, value)
+
+ return content
+
+
+def _iface_start_entry(iface, index):
+ fullname = iface['name']
+ if index != 0:
+ fullname += ":%s" % index
+
+ control = iface['control']
+ if control == "auto":
+ cverb = "auto"
+ elif control in ("hotplug",):
+ cverb = "allow-" + control
+ else:
+ cverb = "# control-" + control
+
+ subst = iface.copy()
+ subst.update({'fullname': fullname, 'cverb': cverb})
+
+ return ("{cverb} {fullname}\n"
+ "iface {fullname} {inet} {mode}\n").format(**subst)
+
+
+def _parse_deb_config_data(ifaces, contents, src_dir, src_path):
+ """Parses the file contents, placing result into ifaces.
+
+ '_source_path' is added to every dictionary entry to define which file
+ the configration information came from.
+
+ :param ifaces: interface dictionary
+ :param contents: contents of interfaces file
+ :param src_dir: directory interfaces file was located
+ :param src_path: file path the `contents` was read
+ """
+ currif = None
+ for line in contents.splitlines():
+ line = line.strip()
+ if line.startswith('#'):
+ continue
+ split = line.split(' ')
+ option = split[0]
+ if option == "source-directory":
+ parsed_src_dir = split[1]
+ if not parsed_src_dir.startswith("/"):
+ parsed_src_dir = os.path.join(src_dir, parsed_src_dir)
+ for expanded_path in glob.glob(parsed_src_dir):
+ dir_contents = os.listdir(expanded_path)
+ dir_contents = [
+ os.path.join(expanded_path, path)
+ for path in dir_contents
+ if (os.path.isfile(os.path.join(expanded_path, path)) and
+ re.match("^[a-zA-Z0-9_-]+$", path) is not None)
+ ]
+ for entry in dir_contents:
+ with open(entry, "r") as fp:
+ src_data = fp.read().strip()
+ abs_entry = os.path.abspath(entry)
+ _parse_deb_config_data(
+ ifaces, src_data,
+ os.path.dirname(abs_entry), abs_entry)
+ elif option == "source":
+ new_src_path = split[1]
+ if not new_src_path.startswith("/"):
+ new_src_path = os.path.join(src_dir, new_src_path)
+ for expanded_path in glob.glob(new_src_path):
+ with open(expanded_path, "r") as fp:
+ src_data = fp.read().strip()
+ abs_path = os.path.abspath(expanded_path)
+ _parse_deb_config_data(
+ ifaces, src_data,
+ os.path.dirname(abs_path), abs_path)
+ elif option == "auto":
+ for iface in split[1:]:
+ if iface not in ifaces:
+ ifaces[iface] = {
+ # Include the source path this interface was found in.
+ "_source_path": src_path
+ }
+ ifaces[iface]['auto'] = True
+ elif option == "iface":
+ iface, family, method = split[1:4]
+ if iface not in ifaces:
+ ifaces[iface] = {
+ # Include the source path this interface was found in.
+ "_source_path": src_path
+ }
+ elif 'family' in ifaces[iface]:
+ raise ParserError(
+ "Interface %s can only be defined once. "
+ "Re-defined in '%s'." % (iface, src_path))
+ ifaces[iface]['family'] = family
+ ifaces[iface]['method'] = method
+ currif = iface
+ elif option == "hwaddress":
+ ifaces[currif]['hwaddress'] = split[1]
+ elif option in NET_CONFIG_OPTIONS:
+ ifaces[currif][option] = split[1]
+ elif option in NET_CONFIG_COMMANDS:
+ if option not in ifaces[currif]:
+ ifaces[currif][option] = []
+ ifaces[currif][option].append(' '.join(split[1:]))
+ elif option.startswith('dns-'):
+ if 'dns' not in ifaces[currif]:
+ ifaces[currif]['dns'] = {}
+ if option == 'dns-search':
+ ifaces[currif]['dns']['search'] = []
+ for domain in split[1:]:
+ ifaces[currif]['dns']['search'].append(domain)
+ elif option == 'dns-nameservers':
+ ifaces[currif]['dns']['nameservers'] = []
+ for server in split[1:]:
+ ifaces[currif]['dns']['nameservers'].append(server)
+ elif option.startswith('bridge_'):
+ if 'bridge' not in ifaces[currif]:
+ ifaces[currif]['bridge'] = {}
+ if option in NET_CONFIG_BRIDGE_OPTIONS:
+ bridge_option = option.replace('bridge_', '', 1)
+ ifaces[currif]['bridge'][bridge_option] = split[1]
+ elif option == "bridge_ports":
+ ifaces[currif]['bridge']['ports'] = []
+ for iface in split[1:]:
+ ifaces[currif]['bridge']['ports'].append(iface)
+ elif option == "bridge_hw" and split[1].lower() == "mac":
+ ifaces[currif]['bridge']['mac'] = split[2]
+ elif option == "bridge_pathcost":
+ if 'pathcost' not in ifaces[currif]['bridge']:
+ ifaces[currif]['bridge']['pathcost'] = {}
+ ifaces[currif]['bridge']['pathcost'][split[1]] = split[2]
+ elif option == "bridge_portprio":
+ if 'portprio' not in ifaces[currif]['bridge']:
+ ifaces[currif]['bridge']['portprio'] = {}
+ ifaces[currif]['bridge']['portprio'][split[1]] = split[2]
+ elif option.startswith('bond-'):
+ if 'bond' not in ifaces[currif]:
+ ifaces[currif]['bond'] = {}
+ bond_option = option.replace('bond-', '', 1)
+ ifaces[currif]['bond'][bond_option] = split[1]
+ for iface in ifaces.keys():
+ if 'auto' not in ifaces[iface]:
+ ifaces[iface]['auto'] = False
+
+
+def _parse_deb_config(path):
+ """Parses a debian network configuration file."""
+ ifaces = {}
+ with open(path, "r") as fp:
+ contents = fp.read().strip()
+ abs_path = os.path.abspath(path)
+ _parse_deb_config_data(
+ ifaces, contents,
+ os.path.dirname(abs_path), abs_path)
+ return ifaces
+
+
+class Renderer(object):
+ """Renders network information in a /etc/network/interfaces format."""
+
+ def _render_persistent_net(self, network_state):
+ """Given state, emit udev rules to map mac to ifname."""
+ content = ""
+ interfaces = network_state.get('interfaces')
+ for iface in interfaces.values():
+ # for physical interfaces write out a persist net udev rule
+ if iface['type'] == 'physical' and \
+ 'name' in iface and iface.get('mac_address'):
+ content += generate_udev_rule(iface['name'],
+ iface['mac_address'])
+
+ return content
+
+ def _render_route(self, route, indent=""):
+ """ When rendering routes for an iface, in some cases applying a route
+ may result in the route command returning non-zero which produces
+ some confusing output for users manually using ifup/ifdown[1]. To
+ that end, we will optionally include an '|| true' postfix to each
+ route line allowing users to work with ifup/ifdown without using
+ --force option.
+
+ We may at somepoint not want to emit this additional postfix, and
+ add a 'strict' flag to this function. When called with strict=True,
+ then we will not append the postfix.
+
+ 1. http://askubuntu.com/questions/168033/
+ how-to-set-static-routes-in-ubuntu-server
+ """
+ content = ""
+ up = indent + "post-up route add"
+ down = indent + "pre-down route del"
+ eol = " || true\n"
+ mapping = {
+ 'network': '-net',
+ 'netmask': 'netmask',
+ 'gateway': 'gw',
+ 'metric': 'metric',
+ }
+ if route['network'] == '0.0.0.0' and route['netmask'] == '0.0.0.0':
+ default_gw = " default gw %s" % route['gateway']
+ content += up + default_gw + eol
+ content += down + default_gw + eol
+ elif route['network'] == '::' and route['netmask'] == 0:
+ # ipv6!
+ default_gw = " -A inet6 default gw %s" % route['gateway']
+ content += up + default_gw + eol
+ content += down + default_gw + eol
+ else:
+ route_line = ""
+ for k in ['network', 'netmask', 'gateway', 'metric']:
+ if k in route:
+ route_line += " %s %s" % (mapping[k], route[k])
+ content += up + route_line + eol
+ content += down + route_line + eol
+
+ return content
+
+ def _render_interfaces(self, network_state):
+ ''' Given state, emit etc/network/interfaces content '''
+
+ content = ""
+ interfaces = network_state.get('interfaces')
+ ''' Apply a sort order to ensure that we write out
+ the physical interfaces first; this is critical for
+ bonding
+ '''
+ order = {
+ 'physical': 0,
+ 'bond': 1,
+ 'bridge': 2,
+ 'vlan': 3,
+ }
+ content += "auto lo\niface lo inet loopback\n"
+ for dnskey, value in network_state.get('dns', {}).items():
+ if len(value):
+ content += " dns-{} {}\n".format(dnskey, " ".join(value))
+
+ for iface in sorted(interfaces.values(),
+ key=lambda k: (order[k['type']], k['name'])):
+
+ if content[-2:] != "\n\n":
+ content += "\n"
+ subnets = iface.get('subnets', {})
+ if subnets:
+ for index, subnet in zip(range(0, len(subnets)), subnets):
+ if content[-2:] != "\n\n":
+ content += "\n"
+ iface['index'] = index
+ iface['mode'] = subnet['type']
+ iface['control'] = subnet.get('control', 'auto')
+ if iface['mode'].endswith('6'):
+ iface['inet'] += '6'
+ elif iface['mode'] == 'static' \
+ and ":" in subnet['address']:
+ iface['inet'] += '6'
+ if iface['mode'].startswith('dhcp'):
+ iface['mode'] = 'dhcp'
+
+ content += _iface_start_entry(iface, index)
+ content += _iface_add_subnet(iface, subnet)
+ content += _iface_add_attrs(iface)
+ else:
+ # ifenslave docs say to auto the slave devices
+ if 'bond-master' in iface:
+ content += "auto {name}\n".format(**iface)
+ content += "iface {name} {inet} {mode}\n".format(**iface)
+ content += _iface_add_attrs(iface)
+
+ for route in network_state.get('routes'):
+ content += self._render_route(route)
+
+ # global replacements until v2 format
+ content = content.replace('mac_address', 'hwaddress')
+ return content
+
+ def render_network_state(self,
+ target, network_state, eni="etc/network/interfaces",
+ links_prefix=LINKS_FNAME_PREFIX,
+ netrules='etc/udev/rules.d/70-persistent-net.rules'):
+
+ fpeni = os.path.join(target, eni)
+ net.write_file(fpeni, self._render_interfaces(network_state))
+
+ if netrules:
+ netrules = os.path.join(target, netrules)
+ net.write_file(netrules,
+ self._render_persistent_net(network_state))
+
+ if links_prefix:
+ self._render_systemd_links(target, network_state, links_prefix)
+
+ def _render_systemd_links(self, target, network_state,
+ links_prefix=LINKS_FNAME_PREFIX):
+ fp_prefix = os.path.sep.join((target, links_prefix))
+ for f in glob.glob(fp_prefix + "*"):
+ os.unlink(f)
+ interfaces = network_state.get('interfaces')
+ for iface in interfaces.values():
+ if (iface['type'] == 'physical' and 'name' in iface and
+ iface.get('mac_address')):
+ fname = fp_prefix + iface['name'] + ".link"
+ content = "\n".join([
+ "[Match]",
+ "MACAddress=" + iface['mac_address'],
+ "",
+ "[Link]",
+ "Name=" + iface['name'],
+ ""
+ ])
+ net.write_file(fname, content)
diff --git a/cloudinit/net/network_state.py b/cloudinit/net/network_state.py
index 4c726ab4..c5aeadb5 100644
--- a/cloudinit/net/network_state.py
+++ b/cloudinit/net/network_state.py
@@ -15,9 +15,12 @@
# You should have received a copy of the GNU Affero General Public License
# along with Curtin. If not, see <http://www.gnu.org/licenses/>.
-from cloudinit import log as logging
-from cloudinit import util
-from cloudinit.util import yaml_dumps as dump_config
+import copy
+import logging
+
+import six
+
+from cloudinit import net
LOG = logging.getLogger(__name__)
@@ -27,39 +30,104 @@ NETWORK_STATE_REQUIRED_KEYS = {
}
+def parse_net_config_data(net_config, skip_broken=True):
+ """Parses the config, returns NetworkState object
+
+ :param net_config: curtin network config dict
+ """
+ state = None
+ if 'version' in net_config and 'config' in net_config:
+ ns = NetworkState(version=net_config.get('version'),
+ config=net_config.get('config'))
+ ns.parse_config(skip_broken=skip_broken)
+ state = ns.network_state
+ return state
+
+
+def parse_net_config(path, skip_broken=True):
+ """Parses a curtin network configuration file and
+ return network state"""
+ ns = None
+ net_config = net.read_yaml_file(path)
+ if 'network' in net_config:
+ ns = parse_net_config_data(net_config.get('network'),
+ skip_broken=skip_broken)
+ return ns
+
+
def from_state_file(state_file):
network_state = None
- state = util.read_conf(state_file)
+ state = net.read_yaml_file(state_file)
network_state = NetworkState()
network_state.load(state)
-
return network_state
+def diff_keys(expected, actual):
+ missing = set(expected)
+ for key in actual:
+ missing.discard(key)
+ return missing
+
+
+class InvalidCommand(Exception):
+ pass
+
+
+def ensure_command_keys(required_keys):
+
+ def wrapper(func):
+
+ @six.wraps(func)
+ def decorator(self, command, *args, **kwargs):
+ if required_keys:
+ missing_keys = diff_keys(required_keys, command)
+ if missing_keys:
+ raise InvalidCommand("Command missing %s of required"
+ " keys %s" % (missing_keys,
+ required_keys))
+ return func(self, command, *args, **kwargs)
+
+ return decorator
+
+ return wrapper
+
+
+class CommandHandlerMeta(type):
+ """Metaclass that dynamically creates a 'command_handlers' attribute.
+
+ This will scan the to-be-created class for methods that start with
+ 'handle_' and on finding those will populate a class attribute mapping
+ so that those methods can be quickly located and called.
+ """
+ def __new__(cls, name, parents, dct):
+ command_handlers = {}
+ for attr_name, attr in six.iteritems(dct):
+ if six.callable(attr) and attr_name.startswith('handle_'):
+ handles_what = attr_name[len('handle_'):]
+ if handles_what:
+ command_handlers[handles_what] = attr
+ dct['command_handlers'] = command_handlers
+ return super(CommandHandlerMeta, cls).__new__(cls, name,
+ parents, dct)
+
+
+@six.add_metaclass(CommandHandlerMeta)
class NetworkState(object):
+
+ initial_network_state = {
+ 'interfaces': {},
+ 'routes': [],
+ 'dns': {
+ 'nameservers': [],
+ 'search': [],
+ }
+ }
+
def __init__(self, version=NETWORK_STATE_VERSION, config=None):
self.version = version
self.config = config
- self.network_state = {
- 'interfaces': {},
- 'routes': [],
- 'dns': {
- 'nameservers': [],
- 'search': [],
- }
- }
- self.command_handlers = self.get_command_handlers()
-
- def get_command_handlers(self):
- METHOD_PREFIX = 'handle_'
- methods = filter(lambda x: callable(getattr(self, x)) and
- x.startswith(METHOD_PREFIX), dir(self))
- handlers = {}
- for m in methods:
- key = m.replace(METHOD_PREFIX, '')
- handlers[key] = getattr(self, m)
-
- return handlers
+ self.network_state = copy.deepcopy(self.initial_network_state)
def dump(self):
state = {
@@ -67,7 +135,7 @@ class NetworkState(object):
'config': self.config,
'network_state': self.network_state,
}
- return dump_config(state)
+ return net.dump_yaml(state)
def load(self, state):
if 'version' not in state:
@@ -75,32 +143,39 @@ class NetworkState(object):
raise Exception('Invalid state, missing version field')
required_keys = NETWORK_STATE_REQUIRED_KEYS[state['version']]
- if not self.valid_command(state, required_keys):
- msg = 'Invalid state, missing keys: {}'.format(required_keys)
+ missing_keys = diff_keys(required_keys, state)
+ if missing_keys:
+ msg = 'Invalid state, missing keys: %s' % (missing_keys)
LOG.error(msg)
- raise Exception(msg)
+ raise ValueError(msg)
# v1 - direct attr mapping, except version
for key in [k for k in required_keys if k not in ['version']]:
setattr(self, key, state[key])
- self.command_handlers = self.get_command_handlers()
def dump_network_state(self):
- return dump_config(self.network_state)
+ return net.dump_yaml(self.network_state)
- def parse_config(self):
+ def parse_config(self, skip_broken=True):
# rebuild network state
for command in self.config:
- handler = self.command_handlers.get(command['type'])
- handler(command)
-
- def valid_command(self, command, required_keys):
- if not required_keys:
- return False
-
- found_keys = [key for key in command.keys() if key in required_keys]
- return len(found_keys) == len(required_keys)
-
+ command_type = command['type']
+ try:
+ handler = self.command_handlers[command_type]
+ except KeyError:
+ raise RuntimeError("No handler found for"
+ " command '%s'" % command_type)
+ try:
+ handler(self, command)
+ except InvalidCommand:
+ if not skip_broken:
+ raise
+ else:
+ LOG.warn("Skipping invalid command: %s", command,
+ exc_info=True)
+ LOG.debug(self.dump_network_state())
+
+ @ensure_command_keys(['name'])
def handle_physical(self, command):
'''
command = {
@@ -112,13 +187,6 @@ class NetworkState(object):
]
}
'''
- required_keys = [
- 'name',
- ]
- if not self.valid_command(command, required_keys):
- LOG.warn('Skipping Invalid command: {}'.format(command))
- LOG.debug(self.dump_network_state())
- return
interfaces = self.network_state.get('interfaces')
iface = interfaces.get(command['name'], {})
@@ -149,6 +217,7 @@ class NetworkState(object):
self.network_state['interfaces'].update({command.get('name'): iface})
self.dump_network_state()
+ @ensure_command_keys(['name', 'vlan_id', 'vlan_link'])
def handle_vlan(self, command):
'''
auto eth0.222
@@ -158,16 +227,6 @@ class NetworkState(object):
hwaddress ether BC:76:4E:06:96:B3
vlan-raw-device eth0
'''
- required_keys = [
- 'name',
- 'vlan_link',
- 'vlan_id',
- ]
- if not self.valid_command(command, required_keys):
- print('Skipping Invalid command: {}'.format(command))
- print(self.dump_network_state())
- return
-
interfaces = self.network_state.get('interfaces')
self.handle_physical(command)
iface = interfaces.get(command.get('name'), {})
@@ -175,6 +234,7 @@ class NetworkState(object):
iface['vlan_id'] = command.get('vlan_id')
interfaces.update({iface['name']: iface})
+ @ensure_command_keys(['name', 'bond_interfaces', 'params'])
def handle_bond(self, command):
'''
#/etc/network/interfaces
@@ -200,15 +260,6 @@ class NetworkState(object):
bond-updelay 200
bond-lacp-rate 4
'''
- required_keys = [
- 'name',
- 'bond_interfaces',
- 'params',
- ]
- if not self.valid_command(command, required_keys):
- print('Skipping Invalid command: {}'.format(command))
- print(self.dump_network_state())
- return
self.handle_physical(command)
interfaces = self.network_state.get('interfaces')
@@ -236,6 +287,7 @@ class NetworkState(object):
bond_if.update({param: val})
self.network_state['interfaces'].update({ifname: bond_if})
+ @ensure_command_keys(['name', 'bridge_interfaces', 'params'])
def handle_bridge(self, command):
'''
auto br0
@@ -263,15 +315,6 @@ class NetworkState(object):
"bridge_waitport",
]
'''
- required_keys = [
- 'name',
- 'bridge_interfaces',
- 'params',
- ]
- if not self.valid_command(command, required_keys):
- print('Skipping Invalid command: {}'.format(command))
- print(self.dump_network_state())
- return
# find one of the bridge port ifaces to get mac_addr
# handle bridge_slaves
@@ -295,15 +338,8 @@ class NetworkState(object):
interfaces.update({iface['name']: iface})
+ @ensure_command_keys(['address'])
def handle_nameserver(self, command):
- required_keys = [
- 'address',
- ]
- if not self.valid_command(command, required_keys):
- print('Skipping Invalid command: {}'.format(command))
- print(self.dump_network_state())
- return
-
dns = self.network_state.get('dns')
if 'address' in command:
addrs = command['address']
@@ -318,15 +354,8 @@ class NetworkState(object):
for path in paths:
dns['search'].append(path)
+ @ensure_command_keys(['destination'])
def handle_route(self, command):
- required_keys = [
- 'destination',
- ]
- if not self.valid_command(command, required_keys):
- print('Skipping Invalid command: {}'.format(command))
- print(self.dump_network_state())
- return
-
routes = self.network_state.get('routes')
network, cidr = command['destination'].split("/")
netmask = cidr2mask(int(cidr))
@@ -376,72 +405,3 @@ def mask2cidr(mask):
return ipv4mask2cidr(mask)
else:
return mask
-
-
-if __name__ == '__main__':
- import random
- import sys
-
- from cloudinit import net
-
- def load_config(nc):
- version = nc.get('version')
- config = nc.get('config')
- return (version, config)
-
- def test_parse(network_config):
- (version, config) = load_config(network_config)
- ns1 = NetworkState(version=version, config=config)
- ns1.parse_config()
- random.shuffle(config)
- ns2 = NetworkState(version=version, config=config)
- ns2.parse_config()
- print("----NS1-----")
- print(ns1.dump_network_state())
- print()
- print("----NS2-----")
- print(ns2.dump_network_state())
- print("NS1 == NS2 ?=> {}".format(
- ns1.network_state == ns2.network_state))
- eni = net.render_interfaces(ns2.network_state)
- print(eni)
- udev_rules = net.render_persistent_net(ns2.network_state)
- print(udev_rules)
-
- def test_dump_and_load(network_config):
- print("Loading network_config into NetworkState")
- (version, config) = load_config(network_config)
- ns1 = NetworkState(version=version, config=config)
- ns1.parse_config()
- print("Dumping state to file")
- ns1_dump = ns1.dump()
- ns1_state = "/tmp/ns1.state"
- with open(ns1_state, "w+") as f:
- f.write(ns1_dump)
-
- print("Loading state from file")
- ns2 = from_state_file(ns1_state)
- print("NS1 == NS2 ?=> {}".format(
- ns1.network_state == ns2.network_state))
-
- def test_output(network_config):
- (version, config) = load_config(network_config)
- ns1 = NetworkState(version=version, config=config)
- ns1.parse_config()
- random.shuffle(config)
- ns2 = NetworkState(version=version, config=config)
- ns2.parse_config()
- print("NS1 == NS2 ?=> {}".format(
- ns1.network_state == ns2.network_state))
- eni_1 = net.render_interfaces(ns1.network_state)
- eni_2 = net.render_interfaces(ns2.network_state)
- print(eni_1)
- print(eni_2)
- print("eni_1 == eni_2 ?=> {}".format(
- eni_1 == eni_2))
-
- y = util.read_conf(sys.argv[1])
- network_config = y.get('network')
- test_parse(network_config)
- test_dump_and_load(network_config)
- test_output(network_config)
diff --git a/cloudinit/serial.py b/cloudinit/serial.py
new file mode 100644
index 00000000..af45c13e
--- /dev/null
+++ b/cloudinit/serial.py
@@ -0,0 +1,50 @@
+# vi: ts=4 expandtab
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 3, as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+
+from __future__ import absolute_import
+
+try:
+ from serial import Serial
+except ImportError:
+ # For older versions of python (ie 2.6) pyserial may not exist and/or
+ # work and/or be installed, so make a dummy/fake serial that blows up
+ # when used...
+ class Serial(object):
+ def __init__(self, *args, **kwargs):
+ pass
+
+ @staticmethod
+ def isOpen():
+ return False
+
+ @staticmethod
+ def write(data):
+ raise IOError("Unable to perform serial `write` operation,"
+ " pyserial not installed.")
+
+ @staticmethod
+ def readline():
+ raise IOError("Unable to perform serial `readline` operation,"
+ " pyserial not installed.")
+
+ @staticmethod
+ def flush():
+ raise IOError("Unable to perform serial `flush` operation,"
+ " pyserial not installed.")
+
+ @staticmethod
+ def read(size=1):
+ raise IOError("Unable to perform serial `read` operation,"
+ " pyserial not installed.")
diff --git a/cloudinit/sources/DataSourceAzure.py b/cloudinit/sources/DataSourceAzure.py
index 2d046600..8c7e8673 100644
--- a/cloudinit/sources/DataSourceAzure.py
+++ b/cloudinit/sources/DataSourceAzure.py
@@ -423,7 +423,7 @@ def write_files(datadir, files, dirmode=None):
elem.text = DEF_PASSWD_REDACTION
return ET.tostring(root)
except Exception:
- LOG.critical("failed to redact userpassword in {}".format(fname))
+ LOG.critical("failed to redact userpassword in %s", fname)
return cnt
if not datadir:
diff --git a/cloudinit/sources/DataSourceConfigDrive.py b/cloudinit/sources/DataSourceConfigDrive.py
index 52a9f543..70373b43 100644
--- a/cloudinit/sources/DataSourceConfigDrive.py
+++ b/cloudinit/sources/DataSourceConfigDrive.py
@@ -61,7 +61,7 @@ class DataSourceConfigDrive(openstack.SourceMixin, sources.DataSource):
mstr += "[source=%s]" % (self.source)
return mstr
- def get_data(self):
+ def get_data(self, skip_first_boot=False):
found = None
md = {}
results = {}
@@ -119,7 +119,8 @@ class DataSourceConfigDrive(openstack.SourceMixin, sources.DataSource):
# instance-id
prev_iid = get_previous_iid(self.paths)
cur_iid = md['instance-id']
- if prev_iid != cur_iid and self.dsmode == "local":
+ if prev_iid != cur_iid and \
+ self.dsmode == "local" and not skip_first_boot:
on_first_boot(results, distro=self.distro)
# dsmode != self.dsmode here if:
@@ -163,7 +164,8 @@ class DataSourceConfigDrive(openstack.SourceMixin, sources.DataSource):
def network_config(self):
if self._network_config is None:
if self.network_json is not None:
- self._network_config = convert_network_data(self.network_json)
+ self._network_config = openstack.convert_net_json(
+ self.network_json)
return self._network_config
@@ -303,122 +305,3 @@ datasources = [
# Return a list of data sources that match this set of dependencies
def get_datasource_list(depends):
return sources.list_from_depends(depends, datasources)
-
-
-# Convert OpenStack ConfigDrive NetworkData json to network_config yaml
-def convert_network_data(network_json=None):
- """Return a dictionary of network_config by parsing provided
- OpenStack ConfigDrive NetworkData json format
-
- OpenStack network_data.json provides a 3 element dictionary
- - "links" (links are network devices, physical or virtual)
- - "networks" (networks are ip network configurations for one or more
- links)
- - services (non-ip services, like dns)
-
- networks and links are combined via network items referencing specific
- links via a 'link_id' which maps to a links 'id' field.
-
- To convert this format to network_config yaml, we first iterate over the
- links and then walk the network list to determine if any of the networks
- utilize the current link; if so we generate a subnet entry for the device
-
- We also need to map network_data.json fields to network_config fields. For
- example, the network_data links 'id' field is equivalent to network_config
- 'name' field for devices. We apply more of this mapping to the various
- link types that we encounter.
-
- There are additional fields that are populated in the network_data.json
- from OpenStack that are not relevant to network_config yaml, so we
- enumerate a dictionary of valid keys for network_yaml and apply filtering
- to drop these superflous keys from the network_config yaml.
- """
- if network_json is None:
- return None
-
- # dict of network_config key for filtering network_json
- valid_keys = {
- 'physical': [
- 'name',
- 'type',
- 'mac_address',
- 'subnets',
- 'params',
- ],
- 'subnet': [
- 'type',
- 'address',
- 'netmask',
- 'broadcast',
- 'metric',
- 'gateway',
- 'pointopoint',
- 'mtu',
- 'scope',
- 'dns_nameservers',
- 'dns_search',
- 'routes',
- ],
- }
-
- links = network_json.get('links', [])
- networks = network_json.get('networks', [])
- services = network_json.get('services', [])
-
- config = []
- for link in links:
- subnets = []
- cfg = {k: v for k, v in link.items()
- if k in valid_keys['physical']}
- cfg.update({'name': link['id']})
- for network in [net for net in networks
- if net['link'] == link['id']]:
- subnet = {k: v for k, v in network.items()
- if k in valid_keys['subnet']}
- if 'dhcp' in network['type']:
- t = 'dhcp6' if network['type'].startswith('ipv6') else 'dhcp4'
- subnet.update({
- 'type': t,
- })
- else:
- subnet.update({
- 'type': 'static',
- 'address': network.get('ip_address'),
- })
- subnets.append(subnet)
- cfg.update({'subnets': subnets})
- if link['type'] in ['ethernet', 'vif', 'ovs', 'phy']:
- cfg.update({
- 'type': 'physical',
- 'mac_address': link['ethernet_mac_address']})
- elif link['type'] in ['bond']:
- params = {}
- for k, v in link.items():
- if k == 'bond_links':
- continue
- elif k.startswith('bond'):
- params.update({k: v})
- cfg.update({
- 'bond_interfaces': copy.deepcopy(link['bond_links']),
- 'params': params,
- })
- elif link['type'] in ['vlan']:
- cfg.update({
- 'name': "%s.%s" % (link['vlan_link'],
- link['vlan_id']),
- 'vlan_link': link['vlan_link'],
- 'vlan_id': link['vlan_id'],
- 'mac_address': link['vlan_mac_address'],
- })
- else:
- raise ValueError(
- 'Unknown network_data link type: %s' % link['type'])
-
- config.append(cfg)
-
- for service in services:
- cfg = service
- cfg.update({'type': 'nameserver'})
- config.append(cfg)
-
- return {'version': 1, 'config': config}
diff --git a/cloudinit/sources/DataSourceSmartOS.py b/cloudinit/sources/DataSourceSmartOS.py
index 6cbd8dfa..c7641eb3 100644
--- a/cloudinit/sources/DataSourceSmartOS.py
+++ b/cloudinit/sources/DataSourceSmartOS.py
@@ -40,13 +40,11 @@ import re
import socket
import stat
-import serial
-
from cloudinit import log as logging
+from cloudinit import serial
from cloudinit import sources
from cloudinit import util
-
LOG = logging.getLogger(__name__)
SMARTOS_ATTRIB_MAP = {
diff --git a/cloudinit/sources/helpers/openstack.py b/cloudinit/sources/helpers/openstack.py
index 156aba6c..1af99118 100644
--- a/cloudinit/sources/helpers/openstack.py
+++ b/cloudinit/sources/helpers/openstack.py
@@ -474,6 +474,131 @@ class MetadataReader(BaseReader):
retries=self.retries)
+def convert_net_json(network_json):
+ """Return a dictionary of network_config by parsing provided
+ OpenStack ConfigDrive NetworkData json format
+
+ OpenStack network_data.json provides a 3 element dictionary
+ - "links" (links are network devices, physical or virtual)
+ - "networks" (networks are ip network configurations for one or more
+ links)
+ - services (non-ip services, like dns)
+
+ networks and links are combined via network items referencing specific
+ links via a 'link_id' which maps to a links 'id' field.
+
+ To convert this format to network_config yaml, we first iterate over the
+ links and then walk the network list to determine if any of the networks
+ utilize the current link; if so we generate a subnet entry for the device
+
+ We also need to map network_data.json fields to network_config fields. For
+ example, the network_data links 'id' field is equivalent to network_config
+ 'name' field for devices. We apply more of this mapping to the various
+ link types that we encounter.
+
+ There are additional fields that are populated in the network_data.json
+ from OpenStack that are not relevant to network_config yaml, so we
+ enumerate a dictionary of valid keys for network_yaml and apply filtering
+ to drop these superflous keys from the network_config yaml.
+ """
+
+ # Dict of network_config key for filtering network_json
+ valid_keys = {
+ 'physical': [
+ 'name',
+ 'type',
+ 'mac_address',
+ 'subnets',
+ 'params',
+ ],
+ 'subnet': [
+ 'type',
+ 'address',
+ 'netmask',
+ 'broadcast',
+ 'metric',
+ 'gateway',
+ 'pointopoint',
+ 'mtu',
+ 'scope',
+ 'dns_nameservers',
+ 'dns_search',
+ 'routes',
+ ],
+ }
+
+ links = network_json.get('links', [])
+ networks = network_json.get('networks', [])
+ services = network_json.get('services', [])
+
+ config = []
+ for link in links:
+ subnets = []
+ cfg = dict((k, v) for k, v in link.items()
+ if k in valid_keys['physical'])
+ cfg.update({'name': link['id']})
+ for network in [net for net in networks
+ if net['link'] == link['id']]:
+ subnet = dict((k, v) for k, v in network.items()
+ if k in valid_keys['subnet'])
+ if 'dhcp' in network['type']:
+ t = 'dhcp6' if network['type'].startswith('ipv6') else 'dhcp4'
+ subnet.update({
+ 'type': t,
+ })
+ else:
+ subnet.update({
+ 'type': 'static',
+ 'address': network.get('ip_address'),
+ })
+ if network['type'] == 'ipv6':
+ subnet['ipv6'] = True
+ else:
+ subnet['ipv4'] = True
+ subnets.append(subnet)
+ cfg.update({'subnets': subnets})
+ if link['type'] in ['ethernet', 'vif', 'ovs', 'phy']:
+ cfg.update({
+ 'type': 'physical',
+ 'mac_address': link['ethernet_mac_address']})
+ elif link['type'] in ['bond']:
+ params = {}
+ for k, v in link.items():
+ if k == 'bond_links':
+ continue
+ elif k.startswith('bond'):
+ params.update({k: v})
+ cfg.update({
+ 'bond_interfaces': copy.deepcopy(link['bond_links']),
+ 'params': params,
+ })
+ elif link['type'] in ['vlan']:
+ cfg.update({
+ 'name': "%s.%s" % (link['vlan_link'],
+ link['vlan_id']),
+ 'vlan_link': link['vlan_link'],
+ 'vlan_id': link['vlan_id'],
+ 'mac_address': link['vlan_mac_address'],
+ })
+ elif link['type'] in ['bridge']:
+ cfg.update({
+ 'type': 'bridge',
+ 'mac_address': link['ethernet_mac_address'],
+ 'mtu': link['mtu']})
+ else:
+ raise ValueError(
+ 'Unknown network_data link type: %s' % link['type'])
+
+ config.append(cfg)
+
+ for service in services:
+ cfg = copy.deepcopy(service)
+ cfg.update({'type': 'nameserver'})
+ config.append(cfg)
+
+ return {'version': 1, 'config': config}
+
+
def convert_vendordata_json(data, recurse=True):
"""data: a loaded json *object* (strings, arrays, dicts).
return something suitable for cloudinit vendordata_raw.
diff --git a/cloudinit/stages.py b/cloudinit/stages.py
index 62d066de..5dd31539 100644
--- a/cloudinit/stages.py
+++ b/cloudinit/stages.py
@@ -45,6 +45,7 @@ from cloudinit import importer
from cloudinit import log as logging
from cloudinit import net
from cloudinit.reporting import events
+from cloudinit.net import cmdline
from cloudinit import sources
from cloudinit import type_utils
from cloudinit import util
@@ -579,7 +580,7 @@ class Init(object):
if os.path.exists(disable_file):
return (None, disable_file)
- cmdline_cfg = ('cmdline', net.read_kernel_cmdline_config())
+ cmdline_cfg = ('cmdline', cmdline.read_kernel_cmdline_config())
dscfg = ('ds', None)
if self.datasource and hasattr(self.datasource, 'network_config'):
dscfg = ('ds', self.datasource.network_config)
diff --git a/cloudinit/util.py b/cloudinit/util.py
index 8d6cbb4b..851c1fcb 100644
--- a/cloudinit/util.py
+++ b/cloudinit/util.py
@@ -171,7 +171,8 @@ class ProcessExecutionError(IOError):
def __init__(self, stdout=None, stderr=None,
exit_code=None, cmd=None,
- description=None, reason=None):
+ description=None, reason=None,
+ errno=None):
if not cmd:
self.cmd = '-'
else:
@@ -202,6 +203,7 @@ class ProcessExecutionError(IOError):
else:
self.reason = '-'
+ self.errno = errno
message = self.MESSAGE_TMPL % {
'description': self.description,
'cmd': self.cmd,
@@ -1147,7 +1149,14 @@ def find_devs_with(criteria=None, oformat='device',
options.append(path)
cmd = blk_id_cmd + options
# See man blkid for why 2 is added
- (out, _err) = subp(cmd, rcs=[0, 2])
+ try:
+ (out, _err) = subp(cmd, rcs=[0, 2])
+ except ProcessExecutionError as e:
+ if e.errno == errno.ENOENT:
+ # blkid not found...
+ out = ""
+ else:
+ raise
entries = []
for line in out.splitlines():
line = line.strip()
@@ -1696,7 +1705,8 @@ def subp(args, data=None, rcs=None, env=None, capture=True, shell=False,
sp = subprocess.Popen(args, **kws)
(out, err) = sp.communicate(data)
except OSError as e:
- raise ProcessExecutionError(cmd=args, reason=e)
+ raise ProcessExecutionError(cmd=args, reason=e,
+ errno=e.errno)
rc = sp.returncode
if rc not in rcs:
raise ProcessExecutionError(stdout=out, stderr=err,
generated by cgit v1.2.3 (git 2.39.1) at 2024-11-10 17:11:29 +0000