diff options
Diffstat (limited to 'cloudinit')
31 files changed, 1238 insertions, 198 deletions
diff --git a/cloudinit/cmd/main.py b/cloudinit/cmd/main.py index 6ff4e1c0..fd221323 100644 --- a/cloudinit/cmd/main.py +++ b/cloudinit/cmd/main.py @@ -680,6 +680,10 @@ def status_wrapper(name, args, data_d=None, link_d=None): return len(v1[mode]['errors']) +def main_features(name, args): + sys.stdout.write('\n'.join(sorted(version.FEATURES)) + '\n') + + def main(sysv_args=None): if sysv_args is not None: parser = argparse.ArgumentParser(prog=sysv_args[0]) @@ -770,6 +774,10 @@ def main(sysv_args=None): ' upon')) parser_dhclient.set_defaults(action=('dhclient_hook', dhclient_hook)) + parser_features = subparsers.add_parser('features', + help=('list defined features')) + parser_features.set_defaults(action=('features', main_features)) + args = parser.parse_args(args=sysv_args) try: @@ -788,6 +796,7 @@ def main(sysv_args=None): if name in ("modules", "init"): functor = status_wrapper + rname = None report_on = True if name == "init": if args.local: @@ -802,10 +811,10 @@ def main(sysv_args=None): rname, rdesc = ("single/%s" % args.name, "running single module %s" % args.name) report_on = args.report - - elif name == 'dhclient_hook': - rname, rdesc = ("dhclient-hook", - "running dhclient-hook module") + else: + rname = name + rdesc = "running 'cloud-init %s'" % name + report_on = False args.reporter = events.ReportEventStack( rname, rdesc, reporting_enabled=report_on) diff --git a/cloudinit/config/cc_apt_configure.py b/cloudinit/config/cc_apt_configure.py index 7f09c919..06804e85 100644 --- a/cloudinit/config/cc_apt_configure.py +++ b/cloudinit/config/cc_apt_configure.py @@ -278,15 +278,29 @@ def handle(name, ocfg, cloud, log, _): raise ValueError("Expected dictionary for 'apt' config, found %s", type(cfg)) - LOG.debug("handling apt (module %s) with apt config '%s'", name, cfg) + apply_debconf_selections(cfg, target) + apply_apt(cfg, cloud, target) + + +def apply_apt(cfg, cloud, target): + # cfg is the 'apt' top level dictionary already in 'v3' format. + if not cfg: + # no config was provided. If apt configuration does not seem + # necessary on this system, then return. + if util.system_is_snappy(): + LOG.debug("Nothing to do: No apt config and running on snappy") + return + if not (util.which('apt-get') or util.which('apt')): + LOG.debug("Nothing to do: No apt config and no apt commands") + return + + LOG.debug("handling apt config: %s", cfg) release = util.lsb_release(target=target)['codename'] arch = util.get_architecture(target) mirrors = find_apt_mirror_info(cfg, cloud, arch=arch) LOG.debug("Apt Mirror info: %s", mirrors) - apply_debconf_selections(cfg, target) - if util.is_false(cfg.get('preserve_sources_list', False)): generate_sources_list(cfg, release, mirrors, cloud) rename_apt_lists(mirrors, target) diff --git a/cloudinit/config/cc_chef.py b/cloudinit/config/cc_chef.py index f6564e5c..2be2532c 100644 --- a/cloudinit/config/cc_chef.py +++ b/cloudinit/config/cc_chef.py @@ -302,7 +302,7 @@ def install_chef(cloud, chef_cfg, log): retries = max(0, util.get_cfg_option_int(chef_cfg, "omnibus_url_retries", default=OMNIBUS_URL_RETRIES)) - content = url_helper.readurl(url=url, retries=retries) + content = url_helper.readurl(url=url, retries=retries).contents with util.tempdir() as tmpd: # Use tmpdir over tmpfile to avoid 'text file busy' on execute tmpf = "%s/chef-omnibus-install" % tmpd diff --git a/cloudinit/config/cc_disk_setup.py b/cloudinit/config/cc_disk_setup.py index 38df13ab..f39f0815 100644 --- a/cloudinit/config/cc_disk_setup.py +++ b/cloudinit/config/cc_disk_setup.py @@ -201,7 +201,7 @@ def update_fs_setup_devices(disk_setup, tformer): if part and 'partition' in definition: definition['_partition'] = definition['partition'] - definition['partition'] = part + definition['partition'] = part def value_splitter(values, start=None): diff --git a/cloudinit/config/cc_growpart.py b/cloudinit/config/cc_growpart.py index 832bb3fd..089693e8 100644 --- a/cloudinit/config/cc_growpart.py +++ b/cloudinit/config/cc_growpart.py @@ -247,7 +247,16 @@ def devent2dev(devent): result = util.get_mount_info(devent) if not result: raise ValueError("Could not determine device of '%s' % dev_ent") - return result[0] + dev = result[0] + + container = util.is_container() + + # Ensure the path is a block device. + if (dev == "/dev/root" and not os.path.exists(dev) and not container): + dev = util.rootdev_from_cmdline(util.get_cmdline()) + if dev is None: + raise ValueError("Unable to find device '/dev/root'") + return dev def resize_devices(resizer, devices): diff --git a/cloudinit/config/cc_resizefs.py b/cloudinit/config/cc_resizefs.py index e028abf4..60e3ab53 100644 --- a/cloudinit/config/cc_resizefs.py +++ b/cloudinit/config/cc_resizefs.py @@ -71,25 +71,6 @@ RESIZE_FS_PREFIXES_CMDS = [ NOBLOCK = "noblock" -def rootdev_from_cmdline(cmdline): - found = None - for tok in cmdline.split(): - if tok.startswith("root="): - found = tok[5:] - break - if found is None: - return None - - if found.startswith("/dev/"): - return found - if found.startswith("LABEL="): - return "/dev/disk/by-label/" + found[len("LABEL="):] - if found.startswith("UUID="): - return "/dev/disk/by-uuid/" + found[len("UUID="):] - - return "/dev/" + found - - def handle(name, cfg, _cloud, log, args): if len(args) != 0: resize_root = args[0] @@ -121,7 +102,7 @@ def handle(name, cfg, _cloud, log, args): # Ensure the path is a block device. if (devpth == "/dev/root" and not os.path.exists(devpth) and not container): - devpth = rootdev_from_cmdline(util.get_cmdline()) + devpth = util.rootdev_from_cmdline(util.get_cmdline()) if devpth is None: log.warn("Unable to find device '/dev/root'") return diff --git a/cloudinit/config/cc_set_passwords.py b/cloudinit/config/cc_set_passwords.py index cf1f59ec..eb0bdab0 100755 --- a/cloudinit/config/cc_set_passwords.py +++ b/cloudinit/config/cc_set_passwords.py @@ -23,7 +23,8 @@ If the ``list`` key is provided, a list of ``username:password`` pairs can be specified. The usernames specified must already exist on the system, or have been created using the ``cc_users_groups`` module. A password can be randomly generated using -``username:RANDOM`` or ``username:R``. Password ssh authentication can be +``username:RANDOM`` or ``username:R``. A hashed password can be specified +using ``username:$6$salt$hash``. Password ssh authentication can be enabled, disabled, or left to system defaults using ``ssh_pwauth``. .. note:: @@ -45,13 +46,25 @@ enabled, disabled, or left to system defaults using ``ssh_pwauth``. expire: <true/false> chpasswd: + list: | + user1:password1 + user2:RANDOM + user3:password3 + user4:R + + ## + # or as yaml list + ## + chpasswd: list: - user1:password1 - - user2:Random + - user2:RANDOM - user3:password3 - user4:R + - user4:$6$rL..$ej... """ +import re import sys from cloudinit.distros import ug_util @@ -79,38 +92,66 @@ def handle(_name, cfg, cloud, log, args): if 'chpasswd' in cfg: chfg = cfg['chpasswd'] - plist = util.get_cfg_option_str(chfg, 'list', plist) + if 'list' in chfg and chfg['list']: + if isinstance(chfg['list'], list): + log.debug("Handling input for chpasswd as list.") + plist = util.get_cfg_option_list(chfg, 'list', plist) + else: + log.debug("Handling input for chpasswd as multiline string.") + plist = util.get_cfg_option_str(chfg, 'list', plist) + if plist: + plist = plist.splitlines() + expire = util.get_cfg_option_bool(chfg, 'expire', expire) if not plist and password: (users, _groups) = ug_util.normalize_users_groups(cfg, cloud.distro) (user, _user_config) = ug_util.extract_default(users) if user: - plist = "%s:%s" % (user, password) + plist = ["%s:%s" % (user, password)] else: log.warn("No default or defined user to change password for.") errors = [] if plist: plist_in = [] + hashed_plist_in = [] + hashed_users = [] randlist = [] users = [] - for line in plist.splitlines(): + prog = re.compile(r'\$[1,2a,2y,5,6](\$.+){2}') + for line in plist: u, p = line.split(':', 1) - if p == "R" or p == "RANDOM": - p = rand_user_password() - randlist.append("%s:%s" % (u, p)) - plist_in.append("%s:%s" % (u, p)) - users.append(u) + if prog.match(p) is not None and ":" not in p: + hashed_plist_in.append("%s:%s" % (u, p)) + hashed_users.append(u) + else: + if p == "R" or p == "RANDOM": + p = rand_user_password() + randlist.append("%s:%s" % (u, p)) + plist_in.append("%s:%s" % (u, p)) + users.append(u) ch_in = '\n'.join(plist_in) + '\n' - try: - log.debug("Changing password for %s:", users) - util.subp(['chpasswd'], ch_in) - except Exception as e: - errors.append(e) - util.logexc(log, "Failed to set passwords with chpasswd for %s", - users) + if users: + try: + log.debug("Changing password for %s:", users) + util.subp(['chpasswd'], ch_in) + except Exception as e: + errors.append(e) + util.logexc( + log, "Failed to set passwords with chpasswd for %s", users) + + hashed_ch_in = '\n'.join(hashed_plist_in) + '\n' + if hashed_users: + try: + log.debug("Setting hashed password for %s:", hashed_users) + util.subp(['chpasswd', '-e'], hashed_ch_in) + except Exception as e: + errors.append(e) + util.logexc( + log, "Failed to set hashed passwords with chpasswd for %s", + hashed_users) if len(randlist): blurb = ("Set the following 'random' passwords\n", diff --git a/cloudinit/distros/__init__.py b/cloudinit/distros/__init__.py index f3d395b9..803ac74e 100755 --- a/cloudinit/distros/__init__.py +++ b/cloudinit/distros/__init__.py @@ -22,6 +22,7 @@ from cloudinit import log as logging from cloudinit import net from cloudinit.net import eni from cloudinit.net import network_state +from cloudinit.net import renderers from cloudinit import ssh_util from cloudinit import type_utils from cloudinit import util @@ -50,6 +51,7 @@ class Distro(object): hostname_conf_fn = "/etc/hostname" tz_zone_dir = "/usr/share/zoneinfo" init_cmd = ['service'] # systemctl, service etc + renderer_configs = {} def __init__(self, name, cfg, paths): self._paths = paths @@ -69,6 +71,17 @@ class Distro(object): def _write_network_config(self, settings): raise NotImplementedError() + def _supported_write_network_config(self, network_config): + priority = util.get_cfg_by_path( + self._cfg, ('network', 'renderers'), None) + + name, render_cls = renderers.select(priority=priority) + LOG.debug("Selected renderer '%s' from priority list: %s", + name, priority) + renderer = render_cls(config=self.renderer_configs.get(name)) + renderer.render_network_config(network_config=network_config) + return [] + def _find_tz_file(self, tz): tz_file = os.path.join(self.tz_zone_dir, str(tz)) if not os.path.isfile(tz_file): diff --git a/cloudinit/distros/debian.py b/cloudinit/distros/debian.py index 48ccec8c..3f0f9d53 100644 --- a/cloudinit/distros/debian.py +++ b/cloudinit/distros/debian.py @@ -13,8 +13,6 @@ import os from cloudinit import distros from cloudinit import helpers from cloudinit import log as logging -from cloudinit.net import eni -from cloudinit.net.network_state import parse_net_config_data from cloudinit import util from cloudinit.distros.parsers.hostname import HostnameConf @@ -38,11 +36,23 @@ ENI_HEADER = """# This file is generated from information provided by # network: {config: disabled} """ +NETWORK_CONF_FN = "/etc/network/interfaces.d/50-cloud-init.cfg" + class Distro(distros.Distro): hostname_conf_fn = "/etc/hostname" locale_conf_fn = "/etc/default/locale" - network_conf_fn = "/etc/network/interfaces.d/50-cloud-init.cfg" + network_conf_fn = { + "eni": "/etc/network/interfaces.d/50-cloud-init.cfg", + "netplan": "/etc/netplan/50-cloud-init.yaml" + } + renderer_configs = { + "eni": {"eni_path": network_conf_fn["eni"], + "eni_header": ENI_HEADER}, + "netplan": {"netplan_path": network_conf_fn["netplan"], + "netplan_header": ENI_HEADER, + "postcmds": True} + } def __init__(self, name, cfg, paths): distros.Distro.__init__(self, name, cfg, paths) @@ -51,12 +61,6 @@ class Distro(distros.Distro): # should only happen say once per instance...) self._runner = helpers.Runners(paths) self.osfamily = 'debian' - self._net_renderer = eni.Renderer({ - 'eni_path': self.network_conf_fn, - 'eni_header': ENI_HEADER, - 'links_path_prefix': None, - 'netrules_path': None, - }) def apply_locale(self, locale, out_fn=None): if not out_fn: @@ -76,14 +80,13 @@ class Distro(distros.Distro): self.package_command('install', pkgs=pkglist) def _write_network(self, settings): - util.write_file(self.network_conf_fn, settings) + # this is a legacy method, it will always write eni + util.write_file(self.network_conf_fn["eni"], settings) return ['all'] def _write_network_config(self, netconfig): - ns = parse_net_config_data(netconfig) - self._net_renderer.render_network_state("/", ns) _maybe_remove_legacy_eth0() - return [] + return self._supported_write_network_config(netconfig) def _bring_up_interfaces(self, device_names): use_all = False diff --git a/cloudinit/distros/parsers/resolv_conf.py b/cloudinit/distros/parsers/resolv_conf.py index ff6ee307..d1f8a042 100644 --- a/cloudinit/distros/parsers/resolv_conf.py +++ b/cloudinit/distros/parsers/resolv_conf.py @@ -6,9 +6,11 @@ from six import StringIO +from cloudinit.distros.parsers import chop_comment +from cloudinit import log as logging from cloudinit import util -from cloudinit.distros.parsers import chop_comment +LOG = logging.getLogger(__name__) # See: man resolv.conf @@ -79,9 +81,10 @@ class ResolvConf(object): if len(new_ns) == len(current_ns): return current_ns if len(current_ns) >= 3: - # Hard restriction on only 3 name servers - raise ValueError(("Adding %r would go beyond the " - "'3' maximum name servers") % (ns)) + LOG.warn("ignoring nameserver %r: adding would " + "exceed the maximum of " + "'3' name servers (see resolv.conf(5))" % (ns)) + return current_ns[:3] self._remove_option('nameserver') for n in new_ns: self._contents.append(('option', ['nameserver', n, ''])) diff --git a/cloudinit/distros/rhel.py b/cloudinit/distros/rhel.py index 7498c63a..372c7d0f 100644 --- a/cloudinit/distros/rhel.py +++ b/cloudinit/distros/rhel.py @@ -11,8 +11,6 @@ from cloudinit import distros from cloudinit import helpers from cloudinit import log as logging -from cloudinit.net.network_state import parse_net_config_data -from cloudinit.net import sysconfig from cloudinit import util from cloudinit.distros import net_util @@ -49,16 +47,13 @@ class Distro(distros.Distro): # should only happen say once per instance...) self._runner = helpers.Runners(paths) self.osfamily = 'redhat' - self._net_renderer = sysconfig.Renderer() cfg['ssh_svcname'] = 'sshd' def install_packages(self, pkglist): self.package_command('install', pkgs=pkglist) def _write_network_config(self, netconfig): - ns = parse_net_config_data(netconfig) - self._net_renderer.render_network_state("/", ns) - return [] + return self._supported_write_network_config(netconfig) def _write_network(self, settings): # TODO(harlowja) fix this... since this is the ubuntu format diff --git a/cloudinit/net/__init__.py b/cloudinit/net/__init__.py index ea649cc2..346be5d3 100755..100644 --- a/cloudinit/net/__init__.py +++ b/cloudinit/net/__init__.py @@ -82,6 +82,10 @@ def is_wireless(devname): return os.path.exists(sys_dev_path(devname, "wireless")) +def is_bridge(devname): + return os.path.exists(sys_dev_path(devname, "bridge")) + + def is_connected(devname): # is_connected isn't really as simple as that. 2 is # 'physically connected'. 3 is 'not connected'. but a wlan interface will @@ -132,7 +136,7 @@ def generate_fallback_config(): for interface in potential_interfaces: if interface.startswith("veth"): continue - if os.path.exists(sys_dev_path(interface, "bridge")): + if is_bridge(interface): # skip any bridges continue carrier = read_sys_net_int(interface, 'carrier') @@ -187,7 +191,11 @@ def apply_network_config_names(netcfg, strict_present=True, strict_busy=True): """read the network config and rename devices accordingly. if strict_present is false, then do not raise exception if no devices match. if strict_busy is false, then do not raise exception if the - device cannot be renamed because it is currently configured.""" + device cannot be renamed because it is currently configured. + + renames are only attempted for interfaces of type 'physical'. It is + expected that the network system will create other devices with the + correct name in place.""" renames = [] for ent in netcfg.get('config', {}): if ent.get('type') != 'physical': @@ -201,13 +209,35 @@ def apply_network_config_names(netcfg, strict_present=True, strict_busy=True): return _rename_interfaces(renames) +def interface_has_own_mac(ifname, strict=False): + """return True if the provided interface has its own address. + + Based on addr_assign_type in /sys. Return true for any interface + that does not have a 'stolen' address. Examples of such devices + are bonds or vlans that inherit their mac from another device. + Possible values are: + 0: permanent address 2: stolen from another device + 1: randomly generated 3: set using dev_set_mac_address""" + + assign_type = read_sys_net_int(ifname, "addr_assign_type") + if strict and assign_type is None: + raise ValueError("%s had no addr_assign_type.") + return assign_type in (0, 1, 3) + + def _get_current_rename_info(check_downable=True): - """Collect information necessary for rename_interfaces.""" - names = get_devicelist() + """Collect information necessary for rename_interfaces. + + returns a dictionary by mac address like: + {mac: + {'name': name + 'up': boolean: is_up(name), + 'downable': None or boolean indicating that the + device has only automatically assigned ip addrs.}} + """ bymac = {} - for n in names: - bymac[get_interface_mac(n)] = { - 'name': n, 'up': is_up(n), 'downable': None} + for mac, name in get_interfaces_by_mac().items(): + bymac[mac] = {'name': name, 'up': is_up(name), 'downable': None} if check_downable: nmatch = re.compile(r"[0-9]+:\s+(\w+)[@:]") @@ -346,22 +376,37 @@ def get_interface_mac(ifname): return read_sys_net_safe(ifname, path) -def get_interfaces_by_mac(devs=None): - """Build a dictionary of tuples {mac: name}""" - if devs is None: - try: - devs = get_devicelist() - except OSError as e: - if e.errno == errno.ENOENT: - devs = [] - else: - raise +def get_interfaces_by_mac(): + """Build a dictionary of tuples {mac: name}. + + Bridges and any devices that have a 'stolen' mac are excluded.""" + try: + devs = get_devicelist() + except OSError as e: + if e.errno == errno.ENOENT: + devs = [] + else: + raise ret = {} for name in devs: + if not interface_has_own_mac(name): + continue + if is_bridge(name): + continue mac = get_interface_mac(name) # some devices may not have a mac (tun0) - if mac: - ret[mac] = name + if not mac: + continue + if mac in ret: + raise RuntimeError( + "duplicate mac found! both '%s' and '%s' have mac '%s'" % + (name, ret[mac], mac)) + ret[mac] = name return ret + +class RendererNotFoundError(RuntimeError): + pass + + # vi: ts=4 expandtab diff --git a/cloudinit/net/eni.py b/cloudinit/net/eni.py index 5b249f1f..9819d4f5 100644 --- a/cloudinit/net/eni.py +++ b/cloudinit/net/eni.py @@ -8,6 +8,7 @@ import re from . import ParserError from . import renderer +from .network_state import subnet_is_ipv6 from cloudinit import util @@ -111,16 +112,6 @@ def _iface_start_entry(iface, index, render_hwaddress=False): return lines -def _subnet_is_ipv6(subnet): - # 'static6' or 'dhcp6' - if subnet['type'].endswith('6'): - # This is a request for DHCPv6. - return True - elif subnet['type'] == 'static' and ":" in subnet['address']: - return True - return False - - def _parse_deb_config_data(ifaces, contents, src_dir, src_path): """Parses the file contents, placing result into ifaces. @@ -273,8 +264,11 @@ def _ifaces_to_net_config_data(ifaces): # devname is 'eth0' for name='eth0:1' devname = name.partition(":")[0] if devname not in devs: - devs[devname] = {'type': 'physical', 'name': devname, - 'subnets': []} + if devname == "lo": + dtype = "loopback" + else: + dtype = "physical" + devs[devname] = {'type': dtype, 'name': devname, 'subnets': []} # this isnt strictly correct, but some might specify # hwaddress on a nic for matching / declaring name. if 'hwaddress' in data: @@ -367,7 +361,7 @@ class Renderer(renderer.Renderer): iface['mode'] = subnet['type'] iface['control'] = subnet.get('control', 'auto') subnet_inet = 'inet' - if _subnet_is_ipv6(subnet): + if subnet_is_ipv6(subnet): subnet_inet += '6' iface['inet'] = subnet_inet if subnet['type'].startswith('dhcp'): @@ -423,10 +417,11 @@ class Renderer(renderer.Renderer): bonding ''' order = { - 'physical': 0, - 'bond': 1, - 'bridge': 2, - 'vlan': 3, + 'loopback': 0, + 'physical': 1, + 'bond': 2, + 'bridge': 3, + 'vlan': 4, } sections = [] @@ -444,14 +439,14 @@ class Renderer(renderer.Renderer): return '\n\n'.join(['\n'.join(s) for s in sections]) + "\n" - def render_network_state(self, target, network_state): - fpeni = os.path.join(target, self.eni_path) + def render_network_state(self, network_state, target=None): + fpeni = util.target_path(target, self.eni_path) util.ensure_dir(os.path.dirname(fpeni)) header = self.eni_header if self.eni_header else "" util.write_file(fpeni, header + self._render_interfaces(network_state)) if self.netrules_path: - netrules = os.path.join(target, self.netrules_path) + netrules = util.target_path(target, self.netrules_path) util.ensure_dir(os.path.dirname(netrules)) util.write_file(netrules, self._render_persistent_net(network_state)) @@ -461,7 +456,7 @@ class Renderer(renderer.Renderer): links_prefix=self.links_path_prefix) def _render_systemd_links(self, target, network_state, links_prefix): - fp_prefix = os.path.join(target, links_prefix) + fp_prefix = util.target_path(target, links_prefix) for f in glob.glob(fp_prefix + "*"): os.unlink(f) for iface in network_state.iter_interfaces(): @@ -482,7 +477,7 @@ class Renderer(renderer.Renderer): def network_state_to_eni(network_state, header=None, render_hwaddress=False): # render the provided network state, return a string of equivalent eni eni_path = 'etc/network/interfaces' - renderer = Renderer({ + renderer = Renderer(config={ 'eni_path': eni_path, 'eni_header': header, 'links_path_prefix': None, @@ -496,4 +491,18 @@ def network_state_to_eni(network_state, header=None, render_hwaddress=False): network_state, render_hwaddress=render_hwaddress) return header + contents + +def available(target=None): + expected = ['ifquery', 'ifup', 'ifdown'] + search = ['/sbin', '/usr/sbin'] + for p in expected: + if not util.which(p, search=search, target=target): + return False + eni = util.target_path(target, 'etc/network/interfaces') + if not os.path.isfile(eni): + return False + + return True + + # vi: ts=4 expandtab diff --git a/cloudinit/net/netplan.py b/cloudinit/net/netplan.py new file mode 100644 index 00000000..825fe831 --- /dev/null +++ b/cloudinit/net/netplan.py @@ -0,0 +1,412 @@ +# This file is part of cloud-init. See LICENSE file ... + +import copy +import os + +from . import renderer +from .network_state import subnet_is_ipv6 + +from cloudinit import log as logging +from cloudinit import util +from cloudinit.net import SYS_CLASS_NET, get_devicelist + +KNOWN_SNAPD_CONFIG = b"""\ +# This is the initial network config. +# It can be overwritten by cloud-init or console-conf. +network: + version: 2 + ethernets: + all-en: + match: + name: "en*" + dhcp4: true + all-eth: + match: + name: "eth*" + dhcp4: true +""" + +LOG = logging.getLogger(__name__) +NET_CONFIG_TO_V2 = { + 'bond': {'bond-ad-select': 'ad-select', + 'bond-arp-interval': 'arp-interval', + 'bond-arp-ip-target': 'arp-ip-target', + 'bond-arp-validate': 'arp-validate', + 'bond-downdelay': 'down-delay', + 'bond-fail-over-mac': 'fail-over-mac-policy', + 'bond-lacp-rate': 'lacp-rate', + 'bond-miimon': 'mii-monitor-interval', + 'bond-min-links': 'min-links', + 'bond-mode': 'mode', + 'bond-num-grat-arp': 'gratuitious-arp', + 'bond-primary-reselect': 'primary-reselect-policy', + 'bond-updelay': 'up-delay', + 'bond-xmit_hash_policy': 'transmit_hash_policy'}, + 'bridge': {'bridge_ageing': 'ageing-time', + 'bridge_bridgeprio': 'priority', + 'bridge_fd': 'forward-delay', + 'bridge_gcint': None, + 'bridge_hello': 'hello-time', + 'bridge_maxage': 'max-age', + 'bridge_maxwait': None, + 'bridge_pathcost': 'path-cost', + 'bridge_portprio': None, + 'bridge_waitport': None}} + + +def _get_params_dict_by_match(config, match): + return dict((key, value) for (key, value) in config.items() + if key.startswith(match)) + + +def _extract_addresses(config, entry): + """This method parse a cloudinit.net.network_state dictionary (config) and + maps netstate keys/values into a dictionary (entry) to represent + netplan yaml. + + An example config dictionary might look like: + + {'mac_address': '52:54:00:12:34:00', + 'name': 'interface0', + 'subnets': [ + {'address': '192.168.1.2/24', + 'mtu': 1501, + 'type': 'static'}, + {'address': '2001:4800:78ff:1b:be76:4eff:fe06:1000", + 'mtu': 1480, + 'netmask': 64, + 'type': 'static'}], + 'type: physical' + } + + An entry dictionary looks like: + + {'set-name': 'interface0', + 'match': {'macaddress': '52:54:00:12:34:00'}, + 'mtu': 1501} + + After modification returns + + {'set-name': 'interface0', + 'match': {'macaddress': '52:54:00:12:34:00'}, + 'mtu': 1501, + 'address': ['192.168.1.2/24', '2001:4800:78ff:1b:be76:4eff:fe06:1000"], + 'mtu6': 1480} + + """ + + def _listify(obj, token=' '): + "Helper to convert strings to list of strings, handle single string" + if not obj or type(obj) not in [str]: + return obj + if token in obj: + return obj.split(token) + else: + return [obj, ] + + addresses = [] + routes = [] + nameservers = [] + searchdomains = [] + subnets = config.get('subnets', []) + if subnets is None: + subnets = [] + for subnet in subnets: + sn_type = subnet.get('type') + if sn_type.startswith('dhcp'): + if sn_type == 'dhcp': + sn_type += '4' + entry.update({sn_type: True}) + elif sn_type in ['static']: + addr = "%s" % subnet.get('address') + if 'netmask' in subnet: + addr += "/%s" % subnet.get('netmask') + if 'gateway' in subnet and subnet.get('gateway'): + gateway = subnet.get('gateway') + if ":" in gateway: + entry.update({'gateway6': gateway}) + else: + entry.update({'gateway4': gateway}) + if 'dns_nameservers' in subnet: + nameservers += _listify(subnet.get('dns_nameservers', [])) + if 'dns_search' in subnet: + searchdomains += _listify(subnet.get('dns_search', [])) + if 'mtu' in subnet: + mtukey = 'mtu' + if subnet_is_ipv6(subnet): + mtukey += '6' + entry.update({mtukey: subnet.get('mtu')}) + for route in subnet.get('routes', []): + to_net = "%s/%s" % (route.get('network'), + route.get('netmask')) + route = { + 'via': route.get('gateway'), + 'to': to_net, + } + if 'metric' in route: + route.update({'metric': route.get('metric', 100)}) + routes.append(route) + + addresses.append(addr) + + if len(addresses) > 0: + entry.update({'addresses': addresses}) + if len(routes) > 0: + entry.update({'routes': routes}) + if len(nameservers) > 0: + ns = {'addresses': nameservers} + entry.update({'nameservers': ns}) + if len(searchdomains) > 0: + ns = entry.get('nameservers', {}) + ns.update({'search': searchdomains}) + entry.update({'nameservers': ns}) + + +def _extract_bond_slaves_by_name(interfaces, entry, bond_master): + bond_slave_names = sorted([name for (name, cfg) in interfaces.items() + if cfg.get('bond-master', None) == bond_master]) + if len(bond_slave_names) > 0: + entry.update({'interfaces': bond_slave_names}) + + +def _clean_default(target=None): + # clean out any known default files and derived files in target + # LP: #1675576 + tpath = util.target_path(target, "etc/netplan/00-snapd-config.yaml") + if not os.path.isfile(tpath): + return + content = util.load_file(tpath, decode=False) + if content != KNOWN_SNAPD_CONFIG: + return + + derived = [util.target_path(target, f) for f in ( + 'run/systemd/network/10-netplan-all-en.network', + 'run/systemd/network/10-netplan-all-eth.network', + 'run/systemd/generator/netplan.stamp')] + existing = [f for f in derived if os.path.isfile(f)] + LOG.debug("removing known config '%s' and derived existing files: %s", + tpath, existing) + + for f in [tpath] + existing: + os.unlink(f) + + +class Renderer(renderer.Renderer): + """Renders network information in a /etc/netplan/network.yaml format.""" + + NETPLAN_GENERATE = ['netplan', 'generate'] + + def __init__(self, config=None): + if not config: + config = {} + self.netplan_path = config.get('netplan_path', + 'etc/netplan/50-cloud-init.yaml') + self.netplan_header = config.get('netplan_header', None) + self._postcmds = config.get('postcmds', False) + self.clean_default = config.get('clean_default', True) + + def render_network_state(self, target, network_state): + # check network state for version + # if v2, then extract network_state.config + # else render_v2_from_state + fpnplan = os.path.join(target, self.netplan_path) + util.ensure_dir(os.path.dirname(fpnplan)) + header = self.netplan_header if self.netplan_header else "" + + # render from state + content = self._render_content(network_state) + + if not header.endswith("\n"): + header += "\n" + util.write_file(fpnplan, header + content) + + if self.clean_default: + _clean_default(target=target) + self._netplan_generate(run=self._postcmds) + self._net_setup_link(run=self._postcmds) + + def _netplan_generate(self, run=False): + if not run: + LOG.debug("netplan generate postcmd disabled") + return + util.subp(self.NETPLAN_GENERATE, capture=True) + + def _net_setup_link(self, run=False): + """To ensure device link properties are applied, we poke + udev to re-evaluate networkd .link files and call + the setup_link udev builtin command + """ + if not run: + LOG.debug("netplan net_setup_link postcmd disabled") + return + setup_lnk = ['udevadm', 'test-builtin', 'net_setup_link'] + for cmd in [setup_lnk + [SYS_CLASS_NET + iface] + for iface in get_devicelist() if + os.path.islink(SYS_CLASS_NET + iface)]: + util.subp(cmd, capture=True) + + def _render_content(self, network_state): + ethernets = {} + wifis = {} + bridges = {} + bonds = {} + vlans = {} + content = [] + + interfaces = network_state._network_state.get('interfaces', []) + + nameservers = network_state.dns_nameservers + searchdomains = network_state.dns_searchdomains + + for config in network_state.iter_interfaces(): + ifname = config.get('name') + # filter None entries up front so we can do simple if key in dict + ifcfg = dict((key, value) for (key, value) in config.items() + if value) + + if_type = ifcfg.get('type') + if if_type == 'physical': + # required_keys = ['name', 'mac_address'] + eth = { + 'set-name': ifname, + 'match': ifcfg.get('match', None), + } + if eth['match'] is None: + macaddr = ifcfg.get('mac_address', None) + if macaddr is not None: + eth['match'] = {'macaddress': macaddr.lower()} + else: + del eth['match'] + del eth['set-name'] + if 'mtu' in ifcfg: + eth['mtu'] = ifcfg.get('mtu') + + _extract_addresses(ifcfg, eth) + ethernets.update({ifname: eth}) + + elif if_type == 'bond': + # required_keys = ['name', 'bond_interfaces'] + bond = {} + bond_config = {} + # extract bond params and drop the bond_ prefix as it's + # redundent in v2 yaml format + v2_bond_map = NET_CONFIG_TO_V2.get('bond') + for match in ['bond_', 'bond-']: + bond_params = _get_params_dict_by_match(ifcfg, match) + for (param, value) in bond_params.items(): + newname = v2_bond_map.get(param) + if newname is None: + continue + bond_config.update({newname: value}) + + if len(bond_config) > 0: + bond.update({'parameters': bond_config}) + slave_interfaces = ifcfg.get('bond-slaves') + if slave_interfaces == 'none': + _extract_bond_slaves_by_name(interfaces, bond, ifname) + _extract_addresses(ifcfg, bond) + bonds.update({ifname: bond}) + + elif if_type == 'bridge': + # required_keys = ['name', 'bridge_ports'] + ports = sorted(copy.copy(ifcfg.get('bridge_ports'))) + bridge = { + 'interfaces': ports, + } + # extract bridge params and drop the bridge prefix as it's + # redundent in v2 yaml format + match_prefix = 'bridge_' + params = _get_params_dict_by_match(ifcfg, match_prefix) + br_config = {} + + # v2 yaml uses different names for the keys + # and at least one value format change + v2_bridge_map = NET_CONFIG_TO_V2.get('bridge') + for (param, value) in params.items(): + newname = v2_bridge_map.get(param) + if newname is None: + continue + br_config.update({newname: value}) + if newname == 'path-cost': + # <interface> <cost> -> <interface>: int(<cost>) + newvalue = {} + for costval in value: + (port, cost) = costval.split() + newvalue[port] = int(cost) + br_config.update({newname: newvalue}) + if len(br_config) > 0: + bridge.update({'parameters': br_config}) + _extract_addresses(ifcfg, bridge) + bridges.update({ifname: bridge}) + + elif if_type == 'vlan': + # required_keys = ['name', 'vlan_id', 'vlan-raw-device'] + vlan = { + 'id': ifcfg.get('vlan_id'), + 'link': ifcfg.get('vlan-raw-device') + } + + _extract_addresses(ifcfg, vlan) + vlans.update({ifname: vlan}) + + # inject global nameserver values under each physical interface + if nameservers: + for _eth, cfg in ethernets.items(): + nscfg = cfg.get('nameservers', {}) + addresses = nscfg.get('addresses', []) + addresses += nameservers + nscfg.update({'addresses': addresses}) + cfg.update({'nameservers': nscfg}) + + if searchdomains: + for _eth, cfg in ethernets.items(): + nscfg = cfg.get('nameservers', {}) + search = nscfg.get('search', []) + search += searchdomains + nscfg.update({'search': search}) + cfg.update({'nameservers': nscfg}) + + # workaround yaml dictionary key sorting when dumping + def _render_section(name, section): + if section: + dump = util.yaml_dumps({name: section}, + explicit_start=False, + explicit_end=False) + txt = util.indent(dump, ' ' * 4) + return [txt] + return [] + + content.append("network:\n version: 2\n") + content += _render_section('ethernets', ethernets) + content += _render_section('wifis', wifis) + content += _render_section('bonds', bonds) + content += _render_section('bridges', bridges) + content += _render_section('vlans', vlans) + + return "".join(content) + + +def available(target=None): + expected = ['netplan'] + search = ['/usr/sbin', '/sbin'] + for p in expected: + if not util.which(p, search=search, target=target): + return False + return True + + +def network_state_to_netplan(network_state, header=None): + # render the provided network state, return a string of equivalent eni + netplan_path = 'etc/network/50-cloud-init.yaml' + renderer = Renderer({ + 'netplan_path': netplan_path, + 'netplan_header': header, + }) + if not header: + header = "" + if not header.endswith("\n"): + header += "\n" + contents = renderer._render_content(network_state) + return header + contents + +# vi: ts=4 expandtab diff --git a/cloudinit/net/network_state.py b/cloudinit/net/network_state.py index 11ef585b..692b6007 100644 --- a/cloudinit/net/network_state.py +++ b/cloudinit/net/network_state.py @@ -1,4 +1,4 @@ -# Copyright (C) 2013-2014 Canonical Ltd. +# Copyright (C) 2017 Canonical Ltd. # # Author: Ryan Harper <ryan.harper@canonical.com> # @@ -18,6 +18,10 @@ NETWORK_STATE_VERSION = 1 NETWORK_STATE_REQUIRED_KEYS = { 1: ['version', 'config', 'network_state'], } +NETWORK_V2_KEY_FILTER = [ + 'addresses', 'dhcp4', 'dhcp6', 'gateway4', 'gateway6', 'interfaces', + 'match', 'mtu', 'nameservers', 'renderer', 'set-name', 'wakeonlan' +] def parse_net_config_data(net_config, skip_broken=True): @@ -26,11 +30,18 @@ def parse_net_config_data(net_config, skip_broken=True): :param net_config: curtin network config dict """ state = None - if 'version' in net_config and 'config' in net_config: - nsi = NetworkStateInterpreter(version=net_config.get('version'), - config=net_config.get('config')) + version = net_config.get('version') + config = net_config.get('config') + if version == 2: + # v2 does not have explicit 'config' key so we + # pass the whole net-config as-is + config = net_config + + if version and config: + nsi = NetworkStateInterpreter(version=version, config=config) nsi.parse_config(skip_broken=skip_broken) - state = nsi.network_state + state = nsi.get_network_state() + return state @@ -106,6 +117,7 @@ class NetworkState(object): def __init__(self, network_state, version=NETWORK_STATE_VERSION): self._network_state = copy.deepcopy(network_state) self._version = version + self.use_ipv6 = network_state.get('use_ipv6', False) @property def version(self): @@ -152,7 +164,8 @@ class NetworkStateInterpreter(object): 'dns': { 'nameservers': [], 'search': [], - } + }, + 'use_ipv6': False, } def __init__(self, version=NETWORK_STATE_VERSION, config=None): @@ -165,6 +178,14 @@ class NetworkStateInterpreter(object): def network_state(self): return NetworkState(self._network_state, version=self._version) + @property + def use_ipv6(self): + return self._network_state.get('use_ipv6') + + @use_ipv6.setter + def use_ipv6(self, val): + self._network_state.update({'use_ipv6': val}) + def dump(self): state = { 'version': self._version, @@ -192,8 +213,22 @@ class NetworkStateInterpreter(object): def dump_network_state(self): return util.yaml_dumps(self._network_state) + def as_dict(self): + return {'version': self._version, 'config': self._config} + + def get_network_state(self): + ns = self.network_state + return ns + def parse_config(self, skip_broken=True): - # rebuild network state + if self._version == 1: + self.parse_config_v1(skip_broken=skip_broken) + self._parsed = True + elif self._version == 2: + self.parse_config_v2(skip_broken=skip_broken) + self._parsed = True + + def parse_config_v1(self, skip_broken=True): for command in self._config: command_type = command['type'] try: @@ -211,6 +246,30 @@ class NetworkStateInterpreter(object): exc_info=True) LOG.debug(self.dump_network_state()) + def parse_config_v2(self, skip_broken=True): + for command_type, command in self._config.items(): + if command_type == 'version': + continue + try: + handler = self.command_handlers[command_type] + except KeyError: + raise RuntimeError("No handler found for" + " command '%s'" % command_type) + try: + handler(self, command) + self._v2_common(command) + except InvalidCommand: + if not skip_broken: + raise + else: + LOG.warn("Skipping invalid command: %s", command, + exc_info=True) + LOG.debug(self.dump_network_state()) + + @ensure_command_keys(['name']) + def handle_loopback(self, command): + return self.handle_physical(command) + @ensure_command_keys(['name']) def handle_physical(self, command): ''' @@ -234,11 +293,16 @@ class NetworkStateInterpreter(object): if subnets: for subnet in subnets: if subnet['type'] == 'static': + if ':' in subnet['address']: + self.use_ipv6 = True if 'netmask' in subnet and ':' in subnet['address']: subnet['netmask'] = mask2cidr(subnet['netmask']) for route in subnet.get('routes', []): if 'netmask' in route: route['netmask'] = mask2cidr(route['netmask']) + elif subnet['type'].endswith('6'): + self.use_ipv6 = True + iface.update({ 'name': command.get('name'), 'type': command.get('type'), @@ -323,7 +387,7 @@ class NetworkStateInterpreter(object): bond_if.update({param: val}) self._network_state['interfaces'].update({ifname: bond_if}) - @ensure_command_keys(['name', 'bridge_interfaces', 'params']) + @ensure_command_keys(['name', 'bridge_interfaces']) def handle_bridge(self, command): ''' auto br0 @@ -369,7 +433,7 @@ class NetworkStateInterpreter(object): self.handle_physical(command) iface = interfaces.get(command.get('name'), {}) iface['bridge_ports'] = command['bridge_interfaces'] - for param, val in command.get('params').items(): + for param, val in command.get('params', {}).items(): iface.update({param: val}) interfaces.update({iface['name']: iface}) @@ -403,6 +467,241 @@ class NetworkStateInterpreter(object): } routes.append(route) + # V2 handlers + def handle_bonds(self, command): + ''' + v2_command = { + bond0: { + 'interfaces': ['interface0', 'interface1'], + 'miimon': 100, + 'mode': '802.3ad', + 'xmit_hash_policy': 'layer3+4'}, + bond1: { + 'bond-slaves': ['interface2', 'interface7'], + 'mode': 1 + } + } + + v1_command = { + 'type': 'bond' + 'name': 'bond0', + 'bond_interfaces': [interface0, interface1], + 'params': { + 'bond-mode': '802.3ad', + 'bond_miimon: 100, + 'bond_xmit_hash_policy': 'layer3+4', + } + } + + ''' + self._handle_bond_bridge(command, cmd_type='bond') + + def handle_bridges(self, command): + + ''' + v2_command = { + br0: { + 'interfaces': ['interface0', 'interface1'], + 'fd': 0, + 'stp': 'off', + 'maxwait': 0, + } + } + + v1_command = { + 'type': 'bridge' + 'name': 'br0', + 'bridge_interfaces': [interface0, interface1], + 'params': { + 'bridge_stp': 'off', + 'bridge_fd: 0, + 'bridge_maxwait': 0 + } + } + + ''' + self._handle_bond_bridge(command, cmd_type='bridge') + + def handle_ethernets(self, command): + ''' + ethernets: + eno1: + match: + macaddress: 00:11:22:33:44:55 + wakeonlan: true + dhcp4: true + dhcp6: false + addresses: + - 192.168.14.2/24 + - 2001:1::1/64 + gateway4: 192.168.14.1 + gateway6: 2001:1::2 + nameservers: + search: [foo.local, bar.local] + addresses: [8.8.8.8, 8.8.4.4] + lom: + match: + driver: ixgbe + set-name: lom1 + dhcp6: true + switchports: + match: + name: enp2* + mtu: 1280 + + command = { + 'type': 'physical', + 'mac_address': 'c0:d6:9f:2c:e8:80', + 'name': 'eth0', + 'subnets': [ + {'type': 'dhcp4'} + ] + } + ''' + for eth, cfg in command.items(): + phy_cmd = { + 'type': 'physical', + 'name': cfg.get('set-name', eth), + } + mac_address = cfg.get('match', {}).get('macaddress', None) + if not mac_address: + LOG.debug('NetworkState Version2: missing "macaddress" info ' + 'in config entry: %s: %s', eth, str(cfg)) + + for key in ['mtu', 'match', 'wakeonlan']: + if key in cfg: + phy_cmd.update({key: cfg.get(key)}) + + subnets = self._v2_to_v1_ipcfg(cfg) + if len(subnets) > 0: + phy_cmd.update({'subnets': subnets}) + + LOG.debug('v2(ethernets) -> v1(physical):\n%s', phy_cmd) + self.handle_physical(phy_cmd) + + def handle_vlans(self, command): + ''' + v2_vlans = { + 'eth0.123': { + 'id': 123, + 'link': 'eth0', + 'dhcp4': True, + } + } + + v1_command = { + 'type': 'vlan', + 'name': 'eth0.123', + 'vlan_link': 'eth0', + 'vlan_id': 123, + 'subnets': [{'type': 'dhcp4'}], + } + ''' + for vlan, cfg in command.items(): + vlan_cmd = { + 'type': 'vlan', + 'name': vlan, + 'vlan_id': cfg.get('id'), + 'vlan_link': cfg.get('link'), + } + subnets = self._v2_to_v1_ipcfg(cfg) + if len(subnets) > 0: + vlan_cmd.update({'subnets': subnets}) + LOG.debug('v2(vlans) -> v1(vlan):\n%s', vlan_cmd) + self.handle_vlan(vlan_cmd) + + def handle_wifis(self, command): + raise NotImplementedError("NetworkState V2: " + "Skipping wifi configuration") + + def _v2_common(self, cfg): + LOG.debug('v2_common: handling config:\n%s', cfg) + if 'nameservers' in cfg: + search = cfg.get('nameservers').get('search', []) + dns = cfg.get('nameservers').get('addresses', []) + name_cmd = {'type': 'nameserver'} + if len(search) > 0: + name_cmd.update({'search': search}) + if len(dns) > 0: + name_cmd.update({'addresses': dns}) + LOG.debug('v2(nameserver) -> v1(nameserver):\n%s', name_cmd) + self.handle_nameserver(name_cmd) + + def _handle_bond_bridge(self, command, cmd_type=None): + """Common handler for bond and bridge types""" + for item_name, item_cfg in command.items(): + item_params = dict((key, value) for (key, value) in + item_cfg.items() if key not in + NETWORK_V2_KEY_FILTER) + v1_cmd = { + 'type': cmd_type, + 'name': item_name, + cmd_type + '_interfaces': item_cfg.get('interfaces'), + 'params': item_params, + } + subnets = self._v2_to_v1_ipcfg(item_cfg) + if len(subnets) > 0: + v1_cmd.update({'subnets': subnets}) + + LOG.debug('v2(%ss) -> v1(%s):\n%s', cmd_type, cmd_type, v1_cmd) + self.handle_bridge(v1_cmd) + + def _v2_to_v1_ipcfg(self, cfg): + """Common ipconfig extraction from v2 to v1 subnets array.""" + + subnets = [] + if 'dhcp4' in cfg: + subnets.append({'type': 'dhcp4'}) + if 'dhcp6' in cfg: + self.use_ipv6 = True + subnets.append({'type': 'dhcp6'}) + + gateway4 = None + gateway6 = None + for address in cfg.get('addresses', []): + subnet = { + 'type': 'static', + 'address': address, + } + + routes = [] + for route in cfg.get('routes', []): + route_addr = route.get('to') + if "/" in route_addr: + route_addr, route_cidr = route_addr.split("/") + route_netmask = cidr2mask(route_cidr) + subnet_route = { + 'address': route_addr, + 'netmask': route_netmask, + 'gateway': route.get('via') + } + routes.append(subnet_route) + if len(routes) > 0: + subnet.update({'routes': routes}) + + if ":" in address: + if 'gateway6' in cfg and gateway6 is None: + gateway6 = cfg.get('gateway6') + subnet.update({'gateway': gateway6}) + else: + if 'gateway4' in cfg and gateway4 is None: + gateway4 = cfg.get('gateway4') + subnet.update({'gateway': gateway4}) + + subnets.append(subnet) + return subnets + + +def subnet_is_ipv6(subnet): + """Common helper for checking network_state subnets for ipv6.""" + # 'static6' or 'dhcp6' + if subnet['type'].endswith('6'): + # This is a request for DHCPv6. + return True + elif subnet['type'] == 'static' and ":" in subnet['address']: + return True + return False + def cidr2mask(cidr): mask = [0, 0, 0, 0] diff --git a/cloudinit/net/renderer.py b/cloudinit/net/renderer.py index 3a192436..c68658dc 100644 --- a/cloudinit/net/renderer.py +++ b/cloudinit/net/renderer.py @@ -5,8 +5,10 @@ # # This file is part of cloud-init. See LICENSE file for license information. +import abc import six +from .network_state import parse_net_config_data from .udev import generate_udev_rule @@ -36,4 +38,12 @@ class Renderer(object): iface['mac_address'])) return content.getvalue() + @abc.abstractmethod + def render_network_state(self, network_state, target=None): + """Render network state.""" + + def render_network_config(self, network_config, target=None): + return self.render_network_state( + network_state=parse_net_config_data(network_config), target=target) + # vi: ts=4 expandtab diff --git a/cloudinit/net/renderers.py b/cloudinit/net/renderers.py new file mode 100644 index 00000000..5117b4a5 --- /dev/null +++ b/cloudinit/net/renderers.py @@ -0,0 +1,53 @@ +# This file is part of cloud-init. See LICENSE file for license information. + +from . import eni +from . import netplan +from . import RendererNotFoundError +from . import sysconfig + +NAME_TO_RENDERER = { + "eni": eni, + "netplan": netplan, + "sysconfig": sysconfig, +} + +DEFAULT_PRIORITY = ["eni", "sysconfig", "netplan"] + + +def search(priority=None, target=None, first=False): + if priority is None: + priority = DEFAULT_PRIORITY + + available = NAME_TO_RENDERER + + unknown = [i for i in priority if i not in available] + if unknown: + raise ValueError( + "Unknown renderers provided in priority list: %s" % unknown) + + found = [] + for name in priority: + render_mod = available[name] + if render_mod.available(target): + cur = (name, render_mod.Renderer) + if first: + return cur + found.append(cur) + + return found + + +def select(priority=None, target=None): + found = search(priority, target=target, first=True) + if not found: + if priority is None: + priority = DEFAULT_PRIORITY + tmsg = "" + if target and target != "/": + tmsg = " in target=%s" % target + raise RendererNotFoundError( + "No available network renderers found%s. Searched " + "through list: %s" % (tmsg, priority)) + return found + +# vi: ts=4 expandtab diff --git a/cloudinit/net/sysconfig.py b/cloudinit/net/sysconfig.py index 6e7739fb..504e4d02 100644 --- a/cloudinit/net/sysconfig.py +++ b/cloudinit/net/sysconfig.py @@ -9,6 +9,7 @@ from cloudinit.distros.parsers import resolv_conf from cloudinit import util from . import renderer +from .network_state import subnet_is_ipv6 def _make_header(sep='#'): @@ -87,7 +88,8 @@ class Route(ConfigMap): def __init__(self, route_name, base_sysconf_dir): super(Route, self).__init__() self.last_idx = 1 - self.has_set_default = False + self.has_set_default_ipv4 = False + self.has_set_default_ipv6 = False self._route_name = route_name self._base_sysconf_dir = base_sysconf_dir @@ -95,7 +97,8 @@ class Route(ConfigMap): r = Route(self._route_name, self._base_sysconf_dir) r._conf = self._conf.copy() r.last_idx = self.last_idx - r.has_set_default = self.has_set_default + r.has_set_default_ipv4 = self.has_set_default_ipv4 + r.has_set_default_ipv6 = self.has_set_default_ipv6 return r @property @@ -119,10 +122,10 @@ class NetInterface(ConfigMap): super(NetInterface, self).__init__() self.children = [] self.routes = Route(iface_name, base_sysconf_dir) - self._kind = kind + self.kind = kind + self._iface_name = iface_name self._conf['DEVICE'] = iface_name - self._conf['TYPE'] = self.iface_types[kind] self._base_sysconf_dir = base_sysconf_dir @property @@ -140,6 +143,8 @@ class NetInterface(ConfigMap): @kind.setter def kind(self, kind): + if kind not in self.iface_types: + raise ValueError(kind) self._kind = kind self._conf['TYPE'] = self.iface_types[kind] @@ -173,7 +178,7 @@ class Renderer(renderer.Renderer): ('BOOTPROTO', 'none'), ]) - # If these keys exist, then there values will be used to form + # If these keys exist, then their values will be used to form # a BONDING_OPTS grouping; otherwise no grouping will be set. bond_tpl_opts = tuple([ ('bond_mode', "mode=%s"), @@ -190,7 +195,7 @@ class Renderer(renderer.Renderer): def __init__(self, config=None): if not config: config = {} - self.sysconf_dir = config.get('sysconf_dir', 'etc/sysconfig/') + self.sysconf_dir = config.get('sysconf_dir', 'etc/sysconfig') self.netrules_path = config.get( 'netrules_path', 'etc/udev/rules.d/70-persistent-net.rules') self.dns_path = config.get('dns_path', 'etc/resolv.conf') @@ -199,6 +204,7 @@ class Renderer(renderer.Renderer): def _render_iface_shared(cls, iface, iface_cfg): for k, v in cls.iface_defaults: iface_cfg[k] = v + for (old_key, new_key) in [('mac_address', 'HWADDR'), ('mtu', 'MTU')]: old_value = iface.get(old_key) if old_value is not None: @@ -215,7 +221,7 @@ class Renderer(renderer.Renderer): iface_cfg['BOOTPROTO'] = 'dhcp' elif subnet_type == 'static': iface_cfg['BOOTPROTO'] = 'static' - if subnet.get('ipv6'): + if subnet_is_ipv6(subnet): iface_cfg['IPV6ADDR'] = subnet['address'] iface_cfg['IPV6INIT'] = True else: @@ -227,10 +233,20 @@ class Renderer(renderer.Renderer): if 'netmask' in subnet: iface_cfg['NETMASK'] = subnet['netmask'] for route in subnet.get('routes', []): + if subnet.get('ipv6'): + gw_cfg = 'IPV6_DEFAULTGW' + else: + gw_cfg = 'GATEWAY' + if _is_default_route(route): - if route_cfg.has_set_default: - raise ValueError("Duplicate declaration of default" - " route found for interface '%s'" + if ( + (subnet.get('ipv4') and + route_cfg.has_set_default_ipv4) or + (subnet.get('ipv6') and + route_cfg.has_set_default_ipv6) + ): + raise ValueError("Duplicate declaration of default " + "route found for interface '%s'" % (iface_cfg.name)) # NOTE(harlowja): ipv6 and ipv4 default gateways gw_key = 'GATEWAY0' @@ -242,7 +258,7 @@ class Renderer(renderer.Renderer): # also provided the default route? iface_cfg['DEFROUTE'] = True if 'gateway' in route: - iface_cfg['GATEWAY'] = route['gateway'] + iface_cfg[gw_cfg] = route['gateway'] route_cfg.has_set_default = True else: gw_key = 'GATEWAY%s' % route_cfg.last_idx @@ -353,6 +369,8 @@ class Renderer(renderer.Renderer): '''Given state, return /etc/sysconfig files + contents''' iface_contents = {} for iface in network_state.iter_interfaces(): + if iface['type'] == "loopback": + continue iface_name = iface['name'] iface_cfg = NetInterface(iface_name, base_sysconf_dir) cls._render_iface_shared(iface, iface_cfg) @@ -372,19 +390,45 @@ class Renderer(renderer.Renderer): contents[iface_cfg.routes.path] = iface_cfg.routes.to_string() return contents - def render_network_state(self, target, network_state): - base_sysconf_dir = os.path.join(target, self.sysconf_dir) + def render_network_state(self, network_state, target=None): + file_mode = 0o644 + base_sysconf_dir = util.target_path(target, self.sysconf_dir) for path, data in self._render_sysconfig(base_sysconf_dir, network_state).items(): - util.write_file(path, data) + util.write_file(path, data, file_mode) if self.dns_path: - dns_path = os.path.join(target, self.dns_path) + dns_path = util.target_path(target, self.dns_path) resolv_content = self._render_dns(network_state, existing_dns_path=dns_path) - util.write_file(dns_path, resolv_content) + util.write_file(dns_path, resolv_content, file_mode) if self.netrules_path: netrules_content = self._render_persistent_net(network_state) - netrules_path = os.path.join(target, self.netrules_path) - util.write_file(netrules_path, netrules_content) + netrules_path = util.target_path(target, self.netrules_path) + util.write_file(netrules_path, netrules_content, file_mode) + + # always write /etc/sysconfig/network configuration + sysconfig_path = util.target_path(target, "etc/sysconfig/network") + netcfg = [_make_header(), 'NETWORKING=yes'] + if network_state.use_ipv6: + netcfg.append('NETWORKING_IPV6=yes') + netcfg.append('IPV6_AUTOCONF=no') + util.write_file(sysconfig_path, "\n".join(netcfg) + "\n", file_mode) + + +def available(target=None): + expected = ['ifup', 'ifdown'] + search = ['/sbin', '/usr/sbin'] + for p in expected: + if not util.which(p, search=search, target=target): + return False + + expected_paths = [ + 'etc/sysconfig/network-scripts/network-functions', + 'etc/sysconfig/network-scripts/ifdown-eth'] + for p in expected_paths: + if not os.path.isfile(util.target_path(target, p)): + return False + return True + # vi: ts=4 expandtab diff --git a/cloudinit/settings.py b/cloudinit/settings.py index 692ff5e5..dbafead5 100644 --- a/cloudinit/settings.py +++ b/cloudinit/settings.py @@ -46,6 +46,7 @@ CFG_BUILTIN = { 'templates_dir': '/etc/cloud/templates/', }, 'distro': 'ubuntu', + 'network': {'renderers': None}, }, 'vendor_data': {'enabled': True, 'prefix': []}, } diff --git a/cloudinit/sources/DataSourceAltCloud.py b/cloudinit/sources/DataSourceAltCloud.py index c2b0eac2..8528fa10 100644 --- a/cloudinit/sources/DataSourceAltCloud.py +++ b/cloudinit/sources/DataSourceAltCloud.py @@ -201,8 +201,7 @@ class DataSourceAltCloud(sources.DataSource): util.logexc(LOG, 'Failed command: %s\n%s', ' '.join(cmd), _err) return False except OSError as _err: - util.logexc(LOG, 'Failed command: %s\n%s', ' '.join(cmd), - _err.message) + util.logexc(LOG, 'Failed command: %s\n%s', ' '.join(cmd), _err) return False try: diff --git a/cloudinit/sources/DataSourceAzure.py b/cloudinit/sources/DataSourceAzure.py index c5af8b84..48a3e1df 100644 --- a/cloudinit/sources/DataSourceAzure.py +++ b/cloudinit/sources/DataSourceAzure.py @@ -111,50 +111,62 @@ class DataSourceAzureNet(sources.DataSource): root = sources.DataSource.__str__(self) return "%s [seed=%s]" % (root, self.seed) - def get_metadata_from_agent(self): - temp_hostname = self.metadata.get('local-hostname') + def bounce_network_with_azure_hostname(self): + # When using cloud-init to provision, we have to set the hostname from + # the metadata and "bounce" the network to force DDNS to update via + # dhclient + azure_hostname = self.metadata.get('local-hostname') + LOG.debug("Hostname in metadata is {}".format(azure_hostname)) hostname_command = self.ds_cfg['hostname_bounce']['hostname_command'] - agent_cmd = self.ds_cfg['agent_command'] - LOG.debug("Getting metadata via agent. hostname=%s cmd=%s", - temp_hostname, agent_cmd) - with temporary_hostname(temp_hostname, self.ds_cfg, + + with temporary_hostname(azure_hostname, self.ds_cfg, hostname_command=hostname_command) \ as previous_hostname: if (previous_hostname is not None and - util.is_true(self.ds_cfg.get('set_hostname'))): + util.is_true(self.ds_cfg.get('set_hostname'))): cfg = self.ds_cfg['hostname_bounce'] + + # "Bouncing" the network try: - perform_hostname_bounce(hostname=temp_hostname, + perform_hostname_bounce(hostname=azure_hostname, cfg=cfg, prev_hostname=previous_hostname) except Exception as e: LOG.warn("Failed publishing hostname: %s", e) util.logexc(LOG, "handling set_hostname failed") - try: - invoke_agent(agent_cmd) - except util.ProcessExecutionError: - # claim the datasource even if the command failed - util.logexc(LOG, "agent command '%s' failed.", - self.ds_cfg['agent_command']) - - ddir = self.ds_cfg['data_dir'] - - fp_files = [] - key_value = None - for pk in self.cfg.get('_pubkeys', []): - if pk.get('value', None): - key_value = pk['value'] - LOG.debug("ssh authentication: using value from fabric") - else: - bname = str(pk['fingerprint'] + ".crt") - fp_files += [os.path.join(ddir, bname)] - LOG.debug("ssh authentication: " - "using fingerprint from fabirc") - - missing = util.log_time(logfunc=LOG.debug, msg="waiting for files", - func=wait_for_files, - args=(fp_files,)) + def get_metadata_from_agent(self): + temp_hostname = self.metadata.get('local-hostname') + agent_cmd = self.ds_cfg['agent_command'] + LOG.debug("Getting metadata via agent. hostname=%s cmd=%s", + temp_hostname, agent_cmd) + + self.bounce_network_with_azure_hostname() + + try: + invoke_agent(agent_cmd) + except util.ProcessExecutionError: + # claim the datasource even if the command failed + util.logexc(LOG, "agent command '%s' failed.", + self.ds_cfg['agent_command']) + + ddir = self.ds_cfg['data_dir'] + + fp_files = [] + key_value = None + for pk in self.cfg.get('_pubkeys', []): + if pk.get('value', None): + key_value = pk['value'] + LOG.debug("ssh authentication: using value from fabric") + else: + bname = str(pk['fingerprint'] + ".crt") + fp_files += [os.path.join(ddir, bname)] + LOG.debug("ssh authentication: " + "using fingerprint from fabirc") + + missing = util.log_time(logfunc=LOG.debug, msg="waiting for files", + func=wait_for_files, + args=(fp_files,)) if len(missing): LOG.warn("Did not find files, but going on: %s", missing) @@ -220,6 +232,8 @@ class DataSourceAzureNet(sources.DataSource): write_files(ddir, files, dirmode=0o700) if self.ds_cfg['agent_command'] == AGENT_START_BUILTIN: + self.bounce_network_with_azure_hostname() + metadata_func = partial(get_metadata_from_fabric, fallback_lease_file=self. dhclient_lease_file) diff --git a/cloudinit/sources/DataSourceBigstep.py b/cloudinit/sources/DataSourceBigstep.py index 5ffdcb25..d7fcd45a 100644 --- a/cloudinit/sources/DataSourceBigstep.py +++ b/cloudinit/sources/DataSourceBigstep.py @@ -27,7 +27,7 @@ class DataSourceBigstep(sources.DataSource): if url is None: return False response = url_helper.readurl(url) - decoded = json.loads(response.contents) + decoded = json.loads(response.contents.decode()) self.metadata = decoded["metadata"] self.vendordata_raw = decoded["vendordata_raw"] self.userdata_raw = decoded["userdata_raw"] diff --git a/cloudinit/sources/DataSourceConfigDrive.py b/cloudinit/sources/DataSourceConfigDrive.py index 8a448dc9..46dd89e0 100644 --- a/cloudinit/sources/DataSourceConfigDrive.py +++ b/cloudinit/sources/DataSourceConfigDrive.py @@ -54,13 +54,16 @@ class DataSourceConfigDrive(openstack.SourceMixin, sources.DataSource): found = None md = {} results = {} - if os.path.isdir(self.seed_dir): + for sdir in (self.seed_dir, "/config-drive"): + if not os.path.isdir(sdir): + continue try: - results = read_config_drive(self.seed_dir) - found = self.seed_dir + results = read_config_drive(sdir) + found = sdir + break except openstack.NonReadable: - util.logexc(LOG, "Failed reading config drive from %s", - self.seed_dir) + util.logexc(LOG, "Failed reading config drive from %s", sdir) + if not found: for dev in find_candidate_devs(): try: diff --git a/cloudinit/sources/DataSourceGCE.py b/cloudinit/sources/DataSourceGCE.py index b1a1c8f2..637c9505 100644 --- a/cloudinit/sources/DataSourceGCE.py +++ b/cloudinit/sources/DataSourceGCE.py @@ -62,6 +62,9 @@ class DataSourceGCE(sources.DataSource): return public_key def get_data(self): + if not platform_reports_gce(): + return False + # url_map: (our-key, path, required, is_text) url_map = [ ('instance-id', ('instance/id',), True, True), @@ -144,6 +147,21 @@ class DataSourceGCE(sources.DataSource): return self.availability_zone.rsplit('-', 1)[0] +def platform_reports_gce(): + pname = util.read_dmi_data('system-product-name') or "N/A" + if pname == "Google Compute Engine": + return True + + # system-product-name is not always guaranteed (LP: #1674861) + serial = util.read_dmi_data('system-serial-number') or "N/A" + if serial.startswith("GoogleCloud-"): + return True + + LOG.debug("Not running on google cloud. product-name=%s serial=%s", + pname, serial) + return False + + # Used to match classes to dependencies datasources = [ (DataSourceGCE, (sources.DEP_FILESYSTEM, sources.DEP_NETWORK)), diff --git a/cloudinit/sources/DataSourceOpenNebula.py b/cloudinit/sources/DataSourceOpenNebula.py index 1f1baf46..cd75e6ea 100644 --- a/cloudinit/sources/DataSourceOpenNebula.py +++ b/cloudinit/sources/DataSourceOpenNebula.py @@ -286,12 +286,12 @@ def parse_shell_config(content, keylist=None, bash=None, asuser=None, output = output[0:-1] # remove trailing null # go through output. First _start_ is for 'preset', second for 'target'. - # Add to target only things were changed and not in volitile + # Add to ret only things were changed and not in excluded. for line in output.split("\x00"): try: (key, val) = line.split("=", 1) if target is preset: - target[key] = val + preset[key] = val elif (key not in excluded and (key in keylist_in or preset.get(key) != val)): ret[key] = val diff --git a/cloudinit/sources/__init__.py b/cloudinit/sources/__init__.py index 3d01072f..5c99437e 100644 --- a/cloudinit/sources/__init__.py +++ b/cloudinit/sources/__init__.py @@ -50,7 +50,7 @@ class DataSource(object): self.distro = distro self.paths = paths self.userdata = None - self.metadata = None + self.metadata = {} self.userdata_raw = None self.vendordata = None self.vendordata_raw = None @@ -210,7 +210,7 @@ class DataSource(object): else: hostname = toks[0] - if fqdn: + if fqdn and domain != defdomain: return "%s.%s" % (hostname, domain) else: return hostname diff --git a/cloudinit/sources/helpers/openstack.py b/cloudinit/sources/helpers/openstack.py index 096062d5..61cd36bd 100644 --- a/cloudinit/sources/helpers/openstack.py +++ b/cloudinit/sources/helpers/openstack.py @@ -52,6 +52,7 @@ OS_VERSIONS = ( PHYSICAL_TYPES = ( None, 'bridge', + 'dvs', 'ethernet', 'hw_veb', 'hyperv', diff --git a/cloudinit/stages.py b/cloudinit/stages.py index 5bed9032..12165433 100644 --- a/cloudinit/stages.py +++ b/cloudinit/stages.py @@ -646,9 +646,13 @@ class Init(object): src, bring_up, netcfg) try: return self.distro.apply_network_config(netcfg, bring_up=bring_up) + except net.RendererNotFoundError as e: + LOG.error("Unable to render networking. Network config is " + "likely broken: %s", e) + return except NotImplementedError: LOG.warn("distro '%s' does not implement apply_network_config. " - "networking may not be configured properly." % + "networking may not be configured properly.", self.distro) return diff --git a/cloudinit/url_helper.py b/cloudinit/url_helper.py index 312b0460..2f6a158e 100644 --- a/cloudinit/url_helper.py +++ b/cloudinit/url_helper.py @@ -45,7 +45,7 @@ try: from distutils.version import LooseVersion import pkg_resources _REQ = pkg_resources.get_distribution('requests') - _REQ_VER = LooseVersion(_REQ.version) + _REQ_VER = LooseVersion(_REQ.version) # pylint: disable=no-member if _REQ_VER >= LooseVersion('0.8.8'): SSL_ENABLED = True if _REQ_VER >= LooseVersion('0.7.0') and _REQ_VER < LooseVersion('1.0.0'): diff --git a/cloudinit/util.py b/cloudinit/util.py index 7196a7ca..17abdf81 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -2099,21 +2099,36 @@ def get_mount_info(path, log=LOG): return parse_mount(path) -def which(program): - # Return path of program for execution if found in path - def is_exe(fpath): - return os.path.isfile(fpath) and os.access(fpath, os.X_OK) - - _fpath, _ = os.path.split(program) - if _fpath: - if is_exe(program): +def is_exe(fpath): + # return boolean indicating if fpath exists and is executable. + return os.path.isfile(fpath) and os.access(fpath, os.X_OK) + + +def which(program, search=None, target=None): + target = target_path(target) + + if os.path.sep in program: + # if program had a '/' in it, then do not search PATH + # 'which' does consider cwd here. (cd / && which bin/ls) = bin/ls + # so effectively we set cwd to / (or target) + if is_exe(target_path(target, program)): return program - else: - for path in os.environ.get("PATH", "").split(os.pathsep): - path = path.strip('"') - exe_file = os.path.join(path, program) - if is_exe(exe_file): - return exe_file + + if search is None: + paths = [p.strip('"') for p in + os.environ.get("PATH", "").split(os.pathsep)] + if target == "/": + search = paths + else: + search = [p for p in paths if p.startswith("/")] + + # normalize path input + search = [os.path.abspath(p) for p in search] + + for path in search: + ppath = os.path.sep.join((path, program)) + if is_exe(target_path(target, ppath)): + return ppath return None @@ -2358,4 +2373,42 @@ def system_is_snappy(): return True return False + +def indent(text, prefix): + """replacement for indent from textwrap that is not available in 2.7.""" + lines = [] + for line in text.splitlines(True): + lines.append(prefix + line) + return ''.join(lines) + + +def rootdev_from_cmdline(cmdline): + found = None + for tok in cmdline.split(): + if tok.startswith("root="): + found = tok[5:] + break + if found is None: + return None + + if found.startswith("/dev/"): + return found + if found.startswith("LABEL="): + return "/dev/disk/by-label/" + found[len("LABEL="):] + if found.startswith("UUID="): + return "/dev/disk/by-uuid/" + found[len("UUID="):] + if found.startswith("PARTUUID="): + disks_path = "/dev/disk/by-partuuid/" + found[len("PARTUUID="):] + if os.path.exists(disks_path): + return disks_path + results = find_devs_with(found) + if results: + return results[0] + # we know this doesn't exist, but for consistency return the path as + # it /would/ exist + return disks_path + + return "/dev/" + found + + # vi: ts=4 expandtab diff --git a/cloudinit/version.py b/cloudinit/version.py index 92bace1a..dff4af04 100644 --- a/cloudinit/version.py +++ b/cloudinit/version.py @@ -6,6 +6,13 @@ __VERSION__ = "0.7.9" +FEATURES = [ + # supports network config version 1 + 'NETWORK_CONFIG_V1', + # supports network config version 2 (netplan) + 'NETWORK_CONFIG_V2', +] + def version_string(): return __VERSION__ |