merge trunk at 0.7.7~bzr1256

8 files changed, 215 insertions, 115 deletions

diff --git a/cloudinit/config/cc_mcollective.py b/cloudinit/config/cc_mcollective.py
index 425420ae..0c84d600 100644
--- a/cloudinit/config/cc_mcollective.py
+++ b/cloudinit/config/cc_mcollective.py
@@ -20,54 +20,49 @@
 #    along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 import six
-from six import StringIO
+from six import BytesIO
 
 # Used since this can maintain comments
 # and doesn't need a top level section
 from configobj import ConfigObj
 
+from cloudinit import log as logging
 from cloudinit import util
 
 PUBCERT_FILE = "/etc/mcollective/ssl/server-public.pem"
 PRICERT_FILE = "/etc/mcollective/ssl/server-private.pem"
 SERVER_CFG = '/etc/mcollective/server.cfg'
 
+LOG = logging.getLogger(__name__)
 
-def handle(name, cfg, cloud, log, _args):
-
-    # If there isn't a mcollective key in the configuration don't do anything
-    if 'mcollective' not in cfg:
-        log.debug(("Skipping module named %s, "
-                   "no 'mcollective' key in configuration"), name)
-        return
-
-    mcollective_cfg = cfg['mcollective']
-
-    # Start by installing the mcollective package ...
-    cloud.distro.install_packages(("mcollective",))
 
-    # ... and then update the mcollective configuration
-    if 'conf' in mcollective_cfg:
-        # Read server.cfg values from the
-        # original file in order to be able to mix the rest up
-        mcollective_config = ConfigObj(SERVER_CFG)
-        # See: http://tiny.cc/jh9agw
-        for (cfg_name, cfg) in mcollective_cfg['conf'].items():
+def configure(config):
+    # Read server.cfg values from the
+    # original file in order to be able to mix the rest up
+    try:
+        mcollective_config = ConfigObj(SERVER_CFG, file_error=True)
+    except IOError:
+        LOG.warn("Did not find file %s", SERVER_CFG)
+        mcollective_config = ConfigObj(config)
+    else:
+        for (cfg_name, cfg) in config.items():
             if cfg_name == 'public-cert':
                 util.write_file(PUBCERT_FILE, cfg, mode=0o644)
-                mcollective_config['plugin.ssl_server_public'] = PUBCERT_FILE
+                mcollective_config[
+                    'plugin.ssl_server_public'] = PUBCERT_FILE
                 mcollective_config['securityprovider'] = 'ssl'
             elif cfg_name == 'private-cert':
                 util.write_file(PRICERT_FILE, cfg, mode=0o600)
-                mcollective_config['plugin.ssl_server_private'] = PRICERT_FILE
+                mcollective_config[
+                    'plugin.ssl_server_private'] = PRICERT_FILE
                 mcollective_config['securityprovider'] = 'ssl'
             else:
                 if isinstance(cfg, six.string_types):
                     # Just set it in the 'main' section
                     mcollective_config[cfg_name] = cfg
                 elif isinstance(cfg, (dict)):
-                    # Iterate through the config items, create a section
-                    # if it is needed and then add/or create items as needed
+                    # Iterate through the config items, create a section if
+                    # it is needed and then add/or create items as needed
                     if cfg_name not in mcollective_config.sections:
                         mcollective_config[cfg_name] = {}
                     for (o, v) in cfg.items():
@@ -78,11 +73,30 @@ def handle(name, cfg, cloud, log, _args):
         # We got all our config as wanted we'll rename
         # the previous server.cfg and create our new one
         util.rename(SERVER_CFG, "%s.old" % (SERVER_CFG))
-        # Now we got the whole file, write to disk...
-        contents = StringIO()
-        mcollective_config.write(contents)
-        contents = contents.getvalue()
-        util.write_file(SERVER_CFG, contents, mode=0o644)
+
+    # Now we got the whole file, write to disk...
+    contents = BytesIO()
+    mcollective_config.write(contents)
+    contents = contents.getvalue()
+    util.write_file(SERVER_CFG, contents, mode=0o644)
+
+
+def handle(name, cfg, cloud, log, _args):
+
+    # If there isn't a mcollective key in the configuration don't do anything
+    if 'mcollective' not in cfg:
+        log.debug(("Skipping module named %s, "
+                   "no 'mcollective' key in configuration"), name)
+        return
+
+    mcollective_cfg = cfg['mcollective']
+
+    # Start by installing the mcollective package ...
+    cloud.distro.install_packages(("mcollective",))
+
+    # ... and then update the mcollective configuration
+    if 'conf' in mcollective_cfg:
+        configure(config=mcollective_cfg['conf'])
 
     # Start mcollective
     util.subp(['service', 'mcollective', 'start'], capture=False)
diff --git a/cloudinit/distros/__init__.py b/cloudinit/distros/__init__.py
index 14b500f8..40af8802 100644
--- a/cloudinit/distros/__init__.py
+++ b/cloudinit/distros/__init__.py
@@ -32,6 +32,8 @@ import stat
 from cloudinit import importer
 from cloudinit import log as logging
 from cloudinit import net
+from cloudinit.net import eni
+from cloudinit.net import network_state
 from cloudinit import ssh_util
 from cloudinit import type_utils
 from cloudinit import util
@@ -138,9 +140,31 @@ class Distro(object):
             return self._bring_up_interfaces(dev_names)
         return False
 
+    def _apply_network_from_network_config(self, netconfig, bring_up=True):
+        distro = self.__class__
+        LOG.warn("apply_network_config is not currently implemented "
+                 "for distribution '%s'.  Attempting to use apply_network",
+                 distro)
+        header = '\n'.join([
+            "# Converted from network_config for distro %s" % distro,
+            "# Implmentation of _write_network_config is needed."
+        ])
+        ns = network_state.parse_net_config_data(netconfig)
+        contents = eni.network_state_to_eni(
+            ns, header=header, render_hwaddress=True)
+        return self.apply_network(contents, bring_up=bring_up)
+
     def apply_network_config(self, netconfig, bring_up=False):
-        # Write it out
-        dev_names = self._write_network_config(netconfig)
+        # apply network config netconfig
+        # This method is preferred to apply_network which only takes
+        # a much less complete network config format (interfaces(5)).
+        try:
+            dev_names = self._write_network_config(netconfig)
+        except NotImplementedError:
+            # backwards compat until all distros have apply_network_config
+            return self._apply_network_from_network_config(
+                netconfig, bring_up=bring_up)
+
         # Now try to bring them up
         if bring_up:
             return self._bring_up_interfaces(dev_names)
diff --git a/cloudinit/distros/debian.py b/cloudinit/distros/debian.py
index 5ae9a509..f9b3b92e 100644
--- a/cloudinit/distros/debian.py
+++ b/cloudinit/distros/debian.py
@@ -55,7 +55,6 @@ class Distro(distros.Distro):
     hostname_conf_fn = "/etc/hostname"
     locale_conf_fn = "/etc/default/locale"
     network_conf_fn = "/etc/network/interfaces.d/50-cloud-init.cfg"
-    links_prefix = "/etc/systemd/network/50-cloud-init-"
 
     def __init__(self, name, cfg, paths):
         distros.Distro.__init__(self, name, cfg, paths)
@@ -67,7 +66,7 @@ class Distro(distros.Distro):
         self._net_renderer = eni.Renderer({
             'eni_path': self.network_conf_fn,
             'eni_header': ENI_HEADER,
-            'links_prefix_path': None,
+            'links_path_prefix': None,
             'netrules_path': None,
         })
 
diff --git a/cloudinit/net/__init__.py b/cloudinit/net/__init__.py
index 63e54f91..21cc602b 100644
--- a/cloudinit/net/__init__.py
+++ b/cloudinit/net/__init__.py
@@ -252,7 +252,8 @@ def _rename_interfaces(renames, strict_present=True, strict_busy=True,
         cur_bymac[mac] = cur
 
     def update_byname(bymac):
-        return {data['name']: data for data in bymac.values()}
+        return dict((data['name'], data)
+                    for data in bymac.values())
 
     def rename(cur, new):
         util.subp(["ip", "link", "set", cur, "name", new], capture=True)
diff --git a/cloudinit/net/eni.py b/cloudinit/net/eni.py
index e5ed10fd..eff5b924 100644
--- a/cloudinit/net/eni.py
+++ b/cloudinit/net/eni.py
@@ -12,6 +12,7 @@
 #    You should have received a copy of the GNU General Public License
 #    along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
+import copy
 import glob
 import os
 import re
@@ -42,7 +43,7 @@ NET_CONFIG_OPTIONS = [
 
 # TODO: switch valid_map based on mode inet/inet6
 def _iface_add_subnet(iface, subnet):
-    content = ""
+    content = []
     valid_map = [
         'address',
         'netmask',
@@ -61,15 +62,21 @@ def _iface_add_subnet(iface, subnet):
                 value = " ".join(value)
             if '_' in key:
                 key = key.replace('_', '-')
-            content += "    {} {}\n".format(key, value)
+            content.append("    {0} {1}".format(key, value))
 
-    return content
+    return sorted(content)
 
 
 # TODO: switch to valid_map for attrs
-
-def _iface_add_attrs(iface):
-    content = ""
+def _iface_add_attrs(iface, index):
+    # If the index is non-zero, this is an alias interface. Alias interfaces
+    # represent additional interface addresses, and should not have additional
+    # attributes. (extra attributes here are almost always either incorrect,
+    # or are applied to the parent interface.) So if this is an alias, stop
+    # right here.
+    if index != 0:
+        return []
+    content = []
     ignore_map = [
         'control',
         'index',
@@ -79,19 +86,21 @@ def _iface_add_attrs(iface):
         'subnets',
         'type',
     ]
+    renames = {'mac_address': 'hwaddress'}
     if iface['type'] not in ['bond', 'bridge', 'vlan']:
         ignore_map.append('mac_address')
 
     for key, value in iface.items():
-        if value and key not in ignore_map:
-            if type(value) == list:
-                value = " ".join(value)
-            content += "    {} {}\n".format(key, value)
+        if not value or key in ignore_map:
+            continue
+        if type(value) == list:
+            value = " ".join(value)
+        content.append("    {0} {1}".format(renames.get(key, key), value))
 
-    return content
+    return sorted(content)
 
 
-def _iface_start_entry(iface, index):
+def _iface_start_entry(iface, index, render_hwaddress=False):
     fullname = iface['name']
     if index != 0:
         fullname += ":%s" % index
@@ -107,8 +116,13 @@ def _iface_start_entry(iface, index):
     subst = iface.copy()
     subst.update({'fullname': fullname, 'cverb': cverb})
 
-    return ("{cverb} {fullname}\n"
-            "iface {fullname} {inet} {mode}\n").format(**subst)
+    lines = [
+        "{cverb} {fullname}".format(**subst),
+        "iface {fullname} {inet} {mode}".format(**subst)]
+    if render_hwaddress and iface.get('mac_address'):
+        lines.append("    hwaddress {mac_address}".format(**subst))
+
+    return lines
 
 
 def _parse_deb_config_data(ifaces, contents, src_dir, src_path):
@@ -262,10 +276,6 @@ def _ifaces_to_net_config_data(ifaces):
     for name, data in ifaces.items():
         # devname is 'eth0' for name='eth0:1'
         devname = name.partition(":")[0]
-        if devname == "lo":
-            # currently provding 'lo' in network config results in duplicate
-            # entries. in rendered interfaces file. so skip it.
-            continue
         if devname not in devs:
             devs[devname] = {'type': 'physical', 'name': devname,
                              'subnets': []}
@@ -324,10 +334,10 @@ class Renderer(renderer.Renderer):
         1. http://askubuntu.com/questions/168033/
                  how-to-set-static-routes-in-ubuntu-server
         """
-        content = ""
+        content = []
         up = indent + "post-up route add"
         down = indent + "pre-down route del"
-        eol = " || true\n"
+        or_true = " || true"
         mapping = {
             'network': '-net',
             'netmask': 'netmask',
@@ -336,34 +346,84 @@ class Renderer(renderer.Renderer):
         }
         if route['network'] == '0.0.0.0' and route['netmask'] == '0.0.0.0':
             default_gw = " default gw %s" % route['gateway']
-            content += up + default_gw + eol
-            content += down + default_gw + eol
+            content.append(up + default_gw + or_true)
+            content.append(down + default_gw + or_true)
         elif route['network'] == '::' and route['netmask'] == 0:
             # ipv6!
             default_gw = " -A inet6 default gw %s" % route['gateway']
-            content += up + default_gw + eol
-            content += down + default_gw + eol
+            content.append(up + default_gw + or_true)
+            content.append(down + default_gw + or_true)
         else:
             route_line = ""
             for k in ['network', 'netmask', 'gateway', 'metric']:
                 if k in route:
                     route_line += " %s %s" % (mapping[k], route[k])
-            content += up + route_line + eol
-            content += down + route_line + eol
+            content.append(up + route_line + or_true)
+            content.append(down + route_line + or_true)
         return content
 
-    def _render_interfaces(self, network_state):
+    def _render_iface(self, iface, render_hwaddress=False):
+        sections = []
+        subnets = iface.get('subnets', {})
+        if subnets:
+            for index, subnet in zip(range(0, len(subnets)), subnets):
+                iface['index'] = index
+                iface['mode'] = subnet['type']
+                iface['control'] = subnet.get('control', 'auto')
+                subnet_inet = 'inet'
+                if iface['mode'].endswith('6'):
+                    # This is a request for DHCPv6.
+                    subnet_inet += '6'
+                elif iface['mode'] == 'static' and ":" in subnet['address']:
+                    # This is a static IPv6 address.
+                    subnet_inet += '6'
+                iface['inet'] = subnet_inet
+                if iface['mode'].startswith('dhcp'):
+                    iface['mode'] = 'dhcp'
+
+                lines = list(
+                    _iface_start_entry(
+                        iface, index, render_hwaddress=render_hwaddress) +
+                    _iface_add_subnet(iface, subnet) +
+                    _iface_add_attrs(iface, index)
+                )
+                for route in subnet.get('routes', []):
+                    lines.extend(self._render_route(route, indent="    "))
+
+                if len(subnets) > 1 and index == 0:
+                    tmpl = "    post-up ifup %s:%s\n"
+                    for i in range(1, len(subnets)):
+                        lines.append(tmpl % (iface['name'], i))
+
+                sections.append(lines)
+        else:
+            # ifenslave docs say to auto the slave devices
+            lines = []
+            if 'bond-master' in iface:
+                lines.append("auto {name}".format(**iface))
+            lines.append("iface {name} {inet} {mode}".format(**iface))
+            lines.extend(_iface_add_attrs(iface, index=0))
+            sections.append(lines)
+        return sections
+
+    def _render_interfaces(self, network_state, render_hwaddress=False):
         '''Given state, emit etc/network/interfaces content.'''
 
-        content = ""
-        content += "auto lo\niface lo inet loopback\n"
+        # handle 'lo' specifically as we need to insert the global dns entries
+        # there (as that is the only interface that will be always up).
+        lo = {'name': 'lo', 'type': 'physical', 'inet': 'inet',
+              'subnets': [{'type': 'loopback', 'control': 'auto'}]}
+        for iface in network_state.iter_interfaces():
+            if iface.get('name') == "lo":
+                lo = copy.deepcopy(iface)
 
         nameservers = network_state.dns_nameservers
         if nameservers:
-            content += "    dns-nameservers %s\n" % (" ".join(nameservers))
+            lo['subnets'][0]["dns_nameservers"] = (" ".join(nameservers))
+
         searchdomains = network_state.dns_searchdomains
         if searchdomains:
-            content += "    dns-search %s\n" % (" ".join(searchdomains))
+            lo['subnets'][0]["dns_search"] = (" ".join(searchdomains))
 
         ''' Apply a sort order to ensure that we write out
             the physical interfaces first; this is critical for
@@ -375,45 +435,21 @@ class Renderer(renderer.Renderer):
             'bridge': 2,
             'vlan': 3,
         }
+
+        sections = []
+        sections.extend(self._render_iface(lo))
         for iface in sorted(network_state.iter_interfaces(),
                             key=lambda k: (order[k['type']], k['name'])):
 
-            if content[-2:] != "\n\n":
-                content += "\n"
-            subnets = iface.get('subnets', {})
-            if subnets:
-                for index, subnet in zip(range(0, len(subnets)), subnets):
-                    if content[-2:] != "\n\n":
-                        content += "\n"
-                    iface['index'] = index
-                    iface['mode'] = subnet['type']
-                    iface['control'] = subnet.get('control', 'auto')
-                    if iface['mode'].endswith('6'):
-                        iface['inet'] += '6'
-                    elif (iface['mode'] == 'static' and
-                          ":" in subnet['address']):
-                        iface['inet'] += '6'
-                    if iface['mode'].startswith('dhcp'):
-                        iface['mode'] = 'dhcp'
-
-                    content += _iface_start_entry(iface, index)
-                    content += _iface_add_subnet(iface, subnet)
-                    content += _iface_add_attrs(iface)
-                    for route in subnet.get('routes', []):
-                        content += self._render_route(route, indent="    ")
-            else:
-                # ifenslave docs say to auto the slave devices
-                if 'bond-master' in iface:
-                    content += "auto {name}\n".format(**iface)
-                content += "iface {name} {inet} {mode}\n".format(**iface)
-                content += _iface_add_attrs(iface)
+            if iface.get('name') == "lo":
+                continue
+            sections.extend(
+                self._render_iface(iface, render_hwaddress=render_hwaddress))
 
         for route in network_state.iter_routes():
-            content += self._render_route(route)
+            sections.append(self._render_route(route))
 
-        # global replacements until v2 format
-        content = content.replace('mac_address', 'hwaddress')
-        return content
+        return '\n\n'.join(['\n'.join(s) for s in sections]) + "\n"
 
     def render_network_state(self, target, network_state):
         fpeni = os.path.join(target, self.eni_path)
@@ -448,3 +484,21 @@ class Renderer(renderer.Renderer):
                     ""
                 ])
                 util.write_file(fname, content)
+
+
+def network_state_to_eni(network_state, header=None, render_hwaddress=False):
+    # render the provided network state, return a string of equivalent eni
+    eni_path = 'etc/network/interfaces'
+    renderer = Renderer({
+        'eni_path': eni_path,
+        'eni_header': header,
+        'links_path_prefix': None,
+        'netrules_path': None,
+    })
+    if not header:
+        header = ""
+    if not header.endswith("\n"):
+        header += "\n"
+    contents = renderer._render_interfaces(
+        network_state, render_hwaddress=render_hwaddress)
+    return header + contents
diff --git a/cloudinit/sources/DataSourceConfigDrive.py b/cloudinit/sources/DataSourceConfigDrive.py
index 3130e618..91d6ff13 100644
--- a/cloudinit/sources/DataSourceConfigDrive.py
+++ b/cloudinit/sources/DataSourceConfigDrive.py
@@ -107,12 +107,19 @@ class DataSourceConfigDrive(openstack.SourceMixin, sources.DataSource):
         if self.dsmode == sources.DSMODE_DISABLED:
             return False
 
-        # This is legacy and sneaky.  If dsmode is 'pass' then write
-        # 'injected files' and apply legacy ENI network format.
         prev_iid = get_previous_iid(self.paths)
         cur_iid = md['instance-id']
-        if prev_iid != cur_iid and self.dsmode == sources.DSMODE_PASS:
-            on_first_boot(results, distro=self.distro)
+        if prev_iid != cur_iid:
+            # better would be to handle this centrally, allowing
+            # the datasource to do something on new instance id
+            # note, networking is only rendered here if dsmode is DSMODE_PASS
+            # which means "DISABLED, but render files and networking"
+            on_first_boot(results, distro=self.distro,
+                          network=self.dsmode == sources.DSMODE_PASS)
+
+        # This is legacy and sneaky.  If dsmode is 'pass' then do not claim
+        # the datasource was used, even though we did run on_first_boot above.
+        if self.dsmode == sources.DSMODE_PASS:
             LOG.debug("%s: not claiming datasource, dsmode=%s", self,
                       self.dsmode)
             return False
@@ -184,15 +191,16 @@ def get_previous_iid(paths):
         return None
 
 
-def on_first_boot(data, distro=None):
+def on_first_boot(data, distro=None, network=True):
     """Performs any first-boot actions using data read from a config-drive."""
     if not isinstance(data, dict):
         raise TypeError("Config-drive data expected to be a dict; not %s"
                         % (type(data)))
-    net_conf = data.get("network_config", '')
-    if net_conf and distro:
-        LOG.warn("Updating network interfaces from config drive")
-        distro.apply_network(net_conf)
+    if network:
+        net_conf = data.get("network_config", '')
+        if net_conf and distro:
+            LOG.warn("Updating network interfaces from config drive")
+            distro.apply_network(net_conf)
     write_injected_files(data.get('files'))
 
 
diff --git a/cloudinit/sources/DataSourceSmartOS.py b/cloudinit/sources/DataSourceSmartOS.py
index 08bc132b..ccc86883 100644
--- a/cloudinit/sources/DataSourceSmartOS.py
+++ b/cloudinit/sources/DataSourceSmartOS.py
@@ -718,8 +718,8 @@ def convert_smartos_network_data(network_data=None):
 
     config = []
     for nic in network_data:
-        cfg = {k: v for k, v in nic.items()
-               if k in valid_keys['physical']}
+        cfg = dict((k, v) for k, v in nic.items()
+                   if k in valid_keys['physical'])
         cfg.update({
             'type': 'physical',
             'name': nic['interface']})
@@ -728,8 +728,8 @@ def convert_smartos_network_data(network_data=None):
 
         subnets = []
         for ip, gw in zip(nic['ips'], nic['gateways']):
-            subnet = {k: v for k, v in nic.items()
-                      if k in valid_keys['subnet']}
+            subnet = dict((k, v) for k, v in nic.items()
+                          if k in valid_keys['subnet'])
             subnet.update({
                 'type': 'static',
                 'address': ip,
diff --git a/cloudinit/sources/helpers/openstack.py b/cloudinit/sources/helpers/openstack.py
index d52cb56a..2e7a1d47 100644
--- a/cloudinit/sources/helpers/openstack.py
+++ b/cloudinit/sources/helpers/openstack.py
@@ -542,8 +542,8 @@ def convert_net_json(network_json=None, known_macs=None):
     config = []
     for link in links:
         subnets = []
-        cfg = {k: v for k, v in link.items()
-               if k in valid_keys['physical']}
+        cfg = dict((k, v) for k, v in link.items()
+                   if k in valid_keys['physical'])
         # 'name' is not in openstack spec yet, but we will support it if it is
         # present.  The 'id' in the spec is currently implemented as the host
         # nic's name, meaning something like 'tap-adfasdffd'.  We do not want
@@ -553,8 +553,8 @@ def convert_net_json(network_json=None, known_macs=None):
 
         for network in [n for n in networks
                         if n['link'] == link['id']]:
-            subnet = {k: v for k, v in network.items()
-                      if k in valid_keys['subnet']}
+            subnet = dict((k, v) for k, v in network.items()
+                          if k in valid_keys['subnet'])
             if 'dhcp' in network['type']:
                 t = 'dhcp6' if network['type'].startswith('ipv6') else 'dhcp4'
                 subnet.update({