summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rw-r--r--ChangeLog9
-rw-r--r--cloudinit/cmd/__init__.py21
-rw-r--r--[-rwxr-xr-x]cloudinit/cmd/main.py (renamed from bin/cloud-init)46
-rw-r--r--cloudinit/config/cc_apt_configure.py151
-rw-r--r--cloudinit/config/cc_growpart.py10
-rw-r--r--cloudinit/cs_utils.py3
-rw-r--r--cloudinit/distros/__init__.py2
-rw-r--r--cloudinit/distros/arch.py4
-rw-r--r--cloudinit/distros/debian.py16
-rw-r--r--cloudinit/distros/rhel.py8
-rw-r--r--cloudinit/gpg.py74
-rw-r--r--cloudinit/net/__init__.py693
-rw-r--r--cloudinit/net/cmdline.py203
-rw-r--r--cloudinit/net/eni.py460
-rw-r--r--cloudinit/net/network_state.py365
-rw-r--r--cloudinit/net/renderer.py48
-rw-r--r--cloudinit/net/sysconfig.py400
-rw-r--r--cloudinit/serial.py50
-rw-r--r--cloudinit/sources/DataSourceAltCloud.py3
-rw-r--r--cloudinit/sources/DataSourceAzure.py2
-rw-r--r--cloudinit/sources/DataSourceConfigDrive.py152
-rw-r--r--cloudinit/sources/DataSourceGCE.py2
-rw-r--r--cloudinit/sources/DataSourceNoCloud.py19
-rw-r--r--cloudinit/sources/DataSourceOVF.py10
-rw-r--r--cloudinit/sources/DataSourceOpenNebula.py2
-rw-r--r--cloudinit/sources/DataSourceSmartOS.py4
-rw-r--r--cloudinit/sources/helpers/openstack.py147
-rw-r--r--cloudinit/stages.py5
-rw-r--r--cloudinit/templater.py16
-rw-r--r--cloudinit/util.py28
-rw-r--r--doc/examples/cloud-config.txt127
-rwxr-xr-xpackages/bddeb7
-rw-r--r--requirements.txt8
-rwxr-xr-xsetup.py9
-rw-r--r--test-requirements.txt1
-rw-r--r--tests/unittests/helpers.py97
-rw-r--r--tests/unittests/test__init__.py17
-rw-r--r--tests/unittests/test_cli.py46
-rw-r--r--tests/unittests/test_cs_util.py27
-rw-r--r--tests/unittests/test_datasource/test_azure.py12
-rw-r--r--tests/unittests/test_datasource/test_azure_helper.py13
-rw-r--r--tests/unittests/test_datasource/test_cloudsigma.py2
-rw-r--r--tests/unittests/test_datasource/test_cloudstack.py11
-rw-r--r--tests/unittests/test_datasource/test_configdrive.py203
-rw-r--r--tests/unittests/test_datasource/test_gce.py2
-rw-r--r--tests/unittests/test_datasource/test_nocloud.py15
-rw-r--r--tests/unittests/test_datasource/test_smartos.py10
-rw-r--r--tests/unittests/test_handler/test_handler_apt_configure_sources_list.py180
-rw-r--r--tests/unittests/test_handler/test_handler_apt_source.py516
-rw-r--r--tests/unittests/test_net.py277
-rw-r--r--tests/unittests/test_reporting.py4
-rw-r--r--tests/unittests/test_rh_subscription.py22
-rwxr-xr-xtools/run-pep85
-rwxr-xr-xtools/run-pyflakes2
-rw-r--r--tox.ini17
55 files changed, 3177 insertions, 1406 deletions
diff --git a/ChangeLog b/ChangeLog
index d6adad85..10bd58b8 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -113,8 +113,17 @@
- settings on the kernel command line (cc:) override all local settings
rather than only those in /etc/cloud/cloud.cfg (LP: #1582323)
- Improve merging documentation [Daniel Watkins]
+ - apt sources: support inserting key/key-id only, custom sources.list,
+ long gpg key fingerprints with spaces, and dictionary format (LP: #1574113)
- SmartOS: datasource improvements and support for metadata service
providing networking information.
+ - Datasources: centrally handle 'dsmode' and no longer require datasources
+ to "pass" if modules_init should be executed with network access.
+ - ConfigDrive: improved support for networking information from
+ a network_data.json or older interfaces formated network_config.
+ - Change missing Cheetah log warning to debug [Andrew Jorgensen]
+ - Remove trailing dot from GCE metadata URL (LP: #1581200) [Phil Roche]
+ - support network rendering to sysconfig (for centos and RHEL)
0.7.6:
- open 0.7.6
diff --git a/cloudinit/cmd/__init__.py b/cloudinit/cmd/__init__.py
new file mode 100644
index 00000000..da124641
--- /dev/null
+++ b/cloudinit/cmd/__init__.py
@@ -0,0 +1,21 @@
+# vi: ts=4 expandtab
+#
+# Copyright (C) 2012 Canonical Ltd.
+# Copyright (C) 2012 Hewlett-Packard Development Company, L.P.
+# Copyright (C) 2012 Yahoo! Inc.
+#
+# Author: Scott Moser <scott.moser@canonical.com>
+# Author: Juerg Haefliger <juerg.haefliger@hp.com>
+# Author: Joshua Harlow <harlowja@yahoo-inc.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 3, as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
diff --git a/bin/cloud-init b/cloudinit/cmd/main.py
index 21c3a684..63621c1d 100755..100644
--- a/bin/cloud-init
+++ b/cloudinit/cmd/main.py
@@ -25,19 +25,12 @@ import argparse
import json
import os
import sys
-import time
import tempfile
+import time
import traceback
-# This is more just for running from the bin folder so that
-# cloud-init binary can find the cloudinit module
-possible_topdir = os.path.normpath(os.path.join(os.path.abspath(
- sys.argv[0]), os.pardir, os.pardir))
-if os.path.exists(os.path.join(possible_topdir, "cloudinit", "__init__.py")):
- sys.path.insert(0, possible_topdir)
-
from cloudinit import patcher
-patcher.patch()
+patcher.patch() # noqa
from cloudinit import log as logging
from cloudinit import netinfo
@@ -46,9 +39,10 @@ from cloudinit import sources
from cloudinit import stages
from cloudinit import templater
from cloudinit import util
+from cloudinit import version
+
from cloudinit import reporting
from cloudinit.reporting import events
-from cloudinit import version
from cloudinit.settings import (PER_INSTANCE, PER_ALWAYS, PER_ONCE,
CLOUD_CONFIG)
@@ -188,7 +182,7 @@ def main_init(name, args):
LOG.debug("Closing stdin")
util.close_stdin()
(outfmt, errfmt) = util.fixup_output(init.cfg, name)
- except:
+ except Exception:
util.logexc(LOG, "Failed to setup output redirection!")
print_exc("Failed to setup output redirection!")
if args.debug:
@@ -325,7 +319,7 @@ def main_init(name, args):
if outfmt_orig != outfmt or errfmt_orig != errfmt:
LOG.warn("Stdout, stderr changing to (%s, %s)", outfmt, errfmt)
(outfmt, errfmt) = util.fixup_output(mods.cfg, name)
- except:
+ except Exception:
util.logexc(LOG, "Failed to re-adjust output redirection!")
logging.setupLogging(mods.cfg)
@@ -367,7 +361,7 @@ def main_modules(action_name, args):
LOG.debug("Closing stdin")
util.close_stdin()
util.fixup_output(mods.cfg, name)
- except:
+ except Exception:
util.logexc(LOG, "Failed to setup output redirection!")
if args.debug:
# Reset so that all the debug handlers are closed out
@@ -430,7 +424,7 @@ def main_single(name, args):
LOG.debug("Closing stdin")
util.close_stdin()
util.fixup_output(mods.cfg, None)
- except:
+ except Exception:
util.logexc(LOG, "Failed to setup output redirection!")
if args.debug:
# Reset so that all the debug handlers are closed out
@@ -510,7 +504,7 @@ def status_wrapper(name, args, data_d=None, link_d=None):
else:
try:
status = json.loads(util.load_file(status_path))
- except:
+ except Exception:
pass
if status is None:
@@ -569,8 +563,12 @@ def status_wrapper(name, args, data_d=None, link_d=None):
return len(v1[mode]['errors'])
-def main():
- parser = argparse.ArgumentParser()
+def main(sysv_args=None):
+ if sysv_args is not None:
+ parser = argparse.ArgumentParser(prog=sysv_args[0])
+ sysv_args = sysv_args[1:]
+ else:
+ parser = argparse.ArgumentParser()
# Top level args
parser.add_argument('--version', '-v', action='version',
@@ -646,7 +644,12 @@ def main():
' pass to this module'))
parser_single.set_defaults(action=('single', main_single))
- args = parser.parse_args()
+ args = parser.parse_args(args=sysv_args)
+
+ try:
+ (name, functor) = args.action
+ except AttributeError:
+ parser.error('too few arguments')
# Setup basic logging to start (until reinitialized)
# iff in debug mode...
@@ -656,9 +659,6 @@ def main():
# Setup signal handlers before running
signal_handler.attach_handlers()
- if not hasattr(args, 'action'):
- parser.error('too few arguments')
- (name, functor) = args.action
if name in ("modules", "init"):
functor = status_wrapper
@@ -683,7 +683,3 @@ def main():
return util.log_time(
logfunc=LOG.debug, msg="cloud-init mode '%s'" % name,
get_uptime=True, func=functor, args=(name, args))
-
-
-if __name__ == '__main__':
- sys.exit(main())
diff --git a/cloudinit/config/cc_apt_configure.py b/cloudinit/config/cc_apt_configure.py
index e3fadc12..05ad4b03 100644
--- a/cloudinit/config/cc_apt_configure.py
+++ b/cloudinit/config/cc_apt_configure.py
@@ -22,6 +22,7 @@ import glob
import os
import re
+from cloudinit import gpg
from cloudinit import templater
from cloudinit import util
@@ -34,21 +35,6 @@ APT_PROXY_FN = "/etc/apt/apt.conf.d/95cloud-init-proxy"
# this will match 'XXX:YYY' (ie, 'cloud-archive:foo' or 'ppa:bar')
ADD_APT_REPO_MATCH = r"^[\w-]+:\w"
-# A temporary shell program to get a given gpg key
-# from a given keyserver
-EXPORT_GPG_KEYID = """
- k=${1} ks=${2};
- exec 2>/dev/null
- [ -n "$k" ] || exit 1;
- armour=$(gpg --list-keys --armour "${k}")
- if [ -z "${armour}" ]; then
- gpg --keyserver ${ks} --recv $k >/dev/null &&
- armour=$(gpg --export --armour "${k}") &&
- gpg --batch --yes --delete-keys "${k}"
- fi
- [ -n "${armour}" ] && echo "${armour}"
-"""
-
def handle(name, cfg, cloud, log, _args):
if util.is_false(cfg.get('apt_configure_enabled', True)):
@@ -70,7 +56,7 @@ def handle(name, cfg, cloud, log, _args):
if not util.get_cfg_option_bool(cfg,
'apt_preserve_sources_list', False):
- generate_sources_list(release, mirrors, cloud, log)
+ generate_sources_list(cfg, release, mirrors, cloud, log)
old_mirrors = cfg.get('apt_old_mirrors',
{"primary": "archive.ubuntu.com/ubuntu",
"security": "security.ubuntu.com/ubuntu"})
@@ -94,8 +80,8 @@ def handle(name, cfg, cloud, log, _args):
def matcher(x):
return False
- errors = add_sources(cfg['apt_sources'], params,
- aa_repo_match=matcher)
+ errors = add_apt_sources(cfg['apt_sources'], params,
+ aa_repo_match=matcher)
for e in errors:
log.warn("Add source error: %s", ':'.join(e))
@@ -108,17 +94,7 @@ def handle(name, cfg, cloud, log, _args):
util.logexc(log, "Failed to run debconf-set-selections")
-# get gpg keyid from keyserver
-def getkeybyid(keyid, keyserver):
- with util.ExtendedTemporaryFile(suffix='.sh', mode="w+", ) as fh:
- fh.write(EXPORT_GPG_KEYID)
- fh.flush()
- cmd = ['/bin/sh', fh.name, keyid, keyserver]
- (stdout, _stderr) = util.subp(cmd)
- return stdout.strip()
-
-
-def mirror2lists_fileprefix(mirror):
+def mirrorurl_to_apt_fileprefix(mirror):
string = mirror
# take off http:// or ftp://
if string.endswith("/"):
@@ -135,8 +111,8 @@ def rename_apt_lists(old_mirrors, new_mirrors, lists_d="/var/lib/apt/lists"):
nmirror = new_mirrors.get(name)
if not nmirror:
continue
- oprefix = os.path.join(lists_d, mirror2lists_fileprefix(omirror))
- nprefix = os.path.join(lists_d, mirror2lists_fileprefix(nmirror))
+ oprefix = os.path.join(lists_d, mirrorurl_to_apt_fileprefix(omirror))
+ nprefix = os.path.join(lists_d, mirrorurl_to_apt_fileprefix(nmirror))
if oprefix == nprefix:
continue
olen = len(oprefix)
@@ -149,7 +125,17 @@ def get_release():
return stdout.strip()
-def generate_sources_list(codename, mirrors, cloud, log):
+def generate_sources_list(cfg, codename, mirrors, cloud, log):
+ params = {'codename': codename}
+ for k in mirrors:
+ params[k] = mirrors[k]
+
+ custtmpl = cfg.get('apt_custom_sources_list', None)
+ if custtmpl is not None:
+ templater.render_string_to_file(custtmpl,
+ '/etc/apt/sources.list', params)
+ return
+
template_fn = cloud.get_template_filename('sources.list.%s' %
(cloud.distro.name))
if not template_fn:
@@ -158,13 +144,61 @@ def generate_sources_list(codename, mirrors, cloud, log):
log.warn("No template found, not rendering /etc/apt/sources.list")
return
- params = {'codename': codename}
- for k in mirrors:
- params[k] = mirrors[k]
templater.render_to_file(template_fn, '/etc/apt/sources.list', params)
-def add_sources(srclist, template_params=None, aa_repo_match=None):
+def add_apt_key_raw(key):
+ """
+ actual adding of a key as defined in key argument
+ to the system
+ """
+ try:
+ util.subp(('apt-key', 'add', '-'), key)
+ except util.ProcessExecutionError:
+ raise ValueError('failed to add apt GPG Key to apt keyring')
+
+
+def add_apt_key(ent):
+ """
+ add key to the system as defined in ent (if any)
+ supports raw keys or keyid's
+ The latter will as a first step fetch the raw key from a keyserver
+ """
+ if 'keyid' in ent and 'key' not in ent:
+ keyserver = "keyserver.ubuntu.com"
+ if 'keyserver' in ent:
+ keyserver = ent['keyserver']
+ ent['key'] = gpg.get_key_by_id(ent['keyid'], keyserver)
+
+ if 'key' in ent:
+ add_apt_key_raw(ent['key'])
+
+
+def convert_to_new_format(srclist):
+ """convert_to_new_format
+ convert the old list based format to the new dict based one
+ """
+ srcdict = {}
+ if isinstance(srclist, list):
+ for srcent in srclist:
+ if 'filename' not in srcent:
+ # file collides for multiple !filename cases for compatibility
+ # yet we need them all processed, so not same dictionary key
+ srcent['filename'] = "cloud_config_sources.list"
+ key = util.rand_dict_key(srcdict, "cloud_config_sources.list")
+ else:
+ # all with filename use that as key (matching new format)
+ key = srcent['filename']
+ srcdict[key] = srcent
+ elif isinstance(srclist, dict):
+ srcdict = srclist
+ else:
+ raise ValueError("unknown apt_sources format")
+
+ return srcdict
+
+
+def add_apt_sources(srclist, template_params=None, aa_repo_match=None):
"""
add entries in /etc/apt/sources.list.d for each abbreviated
sources.list entry in 'srclist'. When rendering template, also
@@ -174,18 +208,34 @@ def add_sources(srclist, template_params=None, aa_repo_match=None):
template_params = {}
if aa_repo_match is None:
- def aa_repo_match(x):
+ def _aa_repo_match(x):
return False
+ aa_repo_match = _aa_repo_match
errorlist = []
- for ent in srclist:
+ srcdict = convert_to_new_format(srclist)
+
+ for filename in srcdict:
+ ent = srcdict[filename]
+ if 'filename' not in ent:
+ ent['filename'] = filename
+
+ # keys can be added without specifying a source
+ try:
+ add_apt_key(ent)
+ except ValueError as detail:
+ errorlist.append([ent, detail])
+
if 'source' not in ent:
errorlist.append(["", "missing source"])
continue
-
source = ent['source']
source = templater.render_string(source, template_params)
+ if not ent['filename'].startswith(os.path.sep):
+ ent['filename'] = os.path.join("/etc/apt/sources.list.d/",
+ ent['filename'])
+
if aa_repo_match(source):
try:
util.subp(["add-apt-repository", source])
@@ -194,29 +244,6 @@ def add_sources(srclist, template_params=None, aa_repo_match=None):
("add-apt-repository failed. " + str(e))])
continue
- if 'filename' not in ent:
- ent['filename'] = 'cloud_config_sources.list'
-
- if not ent['filename'].startswith("/"):
- ent['filename'] = os.path.join("/etc/apt/sources.list.d/",
- ent['filename'])
-
- if ('keyid' in ent and 'key' not in ent):
- ks = "keyserver.ubuntu.com"
- if 'keyserver' in ent:
- ks = ent['keyserver']
- try:
- ent['key'] = getkeybyid(ent['keyid'], ks)
- except Exception:
- errorlist.append([source, "failed to get key from %s" % ks])
- continue
-
- if 'key' in ent:
- try:
- util.subp(('apt-key', 'add', '-'), ent['key'])
- except Exception:
- errorlist.append([source, "failed add key"])
-
try:
contents = "%s\n" % (source)
util.write_file(ent['filename'], contents, omode="ab")
diff --git a/cloudinit/config/cc_growpart.py b/cloudinit/config/cc_growpart.py
index 859d69f1..40560f11 100644
--- a/cloudinit/config/cc_growpart.py
+++ b/cloudinit/config/cc_growpart.py
@@ -36,13 +36,13 @@ DEFAULT_CONFIG = {
}
-def enum(**enums):
- return type('Enum', (), enums)
+class RESIZE(object):
+ SKIPPED = "SKIPPED"
+ CHANGED = "CHANGED"
+ NOCHANGE = "NOCHANGE"
+ FAILED = "FAILED"
-RESIZE = enum(SKIPPED="SKIPPED", CHANGED="CHANGED", NOCHANGE="NOCHANGE",
- FAILED="FAILED")
-
LOG = logging.getLogger(__name__)
diff --git a/cloudinit/cs_utils.py b/cloudinit/cs_utils.py
index 83ac1a0e..412431f2 100644
--- a/cloudinit/cs_utils.py
+++ b/cloudinit/cs_utils.py
@@ -33,7 +33,8 @@ API Docs: http://cloudsigma-docs.readthedocs.org/en/latest/server_context.html
import json
import platform
-import serial
+from cloudinit import serial
+
# these high timeouts are necessary as read may read a lot of data.
READ_TIMEOUT = 60
diff --git a/cloudinit/distros/__init__.py b/cloudinit/distros/__init__.py
index 5c29c804..14b500f8 100644
--- a/cloudinit/distros/__init__.py
+++ b/cloudinit/distros/__init__.py
@@ -454,7 +454,7 @@ class Distro(object):
keys = kwargs['ssh_authorized_keys']
if isinstance(keys, six.string_types):
keys = [keys]
- if isinstance(keys, dict):
+ elif isinstance(keys, dict):
keys = list(keys.values())
if keys is not None:
if not isinstance(keys, (tuple, list, set)):
diff --git a/cloudinit/distros/arch.py b/cloudinit/distros/arch.py
index 93a2e008..66209f22 100644
--- a/cloudinit/distros/arch.py
+++ b/cloudinit/distros/arch.py
@@ -196,6 +196,6 @@ def convert_resolv_conf(settings):
"""Returns a settings string formatted for resolv.conf."""
result = ''
if isinstance(settings, list):
- for ns in list:
+ for ns in settings:
result = result + 'nameserver %s\n' % ns
- return result
+ return result
diff --git a/cloudinit/distros/debian.py b/cloudinit/distros/debian.py
index 32bef1cd..53f3aa4d 100644
--- a/cloudinit/distros/debian.py
+++ b/cloudinit/distros/debian.py
@@ -25,7 +25,8 @@ import os
from cloudinit import distros
from cloudinit import helpers
from cloudinit import log as logging
-from cloudinit import net
+from cloudinit.net import eni
+from cloudinit.net.network_state import parse_net_config_data
from cloudinit import util
from cloudinit.distros.parsers.hostname import HostnameConf
@@ -56,6 +57,11 @@ class Distro(distros.Distro):
# should only happen say once per instance...)
self._runner = helpers.Runners(paths)
self.osfamily = 'debian'
+ self._net_renderer = eni.Renderer({
+ 'eni_path': self.network_conf_fn,
+ 'links_prefix_path': self.links_prefix,
+ 'netrules_path': None,
+ })
def apply_locale(self, locale, out_fn=None):
if not out_fn:
@@ -79,13 +85,9 @@ class Distro(distros.Distro):
return ['all']
def _write_network_config(self, netconfig):
- ns = net.parse_net_config_data(netconfig)
- net.render_network_state(target="/", network_state=ns,
- eni=self.network_conf_fn,
- links_prefix=self.links_prefix,
- netrules=None)
+ ns = parse_net_config_data(netconfig)
+ self._net_renderer.render_network_state("/", ns)
_maybe_remove_legacy_eth0()
-
return []
def _bring_up_interfaces(self, device_names):
diff --git a/cloudinit/distros/rhel.py b/cloudinit/distros/rhel.py
index 812e7002..1aa42d75 100644
--- a/cloudinit/distros/rhel.py
+++ b/cloudinit/distros/rhel.py
@@ -23,6 +23,8 @@
from cloudinit import distros
from cloudinit import helpers
from cloudinit import log as logging
+from cloudinit.net.network_state import parse_net_config_data
+from cloudinit.net import sysconfig
from cloudinit import util
from cloudinit.distros import net_util
@@ -59,10 +61,16 @@ class Distro(distros.Distro):
# should only happen say once per instance...)
self._runner = helpers.Runners(paths)
self.osfamily = 'redhat'
+ self._net_renderer = sysconfig.Renderer()
def install_packages(self, pkglist):
self.package_command('install', pkgs=pkglist)
+ def _write_network_config(self, netconfig):
+ ns = parse_net_config_data(netconfig)
+ self._net_renderer.render_network_state("/", ns)
+ return []
+
def _write_network(self, settings):
# TODO(harlowja) fix this... since this is the ubuntu format
entries = net_util.translate_network(settings)
diff --git a/cloudinit/gpg.py b/cloudinit/gpg.py
new file mode 100644
index 00000000..6a76d785
--- /dev/null
+++ b/cloudinit/gpg.py
@@ -0,0 +1,74 @@
+"""gpg.py - Collection of gpg key related functions"""
+# vi: ts=4 expandtab
+#
+# Copyright (C) 2016 Canonical Ltd.
+#
+# Author: Scott Moser <scott.moser@canonical.com>
+# Author: Christian Ehrhardt <christian.ehrhardt@canonical.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 3, as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+from cloudinit import log as logging
+from cloudinit import util
+
+LOG = logging.getLogger(__name__)
+
+
+def export_armour(key):
+ """Export gpg key, armoured key gets returned"""
+ try:
+ (armour, _) = util.subp(["gpg", "--export", "--armour", key],
+ capture=True)
+ except util.ProcessExecutionError as error:
+ # debug, since it happens for any key not on the system initially
+ LOG.debug('Failed to export armoured key "%s": %s', key, error)
+ armour = None
+ return armour
+
+
+def receive_key(key, keyserver):
+ """Receive gpg key from the specified keyserver"""
+ LOG.debug('Receive gpg key "%s"', key)
+ try:
+ util.subp(["gpg", "--keyserver", keyserver, "--recv-keys", key],
+ capture=True)
+ except util.ProcessExecutionError as error:
+ raise ValueError(('Failed to import key "%s" '
+ 'from server "%s" - error %s') %
+ (key, keyserver, error))
+
+
+def delete_key(key):
+ """Delete the specified key from the local gpg ring"""
+ try:
+ util.subp(["gpg", "--batch", "--yes", "--delete-keys", key],
+ capture=True)
+ except util.ProcessExecutionError as error:
+ LOG.warn('Failed delete key "%s": %s', key, error)
+
+
+def get_key_by_id(keyid, keyserver="keyserver.ubuntu.com"):
+ """get gpg keyid from keyserver"""
+ armour = export_armour(keyid)
+ if not armour:
+ try:
+ receive_key(keyid, keyserver=keyserver)
+ armour = export_armour(keyid)
+ except ValueError:
+ LOG.exception('Failed to obtain gpg key %s', keyid)
+ raise
+ finally:
+ # delete just imported key to leave environment as it was before
+ delete_key(keyid)
+
+ return armour
diff --git a/cloudinit/net/__init__.py b/cloudinit/net/__init__.py
index 0066561e..6959ad34 100644
--- a/cloudinit/net/__init__.py
+++ b/cloudinit/net/__init__.py
@@ -16,42 +16,15 @@
# You should have received a copy of the GNU Affero General Public License
# along with Curtin. If not, see <http://www.gnu.org/licenses/>.
-import base64
-import copy
import errno
-import glob
-import gzip
-import io
+import logging
import os
import re
-import shlex
-from cloudinit import log as logging
-from cloudinit.net import network_state
-from cloudinit.net.udev import generate_udev_rule
from cloudinit import util
LOG = logging.getLogger(__name__)
-
SYS_CLASS_NET = "/sys/class/net/"
-LINKS_FNAME_PREFIX = "etc/systemd/network/50-cloud-init-"
-
-NET_CONFIG_OPTIONS = [
- "address", "netmask", "broadcast", "network", "metric", "gateway",
- "pointtopoint", "media", "mtu", "hostname", "leasehours", "leasetime",
- "vendor", "client", "bootfile", "server", "hwaddr", "provider", "frame",
- "netnum", "endpoint", "local", "ttl",
-]
-
-NET_CONFIG_COMMANDS = [
- "pre-up", "up", "post-up", "down", "pre-down", "post-down",
-]
-
-NET_CONFIG_BRIDGE_OPTIONS = [
- "bridge_ageing", "bridge_bridgeprio", "bridge_fd", "bridge_gcinit",
- "bridge_hello", "bridge_maxage", "bridge_maxwait", "bridge_stp",
-]
-
DEFAULT_PRIMARY_INTERFACE = 'eth0'
@@ -61,23 +34,22 @@ def sys_dev_path(devname, path=""):
def read_sys_net(devname, path, translate=None, enoent=None, keyerror=None):
try:
- contents = ""
- with open(sys_dev_path(devname, path), "r") as fp:
- contents = fp.read().strip()
- if translate is None:
- return contents
-
- try:
- return translate.get(contents)
- except KeyError:
- LOG.debug("found unexpected value '%s' in '%s/%s'", contents,
- devname, path)
- if keyerror is not None:
- return keyerror
- raise
- except OSError as e:
- if e.errno == errno.ENOENT and enoent is not None:
- return enoent
+ contents = util.load_file(sys_dev_path(devname, path))
+ except (OSError, IOError) as e:
+ if getattr(e, 'errno', None) == errno.ENOENT:
+ if enoent is not None:
+ return enoent
+ raise
+ contents = contents.strip()
+ if translate is None:
+ return contents
+ try:
+ return translate.get(contents)
+ except KeyError:
+ LOG.debug("found unexpected value '%s' in '%s/%s'", contents,
+ devname, path)
+ if keyerror is not None:
+ return keyerror
raise
@@ -128,523 +100,7 @@ def get_devicelist():
class ParserError(Exception):
- """Raised when parser has issue parsing the interfaces file."""
-
-
-def parse_deb_config_data(ifaces, contents, src_dir, src_path):
- """Parses the file contents, placing result into ifaces.
-
- '_source_path' is added to every dictionary entry to define which file
- the configration information came from.
-
- :param ifaces: interface dictionary
- :param contents: contents of interfaces file
- :param src_dir: directory interfaces file was located
- :param src_path: file path the `contents` was read
- """
- currif = None
- for line in contents.splitlines():
- line = line.strip()
- if line.startswith('#'):
- continue
- split = line.split(' ')
- option = split[0]
- if option == "source-directory":
- parsed_src_dir = split[1]
- if not parsed_src_dir.startswith("/"):
- parsed_src_dir = os.path.join(src_dir, parsed_src_dir)
- for expanded_path in glob.glob(parsed_src_dir):
- dir_contents = os.listdir(expanded_path)
- dir_contents = [
- os.path.join(expanded_path, path)
- for path in dir_contents
- if (os.path.isfile(os.path.join(expanded_path, path)) and
- re.match("^[a-zA-Z0-9_-]+$", path) is not None)
- ]
- for entry in dir_contents:
- with open(entry, "r") as fp:
- src_data = fp.read().strip()
- abs_entry = os.path.abspath(entry)
- parse_deb_config_data(
- ifaces, src_data,
- os.path.dirname(abs_entry), abs_entry)
- elif option == "source":
- new_src_path = split[1]
- if not new_src_path.startswith("/"):
- new_src_path = os.path.join(src_dir, new_src_path)
- for expanded_path in glob.glob(new_src_path):
- with open(expanded_path, "r") as fp:
- src_data = fp.read().strip()
- abs_path = os.path.abspath(expanded_path)
- parse_deb_config_data(
- ifaces, src_data,
- os.path.dirname(abs_path), abs_path)
- elif option == "auto":
- for iface in split[1:]:
- if iface not in ifaces:
- ifaces[iface] = {
- # Include the source path this interface was found in.
- "_source_path": src_path
- }
- ifaces[iface]['auto'] = True
- elif option == "iface":
- iface, family, method = split[1:4]
- if iface not in ifaces:
- ifaces[iface] = {
- # Include the source path this interface was found in.
- "_source_path": src_path
- }
- elif 'family' in ifaces[iface]:
- raise ParserError(
- "Interface %s can only be defined once. "
- "Re-defined in '%s'." % (iface, src_path))
- ifaces[iface]['family'] = family
- ifaces[iface]['method'] = method
- currif = iface
- elif option == "hwaddress":
- if split[1] == "ether":
- val = split[2]
- else:
- val = split[1]
- ifaces[currif]['hwaddress'] = val
- elif option in NET_CONFIG_OPTIONS:
- ifaces[currif][option] = split[1]
- elif option in NET_CONFIG_COMMANDS:
- if option not in ifaces[currif]:
- ifaces[currif][option] = []
- ifaces[currif][option].append(' '.join(split[1:]))
- elif option.startswith('dns-'):
- if 'dns' not in ifaces[currif]:
- ifaces[currif]['dns'] = {}
- if option == 'dns-search':
- ifaces[currif]['dns']['search'] = []
- for domain in split[1:]:
- ifaces[currif]['dns']['search'].append(domain)
- elif option == 'dns-nameservers':
- ifaces[currif]['dns']['nameservers'] = []
- for server in split[1:]:
- ifaces[currif]['dns']['nameservers'].append(server)
- elif option.startswith('bridge_'):
- if 'bridge' not in ifaces[currif]:
- ifaces[currif]['bridge'] = {}
- if option in NET_CONFIG_BRIDGE_OPTIONS:
- bridge_option = option.replace('bridge_', '', 1)
- ifaces[currif]['bridge'][bridge_option] = split[1]
- elif option == "bridge_ports":
- ifaces[currif]['bridge']['ports'] = []
- for iface in split[1:]:
- ifaces[currif]['bridge']['ports'].append(iface)
- elif option == "bridge_hw" and split[1].lower() == "mac":
- ifaces[currif]['bridge']['mac'] = split[2]
- elif option == "bridge_pathcost":
- if 'pathcost' not in ifaces[currif]['bridge']:
- ifaces[currif]['bridge']['pathcost'] = {}
- ifaces[currif]['bridge']['pathcost'][split[1]] = split[2]
- elif option == "bridge_portprio":
- if 'portprio' not in ifaces[currif]['bridge']:
- ifaces[currif]['bridge']['portprio'] = {}
- ifaces[currif]['bridge']['portprio'][split[1]] = split[2]
- elif option.startswith('bond-'):
- if 'bond' not in ifaces[currif]:
- ifaces[currif]['bond'] = {}
- bond_option = option.replace('bond-', '', 1)
- ifaces[currif]['bond'][bond_option] = split[1]
- for iface in ifaces.keys():
- if 'auto' not in ifaces[iface]:
- ifaces[iface]['auto'] = False
-
-
-def parse_deb_config(path):
- """Parses a debian network configuration file."""
- ifaces = {}
- with open(path, "r") as fp:
- contents = fp.read().strip()
- abs_path = os.path.abspath(path)
- parse_deb_config_data(
- ifaces, contents,
- os.path.dirname(abs_path), abs_path)
- return ifaces
-
-
-def parse_net_config_data(net_config):
- """Parses the config, returns NetworkState dictionary
-
- :param net_config: curtin network config dict
- """
- state = None
- if 'version' in net_config and 'config' in net_config:
- ns = network_state.NetworkState(version=net_config.get('version'),
- config=net_config.get('config'))
- ns.parse_config()
- state = ns.network_state
-
- return state
-
-
-def parse_net_config(path):
- """Parses a curtin network configuration file and
- return network state"""
- ns = None
- net_config = util.read_conf(path)
- if 'network' in net_config:
- ns = parse_net_config_data(net_config.get('network'))
-
- return ns
-
-
-def _load_shell_content(content, add_empty=False, empty_val=None):
- """Given shell like syntax (key=value\nkey2=value2\n) in content
- return the data in dictionary form. If 'add_empty' is True
- then add entries in to the returned dictionary for 'VAR='
- variables. Set their value to empty_val."""
- data = {}
- for line in shlex.split(content):
- key, value = line.split("=", 1)
- if not value:
- value = empty_val
- if add_empty or value:
- data[key] = value
-
- return data
-
-
-def _klibc_to_config_entry(content, mac_addrs=None):
- """Convert a klibc writtent shell content file to a 'config' entry
- When ip= is seen on the kernel command line in debian initramfs
- and networking is brought up, ipconfig will populate
- /run/net-<name>.cfg.
-
- The files are shell style syntax, and examples are in the tests
- provided here. There is no good documentation on this unfortunately.
-
- DEVICE=<name> is expected/required and PROTO should indicate if
- this is 'static' or 'dhcp'.
- """
-
- if mac_addrs is None:
- mac_addrs = {}
-
- data = _load_shell_content(content)
- try:
- name = data['DEVICE']
- except KeyError:
- raise ValueError("no 'DEVICE' entry in data")
-
- # ipconfig on precise does not write PROTO
- proto = data.get('PROTO')
- if not proto:
- if data.get('filename'):
- proto = 'dhcp'
- else:
- proto = 'static'
-
- if proto not in ('static', 'dhcp'):
- raise ValueError("Unexpected value for PROTO: %s" % proto)
-
- iface = {
- 'type': 'physical',
- 'name': name,
- 'subnets': [],
- }
-
- if name in mac_addrs:
- iface['mac_address'] = mac_addrs[name]
-
- # originally believed there might be IPV6* values
- for v, pre in (('ipv4', 'IPV4'),):
- # if no IPV4ADDR or IPV6ADDR, then go on.
- if pre + "ADDR" not in data:
- continue
- subnet = {'type': proto, 'control': 'manual'}
-
- # these fields go right on the subnet
- for key in ('NETMASK', 'BROADCAST', 'GATEWAY'):
- if pre + key in data:
- subnet[key.lower()] = data[pre + key]
-
- dns = []
- # handle IPV4DNS0 or IPV6DNS0
- for nskey in ('DNS0', 'DNS1'):
- ns = data.get(pre + nskey)
- # verify it has something other than 0.0.0.0 (or ipv6)
- if ns and len(ns.strip(":.0")):
- dns.append(data[pre + nskey])
- if dns:
- subnet['dns_nameservers'] = dns
- # add search to both ipv4 and ipv6, as it has no namespace
- search = data.get('DOMAINSEARCH')
- if search:
- if ',' in search:
- subnet['dns_search'] = search.split(",")
- else:
- subnet['dns_search'] = search.split()
-
- iface['subnets'].append(subnet)
-
- return name, iface
-
-
-def config_from_klibc_net_cfg(files=None, mac_addrs=None):
- if files is None:
- files = glob.glob('/run/net*.conf')
-
- entries = []
- names = {}
- for cfg_file in files:
- name, entry = _klibc_to_config_entry(util.load_file(cfg_file),
- mac_addrs=mac_addrs)
- if name in names:
- raise ValueError(
- "device '%s' defined multiple times: %s and %s" % (
- name, names[name], cfg_file))
-
- names[name] = cfg_file
- entries.append(entry)
- return {'config': entries, 'version': 1}
-
-
-def render_persistent_net(network_state):
- '''Given state, emit udev rules to map mac to ifname.'''
- content = ""
- interfaces = network_state.get('interfaces')
- for iface in interfaces.values():
- # for physical interfaces write out a persist net udev rule
- if iface['type'] == 'physical' and \
- 'name' in iface and iface.get('mac_address'):
- content += generate_udev_rule(iface['name'],
- iface['mac_address'])
-
- return content
-
-
-# TODO: switch valid_map based on mode inet/inet6
-def iface_add_subnet(iface, subnet):
- content = []
- valid_map = [
- 'address',
- 'netmask',
- 'broadcast',
- 'metric',
- 'gateway',
- 'pointopoint',
- 'mtu',
- 'scope',
- 'dns_search',
- 'dns_nameservers',
- ]
- for key, value in subnet.items():
- if value and key in valid_map:
- if type(value) == list:
- value = " ".join(value)
- if '_' in key:
- key = key.replace('_', '-')
- content.append(" {} {}".format(key, value))
-
- return content
-
-
-# TODO: switch to valid_map for attrs
-def iface_add_attrs(iface):
- content = []
- ignore_map = [
- 'control',
- 'index',
- 'inet',
- 'mode',
- 'name',
- 'subnets',
- 'type',
- ]
- if iface['type'] not in ['bond', 'bridge', 'vlan']:
- ignore_map.append('mac_address')
-
- for key, value in iface.items():
- if value and key not in ignore_map:
- if type(value) == list:
- value = " ".join(value)
- content.append(" {} {}".format(key, value))
-
- return content
-
-
-def render_route(route, indent=""):
- """When rendering routes for an iface, in some cases applying a route
- may result in the route command returning non-zero which produces
- some confusing output for users manually using ifup/ifdown[1]. To
- that end, we will optionally include an '|| true' postfix to each
- route line allowing users to work with ifup/ifdown without using
- --force option.
-
- We may at somepoint not want to emit this additional postfix, and
- add a 'strict' flag to this function. When called with strict=True,
- then we will not append the postfix.
-
- 1. http://askubuntu.com/questions/168033/
- how-to-set-static-routes-in-ubuntu-server
- """
- content = []
- up = indent + "post-up route add"
- down = indent + "pre-down route del"
- eol = " || true"
- mapping = {
- 'network': '-net',
- 'netmask': 'netmask',
- 'gateway': 'gw',
- 'metric': 'metric',
- }
- if route['network'] == '0.0.0.0' and route['netmask'] == '0.0.0.0':
- default_gw = " default gw %s" % route['gateway']
- content.append(up + default_gw + eol)
- content.append(down + default_gw + eol)
- elif route['network'] == '::' and route['netmask'] == 0:
- # ipv6!
- default_gw = " -A inet6 default gw %s" % route['gateway']
- content.append(up + default_gw + eol)
- content.append(down + default_gw + eol)
- else:
- route_line = ""
- for k in ['network', 'netmask', 'gateway', 'metric']:
- if k in route:
- route_line += " %s %s" % (mapping[k], route[k])
- content.append(up + route_line + eol)
- content.append(down + route_line + eol)
-
- return content
-
-
-def iface_start_entry(iface, index):
- fullname = iface['name']
- if index != 0:
- fullname += ":%s" % index
-
- control = iface['control']
- if control == "auto":
- cverb = "auto"
- elif control in ("hotplug",):
- cverb = "allow-" + control
- else:
- cverb = "# control-" + control
-
- subst = iface.copy()
- subst.update({'fullname': fullname, 'cverb': cverb})
-
- if 'inet' not in subst:
- print("bug....iface: %s" % iface)
- return ["{cverb} {fullname}".format(**subst),
- "iface {fullname} {inet} {mode}".format(**subst)]
-
-
-def _render_iface(iface):
- lines = []
- subnets = iface.get('subnets', {})
- if subnets:
- for index, subnet in zip(range(0, len(subnets)), subnets):
- iface['index'] = index
- iface['mode'] = subnet['type']
- iface['control'] = subnet.get('control', 'auto')
- if iface['mode'].endswith('6'):
- iface['inet'] += '6'
- elif iface['mode'] == 'static' and ":" in subnet['address']:
- iface['inet'] += '6'
- if iface['mode'].startswith('dhcp'):
- iface['mode'] = 'dhcp'
-
- lines.extend(iface_start_entry(iface, index))
- lines.extend(iface_add_subnet(iface, subnet))
- lines.extend(iface_add_attrs(iface))
- lines.append("")
- else:
- # ifenslave docs say to auto the slave devices
- if 'bond-master' in iface:
- lines.append("auto {name}".format(**iface))
- lines.append("iface {name} {inet} {mode}".format(**iface))
- lines.extend(iface_add_attrs(iface))
- return lines
-
-
-def render_interfaces(network_state):
- '''Given state, emit etc/network/interfaces content.'''
-
- content = ""
- interfaces = network_state.get('interfaces')
- ''' Apply a sort order to ensure that we write out
- the physical interfaces first; this is critical for
- bonding
- '''
- order = {
- 'physical': 0,
- 'bond': 1,
- 'bridge': 2,
- 'vlan': 3,
- }
-
- # handle 'lo' specifically as we need to insert the global dns entries
- # there (as that is the only interface) that will be always up.
- lo = {'name': 'lo', 'type': 'physical', 'inet': 'inet',
- 'subnets': [{'type': 'loopback', 'control': 'auto'}]}
- for iface in interfaces.values():
- if iface.get('name') == "lo":
- lo = copy.deepcopy(iface)
- for dnskey, value in network_state.get('dns', {}).items():
- if len(value):
- lo['subnets'][0]["dns_" + dnskey] = value
-
- sections = [_render_iface(lo)]
-
- for iface in sorted(interfaces.values(),
- key=lambda k: (order[k['type']], k['name'])):
- if iface.get('name') == "lo":
- continue
- sections.append(_render_iface(iface))
-
- for route in network_state.get('routes'):
- sections.append(render_route(route))
-
- content = ''.join(['\n'.join(s) + '\n\n' for s in sections])
- # global replacements until v2 format
- content = content.replace('mac_address', 'hwaddress')
- return content
-
-
-def render_network_state(target, network_state, eni="etc/network/interfaces",
- links_prefix=LINKS_FNAME_PREFIX,
- netrules='etc/udev/rules.d/70-persistent-net.rules'):
-
- fpeni = os.path.sep.join((target, eni,))
- util.ensure_dir(os.path.dirname(fpeni))
- with open(fpeni, 'w+') as f:
- f.write(render_interfaces(network_state))
-
- if netrules:
- netrules = os.path.sep.join((target, netrules,))
- util.ensure_dir(os.path.dirname(netrules))
- with open(netrules, 'w+') as f:
- f.write(render_persistent_net(network_state))
-
- if links_prefix:
- render_systemd_links(target, network_state, links_prefix)
-
-
-def render_systemd_links(target, network_state,
- links_prefix=LINKS_FNAME_PREFIX):
- fp_prefix = os.path.sep.join((target, links_prefix))
- for f in glob.glob(fp_prefix + "*"):
- os.unlink(f)
-
- interfaces = network_state.get('interfaces')
- for iface in interfaces.values():
- if (iface['type'] == 'physical' and 'name' in iface and
- iface.get('mac_address')):
- fname = fp_prefix + iface['name'] + ".link"
- with open(fname, "w") as fp:
- fp.write("\n".join([
- "[Match]",
- "MACAddress=" + iface['mac_address'],
- "",
- "[Link]",
- "Name=" + iface['name'],
- ""
- ]))
+ """Raised when a parser has issue parsing a file/content."""
def is_disabled_cfg(cfg):
@@ -657,7 +113,6 @@ def sys_netdev_info(name, field):
if not os.path.exists(os.path.join(SYS_CLASS_NET, name)):
raise OSError("%s: interface does not exist in %s" %
(name, SYS_CLASS_NET))
-
fname = os.path.join(SYS_CLASS_NET, name, field)
if not os.path.exists(fname):
raise OSError("%s: could not find sysfs entry: %s" % (name, fname))
@@ -737,104 +192,6 @@ def generate_fallback_config():
return nconf
-def _decomp_gzip(blob, strict=True):
- # decompress blob. raise exception if not compressed unless strict=False.
- with io.BytesIO(blob) as iobuf:
- gzfp = None
- try:
- gzfp = gzip.GzipFile(mode="rb", fileobj=iobuf)
- return gzfp.read()
- except IOError:
- if strict:
- raise
- return blob
- finally:
- if gzfp:
- gzfp.close()
-
-
-def _b64dgz(b64str, gzipped="try"):
- # decode a base64 string. If gzipped is true, transparently uncompresss
- # if gzipped is 'try', then try gunzip, returning the original on fail.
- try:
- blob = base64.b64decode(b64str)
- except TypeError:
- raise ValueError("Invalid base64 text: %s" % b64str)
-
- if not gzipped:
- return blob
-
- return _decomp_gzip(blob, strict=gzipped != "try")
-
-
-def read_kernel_cmdline_config(files=None, mac_addrs=None, cmdline=None):
- if cmdline is None:
- cmdline = util.get_cmdline()
-
- if 'network-config=' in cmdline:
- data64 = None
- for tok in cmdline.split():
- if tok.startswith("network-config="):
- data64 = tok.split("=", 1)[1]
- if data64:
- return util.load_yaml(_b64dgz(data64))
-
- if 'ip=' not in cmdline:
- return None
-
- if mac_addrs is None:
- mac_addrs = {k: sys_netdev_info(k, 'address')
- for k in get_devicelist()}
-
- return config_from_klibc_net_cfg(files=files, mac_addrs=mac_addrs)
-
-
-def convert_eni_data(eni_data):
- # return a network config representation of what is in eni_data
- ifaces = {}
- parse_deb_config_data(ifaces, eni_data, src_dir=None, src_path=None)
- return _ifaces_to_net_config_data(ifaces)
-
-
-def _ifaces_to_net_config_data(ifaces):
- """Return network config that represents the ifaces data provided.
- ifaces = parse_deb_config("/etc/network/interfaces")
- config = ifaces_to_net_config_data(ifaces)
- state = parse_net_config_data(config)."""
- devs = {}
- for name, data in ifaces.items():
- # devname is 'eth0' for name='eth0:1'
- devname = name.partition(":")[0]
- if devname not in devs:
- devs[devname] = {'type': 'physical', 'name': devname,
- 'subnets': []}
- # this isnt strictly correct, but some might specify
- # hwaddress on a nic for matching / declaring name.
- if 'hwaddress' in data:
- devs[devname]['mac_address'] = data['hwaddress']
- subnet = {'_orig_eni_name': name, 'type': data['method']}
- if data.get('auto'):
- subnet['control'] = 'auto'
- else:
- subnet['control'] = 'manual'
-
- if data.get('method') == 'static':
- subnet['address'] = data['address']
-
- for copy_key in ('netmask', 'gateway', 'broadcast'):
- if copy_key in data:
- subnet[copy_key] = data[copy_key]
-
- if 'dns' in data:
- for n in ('nameservers', 'search'):
- if n in data['dns'] and data['dns'][n]:
- subnet['dns_' + n] = data['dns'][n]
- devs[devname]['subnets'].append(subnet)
-
- return {'version': 1,
- 'config': [devs[d] for d in sorted(devs)]}
-
-
def apply_network_config_names(netcfg, strict_present=True, strict_busy=True):
"""read the network config and rename devices accordingly.
if strict_present is false, then do not raise exception if no devices
@@ -850,7 +207,7 @@ def apply_network_config_names(netcfg, strict_present=True, strict_busy=True):
continue
renames.append([mac, name])
- return rename_interfaces(renames)
+ return _rename_interfaces(renames)
def _get_current_rename_info(check_downable=True):
@@ -878,8 +235,8 @@ def _get_current_rename_info(check_downable=True):
return bymac
-def rename_interfaces(renames, strict_present=True, strict_busy=True,
- current_info=None):
+def _rename_interfaces(renames, strict_present=True, strict_busy=True,
+ current_info=None):
if current_info is None:
current_info = _get_current_rename_info()
@@ -990,7 +347,13 @@ def get_interface_mac(ifname):
def get_interfaces_by_mac(devs=None):
"""Build a dictionary of tuples {mac: name}"""
if devs is None:
- devs = get_devicelist()
+ try:
+ devs = get_devicelist()
+ except OSError as e:
+ if e.errno == errno.ENOENT:
+ devs = []
+ else:
+ raise
ret = {}
for name in devs:
mac = get_interface_mac(name)
diff --git a/cloudinit/net/cmdline.py b/cloudinit/net/cmdline.py
new file mode 100644
index 00000000..822a020b
--- /dev/null
+++ b/cloudinit/net/cmdline.py
@@ -0,0 +1,203 @@
+# Copyright (C) 2013-2014 Canonical Ltd.
+#
+# Author: Scott Moser <scott.moser@canonical.com>
+# Author: Blake Rouse <blake.rouse@canonical.com>
+#
+# Curtin is free software: you can redistribute it and/or modify it under
+# the terms of the GNU Affero General Public License as published by the
+# Free Software Foundation, either version 3 of the License, or (at your
+# option) any later version.
+#
+# Curtin is distributed in the hope that it will be useful, but WITHOUT ANY
+# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for
+# more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with Curtin. If not, see <http://www.gnu.org/licenses/>.
+
+import base64
+import glob
+import gzip
+import io
+import shlex
+import sys
+
+import six
+
+from . import get_devicelist
+from . import sys_netdev_info
+
+from cloudinit import util
+
+PY26 = sys.version_info[0:2] == (2, 6)
+
+
+def _shlex_split(blob):
+ if PY26 and isinstance(blob, six.text_type):
+ # Older versions don't support unicode input
+ blob = blob.encode("utf8")
+ return shlex.split(blob)
+
+
+def _load_shell_content(content, add_empty=False, empty_val=None):
+ """Given shell like syntax (key=value\nkey2=value2\n) in content
+ return the data in dictionary form. If 'add_empty' is True
+ then add entries in to the returned dictionary for 'VAR='
+ variables. Set their value to empty_val."""
+ data = {}
+ for line in _shlex_split(content):
+ key, value = line.split("=", 1)
+ if not value:
+ value = empty_val
+ if add_empty or value:
+ data[key] = value
+
+ return data
+
+
+def _klibc_to_config_entry(content, mac_addrs=None):
+ """Convert a klibc writtent shell content file to a 'config' entry
+ When ip= is seen on the kernel command line in debian initramfs
+ and networking is brought up, ipconfig will populate
+ /run/net-<name>.cfg.
+
+ The files are shell style syntax, and examples are in the tests
+ provided here. There is no good documentation on this unfortunately.
+
+ DEVICE=<name> is expected/required and PROTO should indicate if
+ this is 'static' or 'dhcp'.
+ """
+
+ if mac_addrs is None:
+ mac_addrs = {}
+
+ data = _load_shell_content(content)
+ try:
+ name = data['DEVICE']
+ except KeyError:
+ raise ValueError("no 'DEVICE' entry in data")
+
+ # ipconfig on precise does not write PROTO
+ proto = data.get('PROTO')
+ if not proto:
+ if data.get('filename'):
+ proto = 'dhcp'
+ else:
+ proto = 'static'
+
+ if proto not in ('static', 'dhcp'):
+ raise ValueError("Unexpected value for PROTO: %s" % proto)
+
+ iface = {
+ 'type': 'physical',
+ 'name': name,
+ 'subnets': [],
+ }
+
+ if name in mac_addrs:
+ iface['mac_address'] = mac_addrs[name]
+
+ # originally believed there might be IPV6* values
+ for v, pre in (('ipv4', 'IPV4'),):
+ # if no IPV4ADDR or IPV6ADDR, then go on.
+ if pre + "ADDR" not in data:
+ continue
+ subnet = {'type': proto, 'control': 'manual'}
+
+ # these fields go right on the subnet
+ for key in ('NETMASK', 'BROADCAST', 'GATEWAY'):
+ if pre + key in data:
+ subnet[key.lower()] = data[pre + key]
+
+ dns = []
+ # handle IPV4DNS0 or IPV6DNS0
+ for nskey in ('DNS0', 'DNS1'):
+ ns = data.get(pre + nskey)
+ # verify it has something other than 0.0.0.0 (or ipv6)
+ if ns and len(ns.strip(":.0")):
+ dns.append(data[pre + nskey])
+ if dns:
+ subnet['dns_nameservers'] = dns
+ # add search to both ipv4 and ipv6, as it has no namespace
+ search = data.get('DOMAINSEARCH')
+ if search:
+ if ',' in search:
+ subnet['dns_search'] = search.split(",")
+ else:
+ subnet['dns_search'] = search.split()
+
+ iface['subnets'].append(subnet)
+
+ return name, iface
+
+
+def config_from_klibc_net_cfg(files=None, mac_addrs=None):
+ if files is None:
+ files = glob.glob('/run/net*.conf')
+
+ entries = []
+ names = {}
+ for cfg_file in files:
+ name, entry = _klibc_to_config_entry(util.load_file(cfg_file),
+ mac_addrs=mac_addrs)
+ if name in names:
+ raise ValueError(
+ "device '%s' defined multiple times: %s and %s" % (
+ name, names[name], cfg_file))
+
+ names[name] = cfg_file
+ entries.append(entry)
+ return {'config': entries, 'version': 1}
+
+
+def _decomp_gzip(blob, strict=True):
+ # decompress blob. raise exception if not compressed unless strict=False.
+ with io.BytesIO(blob) as iobuf:
+ gzfp = None
+ try:
+ gzfp = gzip.GzipFile(mode="rb", fileobj=iobuf)
+ return gzfp.read()
+ except IOError:
+ if strict:
+ raise
+ return blob
+ finally:
+ if gzfp:
+ gzfp.close()
+
+
+def _b64dgz(b64str, gzipped="try"):
+ # decode a base64 string. If gzipped is true, transparently uncompresss
+ # if gzipped is 'try', then try gunzip, returning the original on fail.
+ try:
+ blob = base64.b64decode(b64str)
+ except TypeError:
+ raise ValueError("Invalid base64 text: %s" % b64str)
+
+ if not gzipped:
+ return blob
+
+ return _decomp_gzip(blob, strict=gzipped != "try")
+
+
+def read_kernel_cmdline_config(files=None, mac_addrs=None, cmdline=None):
+ if cmdline is None:
+ cmdline = util.get_cmdline()
+
+ if 'network-config=' in cmdline:
+ data64 = None
+ for tok in cmdline.split():
+ if tok.startswith("network-config="):
+ data64 = tok.split("=", 1)[1]
+ if data64:
+ return util.load_yaml(_b64dgz(data64))
+
+ if 'ip=' not in cmdline:
+ return None
+
+ if mac_addrs is None:
+ mac_addrs = dict((k, sys_netdev_info(k, 'address'))
+ for k in get_devicelist())
+
+ return config_from_klibc_net_cfg(files=files, mac_addrs=mac_addrs)
diff --git a/cloudinit/net/eni.py b/cloudinit/net/eni.py
new file mode 100644
index 00000000..5a91fcf2
--- /dev/null
+++ b/cloudinit/net/eni.py
@@ -0,0 +1,460 @@
+# vi: ts=4 expandtab
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 3, as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+import copy
+import glob
+import os
+import re
+
+from . import ParserError
+
+from . import renderer
+
+from cloudinit import util
+
+
+NET_CONFIG_COMMANDS = [
+ "pre-up", "up", "post-up", "down", "pre-down", "post-down",
+]
+
+NET_CONFIG_BRIDGE_OPTIONS = [
+ "bridge_ageing", "bridge_bridgeprio", "bridge_fd", "bridge_gcinit",
+ "bridge_hello", "bridge_maxage", "bridge_maxwait", "bridge_stp",
+]
+
+NET_CONFIG_OPTIONS = [
+ "address", "netmask", "broadcast", "network", "metric", "gateway",
+ "pointtopoint", "media", "mtu", "hostname", "leasehours", "leasetime",
+ "vendor", "client", "bootfile", "server", "hwaddr", "provider", "frame",
+ "netnum", "endpoint", "local", "ttl",
+]
+
+
+# TODO: switch valid_map based on mode inet/inet6
+def _iface_add_subnet(iface, subnet):
+ content = []
+ valid_map = [
+ 'address',
+ 'netmask',
+ 'broadcast',
+ 'metric',
+ 'gateway',
+ 'pointopoint',
+ 'mtu',
+ 'scope',
+ 'dns_search',
+ 'dns_nameservers',
+ ]
+ for key, value in subnet.items():
+ if value and key in valid_map:
+ if type(value) == list:
+ value = " ".join(value)
+ if '_' in key:
+ key = key.replace('_', '-')
+ content.append(" {} {}".format(key, value))
+
+ return content
+
+
+# TODO: switch to valid_map for attrs
+
+def _iface_add_attrs(iface):
+ content = []
+ ignore_map = [
+ 'control',
+ 'index',
+ 'inet',
+ 'mode',
+ 'name',
+ 'subnets',
+ 'type',
+ ]
+ if iface['type'] not in ['bond', 'bridge', 'vlan']:
+ ignore_map.append('mac_address')
+
+ for key, value in iface.items():
+ if value and key not in ignore_map:
+ if type(value) == list:
+ value = " ".join(value)
+ content.append(" {} {}".format(key, value))
+
+ return content
+
+
+def _iface_start_entry(iface, index):
+ fullname = iface['name']
+ if index != 0:
+ fullname += ":%s" % index
+
+ control = iface['control']
+ if control == "auto":
+ cverb = "auto"
+ elif control in ("hotplug",):
+ cverb = "allow-" + control
+ else:
+ cverb = "# control-" + control
+
+ subst = iface.copy()
+ subst.update({'fullname': fullname, 'cverb': cverb})
+
+ return ["{cverb} {fullname}".format(**subst),
+ "iface {fullname} {inet} {mode}".format(**subst)]
+
+
+def _parse_deb_config_data(ifaces, contents, src_dir, src_path):
+ """Parses the file contents, placing result into ifaces.
+
+ '_source_path' is added to every dictionary entry to define which file
+ the configration information came from.
+
+ :param ifaces: interface dictionary
+ :param contents: contents of interfaces file
+ :param src_dir: directory interfaces file was located
+ :param src_path: file path the `contents` was read
+ """
+ currif = None
+ for line in contents.splitlines():
+ line = line.strip()
+ if line.startswith('#'):
+ continue
+ split = line.split(' ')
+ option = split[0]
+ if option == "source-directory":
+ parsed_src_dir = split[1]
+ if not parsed_src_dir.startswith("/"):
+ parsed_src_dir = os.path.join(src_dir, parsed_src_dir)
+ for expanded_path in glob.glob(parsed_src_dir):
+ dir_contents = os.listdir(expanded_path)
+ dir_contents = [
+ os.path.join(expanded_path, path)
+ for path in dir_contents
+ if (os.path.isfile(os.path.join(expanded_path, path)) and
+ re.match("^[a-zA-Z0-9_-]+$", path) is not None)
+ ]
+ for entry in dir_contents:
+ with open(entry, "r") as fp:
+ src_data = fp.read().strip()
+ abs_entry = os.path.abspath(entry)
+ _parse_deb_config_data(
+ ifaces, src_data,
+ os.path.dirname(abs_entry), abs_entry)
+ elif option == "source":
+ new_src_path = split[1]
+ if not new_src_path.startswith("/"):
+ new_src_path = os.path.join(src_dir, new_src_path)
+ for expanded_path in glob.glob(new_src_path):
+ with open(expanded_path, "r") as fp:
+ src_data = fp.read().strip()
+ abs_path = os.path.abspath(expanded_path)
+ _parse_deb_config_data(
+ ifaces, src_data,
+ os.path.dirname(abs_path), abs_path)
+ elif option == "auto":
+ for iface in split[1:]:
+ if iface not in ifaces:
+ ifaces[iface] = {
+ # Include the source path this interface was found in.
+ "_source_path": src_path
+ }
+ ifaces[iface]['auto'] = True
+ elif option == "iface":
+ iface, family, method = split[1:4]
+ if iface not in ifaces:
+ ifaces[iface] = {
+ # Include the source path this interface was found in.
+ "_source_path": src_path
+ }
+ elif 'family' in ifaces[iface]:
+ raise ParserError(
+ "Interface %s can only be defined once. "
+ "Re-defined in '%s'." % (iface, src_path))
+ ifaces[iface]['family'] = family
+ ifaces[iface]['method'] = method
+ currif = iface
+ elif option == "hwaddress":
+ if split[1] == "ether":
+ val = split[2]
+ else:
+ val = split[1]
+ ifaces[currif]['hwaddress'] = val
+ elif option in NET_CONFIG_OPTIONS:
+ ifaces[currif][option] = split[1]
+ elif option in NET_CONFIG_COMMANDS:
+ if option not in ifaces[currif]:
+ ifaces[currif][option] = []
+ ifaces[currif][option].append(' '.join(split[1:]))
+ elif option.startswith('dns-'):
+ if 'dns' not in ifaces[currif]:
+ ifaces[currif]['dns'] = {}
+ if option == 'dns-search':
+ ifaces[currif]['dns']['search'] = []
+ for domain in split[1:]:
+ ifaces[currif]['dns']['search'].append(domain)
+ elif option == 'dns-nameservers':
+ ifaces[currif]['dns']['nameservers'] = []
+ for server in split[1:]:
+ ifaces[currif]['dns']['nameservers'].append(server)
+ elif option.startswith('bridge_'):
+ if 'bridge' not in ifaces[currif]:
+ ifaces[currif]['bridge'] = {}
+ if option in NET_CONFIG_BRIDGE_OPTIONS:
+ bridge_option = option.replace('bridge_', '', 1)
+ ifaces[currif]['bridge'][bridge_option] = split[1]
+ elif option == "bridge_ports":
+ ifaces[currif]['bridge']['ports'] = []
+ for iface in split[1:]:
+ ifaces[currif]['bridge']['ports'].append(iface)
+ elif option == "bridge_hw" and split[1].lower() == "mac":
+ ifaces[currif]['bridge']['mac'] = split[2]
+ elif option == "bridge_pathcost":
+ if 'pathcost' not in ifaces[currif]['bridge']:
+ ifaces[currif]['bridge']['pathcost'] = {}
+ ifaces[currif]['bridge']['pathcost'][split[1]] = split[2]
+ elif option == "bridge_portprio":
+ if 'portprio' not in ifaces[currif]['bridge']:
+ ifaces[currif]['bridge']['portprio'] = {}
+ ifaces[currif]['bridge']['portprio'][split[1]] = split[2]
+ elif option.startswith('bond-'):
+ if 'bond' not in ifaces[currif]:
+ ifaces[currif]['bond'] = {}
+ bond_option = option.replace('bond-', '', 1)
+ ifaces[currif]['bond'][bond_option] = split[1]
+ for iface in ifaces.keys():
+ if 'auto' not in ifaces[iface]:
+ ifaces[iface]['auto'] = False
+
+
+def parse_deb_config(path):
+ """Parses a debian network configuration file."""
+ ifaces = {}
+ with open(path, "r") as fp:
+ contents = fp.read().strip()
+ abs_path = os.path.abspath(path)
+ _parse_deb_config_data(
+ ifaces, contents,
+ os.path.dirname(abs_path), abs_path)
+ return ifaces
+
+
+def convert_eni_data(eni_data):
+ # return a network config representation of what is in eni_data
+ ifaces = {}
+ _parse_deb_config_data(ifaces, eni_data, src_dir=None, src_path=None)
+ return _ifaces_to_net_config_data(ifaces)
+
+
+def _ifaces_to_net_config_data(ifaces):
+ """Return network config that represents the ifaces data provided.
+ ifaces = parse_deb_config("/etc/network/interfaces")
+ config = ifaces_to_net_config_data(ifaces)
+ state = parse_net_config_data(config)."""
+ devs = {}
+ for name, data in ifaces.items():
+ # devname is 'eth0' for name='eth0:1'
+ devname = name.partition(":")[0]
+ if devname == "lo":
+ # currently provding 'lo' in network config results in duplicate
+ # entries. in rendered interfaces file. so skip it.
+ continue
+ if devname not in devs:
+ devs[devname] = {'type': 'physical', 'name': devname,
+ 'subnets': []}
+ # this isnt strictly correct, but some might specify
+ # hwaddress on a nic for matching / declaring name.
+ if 'hwaddress' in data:
+ devs[devname]['mac_address'] = data['hwaddress']
+ subnet = {'_orig_eni_name': name, 'type': data['method']}
+ if data.get('auto'):
+ subnet['control'] = 'auto'
+ else:
+ subnet['control'] = 'manual'
+
+ if data.get('method') == 'static':
+ subnet['address'] = data['address']
+
+ for copy_key in ('netmask', 'gateway', 'broadcast'):
+ if copy_key in data:
+ subnet[copy_key] = data[copy_key]
+
+ if 'dns' in data:
+ for n in ('nameservers', 'search'):
+ if n in data['dns'] and data['dns'][n]:
+ subnet['dns_' + n] = data['dns'][n]
+ devs[devname]['subnets'].append(subnet)
+
+ return {'version': 1,
+ 'config': [devs[d] for d in sorted(devs)]}
+
+
+class Renderer(renderer.Renderer):
+ """Renders network information in a /etc/network/interfaces format."""
+
+ def __init__(self, config=None):
+ if not config:
+ config = {}
+ self.eni_path = config.get('eni_path', 'etc/network/interfaces')
+ self.links_path_prefix = config.get(
+ 'links_path_prefix', 'etc/systemd/network/50-cloud-init-')
+ self.netrules_path = config.get(
+ 'netrules_path', 'etc/udev/rules.d/70-persistent-net.rules')
+
+ def _render_route(self, route, indent=""):
+ """When rendering routes for an iface, in some cases applying a route
+ may result in the route command returning non-zero which produces
+ some confusing output for users manually using ifup/ifdown[1]. To
+ that end, we will optionally include an '|| true' postfix to each
+ route line allowing users to work with ifup/ifdown without using
+ --force option.
+
+ We may at somepoint not want to emit this additional postfix, and
+ add a 'strict' flag to this function. When called with strict=True,
+ then we will not append the postfix.
+
+ 1. http://askubuntu.com/questions/168033/
+ how-to-set-static-routes-in-ubuntu-server
+ """
+ content = []
+ up = indent + "post-up route add"
+ down = indent + "pre-down route del"
+ or_true = " || true"
+ mapping = {
+ 'network': '-net',
+ 'netmask': 'netmask',
+ 'gateway': 'gw',
+ 'metric': 'metric',
+ }
+ if route['network'] == '0.0.0.0' and route['netmask'] == '0.0.0.0':
+ default_gw = " default gw %s" % route['gateway']
+ content.append(up + default_gw + or_true)
+ content.append(down + default_gw + or_true)
+ elif route['network'] == '::' and route['netmask'] == 0:
+ # ipv6!
+ default_gw = " -A inet6 default gw %s" % route['gateway']
+ content.append(up + default_gw + or_true)
+ content.append(down + default_gw + or_true)
+ else:
+ route_line = ""
+ for k in ['network', 'netmask', 'gateway', 'metric']:
+ if k in route:
+ route_line += " %s %s" % (mapping[k], route[k])
+ content.append(up + route_line + or_true)
+ content.append(down + route_line + or_true)
+ return content
+
+ def _render_iface(self, iface):
+ lines = []
+ subnets = iface.get('subnets', {})
+ if subnets:
+ for index, subnet in zip(range(0, len(subnets)), subnets):
+ iface['index'] = index
+ iface['mode'] = subnet['type']
+ iface['control'] = subnet.get('control', 'auto')
+ if iface['mode'].endswith('6'):
+ iface['inet'] += '6'
+ elif iface['mode'] == 'static' and ":" in subnet['address']:
+ iface['inet'] += '6'
+ if iface['mode'].startswith('dhcp'):
+ iface['mode'] = 'dhcp'
+
+ lines.extend(_iface_start_entry(iface, index))
+ lines.extend(_iface_add_subnet(iface, subnet))
+ lines.extend(_iface_add_attrs(iface))
+ lines.append("")
+ else:
+ # ifenslave docs say to auto the slave devices
+ if 'bond-master' in iface:
+ lines.append("auto {name}".format(**iface))
+ lines.append("iface {name} {inet} {mode}".format(**iface))
+ lines.extend(_iface_add_attrs(iface))
+ return lines
+
+ def _render_interfaces(self, network_state):
+ '''Given state, emit etc/network/interfaces content.'''
+
+ content = ""
+
+ # handle 'lo' specifically as we need to insert the global dns entries
+ # there (as that is the only interface) that will be always up.
+ lo = {'name': 'lo', 'type': 'physical', 'inet': 'inet',
+ 'subnets': [{'type': 'loopback', 'control': 'auto'}]}
+ for iface in network_state.iter_interfaces():
+ if iface.get('name') == "lo":
+ lo = copy.deepcopy(iface)
+
+ nameservers = network_state.dns_nameservers
+ if nameservers:
+ lo['subnets'][0]["dns_nameservers"] = (" ".join(nameservers))
+
+ searchdomains = network_state.dns_searchdomains
+ if searchdomains:
+ lo['subnets'][0]["dns_search"] = (" ".join(searchdomains))
+
+ ''' Apply a sort order to ensure that we write out
+ the physical interfaces first; this is critical for
+ bonding
+ '''
+ order = {
+ 'physical': 0,
+ 'bond': 1,
+ 'bridge': 2,
+ 'vlan': 3,
+ }
+
+ sections = [self._render_iface(lo)]
+ for iface in sorted(network_state.iter_interfaces(),
+ key=lambda k: (order[k['type']], k['name'])):
+
+ if iface.get('name') == "lo":
+ continue
+ sections.append(self._render_iface(iface))
+
+ for route in network_state.iter_routes():
+ sections.append(self._render_route(route))
+
+ # global replacements until v2 format
+ content = ''.join(['\n'.join(s) + '\n\n' for s in sections])
+ return content
+
+ def render_network_state(self, target, network_state):
+ fpeni = os.path.join(target, self.eni_path)
+ util.ensure_dir(os.path.dirname(fpeni))
+ util.write_file(fpeni, self._render_interfaces(network_state))
+
+ if self.netrules_path:
+ netrules = os.path.join(target, self.netrules_path)
+ util.ensure_dir(os.path.dirname(netrules))
+ util.write_file(netrules,
+ self._render_persistent_net(network_state))
+
+ if self.links_path_prefix:
+ self._render_systemd_links(target, network_state,
+ links_prefix=self.links_path_prefix)
+
+ def _render_systemd_links(self, target, network_state, links_prefix):
+ fp_prefix = os.path.join(target, links_prefix)
+ for f in glob.glob(fp_prefix + "*"):
+ os.unlink(f)
+ for iface in network_state.iter_interfaces():
+ if (iface['type'] == 'physical' and 'name' in iface and
+ iface.get('mac_address')):
+ fname = fp_prefix + iface['name'] + ".link"
+ content = "\n".join([
+ "[Match]",
+ "MACAddress=" + iface['mac_address'],
+ "",
+ "[Link]",
+ "Name=" + iface['name'],
+ ""
+ ])
+ util.write_file(fname, content)
diff --git a/cloudinit/net/network_state.py b/cloudinit/net/network_state.py
index 4c726ab4..8ca5106f 100644
--- a/cloudinit/net/network_state.py
+++ b/cloudinit/net/network_state.py
@@ -15,9 +15,13 @@
# You should have received a copy of the GNU Affero General Public License
# along with Curtin. If not, see <http://www.gnu.org/licenses/>.
-from cloudinit import log as logging
+import copy
+import functools
+import logging
+
+import six
+
from cloudinit import util
-from cloudinit.util import yaml_dumps as dump_config
LOG = logging.getLogger(__name__)
@@ -27,80 +31,198 @@ NETWORK_STATE_REQUIRED_KEYS = {
}
+def parse_net_config_data(net_config, skip_broken=True):
+ """Parses the config, returns NetworkState object
+
+ :param net_config: curtin network config dict
+ """
+ state = None
+ if 'version' in net_config and 'config' in net_config:
+ nsi = NetworkStateInterpreter(version=net_config.get('version'),
+ config=net_config.get('config'))
+ nsi.parse_config(skip_broken=skip_broken)
+ state = nsi.network_state
+ return state
+
+
+def parse_net_config(path, skip_broken=True):
+ """Parses a curtin network configuration file and
+ return network state"""
+ ns = None
+ net_config = util.read_conf(path)
+ if 'network' in net_config:
+ ns = parse_net_config_data(net_config.get('network'),
+ skip_broken=skip_broken)
+ return ns
+
+
def from_state_file(state_file):
- network_state = None
state = util.read_conf(state_file)
- network_state = NetworkState()
- network_state.load(state)
+ nsi = NetworkStateInterpreter()
+ nsi.load(state)
+ return nsi
+
+
+def diff_keys(expected, actual):
+ missing = set(expected)
+ for key in actual:
+ missing.discard(key)
+ return missing
+
+
+class InvalidCommand(Exception):
+ pass
+
+
+def ensure_command_keys(required_keys):
+
+ def wrapper(func):
+
+ @functools.wraps(func)
+ def decorator(self, command, *args, **kwargs):
+ if required_keys:
+ missing_keys = diff_keys(required_keys, command)
+ if missing_keys:
+ raise InvalidCommand("Command missing %s of required"
+ " keys %s" % (missing_keys,
+ required_keys))
+ return func(self, command, *args, **kwargs)
+
+ return decorator
+
+ return wrapper
+
+
+class CommandHandlerMeta(type):
+ """Metaclass that dynamically creates a 'command_handlers' attribute.
- return network_state
+ This will scan the to-be-created class for methods that start with
+ 'handle_' and on finding those will populate a class attribute mapping
+ so that those methods can be quickly located and called.
+ """
+ def __new__(cls, name, parents, dct):
+ command_handlers = {}
+ for attr_name, attr in dct.items():
+ if callable(attr) and attr_name.startswith('handle_'):
+ handles_what = attr_name[len('handle_'):]
+ if handles_what:
+ command_handlers[handles_what] = attr
+ dct['command_handlers'] = command_handlers
+ return super(CommandHandlerMeta, cls).__new__(cls, name,
+ parents, dct)
class NetworkState(object):
- def __init__(self, version=NETWORK_STATE_VERSION, config=None):
- self.version = version
- self.config = config
- self.network_state = {
- 'interfaces': {},
- 'routes': [],
- 'dns': {
- 'nameservers': [],
- 'search': [],
- }
+
+ def __init__(self, network_state, version=NETWORK_STATE_VERSION):
+ self._network_state = copy.deepcopy(network_state)
+ self._version = version
+
+ @property
+ def version(self):
+ return self._version
+
+ def iter_routes(self, filter_func=None):
+ for route in self._network_state.get('routes', []):
+ if filter_func is not None:
+ if filter_func(route):
+ yield route
+ else:
+ yield route
+
+ @property
+ def dns_nameservers(self):
+ try:
+ return self._network_state['dns']['nameservers']
+ except KeyError:
+ return []
+
+ @property
+ def dns_searchdomains(self):
+ try:
+ return self._network_state['dns']['search']
+ except KeyError:
+ return []
+
+ def iter_interfaces(self, filter_func=None):
+ ifaces = self._network_state.get('interfaces', {})
+ for iface in six.itervalues(ifaces):
+ if filter_func is None:
+ yield iface
+ else:
+ if filter_func(iface):
+ yield iface
+
+
+@six.add_metaclass(CommandHandlerMeta)
+class NetworkStateInterpreter(object):
+
+ initial_network_state = {
+ 'interfaces': {},
+ 'routes': [],
+ 'dns': {
+ 'nameservers': [],
+ 'search': [],
}
- self.command_handlers = self.get_command_handlers()
+ }
- def get_command_handlers(self):
- METHOD_PREFIX = 'handle_'
- methods = filter(lambda x: callable(getattr(self, x)) and
- x.startswith(METHOD_PREFIX), dir(self))
- handlers = {}
- for m in methods:
- key = m.replace(METHOD_PREFIX, '')
- handlers[key] = getattr(self, m)
+ def __init__(self, version=NETWORK_STATE_VERSION, config=None):
+ self._version = version
+ self._config = config
+ self._network_state = copy.deepcopy(self.initial_network_state)
+ self._parsed = False
- return handlers
+ @property
+ def network_state(self):
+ return NetworkState(self._network_state, version=self._version)
def dump(self):
state = {
- 'version': self.version,
- 'config': self.config,
- 'network_state': self.network_state,
+ 'version': self._version,
+ 'config': self._config,
+ 'network_state': self._network_state,
}
- return dump_config(state)
+ return util.yaml_dumps(state)
def load(self, state):
if 'version' not in state:
LOG.error('Invalid state, missing version field')
- raise Exception('Invalid state, missing version field')
+ raise ValueError('Invalid state, missing version field')
required_keys = NETWORK_STATE_REQUIRED_KEYS[state['version']]
- if not self.valid_command(state, required_keys):
- msg = 'Invalid state, missing keys: {}'.format(required_keys)
+ missing_keys = diff_keys(required_keys, state)
+ if missing_keys:
+ msg = 'Invalid state, missing keys: %s' % (missing_keys)
LOG.error(msg)
- raise Exception(msg)
+ raise ValueError(msg)
# v1 - direct attr mapping, except version
for key in [k for k in required_keys if k not in ['version']]:
setattr(self, key, state[key])
- self.command_handlers = self.get_command_handlers()
def dump_network_state(self):
- return dump_config(self.network_state)
+ return util.yaml_dumps(self._network_state)
- def parse_config(self):
+ def parse_config(self, skip_broken=True):
# rebuild network state
- for command in self.config:
- handler = self.command_handlers.get(command['type'])
- handler(command)
-
- def valid_command(self, command, required_keys):
- if not required_keys:
- return False
-
- found_keys = [key for key in command.keys() if key in required_keys]
- return len(found_keys) == len(required_keys)
-
+ for command in self._config:
+ command_type = command['type']
+ try:
+ handler = self.command_handlers[command_type]
+ except KeyError:
+ raise RuntimeError("No handler found for"
+ " command '%s'" % command_type)
+ try:
+ handler(self, command)
+ except InvalidCommand:
+ if not skip_broken:
+ raise
+ else:
+ LOG.warn("Skipping invalid command: %s", command,
+ exc_info=True)
+ LOG.debug(self.dump_network_state())
+
+ @ensure_command_keys(['name'])
def handle_physical(self, command):
'''
command = {
@@ -112,15 +234,8 @@ class NetworkState(object):
]
}
'''
- required_keys = [
- 'name',
- ]
- if not self.valid_command(command, required_keys):
- LOG.warn('Skipping Invalid command: {}'.format(command))
- LOG.debug(self.dump_network_state())
- return
- interfaces = self.network_state.get('interfaces')
+ interfaces = self._network_state.get('interfaces', {})
iface = interfaces.get(command['name'], {})
for param, val in command.get('params', {}).items():
iface.update({param: val})
@@ -146,9 +261,10 @@ class NetworkState(object):
'gateway': None,
'subnets': subnets,
})
- self.network_state['interfaces'].update({command.get('name'): iface})
+ self._network_state['interfaces'].update({command.get('name'): iface})
self.dump_network_state()
+ @ensure_command_keys(['name', 'vlan_id', 'vlan_link'])
def handle_vlan(self, command):
'''
auto eth0.222
@@ -158,23 +274,14 @@ class NetworkState(object):
hwaddress ether BC:76:4E:06:96:B3
vlan-raw-device eth0
'''
- required_keys = [
- 'name',
- 'vlan_link',
- 'vlan_id',
- ]
- if not self.valid_command(command, required_keys):
- print('Skipping Invalid command: {}'.format(command))
- print(self.dump_network_state())
- return
-
- interfaces = self.network_state.get('interfaces')
+ interfaces = self._network_state.get('interfaces', {})
self.handle_physical(command)
iface = interfaces.get(command.get('name'), {})
iface['vlan-raw-device'] = command.get('vlan_link')
iface['vlan_id'] = command.get('vlan_id')
interfaces.update({iface['name']: iface})
+ @ensure_command_keys(['name', 'bond_interfaces', 'params'])
def handle_bond(self, command):
'''
#/etc/network/interfaces
@@ -200,23 +307,14 @@ class NetworkState(object):
bond-updelay 200
bond-lacp-rate 4
'''
- required_keys = [
- 'name',
- 'bond_interfaces',
- 'params',
- ]
- if not self.valid_command(command, required_keys):
- print('Skipping Invalid command: {}'.format(command))
- print(self.dump_network_state())
- return
self.handle_physical(command)
- interfaces = self.network_state.get('interfaces')
+ interfaces = self._network_state.get('interfaces')
iface = interfaces.get(command.get('name'), {})
for param, val in command.get('params').items():
iface.update({param: val})
iface.update({'bond-slaves': 'none'})
- self.network_state['interfaces'].update({iface['name']: iface})
+ self._network_state['interfaces'].update({iface['name']: iface})
# handle bond slaves
for ifname in command.get('bond_interfaces'):
@@ -228,14 +326,15 @@ class NetworkState(object):
# inject placeholder
self.handle_physical(cmd)
- interfaces = self.network_state.get('interfaces')
+ interfaces = self._network_state.get('interfaces', {})
bond_if = interfaces.get(ifname)
bond_if['bond-master'] = command.get('name')
# copy in bond config into slave
for param, val in command.get('params').items():
bond_if.update({param: val})
- self.network_state['interfaces'].update({ifname: bond_if})
+ self._network_state['interfaces'].update({ifname: bond_if})
+ @ensure_command_keys(['name', 'bridge_interfaces', 'params'])
def handle_bridge(self, command):
'''
auto br0
@@ -263,19 +362,10 @@ class NetworkState(object):
"bridge_waitport",
]
'''
- required_keys = [
- 'name',
- 'bridge_interfaces',
- 'params',
- ]
- if not self.valid_command(command, required_keys):
- print('Skipping Invalid command: {}'.format(command))
- print(self.dump_network_state())
- return
# find one of the bridge port ifaces to get mac_addr
# handle bridge_slaves
- interfaces = self.network_state.get('interfaces')
+ interfaces = self._network_state.get('interfaces', {})
for ifname in command.get('bridge_interfaces'):
if ifname in interfaces:
continue
@@ -286,7 +376,7 @@ class NetworkState(object):
# inject placeholder
self.handle_physical(cmd)
- interfaces = self.network_state.get('interfaces')
+ interfaces = self._network_state.get('interfaces', {})
self.handle_physical(command)
iface = interfaces.get(command.get('name'), {})
iface['bridge_ports'] = command['bridge_interfaces']
@@ -295,16 +385,9 @@ class NetworkState(object):
interfaces.update({iface['name']: iface})
+ @ensure_command_keys(['address'])
def handle_nameserver(self, command):
- required_keys = [
- 'address',
- ]
- if not self.valid_command(command, required_keys):
- print('Skipping Invalid command: {}'.format(command))
- print(self.dump_network_state())
- return
-
- dns = self.network_state.get('dns')
+ dns = self._network_state.get('dns')
if 'address' in command:
addrs = command['address']
if not type(addrs) == list:
@@ -318,16 +401,9 @@ class NetworkState(object):
for path in paths:
dns['search'].append(path)
+ @ensure_command_keys(['destination'])
def handle_route(self, command):
- required_keys = [
- 'destination',
- ]
- if not self.valid_command(command, required_keys):
- print('Skipping Invalid command: {}'.format(command))
- print(self.dump_network_state())
- return
-
- routes = self.network_state.get('routes')
+ routes = self._network_state.get('routes', [])
network, cidr = command['destination'].split("/")
netmask = cidr2mask(int(cidr))
route = {
@@ -376,72 +452,3 @@ def mask2cidr(mask):
return ipv4mask2cidr(mask)
else:
return mask
-
-
-if __name__ == '__main__':
- import random
- import sys
-
- from cloudinit import net
-
- def load_config(nc):
- version = nc.get('version')
- config = nc.get('config')
- return (version, config)
-
- def test_parse(network_config):
- (version, config) = load_config(network_config)
- ns1 = NetworkState(version=version, config=config)
- ns1.parse_config()
- random.shuffle(config)
- ns2 = NetworkState(version=version, config=config)
- ns2.parse_config()
- print("----NS1-----")
- print(ns1.dump_network_state())
- print()
- print("----NS2-----")
- print(ns2.dump_network_state())
- print("NS1 == NS2 ?=> {}".format(
- ns1.network_state == ns2.network_state))
- eni = net.render_interfaces(ns2.network_state)
- print(eni)
- udev_rules = net.render_persistent_net(ns2.network_state)
- print(udev_rules)
-
- def test_dump_and_load(network_config):
- print("Loading network_config into NetworkState")
- (version, config) = load_config(network_config)
- ns1 = NetworkState(version=version, config=config)
- ns1.parse_config()
- print("Dumping state to file")
- ns1_dump = ns1.dump()
- ns1_state = "/tmp/ns1.state"
- with open(ns1_state, "w+") as f:
- f.write(ns1_dump)
-
- print("Loading state from file")
- ns2 = from_state_file(ns1_state)
- print("NS1 == NS2 ?=> {}".format(
- ns1.network_state == ns2.network_state))
-
- def test_output(network_config):
- (version, config) = load_config(network_config)
- ns1 = NetworkState(version=version, config=config)
- ns1.parse_config()
- random.shuffle(config)
- ns2 = NetworkState(version=version, config=config)
- ns2.parse_config()
- print("NS1 == NS2 ?=> {}".format(
- ns1.network_state == ns2.network_state))
- eni_1 = net.render_interfaces(ns1.network_state)
- eni_2 = net.render_interfaces(ns2.network_state)
- print(eni_1)
- print(eni_2)
- print("eni_1 == eni_2 ?=> {}".format(
- eni_1 == eni_2))
-
- y = util.read_conf(sys.argv[1])
- network_config = y.get('network')
- test_parse(network_config)
- test_dump_and_load(network_config)
- test_output(network_config)
diff --git a/cloudinit/net/renderer.py b/cloudinit/net/renderer.py
new file mode 100644
index 00000000..310cbe0d
--- /dev/null
+++ b/cloudinit/net/renderer.py
@@ -0,0 +1,48 @@
+# Copyright (C) 2013-2014 Canonical Ltd.
+#
+# Author: Scott Moser <scott.moser@canonical.com>
+# Author: Blake Rouse <blake.rouse@canonical.com>
+#
+# Curtin is free software: you can redistribute it and/or modify it under
+# the terms of the GNU Affero General Public License as published by the
+# Free Software Foundation, either version 3 of the License, or (at your
+# option) any later version.
+#
+# Curtin is distributed in the hope that it will be useful, but WITHOUT ANY
+# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for
+# more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with Curtin. If not, see <http://www.gnu.org/licenses/>.
+
+import six
+
+from .udev import generate_udev_rule
+
+
+def filter_by_type(match_type):
+ return lambda iface: match_type == iface['type']
+
+
+def filter_by_name(match_name):
+ return lambda iface: match_name == iface['name']
+
+
+filter_by_physical = filter_by_type('physical')
+
+
+class Renderer(object):
+
+ @staticmethod
+ def _render_persistent_net(network_state):
+ """Given state, emit udev rules to map mac to ifname."""
+ # TODO(harlowja): this seems shared between eni renderer and
+ # this, so move it to a shared location.
+ content = six.StringIO()
+ for iface in network_state.iter_interfaces(filter_by_physical):
+ # for physical interfaces write out a persist net udev rule
+ if 'name' in iface and iface.get('mac_address'):
+ content.write(generate_udev_rule(iface['name'],
+ iface['mac_address']))
+ return content.getvalue()
diff --git a/cloudinit/net/sysconfig.py b/cloudinit/net/sysconfig.py
new file mode 100644
index 00000000..c53acf71
--- /dev/null
+++ b/cloudinit/net/sysconfig.py
@@ -0,0 +1,400 @@
+# vi: ts=4 expandtab
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 3, as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+import os
+import re
+
+import six
+
+from cloudinit.distros.parsers import resolv_conf
+from cloudinit import util
+
+from . import renderer
+
+
+def _make_header(sep='#'):
+ lines = [
+ "Created by cloud-init on instance boot automatically, do not edit.",
+ "",
+ ]
+ for i in range(0, len(lines)):
+ if lines[i]:
+ lines[i] = sep + " " + lines[i]
+ else:
+ lines[i] = sep
+ return "\n".join(lines)
+
+
+def _is_default_route(route):
+ if route['network'] == '::' and route['netmask'] == 0:
+ return True
+ if route['network'] == '0.0.0.0' and route['netmask'] == '0.0.0.0':
+ return True
+ return False
+
+
+def _quote_value(value):
+ if re.search(r"\s", value):
+ # This doesn't handle complex cases...
+ if value.startswith('"') and value.endswith('"'):
+ return value
+ else:
+ return '"%s"' % value
+ else:
+ return value
+
+
+class ConfigMap(object):
+ """Sysconfig like dictionary object."""
+
+ # Why does redhat prefer yes/no to true/false??
+ _bool_map = {
+ True: 'yes',
+ False: 'no',
+ }
+
+ def __init__(self):
+ self._conf = {}
+
+ def __setitem__(self, key, value):
+ self._conf[key] = value
+
+ def drop(self, key):
+ self._conf.pop(key, None)
+
+ def __len__(self):
+ return len(self._conf)
+
+ def to_string(self):
+ buf = six.StringIO()
+ buf.write(_make_header())
+ if self._conf:
+ buf.write("\n")
+ for key in sorted(self._conf.keys()):
+ value = self._conf[key]
+ if isinstance(value, bool):
+ value = self._bool_map[value]
+ if not isinstance(value, six.string_types):
+ value = str(value)
+ buf.write("%s=%s\n" % (key, _quote_value(value)))
+ return buf.getvalue()
+
+
+class Route(ConfigMap):
+ """Represents a route configuration."""
+
+ route_fn_tpl = '%(base)s/network-scripts/route-%(name)s'
+
+ def __init__(self, route_name, base_sysconf_dir):
+ super(Route, self).__init__()
+ self.last_idx = 1
+ self.has_set_default = False
+ self._route_name = route_name
+ self._base_sysconf_dir = base_sysconf_dir
+
+ def copy(self):
+ r = Route(self._route_name, self._base_sysconf_dir)
+ r._conf = self._conf.copy()
+ r.last_idx = self.last_idx
+ r.has_set_default = self.has_set_default
+ return r
+
+ @property
+ def path(self):
+ return self.route_fn_tpl % ({'base': self._base_sysconf_dir,
+ 'name': self._route_name})
+
+
+class NetInterface(ConfigMap):
+ """Represents a sysconfig/networking-script (and its config + children)."""
+
+ iface_fn_tpl = '%(base)s/network-scripts/ifcfg-%(name)s'
+
+ iface_types = {
+ 'ethernet': 'Ethernet',
+ 'bond': 'Bond',
+ 'bridge': 'Bridge',
+ }
+
+ def __init__(self, iface_name, base_sysconf_dir, kind='ethernet'):
+ super(NetInterface, self).__init__()
+ self.children = []
+ self.routes = Route(iface_name, base_sysconf_dir)
+ self._kind = kind
+ self._iface_name = iface_name
+ self._conf['DEVICE'] = iface_name
+ self._conf['TYPE'] = self.iface_types[kind]
+ self._base_sysconf_dir = base_sysconf_dir
+
+ @property
+ def name(self):
+ return self._iface_name
+
+ @name.setter
+ def name(self, iface_name):
+ self._iface_name = iface_name
+ self._conf['DEVICE'] = iface_name
+
+ @property
+ def kind(self):
+ return self._kind
+
+ @kind.setter
+ def kind(self, kind):
+ self._kind = kind
+ self._conf['TYPE'] = self.iface_types[kind]
+
+ @property
+ def path(self):
+ return self.iface_fn_tpl % ({'base': self._base_sysconf_dir,
+ 'name': self.name})
+
+ def copy(self, copy_children=False, copy_routes=False):
+ c = NetInterface(self.name, self._base_sysconf_dir, kind=self._kind)
+ c._conf = self._conf.copy()
+ if copy_children:
+ c.children = list(self.children)
+ if copy_routes:
+ c.routes = self.routes.copy()
+ return c
+
+
+class Renderer(renderer.Renderer):
+ """Renders network information in a /etc/sysconfig format."""
+
+ # See: https://access.redhat.com/documentation/en-US/\
+ # Red_Hat_Enterprise_Linux/6/html/Deployment_Guide/\
+ # s1-networkscripts-interfaces.html (or other docs for
+ # details about this)
+
+ iface_defaults = tuple([
+ ('ONBOOT', True),
+ ('USERCTL', False),
+ ('NM_CONTROLLED', False),
+ ('BOOTPROTO', 'none'),
+ ])
+
+ # If these keys exist, then there values will be used to form
+ # a BONDING_OPTS grouping; otherwise no grouping will be set.
+ bond_tpl_opts = tuple([
+ ('bond_mode', "mode=%s"),
+ ('bond_xmit_hash_policy', "xmit_hash_policy=%s"),
+ ('bond_miimon', "miimon=%s"),
+ ])
+
+ bridge_opts_keys = tuple([
+ ('bridge_stp', 'STP'),
+ ('bridge_ageing', 'AGEING'),
+ ('bridge_bridgeprio', 'PRIO'),
+ ])
+
+ def __init__(self, config=None):
+ if not config:
+ config = {}
+ self.sysconf_dir = config.get('sysconf_dir', 'etc/sysconfig/')
+ self.netrules_path = config.get(
+ 'netrules_path', 'etc/udev/rules.d/70-persistent-net.rules')
+ self.dns_path = config.get('dns_path', 'etc/resolv.conf')
+
+ @classmethod
+ def _render_iface_shared(cls, iface, iface_cfg):
+ for k, v in cls.iface_defaults:
+ iface_cfg[k] = v
+ for (old_key, new_key) in [('mac_address', 'HWADDR'), ('mtu', 'MTU')]:
+ old_value = iface.get(old_key)
+ if old_value is not None:
+ iface_cfg[new_key] = old_value
+
+ @classmethod
+ def _render_subnet(cls, iface_cfg, route_cfg, subnet):
+ subnet_type = subnet.get('type')
+ if subnet_type == 'dhcp6':
+ iface_cfg['DHCPV6C'] = True
+ iface_cfg['IPV6INIT'] = True
+ iface_cfg['BOOTPROTO'] = 'dhcp'
+ elif subnet_type in ['dhcp4', 'dhcp']:
+ iface_cfg['BOOTPROTO'] = 'dhcp'
+ elif subnet_type == 'static':
+ iface_cfg['BOOTPROTO'] = 'static'
+ if subnet.get('ipv6'):
+ iface_cfg['IPV6ADDR'] = subnet['address']
+ iface_cfg['IPV6INIT'] = True
+ else:
+ iface_cfg['IPADDR'] = subnet['address']
+ else:
+ raise ValueError("Unknown subnet type '%s' found"
+ " for interface '%s'" % (subnet_type,
+ iface_cfg.name))
+ if 'netmask' in subnet:
+ iface_cfg['NETMASK'] = subnet['netmask']
+ for route in subnet.get('routes', []):
+ if _is_default_route(route):
+ if route_cfg.has_set_default:
+ raise ValueError("Duplicate declaration of default"
+ " route found for interface '%s'"
+ % (iface_cfg.name))
+ # NOTE(harlowja): ipv6 and ipv4 default gateways
+ gw_key = 'GATEWAY0'
+ nm_key = 'NETMASK0'
+ addr_key = 'ADDRESS0'
+ # The owning interface provides the default route.
+ #
+ # TODO(harlowja): add validation that no other iface has
+ # also provided the default route?
+ iface_cfg['DEFROUTE'] = True
+ if 'gateway' in route:
+ iface_cfg['GATEWAY'] = route['gateway']
+ route_cfg.has_set_default = True
+ else:
+ gw_key = 'GATEWAY%s' % route_cfg.last_idx
+ nm_key = 'NETMASK%s' % route_cfg.last_idx
+ addr_key = 'ADDRESS%s' % route_cfg.last_idx
+ route_cfg.last_idx += 1
+ for (old_key, new_key) in [('gateway', gw_key),
+ ('netmask', nm_key),
+ ('network', addr_key)]:
+ if old_key in route:
+ route_cfg[new_key] = route[old_key]
+
+ @classmethod
+ def _render_bonding_opts(cls, iface_cfg, iface):
+ bond_opts = []
+ for (bond_key, value_tpl) in cls.bond_tpl_opts:
+ # Seems like either dash or underscore is possible?
+ bond_keys = [bond_key, bond_key.replace("_", "-")]
+ for bond_key in bond_keys:
+ if bond_key in iface:
+ bond_value = iface[bond_key]
+ if isinstance(bond_value, (tuple, list)):
+ bond_value = " ".join(bond_value)
+ bond_opts.append(value_tpl % (bond_value))
+ break
+ if bond_opts:
+ iface_cfg['BONDING_OPTS'] = " ".join(bond_opts)
+
+ @classmethod
+ def _render_physical_interfaces(cls, network_state, iface_contents):
+ physical_filter = renderer.filter_by_physical
+ for iface in network_state.iter_interfaces(physical_filter):
+ iface_name = iface['name']
+ iface_subnets = iface.get("subnets", [])
+ iface_cfg = iface_contents[iface_name]
+ route_cfg = iface_cfg.routes
+ if len(iface_subnets) == 1:
+ cls._render_subnet(iface_cfg, route_cfg, iface_subnets[0])
+ elif len(iface_subnets) > 1:
+ for i, iface_subnet in enumerate(iface_subnets,
+ start=len(iface.children)):
+ iface_sub_cfg = iface_cfg.copy()
+ iface_sub_cfg.name = "%s:%s" % (iface_name, i)
+ iface.children.append(iface_sub_cfg)
+ cls._render_subnet(iface_sub_cfg, route_cfg, iface_subnet)
+
+ @classmethod
+ def _render_bond_interfaces(cls, network_state, iface_contents):
+ bond_filter = renderer.filter_by_type('bond')
+ for iface in network_state.iter_interfaces(bond_filter):
+ iface_name = iface['name']
+ iface_cfg = iface_contents[iface_name]
+ cls._render_bonding_opts(iface_cfg, iface)
+ iface_master_name = iface['bond-master']
+ iface_cfg['MASTER'] = iface_master_name
+ iface_cfg['SLAVE'] = True
+ # Ensure that the master interface (and any of its children)
+ # are actually marked as being bond types...
+ master_cfg = iface_contents[iface_master_name]
+ master_cfgs = [master_cfg]
+ master_cfgs.extend(master_cfg.children)
+ for master_cfg in master_cfgs:
+ master_cfg['BONDING_MASTER'] = True
+ master_cfg.kind = 'bond'
+
+ @staticmethod
+ def _render_vlan_interfaces(network_state, iface_contents):
+ vlan_filter = renderer.filter_by_type('vlan')
+ for iface in network_state.iter_interfaces(vlan_filter):
+ iface_name = iface['name']
+ iface_cfg = iface_contents[iface_name]
+ iface_cfg['VLAN'] = True
+ iface_cfg['PHYSDEV'] = iface_name[:iface_name.rfind('.')]
+
+ @staticmethod
+ def _render_dns(network_state, existing_dns_path=None):
+ content = resolv_conf.ResolvConf("")
+ if existing_dns_path and os.path.isfile(existing_dns_path):
+ content = resolv_conf.ResolvConf(util.load_file(existing_dns_path))
+ for nameserver in network_state.dns_nameservers:
+ content.add_nameserver(nameserver)
+ for searchdomain in network_state.dns_searchdomains:
+ content.add_search_domain(searchdomain)
+ return "\n".join([_make_header(';'), str(content)])
+
+ @classmethod
+ def _render_bridge_interfaces(cls, network_state, iface_contents):
+ bridge_filter = renderer.filter_by_type('bridge')
+ for iface in network_state.iter_interfaces(bridge_filter):
+ iface_name = iface['name']
+ iface_cfg = iface_contents[iface_name]
+ iface_cfg.kind = 'bridge'
+ for old_key, new_key in cls.bridge_opts_keys:
+ if old_key in iface:
+ iface_cfg[new_key] = iface[old_key]
+ # Is this the right key to get all the connected interfaces?
+ for bridged_iface_name in iface.get('bridge_ports', []):
+ # Ensure all bridged interfaces are correctly tagged
+ # as being bridged to this interface.
+ bridged_cfg = iface_contents[bridged_iface_name]
+ bridged_cfgs = [bridged_cfg]
+ bridged_cfgs.extend(bridged_cfg.children)
+ for bridge_cfg in bridged_cfgs:
+ bridge_cfg['BRIDGE'] = iface_name
+
+ @classmethod
+ def _render_sysconfig(cls, base_sysconf_dir, network_state):
+ '''Given state, return /etc/sysconfig files + contents'''
+ iface_contents = {}
+ for iface in network_state.iter_interfaces():
+ iface_name = iface['name']
+ iface_cfg = NetInterface(iface_name, base_sysconf_dir)
+ cls._render_iface_shared(iface, iface_cfg)
+ iface_contents[iface_name] = iface_cfg
+ cls._render_physical_interfaces(network_state, iface_contents)
+ cls._render_bond_interfaces(network_state, iface_contents)
+ cls._render_vlan_interfaces(network_state, iface_contents)
+ cls._render_bridge_interfaces(network_state, iface_contents)
+ contents = {}
+ for iface_name, iface_cfg in iface_contents.items():
+ if iface_cfg or iface_cfg.children:
+ contents[iface_cfg.path] = iface_cfg.to_string()
+ for iface_cfg in iface_cfg.children:
+ if iface_cfg:
+ contents[iface_cfg.path] = iface_cfg.to_string()
+ if iface_cfg.routes:
+ contents[iface_cfg.routes.path] = iface_cfg.routes.to_string()
+ return contents
+
+ def render_network_state(self, target, network_state):
+ base_sysconf_dir = os.path.join(target, self.sysconf_dir)
+ for path, data in self._render_sysconfig(base_sysconf_dir,
+ network_state).items():
+ util.write_file(path, data)
+ if self.dns_path:
+ dns_path = os.path.join(target, self.dns_path)
+ resolv_content = self._render_dns(network_state,
+ existing_dns_path=dns_path)
+ util.write_file(dns_path, resolv_content)
+ if self.netrules_path:
+ netrules_content = self._render_persistent_net(network_state)
+ netrules_path = os.path.join(target, self.netrules_path)
+ util.write_file(netrules_path, netrules_content)
diff --git a/cloudinit/serial.py b/cloudinit/serial.py
new file mode 100644
index 00000000..af45c13e
--- /dev/null
+++ b/cloudinit/serial.py
@@ -0,0 +1,50 @@
+# vi: ts=4 expandtab
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 3, as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+
+from __future__ import absolute_import
+
+try:
+ from serial import Serial
+except ImportError:
+ # For older versions of python (ie 2.6) pyserial may not exist and/or
+ # work and/or be installed, so make a dummy/fake serial that blows up
+ # when used...
+ class Serial(object):
+ def __init__(self, *args, **kwargs):
+ pass
+
+ @staticmethod
+ def isOpen():
+ return False
+
+ @staticmethod
+ def write(data):
+ raise IOError("Unable to perform serial `write` operation,"
+ " pyserial not installed.")
+
+ @staticmethod
+ def readline():
+ raise IOError("Unable to perform serial `readline` operation,"
+ " pyserial not installed.")
+
+ @staticmethod
+ def flush():
+ raise IOError("Unable to perform serial `flush` operation,"
+ " pyserial not installed.")
+
+ @staticmethod
+ def read(size=1):
+ raise IOError("Unable to perform serial `read` operation,"
+ " pyserial not installed.")
diff --git a/cloudinit/sources/DataSourceAltCloud.py b/cloudinit/sources/DataSourceAltCloud.py
index cd61df31..a3529609 100644
--- a/cloudinit/sources/DataSourceAltCloud.py
+++ b/cloudinit/sources/DataSourceAltCloud.py
@@ -205,8 +205,7 @@ class DataSourceAltCloud(sources.DataSource):
_err.message)
return False
except OSError as _err:
- util.logexc(LOG, 'Failed command: %s\n%s', ' '.join(cmd),
- _err.message)
+ util.logexc(LOG, 'Failed command: %s\n%s', ' '.join(cmd), _err)
return False
floppy_dev = '/dev/fd0'
diff --git a/cloudinit/sources/DataSourceAzure.py b/cloudinit/sources/DataSourceAzure.py
index 2d046600..8c7e8673 100644
--- a/cloudinit/sources/DataSourceAzure.py
+++ b/cloudinit/sources/DataSourceAzure.py
@@ -423,7 +423,7 @@ def write_files(datadir, files, dirmode=None):
elem.text = DEF_PASSWD_REDACTION
return ET.tostring(root)
except Exception:
- LOG.critical("failed to redact userpassword in {}".format(fname))
+ LOG.critical("failed to redact userpassword in %s", fname)
return cnt
if not datadir:
diff --git a/cloudinit/sources/DataSourceConfigDrive.py b/cloudinit/sources/DataSourceConfigDrive.py
index 3cc9155d..3130e618 100644
--- a/cloudinit/sources/DataSourceConfigDrive.py
+++ b/cloudinit/sources/DataSourceConfigDrive.py
@@ -18,14 +18,14 @@
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
-import copy
import os
from cloudinit import log as logging
-from cloudinit import net
from cloudinit import sources
from cloudinit import util
+from cloudinit.net import eni
+
from cloudinit.sources.helpers import openstack
LOG = logging.getLogger(__name__)
@@ -53,6 +53,7 @@ class DataSourceConfigDrive(openstack.SourceMixin, sources.DataSource):
self._network_config = None
self.network_json = None
self.network_eni = None
+ self.known_macs = None
self.files = {}
def __str__(self):
@@ -147,9 +148,10 @@ class DataSourceConfigDrive(openstack.SourceMixin, sources.DataSource):
if self._network_config is None:
if self.network_json is not None:
LOG.debug("network config provided via network_json")
- self._network_config = convert_network_data(self.network_json)
+ self._network_config = openstack.convert_net_json(
+ self.network_json, known_macs=self.known_macs)
elif self.network_eni is not None:
- self._network_config = net.convert_eni_data(self.network_eni)
+ self._network_config = eni.convert_eni_data(self.network_eni)
LOG.debug("network config provided via converted eni data")
else:
LOG.debug("no network configuration available")
@@ -254,152 +256,12 @@ def find_candidate_devs(probe_optical=True):
return devices
-# Convert OpenStack ConfigDrive NetworkData json to network_config yaml
-def convert_network_data(network_json=None, known_macs=None):
- """Return a dictionary of network_config by parsing provided
- OpenStack ConfigDrive NetworkData json format
-
- OpenStack network_data.json provides a 3 element dictionary
- - "links" (links are network devices, physical or virtual)
- - "networks" (networks are ip network configurations for one or more
- links)
- - services (non-ip services, like dns)
-
- networks and links are combined via network items referencing specific
- links via a 'link_id' which maps to a links 'id' field.
-
- To convert this format to network_config yaml, we first iterate over the
- links and then walk the network list to determine if any of the networks
- utilize the current link; if so we generate a subnet entry for the device
-
- We also need to map network_data.json fields to network_config fields. For
- example, the network_data links 'id' field is equivalent to network_config
- 'name' field for devices. We apply more of this mapping to the various
- link types that we encounter.
-
- There are additional fields that are populated in the network_data.json
- from OpenStack that are not relevant to network_config yaml, so we
- enumerate a dictionary of valid keys for network_yaml and apply filtering
- to drop these superflous keys from the network_config yaml.
- """
- if network_json is None:
- return None
-
- # dict of network_config key for filtering network_json
- valid_keys = {
- 'physical': [
- 'name',
- 'type',
- 'mac_address',
- 'subnets',
- 'params',
- ],
- 'subnet': [
- 'type',
- 'address',
- 'netmask',
- 'broadcast',
- 'metric',
- 'gateway',
- 'pointopoint',
- 'mtu',
- 'scope',
- 'dns_nameservers',
- 'dns_search',
- 'routes',
- ],
- }
-
- links = network_json.get('links', [])
- networks = network_json.get('networks', [])
- services = network_json.get('services', [])
-
- config = []
- for link in links:
- subnets = []
- cfg = {k: v for k, v in link.items()
- if k in valid_keys['physical']}
- # 'name' is not in openstack spec yet, but we will support it if it is
- # present. The 'id' in the spec is currently implemented as the host
- # nic's name, meaning something like 'tap-adfasdffd'. We do not want
- # to name guest devices with such ugly names.
- if 'name' in link:
- cfg['name'] = link['name']
-
- for network in [n for n in networks
- if n['link'] == link['id']]:
- subnet = {k: v for k, v in network.items()
- if k in valid_keys['subnet']}
- if 'dhcp' in network['type']:
- t = 'dhcp6' if network['type'].startswith('ipv6') else 'dhcp4'
- subnet.update({
- 'type': t,
- })
- else:
- subnet.update({
- 'type': 'static',
- 'address': network.get('ip_address'),
- })
- subnets.append(subnet)
- cfg.update({'subnets': subnets})
- if link['type'] in ['ethernet', 'vif', 'ovs', 'phy']:
- cfg.update({
- 'type': 'physical',
- 'mac_address': link['ethernet_mac_address']})
- elif link['type'] in ['bond']:
- params = {}
- for k, v in link.items():
- if k == 'bond_links':
- continue
- elif k.startswith('bond'):
- params.update({k: v})
- cfg.update({
- 'bond_interfaces': copy.deepcopy(link['bond_links']),
- 'params': params,
- })
- elif link['type'] in ['vlan']:
- cfg.update({
- 'name': "%s.%s" % (link['vlan_link'],
- link['vlan_id']),
- 'vlan_link': link['vlan_link'],
- 'vlan_id': link['vlan_id'],
- 'mac_address': link['vlan_mac_address'],
- })
- else:
- raise ValueError(
- 'Unknown network_data link type: %s' % link['type'])
-
- config.append(cfg)
-
- need_names = [d for d in config
- if d.get('type') == 'physical' and 'name' not in d]
-
- if need_names:
- if known_macs is None:
- known_macs = net.get_interfaces_by_mac()
-
- for d in need_names:
- mac = d.get('mac_address')
- if not mac:
- raise ValueError("No mac_address or name entry for %s" % d)
- if mac not in known_macs:
- raise ValueError("Unable to find a system nic for %s" % d)
- d['name'] = known_macs[mac]
-
- for service in services:
- cfg = service
- cfg.update({'type': 'nameserver'})
- config.append(cfg)
-
- return {'version': 1, 'config': config}
-
-
# Legacy: Must be present in case we load an old pkl object
DataSourceConfigDriveNet = DataSourceConfigDrive
# Used to match classes to dependencies
datasources = [
- (DataSourceConfigDrive, (sources.DEP_FILESYSTEM, )),
+ (DataSourceConfigDrive, (sources.DEP_FILESYSTEM,)),
]
diff --git a/cloudinit/sources/DataSourceGCE.py b/cloudinit/sources/DataSourceGCE.py
index 9234d1f8..c660a350 100644
--- a/cloudinit/sources/DataSourceGCE.py
+++ b/cloudinit/sources/DataSourceGCE.py
@@ -25,7 +25,7 @@ from cloudinit import util
LOG = logging.getLogger(__name__)
BUILTIN_DS_CONFIG = {
- 'metadata_url': 'http://metadata.google.internal./computeMetadata/v1/'
+ 'metadata_url': 'http://metadata.google.internal/computeMetadata/v1/'
}
REQUIRED_FIELDS = ('instance-id', 'availability-zone', 'local-hostname')
diff --git a/cloudinit/sources/DataSourceNoCloud.py b/cloudinit/sources/DataSourceNoCloud.py
index 7e30118c..cdc9eef5 100644
--- a/cloudinit/sources/DataSourceNoCloud.py
+++ b/cloudinit/sources/DataSourceNoCloud.py
@@ -24,7 +24,7 @@ import errno
import os
from cloudinit import log as logging
-from cloudinit import net
+from cloudinit.net import eni
from cloudinit import sources
from cloudinit import util
@@ -34,7 +34,6 @@ LOG = logging.getLogger(__name__)
class DataSourceNoCloud(sources.DataSource):
def __init__(self, sys_cfg, distro, paths):
sources.DataSource.__init__(self, sys_cfg, distro, paths)
- self.dsmode = 'local'
self.seed = None
self.seed_dirs = [os.path.join(paths.seed_dir, 'nocloud'),
os.path.join(paths.seed_dir, 'nocloud-net')]
@@ -194,8 +193,7 @@ class DataSourceNoCloud(sources.DataSource):
# LP: #1568150 need getattr in the case that an old class object
# has been loaded from a pickled file and now executing new source.
dirs = getattr(self, 'seed_dirs', [self.seed_dir])
- quick_id = _quick_read_instance_id(cmdline_id=self.cmdline_id,
- dirs=dirs)
+ quick_id = _quick_read_instance_id(dirs=dirs)
if not quick_id:
return None
return quick_id == current
@@ -203,20 +201,19 @@ class DataSourceNoCloud(sources.DataSource):
@property
def network_config(self):
if self._network_config is None:
- if self.network_eni is not None:
- self._network_config = net.convert_eni_data(self.network_eni)
+ if self._network_eni is not None:
+ self._network_config = eni.convert_eni_data(self._network_eni)
return self._network_config
-def _quick_read_instance_id(cmdline_id, dirs=None):
+def _quick_read_instance_id(dirs=None):
if dirs is None:
dirs = []
iid_key = 'instance-id'
- if cmdline_id is None:
- fill = {}
- if parse_cmdline_data(cmdline_id, fill) and iid_key in fill:
- return fill[iid_key]
+ fill = {}
+ if load_cmdline_data(fill) and iid_key in fill:
+ return fill[iid_key]
for d in dirs:
if d is None:
diff --git a/cloudinit/sources/DataSourceOVF.py b/cloudinit/sources/DataSourceOVF.py
index f2bb9366..43347cfb 100644
--- a/cloudinit/sources/DataSourceOVF.py
+++ b/cloudinit/sources/DataSourceOVF.py
@@ -37,16 +37,16 @@ from cloudinit.sources.helpers.vmware.imc.config_file \
import ConfigFile
from cloudinit.sources.helpers.vmware.imc.config_nic \
import NicConfigurator
+from cloudinit.sources.helpers.vmware.imc.guestcust_error \
+ import GuestCustErrorEnum
from cloudinit.sources.helpers.vmware.imc.guestcust_event \
import GuestCustEventEnum
from cloudinit.sources.helpers.vmware.imc.guestcust_state \
import GuestCustStateEnum
-from cloudinit.sourceshelpers.vmware.imc.guestcust_error \
- import GuestCustErrorEnum
-from cloudinit.sourceshelpers.vmware.imc.guestcust_util import (
- set_customization_status,
+from cloudinit.sources.helpers.vmware.imc.guestcust_util import (
+ enable_nics,
get_nics_to_enable,
- enable_nics
+ set_customization_status
)
LOG = logging.getLogger(__name__)
diff --git a/cloudinit/sources/DataSourceOpenNebula.py b/cloudinit/sources/DataSourceOpenNebula.py
index 8f85b115..7b3a76b9 100644
--- a/cloudinit/sources/DataSourceOpenNebula.py
+++ b/cloudinit/sources/DataSourceOpenNebula.py
@@ -104,7 +104,7 @@ class DataSourceOpenNebula(sources.DataSource):
def get_hostname(self, fqdn=False, resolve_ip=None):
if resolve_ip is None:
- if self.dsmode == sources.DSMODE_NET:
+ if self.dsmode == sources.DSMODE_NETWORK:
resolve_ip = True
else:
resolve_ip = False
diff --git a/cloudinit/sources/DataSourceSmartOS.py b/cloudinit/sources/DataSourceSmartOS.py
index 0e03b04f..08bc132b 100644
--- a/cloudinit/sources/DataSourceSmartOS.py
+++ b/cloudinit/sources/DataSourceSmartOS.py
@@ -40,13 +40,11 @@ import random
import re
import socket
-import serial
-
from cloudinit import log as logging
+from cloudinit import serial
from cloudinit import sources
from cloudinit import util
-
LOG = logging.getLogger(__name__)
SMARTOS_ATTRIB_MAP = {
diff --git a/cloudinit/sources/helpers/openstack.py b/cloudinit/sources/helpers/openstack.py
index 3ccb11d3..d52cb56a 100644
--- a/cloudinit/sources/helpers/openstack.py
+++ b/cloudinit/sources/helpers/openstack.py
@@ -28,6 +28,7 @@ import six
from cloudinit import ec2_utils
from cloudinit import log as logging
+from cloudinit import net
from cloudinit import sources
from cloudinit import url_helper
from cloudinit import util
@@ -156,7 +157,7 @@ class BaseReader(object):
pass
@abc.abstractmethod
- def _path_read(self, path):
+ def _path_read(self, path, decode=False):
pass
@abc.abstractmethod
@@ -478,6 +479,150 @@ class MetadataReader(BaseReader):
retries=self.retries)
+# Convert OpenStack ConfigDrive NetworkData json to network_config yaml
+def convert_net_json(network_json=None, known_macs=None):
+ """Return a dictionary of network_config by parsing provided
+ OpenStack ConfigDrive NetworkData json format
+
+ OpenStack network_data.json provides a 3 element dictionary
+ - "links" (links are network devices, physical or virtual)
+ - "networks" (networks are ip network configurations for one or more
+ links)
+ - services (non-ip services, like dns)
+
+ networks and links are combined via network items referencing specific
+ links via a 'link_id' which maps to a links 'id' field.
+
+ To convert this format to network_config yaml, we first iterate over the
+ links and then walk the network list to determine if any of the networks
+ utilize the current link; if so we generate a subnet entry for the device
+
+ We also need to map network_data.json fields to network_config fields. For
+ example, the network_data links 'id' field is equivalent to network_config
+ 'name' field for devices. We apply more of this mapping to the various
+ link types that we encounter.
+
+ There are additional fields that are populated in the network_data.json
+ from OpenStack that are not relevant to network_config yaml, so we
+ enumerate a dictionary of valid keys for network_yaml and apply filtering
+ to drop these superflous keys from the network_config yaml.
+ """
+ if network_json is None:
+ return None
+
+ # dict of network_config key for filtering network_json
+ valid_keys = {
+ 'physical': [
+ 'name',
+ 'type',
+ 'mac_address',
+ 'subnets',
+ 'params',
+ 'mtu',
+ ],
+ 'subnet': [
+ 'type',
+ 'address',
+ 'netmask',
+ 'broadcast',
+ 'metric',
+ 'gateway',
+ 'pointopoint',
+ 'scope',
+ 'dns_nameservers',
+ 'dns_search',
+ 'routes',
+ ],
+ }
+
+ links = network_json.get('links', [])
+ networks = network_json.get('networks', [])
+ services = network_json.get('services', [])
+
+ config = []
+ for link in links:
+ subnets = []
+ cfg = {k: v for k, v in link.items()
+ if k in valid_keys['physical']}
+ # 'name' is not in openstack spec yet, but we will support it if it is
+ # present. The 'id' in the spec is currently implemented as the host
+ # nic's name, meaning something like 'tap-adfasdffd'. We do not want
+ # to name guest devices with such ugly names.
+ if 'name' in link:
+ cfg['name'] = link['name']
+
+ for network in [n for n in networks
+ if n['link'] == link['id']]:
+ subnet = {k: v for k, v in network.items()
+ if k in valid_keys['subnet']}
+ if 'dhcp' in network['type']:
+ t = 'dhcp6' if network['type'].startswith('ipv6') else 'dhcp4'
+ subnet.update({
+ 'type': t,
+ })
+ else:
+ subnet.update({
+ 'type': 'static',
+ 'address': network.get('ip_address'),
+ })
+ if network['type'] == 'ipv4':
+ subnet['ipv4'] = True
+ if network['type'] == 'ipv6':
+ subnet['ipv6'] = True
+ subnets.append(subnet)
+ cfg.update({'subnets': subnets})
+ if link['type'] in ['ethernet', 'vif', 'ovs', 'phy', 'bridge']:
+ cfg.update({
+ 'type': 'physical',
+ 'mac_address': link['ethernet_mac_address']})
+ elif link['type'] in ['bond']:
+ params = {}
+ for k, v in link.items():
+ if k == 'bond_links':
+ continue
+ elif k.startswith('bond'):
+ params.update({k: v})
+ cfg.update({
+ 'bond_interfaces': copy.deepcopy(link['bond_links']),
+ 'params': params,
+ })
+ elif link['type'] in ['vlan']:
+ cfg.update({
+ 'name': "%s.%s" % (link['vlan_link'],
+ link['vlan_id']),
+ 'vlan_link': link['vlan_link'],
+ 'vlan_id': link['vlan_id'],
+ 'mac_address': link['vlan_mac_address'],
+ })
+ else:
+ raise ValueError(
+ 'Unknown network_data link type: %s' % link['type'])
+
+ config.append(cfg)
+
+ need_names = [d for d in config
+ if d.get('type') == 'physical' and 'name' not in d]
+
+ if need_names:
+ if known_macs is None:
+ known_macs = net.get_interfaces_by_mac()
+
+ for d in need_names:
+ mac = d.get('mac_address')
+ if not mac:
+ raise ValueError("No mac_address or name entry for %s" % d)
+ if mac not in known_macs:
+ raise ValueError("Unable to find a system nic for %s" % d)
+ d['name'] = known_macs[mac]
+
+ for service in services:
+ cfg = service
+ cfg.update({'type': 'nameserver'})
+ config.append(cfg)
+
+ return {'version': 1, 'config': config}
+
+
def convert_vendordata_json(data, recurse=True):
"""data: a loaded json *object* (strings, arrays, dicts).
return something suitable for cloudinit vendordata_raw.
diff --git a/cloudinit/stages.py b/cloudinit/stages.py
index 5756e74d..47deac6e 100644
--- a/cloudinit/stages.py
+++ b/cloudinit/stages.py
@@ -44,6 +44,7 @@ from cloudinit import helpers
from cloudinit import importer
from cloudinit import log as logging
from cloudinit import net
+from cloudinit.net import cmdline
from cloudinit.reporting import events
from cloudinit import sources
from cloudinit import type_utils
@@ -612,13 +613,13 @@ class Init(object):
if os.path.exists(disable_file):
return (None, disable_file)
- cmdline_cfg = ('cmdline', net.read_kernel_cmdline_config())
+ cmdline_cfg = ('cmdline', cmdline.read_kernel_cmdline_config())
dscfg = ('ds', None)
if self.datasource and hasattr(self.datasource, 'network_config'):
dscfg = ('ds', self.datasource.network_config)
sys_cfg = ('system_cfg', self.cfg.get('network'))
- for loc, ncfg in (cmdline_cfg, dscfg, sys_cfg):
+ for loc, ncfg in (cmdline_cfg, sys_cfg, dscfg):
if net.is_disabled_cfg(ncfg):
LOG.debug("network config disabled by %s", loc)
return (None, loc)
diff --git a/cloudinit/templater.py b/cloudinit/templater.py
index a9231482..41ef27e3 100644
--- a/cloudinit/templater.py
+++ b/cloudinit/templater.py
@@ -3,10 +3,12 @@
# Copyright (C) 2012 Canonical Ltd.
# Copyright (C) 2012 Hewlett-Packard Development Company, L.P.
# Copyright (C) 2012 Yahoo! Inc.
+# Copyright (C) 2016 Amazon.com, Inc. or its affiliates.
#
# Author: Scott Moser <scott.moser@canonical.com>
# Author: Juerg Haefliger <juerg.haefliger@hp.com>
# Author: Joshua Harlow <harlowja@yahoo-inc.com>
+# Author: Andrew Jorgensen <ajorgens@amazon.com>
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 3, as
@@ -102,12 +104,11 @@ def detect_template(text):
rest = ''
type_match = TYPE_MATCHER.match(ident)
if not type_match:
- if not CHEETAH_AVAILABLE:
- LOG.warn("Cheetah not available as the default renderer for"
- " unknown template, reverting to the basic renderer.")
- return ('basic', basic_render, text)
- else:
+ if CHEETAH_AVAILABLE:
+ LOG.debug("Using Cheetah as the renderer for unknown template.")
return ('cheetah', cheetah_render, text)
+ else:
+ return ('basic', basic_render, text)
else:
template_type = type_match.group(1).lower().strip()
if template_type not in ('jinja', 'cheetah', 'basic'):
@@ -142,6 +143,11 @@ def render_to_file(fn, outfn, params, mode=0o644):
util.write_file(outfn, contents, mode=mode)
+def render_string_to_file(content, outfn, params, mode=0o644):
+ contents = render_string(content, params)
+ util.write_file(outfn, contents, mode=mode)
+
+
def render_string(content, params):
if not params:
params = {}
diff --git a/cloudinit/util.py b/cloudinit/util.py
index 8d6cbb4b..e5dd61a0 100644
--- a/cloudinit/util.py
+++ b/cloudinit/util.py
@@ -171,7 +171,8 @@ class ProcessExecutionError(IOError):
def __init__(self, stdout=None, stderr=None,
exit_code=None, cmd=None,
- description=None, reason=None):
+ description=None, reason=None,
+ errno=None):
if not cmd:
self.cmd = '-'
else:
@@ -202,6 +203,7 @@ class ProcessExecutionError(IOError):
else:
self.reason = '-'
+ self.errno = errno
message = self.MESSAGE_TMPL % {
'description': self.description,
'cmd': self.cmd,
@@ -336,6 +338,16 @@ def rand_str(strlen=32, select_from=None):
return "".join([random.choice(select_from) for _x in range(0, strlen)])
+def rand_dict_key(dictionary, postfix=None):
+ if not postfix:
+ postfix = ""
+ while True:
+ newkey = rand_str(strlen=8) + "_" + postfix
+ if newkey not in dictionary:
+ break
+ return newkey
+
+
def read_conf(fname):
try:
return load_yaml(load_file(fname), default={})
@@ -1147,7 +1159,14 @@ def find_devs_with(criteria=None, oformat='device',
options.append(path)
cmd = blk_id_cmd + options
# See man blkid for why 2 is added
- (out, _err) = subp(cmd, rcs=[0, 2])
+ try:
+ (out, _err) = subp(cmd, rcs=[0, 2])
+ except ProcessExecutionError as e:
+ if e.errno == errno.ENOENT:
+ # blkid not found...
+ out = ""
+ else:
+ raise
entries = []
for line in out.splitlines():
line = line.strip()
@@ -1696,7 +1715,8 @@ def subp(args, data=None, rcs=None, env=None, capture=True, shell=False,
sp = subprocess.Popen(args, **kws)
(out, err) = sp.communicate(data)
except OSError as e:
- raise ProcessExecutionError(cmd=args, reason=e)
+ raise ProcessExecutionError(cmd=args, reason=e,
+ errno=e.errno)
rc = sp.returncode
if rc not in rcs:
raise ProcessExecutionError(stdout=out, stderr=err,
@@ -2190,7 +2210,7 @@ def _call_dmidecode(key, dmidecode_path):
return ""
return result
except (IOError, OSError) as _err:
- LOG.debug('failed dmidecode cmd: %s\n%s', cmd, _err.message)
+ LOG.debug('failed dmidecode cmd: %s\n%s', cmd, _err)
return None
diff --git a/doc/examples/cloud-config.txt b/doc/examples/cloud-config.txt
index 1236796c..3cc9c055 100644
--- a/doc/examples/cloud-config.txt
+++ b/doc/examples/cloud-config.txt
@@ -72,14 +72,87 @@ apt_pipelining: False
# then apt_mirror above will have no effect
apt_preserve_sources_list: true
+# Provide a custom template for rendering sources.list
+# Default: a default template for Ubuntu/Debain will be used as packaged in
+# Ubuntu: /etc/cloud/templates/sources.list.ubuntu.tmpl
+# Debian: /etc/cloud/templates/sources.list.debian.tmpl
+# Others: n/a
+# This will follow the normal mirror/codename replacement rules before
+# being written to disk.
+apt_custom_sources_list: |
+ ## template:jinja
+ ## Note, this file is written by cloud-init on first boot of an instance
+ ## modifications made here will not survive a re-bundle.
+ ## if you wish to make changes you can:
+ ## a.) add 'apt_preserve_sources_list: true' to /etc/cloud/cloud.cfg
+ ## or do the same in user-data
+ ## b.) add sources in /etc/apt/sources.list.d
+ ## c.) make changes to template file /etc/cloud/templates/sources.list.tmpl
+ deb {{mirror}} {{codename}} main restricted
+ deb-src {{mirror}} {{codename}} main restricted
+
+ # could drop some of the usually used entries
+
+ # could refer to other mirrors
+ deb http://ddebs.ubuntu.com {{codename}} main restricted universe multiverse
+ deb http://ddebs.ubuntu.com {{codename}}-updates main restricted universe multiverse
+ deb http://ddebs.ubuntu.com {{codename}}-proposed main restricted universe multiverse
+
+ # or even more uncommon examples like local or NFS mounted repos,
+ # eventually whatever is compatible with sources.list syntax
+ deb file:/home/apt/debian unstable main contrib non-free
+
# 'source' entries in apt-sources that match this python regex
# expression will be passed to add-apt-repository
add_apt_repo_match: '^[\w-]+:\w'
+# 'apt_sources' is a dictionary
+# The key is the filename and will be prepended by /etc/apt/sources.list.d/ if
+# it doesn't start with a '/'.
+# There are certain cases - where no content is written into a source.list file
+# where the filename will be ignored - yet it can still be used as index for
+# merging.
+# The value it maps to is a dictionary with the following optional entries:
+# source: a sources.list entry (some variable replacements apply)
+# keyid: providing a key to import via shortid or fingerprint
+# key: providing a raw PGP key
+# keyserver: keyserver to fetch keys from, default is keyserver.ubuntu.com
+# filename: for compatibility with the older format (now the key to this
+# dictionary is the filename). If specified this overwrites the
+# filename given as key.
+
+# the new "filename: {specification-dictionary}, filename2: ..." format allows
+# better merging between multiple input files than a list like:
+# cloud-config1
+# sources:
+# s1: {'key': 'key1', 'source': 'source1'}
+# cloud-config2
+# sources:
+# s2: {'key': 'key2'}
+# s1: {filename: 'foo'}
+# this would be merged to
+#sources:
+# s1:
+# filename: foo
+# key: key1
+# source: source1
+# s2:
+# key: key2
+# Be aware that this style of merging is not the default (for backward
+# compatibility reasons). You should specify the following merge_how to get
+# this more complete and modern merging behaviour:
+# merge_how: "list()+dict()+str()"
+# This would then also be equivalent to the config merging used in curtin
+# (https://launchpad.net/curtin).
+
+# for more details see below in the various examples
+
apt_sources:
- - source: "deb http://ppa.launchpad.net/byobu/ppa/ubuntu karmic main"
+ byobu-ppa.list:
+ source: "deb http://ppa.launchpad.net/byobu/ppa/ubuntu karmic main"
keyid: F430BBA5 # GPG key ID published on a key server
- filename: byobu-ppa.list
+ # adding a source.list line, importing a gpg key for a given key id and
+ # storing it in the file /etc/apt/sources.list.d/byobu-ppa.list
# PPA shortcut:
# * Setup correct apt sources.list line
@@ -87,7 +160,9 @@ apt_sources:
#
# See https://help.launchpad.net/Packaging/PPA for more information
# this requires 'add-apt-repository'
- - source: "ppa:smoser/ppa" # Quote the string
+ # due to that the filename key is ignored in this case
+ ignored1:
+ source: "ppa:smoser/ppa" # Quote the string
# Custom apt repository:
# * all that is required is 'source'
@@ -95,29 +170,60 @@ apt_sources:
# * [optional] Import the apt signing key from the keyserver
# * Defaults:
# + keyserver: keyserver.ubuntu.com
- # + filename: cloud_config_sources.list
#
# See sources.list man page for more information about the format
- - source: deb http://archive.ubuntu.com/ubuntu karmic-backports main universe multiverse restricted
+ my-repo.list:
+ source: deb http://archive.ubuntu.com/ubuntu karmic-backports main universe multiverse restricted
# sources can use $MIRROR and $RELEASE and they will be replaced
# with the local mirror for this cloud, and the running release
# the entry below would be possibly turned into:
- # - source: deb http://us-east-1.ec2.archive.ubuntu.com/ubuntu natty multiverse
- - source: deb $MIRROR $RELEASE multiverse
+ # source: deb http://us-east-1.ec2.archive.ubuntu.com/ubuntu natty multiverse
+ my-repo.list:
+ source: deb $MIRROR $RELEASE multiverse
# this would have the same end effect as 'ppa:byobu/ppa'
- - source: "deb http://ppa.launchpad.net/byobu/ppa/ubuntu karmic main"
+ my-repo.list:
+ source: "deb http://ppa.launchpad.net/byobu/ppa/ubuntu karmic main"
keyid: F430BBA5 # GPG key ID published on a key server
filename: byobu-ppa.list
+ # this would only import the key without adding a ppa or other source spec
+ # since this doesn't generate a source.list file the filename key is ignored
+ ignored2:
+ keyid: F430BBA5 # GPG key ID published on a key server
+
+ # In general keyid's can also be specified via their long fingerprints
+ # since this doesn't generate a source.list file the filename key is ignored
+ ignored3:
+ keyid: B59D 5F15 97A5 04B7 E230 6DCA 0620 BBCF 0368 3F77
+
# Custom apt repository:
# * The apt signing key can also be specified
# by providing a pgp public key block
- # * Providing the PBG key here is the most robust method for
+ # * Providing the PGP key here is the most robust method for
# specifying a key, as it removes dependency on a remote key server
+ my-repo.list:
+ source: deb http://ppa.launchpad.net/alestic/ppa/ubuntu karmic main
+ key: | # The value needs to start with -----BEGIN PGP PUBLIC KEY BLOCK-----
+ -----BEGIN PGP PUBLIC KEY BLOCK-----
+ Version: SKS 1.0.10
+
+ mI0ESpA3UQEEALdZKVIMq0j6qWAXAyxSlF63SvPVIgxHPb9Nk0DZUixn+akqytxG4zKCONz6
+ qLjoBBfHnynyVLfT4ihg9an1PqxRnTO+JKQxl8NgKGz6Pon569GtAOdWNKw15XKinJTDLjnj
+ 9y96ljJqRcpV9t/WsIcdJPcKFR5voHTEoABE2aEXABEBAAG0GUxhdW5jaHBhZCBQUEEgZm9y
+ IEFsZXN0aWOItgQTAQIAIAUCSpA3UQIbAwYLCQgHAwIEFQIIAwQWAgMBAh4BAheAAAoJEA7H
+ 5Qi+CcVxWZ8D/1MyYvfj3FJPZUm2Yo1zZsQ657vHI9+pPouqflWOayRR9jbiyUFIn0VdQBrP
+ t0FwvnOFArUovUWoKAEdqR8hPy3M3APUZjl5K4cMZR/xaMQeQRZ5CHpS4DBKURKAHC0ltS5o
+ uBJKQOZm5iltJp15cgyIkBkGe8Mx18VFyVglAZey
+ =Y2oI
+ -----END PGP PUBLIC KEY BLOCK-----
- - source: deb http://ppa.launchpad.net/alestic/ppa/ubuntu karmic main
+ # Custom gpg key:
+ # * As with keyid, a key may also be specified without a related source.
+ # * all other facts mentioned above still apply
+ # since this doesn't generate a source.list file the filename key is ignored
+ ignored4:
key: | # The value needs to start with -----BEGIN PGP PUBLIC KEY BLOCK-----
-----BEGIN PGP PUBLIC KEY BLOCK-----
Version: SKS 1.0.10
@@ -132,6 +238,7 @@ apt_sources:
=Y2oI
-----END PGP PUBLIC KEY BLOCK-----
+
## apt config via system_info:
# under the 'system_info', you can further customize cloud-init's interaction
# with apt.
diff --git a/packages/bddeb b/packages/bddeb
index 1b0f642c..3c77ce1d 100755
--- a/packages/bddeb
+++ b/packages/bddeb
@@ -42,6 +42,7 @@ STD_NAMED_PACKAGES = [
'setuptools',
'flake8',
'hacking',
+ 'unittest2',
]
NONSTD_NAMED_PACKAGES = {
'argparse': ('python-argparse', None),
@@ -148,11 +149,17 @@ def main():
parser.add_argument("--sign", default=False, action='store_true',
help="sign result. do not pass -us -uc to debuild")
+ parser.add_argument("--signuser", default=False, action='store',
+ help="user to sign, see man dpkg-genchanges")
+
args = parser.parse_args()
if not args.sign:
args.debuild_args.extend(['-us', '-uc'])
+ if args.signuser:
+ args.debuild_args.extend(['-e%s' % args.signuser])
+
os.environ['INIT_SYSTEM'] = args.init_system
capture = True
diff --git a/requirements.txt b/requirements.txt
index 19c88857..cc1dc05f 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -11,8 +11,12 @@ PrettyTable
oauthlib
# This one is currently used only by the CloudSigma and SmartOS datasources.
-# If these datasources are removed, this is no longer needed
-pyserial
+# If these datasources are removed, this is no longer needed.
+#
+# This will not work in py2.6 so it is only optionally installed on
+# python 2.7 and later.
+#
+# pyserial
# This is only needed for places where we need to support configs in a manner
# that the built-in config parser is not sufficent (ie
diff --git a/setup.py b/setup.py
index 57d946ca..0af576a9 100755
--- a/setup.py
+++ b/setup.py
@@ -196,7 +196,6 @@ requirements = read_requires()
if sys.version_info < (3,):
requirements.append('cheetah')
-
setuptools.setup(
name='cloud-init',
version=get_version(),
@@ -205,10 +204,14 @@ setuptools.setup(
author_email='scott.moser@canonical.com',
url='http://launchpad.net/cloud-init/',
packages=setuptools.find_packages(exclude=['tests']),
- scripts=['bin/cloud-init',
- 'tools/cloud-init-per'],
+ scripts=['tools/cloud-init-per'],
license='GPLv3',
data_files=data_files,
install_requires=requirements,
cmdclass=cmdclass,
+ entry_points={
+ 'console_scripts': [
+ 'cloud-init = cloudinit.cmd.main:main'
+ ],
+ }
)
diff --git a/test-requirements.txt b/test-requirements.txt
index 651af11b..6bf38940 100644
--- a/test-requirements.txt
+++ b/test-requirements.txt
@@ -2,6 +2,7 @@
httpretty>=0.7.1
mock
nose
+unittest2
# Only needed if you want to know the test times
# nose-timer
diff --git a/tests/unittests/helpers.py b/tests/unittests/helpers.py
index 50b2bd72..972245df 100644
--- a/tests/unittests/helpers.py
+++ b/tests/unittests/helpers.py
@@ -7,13 +7,11 @@ import sys
import tempfile
import unittest
+import mock
import six
+import unittest2
try:
- from unittest import mock
-except ImportError:
- import mock
-try:
from contextlib import ExitStack
except ImportError:
from contextlib2 import ExitStack
@@ -21,6 +19,9 @@ except ImportError:
from cloudinit import helpers as ch
from cloudinit import util
+# Used for skipping tests
+SkipTest = unittest2.SkipTest
+
# Used for detecting different python versions
PY2 = False
PY26 = False
@@ -44,78 +45,6 @@ else:
if _PY_MINOR == 4 and _PY_MICRO < 3:
FIX_HTTPRETTY = True
-if PY26:
- # For now add these on, taken from python 2.7 + slightly adjusted. Drop
- # all this once Python 2.6 is dropped as a minimum requirement.
- class TestCase(unittest.TestCase):
- def setUp(self):
- super(TestCase, self).setUp()
- self.__all_cleanups = ExitStack()
-
- def tearDown(self):
- self.__all_cleanups.close()
- unittest.TestCase.tearDown(self)
-
- def addCleanup(self, function, *args, **kws):
- self.__all_cleanups.callback(function, *args, **kws)
-
- def assertIs(self, expr1, expr2, msg=None):
- if expr1 is not expr2:
- standardMsg = '%r is not %r' % (expr1, expr2)
- self.fail(self._formatMessage(msg, standardMsg))
-
- def assertIn(self, member, container, msg=None):
- if member not in container:
- standardMsg = '%r not found in %r' % (member, container)
- self.fail(self._formatMessage(msg, standardMsg))
-
- def assertNotIn(self, member, container, msg=None):
- if member in container:
- standardMsg = '%r unexpectedly found in %r'
- standardMsg = standardMsg % (member, container)
- self.fail(self._formatMessage(msg, standardMsg))
-
- def assertIsNone(self, value, msg=None):
- if value is not None:
- standardMsg = '%r is not None'
- standardMsg = standardMsg % (value)
- self.fail(self._formatMessage(msg, standardMsg))
-
- def assertIsInstance(self, obj, cls, msg=None):
- """Same as self.assertTrue(isinstance(obj, cls)), with a nicer
- default message."""
- if not isinstance(obj, cls):
- standardMsg = '%s is not an instance of %r' % (repr(obj), cls)
- self.fail(self._formatMessage(msg, standardMsg))
-
- def assertDictContainsSubset(self, expected, actual, msg=None):
- missing = []
- mismatched = []
- for k, v in expected.items():
- if k not in actual:
- missing.append(k)
- elif actual[k] != v:
- mismatched.append('%r, expected: %r, actual: %r'
- % (k, v, actual[k]))
-
- if len(missing) == 0 and len(mismatched) == 0:
- return
-
- standardMsg = ''
- if missing:
- standardMsg = 'Missing: %r' % ','.join(m for m in missing)
- if mismatched:
- if standardMsg:
- standardMsg += '; '
- standardMsg += 'Mismatched values: %s' % ','.join(mismatched)
-
- self.fail(self._formatMessage(msg, standardMsg))
-
-
-else:
- class TestCase(unittest.TestCase):
- pass
-
# Makes the old path start
# with new base instead of whatever
@@ -151,6 +80,10 @@ def retarget_many_wrapper(new_base, am, old_func):
return wrapper
+class TestCase(unittest2.TestCase):
+ pass
+
+
class ResourceUsingTestCase(TestCase):
def setUp(self):
super(ResourceUsingTestCase, self).setUp()
@@ -331,6 +264,18 @@ def populate_dir(path, files):
fp.close()
+def dir2dict(startdir, prefix=None):
+ flist = {}
+ if prefix is None:
+ prefix = startdir
+ for root, dirs, files in os.walk(startdir):
+ for fname in files:
+ fpath = os.path.join(root, fname)
+ key = fpath[len(prefix):]
+ flist[key] = util.load_file(fpath)
+ return flist
+
+
try:
skipIf = unittest.skipIf
except AttributeError:
diff --git a/tests/unittests/test__init__.py b/tests/unittests/test__init__.py
index 153f1658..0154784a 100644
--- a/tests/unittests/test__init__.py
+++ b/tests/unittests/test__init__.py
@@ -1,16 +1,6 @@
import os
import shutil
import tempfile
-import unittest
-
-try:
- from unittest import mock
-except ImportError:
- import mock
-try:
- from contextlib import ExitStack
-except ImportError:
- from contextlib2 import ExitStack
from cloudinit import handlers
from cloudinit import helpers
@@ -18,7 +8,7 @@ from cloudinit import settings
from cloudinit import url_helper
from cloudinit import util
-from .helpers import TestCase
+from .helpers import TestCase, ExitStack, mock
class FakeModule(handlers.Handler):
@@ -99,9 +89,10 @@ class TestWalkerHandleHandler(TestCase):
self.assertEqual(self.data['handlercount'], 0)
-class TestHandlerHandlePart(unittest.TestCase):
+class TestHandlerHandlePart(TestCase):
def setUp(self):
+ super(TestHandlerHandlePart, self).setUp()
self.data = "fake data"
self.ctype = "fake ctype"
self.filename = "fake filename"
@@ -177,7 +168,7 @@ class TestHandlerHandlePart(unittest.TestCase):
self.data, self.ctype, self.filename, self.payload)
-class TestCmdlineUrl(unittest.TestCase):
+class TestCmdlineUrl(TestCase):
def test_invalid_content(self):
url = "http://example.com/foo"
key = "mykey"
diff --git a/tests/unittests/test_cli.py b/tests/unittests/test_cli.py
index f537bd83..5fa252f7 100644
--- a/tests/unittests/test_cli.py
+++ b/tests/unittests/test_cli.py
@@ -1,17 +1,10 @@
-import imp
-import os
import six
-import sys
from . import helpers as test_helpers
-try:
- from unittest import mock
-except ImportError:
- import mock
+from cloudinit.cmd import main as cli
-
-BIN_CLOUDINIT = "bin/cloud-init"
+mock = test_helpers.mock
class TestCLI(test_helpers.FilesystemMockingTestCase):
@@ -20,35 +13,22 @@ class TestCLI(test_helpers.FilesystemMockingTestCase):
super(TestCLI, self).setUp()
self.stderr = six.StringIO()
self.patchStdoutAndStderr(stderr=self.stderr)
- self.sys_exit = mock.MagicMock()
- self.patched_funcs.enter_context(
- mock.patch.object(sys, 'exit', self.sys_exit))
-
- def _call_main(self):
- self.patched_funcs.enter_context(
- mock.patch.object(sys, 'argv', ['cloud-init']))
- cli = imp.load_module(
- 'cli', open(BIN_CLOUDINIT), '', ('', 'r', imp.PY_SOURCE))
+
+ def _call_main(self, sysv_args=None):
+ if not sysv_args:
+ sysv_args = ['cloud-init']
try:
- return cli.main()
- except Exception:
- pass
+ return cli.main(sysv_args=sysv_args)
+ except SystemExit as e:
+ return e.code
- @test_helpers.skipIf(not os.path.isfile(BIN_CLOUDINIT), "no bin/cloudinit")
def test_no_arguments_shows_usage(self):
- self._call_main()
- self.assertIn('usage: cloud-init', self.stderr.getvalue())
-
- @test_helpers.skipIf(not os.path.isfile(BIN_CLOUDINIT), "no bin/cloudinit")
- def test_no_arguments_exits_2(self):
exit_code = self._call_main()
- if self.sys_exit.call_count:
- self.assertEqual(mock.call(2), self.sys_exit.call_args)
- else:
- self.assertEqual(2, exit_code)
+ self.assertIn('usage: cloud-init', self.stderr.getvalue())
+ self.assertEqual(2, exit_code)
- @test_helpers.skipIf(not os.path.isfile(BIN_CLOUDINIT), "no bin/cloudinit")
def test_no_arguments_shows_error_message(self):
- self._call_main()
+ exit_code = self._call_main()
self.assertIn('cloud-init: error: too few arguments',
self.stderr.getvalue())
+ self.assertEqual(2, exit_code)
diff --git a/tests/unittests/test_cs_util.py b/tests/unittests/test_cs_util.py
index d7273035..56c9ce9e 100644
--- a/tests/unittests/test_cs_util.py
+++ b/tests/unittests/test_cs_util.py
@@ -1,21 +1,9 @@
from __future__ import print_function
-import sys
-import unittest
+from . import helpers as test_helpers
from cloudinit.cs_utils import Cepko
-try:
- skip = unittest.skip
-except AttributeError:
- # Python 2.6. Doesn't have to be high fidelity.
- def skip(reason):
- def decorator(func):
- def wrapper(*args, **kws):
- print(reason, file=sys.stderr)
- return wrapper
- return decorator
-
SERVER_CONTEXT = {
"cpu": 1000,
@@ -43,18 +31,9 @@ class CepkoMock(Cepko):
# 2015-01-22 BAW: This test is completely useless because it only ever tests
# the CepkoMock object. Even in its original form, I don't think it ever
# touched the underlying Cepko class methods.
-@skip('This test is completely useless')
-class CepkoResultTests(unittest.TestCase):
+class CepkoResultTests(test_helpers.TestCase):
def setUp(self):
- pass
- # self.mocked = self.mocker.replace("cloudinit.cs_utils.Cepko",
- # spec=CepkoMock,
- # count=False,
- # passthrough=False)
- # self.mocked()
- # self.mocker.result(CepkoMock())
- # self.mocker.replay()
- # self.c = Cepko()
+ raise test_helpers.SkipTest('This test is completely useless')
def test_getitem(self):
result = self.c.all()
diff --git a/tests/unittests/test_datasource/test_azure.py b/tests/unittests/test_datasource/test_azure.py
index 5f3eb31f..e90e903c 100644
--- a/tests/unittests/test_datasource/test_azure.py
+++ b/tests/unittests/test_datasource/test_azure.py
@@ -1,16 +1,8 @@
from cloudinit import helpers
from cloudinit.util import b64e, decode_binary, load_file
from cloudinit.sources import DataSourceAzure
-from ..helpers import TestCase, populate_dir
-try:
- from unittest import mock
-except ImportError:
- import mock
-try:
- from contextlib import ExitStack
-except ImportError:
- from contextlib2 import ExitStack
+from ..helpers import TestCase, populate_dir, mock, ExitStack, PY26, SkipTest
import crypt
import os
@@ -83,6 +75,8 @@ class TestAzureDataSource(TestCase):
def setUp(self):
super(TestAzureDataSource, self).setUp()
+ if PY26:
+ raise SkipTest("Does not work on python 2.6")
self.tmp = tempfile.mkdtemp()
self.addCleanup(shutil.rmtree, self.tmp)
diff --git a/tests/unittests/test_datasource/test_azure_helper.py b/tests/unittests/test_datasource/test_azure_helper.py
index d07a1f07..65202ff0 100644
--- a/tests/unittests/test_datasource/test_azure_helper.py
+++ b/tests/unittests/test_datasource/test_azure_helper.py
@@ -2,17 +2,7 @@ import os
from cloudinit.sources.helpers import azure as azure_helper
-from ..helpers import TestCase
-
-try:
- from unittest import mock
-except ImportError:
- import mock
-
-try:
- from contextlib import ExitStack
-except ImportError:
- from contextlib2 import ExitStack
+from ..helpers import ExitStack, mock, TestCase
GOAL_STATE_TEMPLATE = """\
@@ -288,6 +278,7 @@ class TestOpenSSLManager(TestCase):
self.subp.side_effect = capture_directory
manager = azure_helper.OpenSSLManager()
self.assertEqual(manager.tmpdir, subp_directory['path'])
+ manager.clean_up()
@mock.patch.object(azure_helper, 'cd', mock.MagicMock())
@mock.patch.object(azure_helper.tempfile, 'mkdtemp', mock.MagicMock())
diff --git a/tests/unittests/test_datasource/test_cloudsigma.py b/tests/unittests/test_datasource/test_cloudsigma.py
index 772d189a..2a42ce0c 100644
--- a/tests/unittests/test_datasource/test_cloudsigma.py
+++ b/tests/unittests/test_datasource/test_cloudsigma.py
@@ -1,4 +1,5 @@
# coding: utf-8
+
import copy
from cloudinit.cs_utils import Cepko
@@ -6,7 +7,6 @@ from cloudinit.sources import DataSourceCloudSigma
from .. import helpers as test_helpers
-
SERVER_CONTEXT = {
"cpu": 1000,
"cpus_instead_of_cores": False,
diff --git a/tests/unittests/test_datasource/test_cloudstack.py b/tests/unittests/test_datasource/test_cloudstack.py
index 974b3704..b1aab17b 100644
--- a/tests/unittests/test_datasource/test_cloudstack.py
+++ b/tests/unittests/test_datasource/test_cloudstack.py
@@ -1,16 +1,7 @@
from cloudinit import helpers
from cloudinit.sources.DataSourceCloudStack import DataSourceCloudStack
-from ..helpers import TestCase
-
-try:
- from unittest import mock
-except ImportError:
- import mock
-try:
- from contextlib import ExitStack
-except ImportError:
- from contextlib2 import ExitStack
+from ..helpers import TestCase, mock, ExitStack
class TestCloudStackPasswordFetching(TestCase):
diff --git a/tests/unittests/test_datasource/test_configdrive.py b/tests/unittests/test_datasource/test_configdrive.py
index 1364b39d..18551b92 100644
--- a/tests/unittests/test_datasource/test_configdrive.py
+++ b/tests/unittests/test_datasource/test_configdrive.py
@@ -5,22 +5,15 @@ import shutil
import six
import tempfile
-try:
- from unittest import mock
-except ImportError:
- import mock
-try:
- from contextlib import ExitStack
-except ImportError:
- from contextlib2 import ExitStack
-
from cloudinit import helpers
+from cloudinit.net import eni
+from cloudinit.net import network_state
from cloudinit import settings
from cloudinit.sources import DataSourceConfigDrive as ds
from cloudinit.sources.helpers import openstack
from cloudinit import util
-from ..helpers import TestCase
+from ..helpers import TestCase, ExitStack, mock
PUBKEY = u'ssh-rsa AAAAB3NzaC1....sIkJhq8wdX+4I3A4cYbYP ubuntu@server-460\n'
@@ -88,9 +81,34 @@ NETWORK_DATA = {
]
}
+NETWORK_DATA_2 = {
+ "services": [
+ {"type": "dns", "address": "1.1.1.191"},
+ {"type": "dns", "address": "1.1.1.4"}],
+ "networks": [
+ {"network_id": "d94bbe94-7abc-48d4-9c82-4628ea26164a", "type": "ipv4",
+ "netmask": "255.255.255.248", "link": "eth0",
+ "routes": [{"netmask": "0.0.0.0", "network": "0.0.0.0",
+ "gateway": "2.2.2.9"}],
+ "ip_address": "2.2.2.10", "id": "network0-ipv4"},
+ {"network_id": "ca447c83-6409-499b-aaef-6ad1ae995348", "type": "ipv4",
+ "netmask": "255.255.255.224", "link": "eth1",
+ "routes": [], "ip_address": "3.3.3.24", "id": "network1-ipv4"}],
+ "links": [
+ {"ethernet_mac_address": "fa:16:3e:dd:50:9a", "mtu": 1500,
+ "type": "vif", "id": "eth0", "vif_id": "vif-foo1"},
+ {"ethernet_mac_address": "fa:16:3e:a8:14:69", "mtu": 1500,
+ "type": "vif", "id": "eth1", "vif_id": "vif-foo2"}]
+}
+
+
KNOWN_MACS = {
'fa:16:3e:69:b0:58': 'enp0s1',
- 'fa:16:3e:d4:57:ad': 'enp0s2'}
+ 'fa:16:3e:d4:57:ad': 'enp0s2',
+ 'fa:16:3e:dd:50:9a': 'foo1',
+ 'fa:16:3e:a8:14:69': 'foo2',
+ 'fa:16:3e:ed:9a:59': 'foo3',
+}
CFG_DRIVE_FILES_V2 = {
'ec2/2009-04-04/meta-data.json': json.dumps(EC2_META),
@@ -352,35 +370,150 @@ class TestConfigDriveDataSource(TestCase):
util.find_devs_with = orig_find_devs_with
util.is_partition = orig_is_partition
- def test_pubkeys_v2(self):
+ @mock.patch('cloudinit.sources.DataSourceConfigDrive.on_first_boot')
+ def test_pubkeys_v2(self, on_first_boot):
"""Verify that public-keys work in config-drive-v2."""
populate_dir(self.tmp, CFG_DRIVE_FILES_V2)
myds = cfg_ds_from_dir(self.tmp)
self.assertEqual(myds.get_public_ssh_keys(),
[OSTACK_META['public_keys']['mykey']])
- def test_network_data_is_found(self):
+
+class TestNetJson(TestCase):
+ def setUp(self):
+ super(TestNetJson, self).setUp()
+ self.tmp = tempfile.mkdtemp()
+ self.addCleanup(shutil.rmtree, self.tmp)
+ self.maxDiff = None
+
+ @mock.patch('cloudinit.sources.DataSourceConfigDrive.on_first_boot')
+ def test_network_data_is_found(self, on_first_boot):
"""Verify that network_data is present in ds in config-drive-v2."""
populate_dir(self.tmp, CFG_DRIVE_FILES_V2)
myds = cfg_ds_from_dir(self.tmp)
- self.assertEqual(myds.network_json, NETWORK_DATA)
+ self.assertIsNotNone(myds.network_json)
- def test_network_config_is_converted(self):
+ @mock.patch('cloudinit.sources.DataSourceConfigDrive.on_first_boot')
+ def test_network_config_is_converted(self, on_first_boot):
"""Verify that network_data is converted and present on ds object."""
populate_dir(self.tmp, CFG_DRIVE_FILES_V2)
myds = cfg_ds_from_dir(self.tmp)
- network_config = ds.convert_network_data(NETWORK_DATA,
- known_macs=KNOWN_MACS)
+ network_config = openstack.convert_net_json(NETWORK_DATA,
+ known_macs=KNOWN_MACS)
self.assertEqual(myds.network_config, network_config)
+ def test_network_config_conversions(self):
+ """Tests a bunch of input network json and checks the
+ expected conversions."""
+ in_datas = [
+ NETWORK_DATA,
+ {
+ 'services': [{'type': 'dns', 'address': '172.19.0.12'}],
+ 'networks': [{
+ 'network_id': 'dacd568d-5be6-4786-91fe-750c374b78b4',
+ 'type': 'ipv4',
+ 'netmask': '255.255.252.0',
+ 'link': 'tap1a81968a-79',
+ 'routes': [{
+ 'netmask': '0.0.0.0',
+ 'network': '0.0.0.0',
+ 'gateway': '172.19.3.254',
+ }],
+ 'ip_address': '172.19.1.34',
+ 'id': 'network0',
+ }],
+ 'links': [{
+ 'type': 'bridge',
+ 'vif_id': '1a81968a-797a-400f-8a80-567f997eb93f',
+ 'ethernet_mac_address': 'fa:16:3e:ed:9a:59',
+ 'id': 'tap1a81968a-79',
+ 'mtu': None,
+ }],
+ },
+ ]
+ out_datas = [
+ {
+ 'version': 1,
+ 'config': [
+ {
+ 'subnets': [{'type': 'dhcp4'}],
+ 'type': 'physical',
+ 'mac_address': 'fa:16:3e:69:b0:58',
+ 'name': 'enp0s1',
+ 'mtu': None,
+ },
+ {
+ 'subnets': [{'type': 'dhcp4'}],
+ 'type': 'physical',
+ 'mac_address': 'fa:16:3e:d4:57:ad',
+ 'name': 'enp0s2',
+ 'mtu': None,
+ },
+ {
+ 'subnets': [{'type': 'dhcp4'}],
+ 'type': 'physical',
+ 'mac_address': 'fa:16:3e:05:30:fe',
+ 'name': 'nic0',
+ 'mtu': None,
+ },
+ {
+ 'type': 'nameserver',
+ 'address': '199.204.44.24',
+ },
+ {
+ 'type': 'nameserver',
+ 'address': '199.204.47.54',
+ }
+ ],
+
+ },
+ {
+ 'version': 1,
+ 'config': [
+ {
+ 'name': 'foo3',
+ 'mac_address': 'fa:16:3e:ed:9a:59',
+ 'mtu': None,
+ 'type': 'physical',
+ 'subnets': [
+ {
+ 'address': '172.19.1.34',
+ 'netmask': '255.255.252.0',
+ 'type': 'static',
+ 'ipv4': True,
+ 'routes': [{
+ 'gateway': '172.19.3.254',
+ 'netmask': '0.0.0.0',
+ 'network': '0.0.0.0',
+ }],
+ }
+ ]
+ },
+ {
+ 'type': 'nameserver',
+ 'address': '172.19.0.12',
+ }
+ ],
+ },
+ ]
+ for in_data, out_data in zip(in_datas, out_datas):
+ conv_data = openstack.convert_net_json(in_data,
+ known_macs=KNOWN_MACS)
+ self.assertEqual(out_data, conv_data)
+
class TestConvertNetworkData(TestCase):
+ def setUp(self):
+ super(TestConvertNetworkData, self).setUp()
+ self.tmp = tempfile.mkdtemp()
+ self.addCleanup(shutil.rmtree, self.tmp)
+
def _getnames_in_config(self, ncfg):
return set([n['name'] for n in ncfg['config']
if n['type'] == 'physical'])
def test_conversion_fills_names(self):
- ncfg = ds.convert_network_data(NETWORK_DATA, known_macs=KNOWN_MACS)
+ ncfg = openstack.convert_net_json(NETWORK_DATA, known_macs=KNOWN_MACS)
expected = set(['nic0', 'enp0s1', 'enp0s2'])
found = self._getnames_in_config(ncfg)
self.assertEqual(found, expected)
@@ -392,22 +525,45 @@ class TestConvertNetworkData(TestCase):
'fa:16:3e:69:b0:58': 'ens1'})
get_interfaces_by_mac.return_value = macs
- ncfg = ds.convert_network_data(NETWORK_DATA)
+ ncfg = openstack.convert_net_json(NETWORK_DATA)
expected = set(['nic0', 'ens1', 'enp0s2'])
found = self._getnames_in_config(ncfg)
self.assertEqual(found, expected)
def test_convert_raises_value_error_on_missing_name(self):
macs = {'aa:aa:aa:aa:aa:00': 'ens1'}
- self.assertRaises(ValueError, ds.convert_network_data,
+ self.assertRaises(ValueError, openstack.convert_net_json,
NETWORK_DATA, known_macs=macs)
+ def test_conversion_with_route(self):
+ ncfg = openstack.convert_net_json(NETWORK_DATA_2,
+ known_macs=KNOWN_MACS)
+ # not the best test, but see that we get a route in the
+ # network config and that it gets rendered to an ENI file
+ routes = []
+ for n in ncfg['config']:
+ for s in n.get('subnets', []):
+ routes.extend(s.get('routes', []))
+ self.assertIn(
+ {'network': '0.0.0.0', 'netmask': '0.0.0.0', 'gateway': '2.2.2.9'},
+ routes)
+ eni_renderer = eni.Renderer()
+ eni_renderer.render_network_state(
+ self.tmp, network_state.parse_net_config_data(ncfg))
+ with open(os.path.join(self.tmp, "etc",
+ "network", "interfaces"), 'r') as f:
+ eni_rendering = f.read()
+ self.assertIn("route add default gw 2.2.2.9", eni_rendering)
+
def cfg_ds_from_dir(seed_d):
- found = ds.read_config_drive(seed_d)
cfg_ds = ds.DataSourceConfigDrive(settings.CFG_BUILTIN, None,
helpers.Paths({}))
- populate_ds_from_read_config(cfg_ds, seed_d, found)
+ cfg_ds.seed_dir = seed_d
+ cfg_ds.known_macs = KNOWN_MACS.copy()
+ if not cfg_ds.get_data():
+ raise RuntimeError("Data source did not extract itself from"
+ " seed directory %s" % seed_d)
return cfg_ds
@@ -421,7 +577,7 @@ def populate_ds_from_read_config(cfg_ds, source, results):
cfg_ds.userdata_raw = results.get('userdata')
cfg_ds.version = results.get('version')
cfg_ds.network_json = results.get('networkdata')
- cfg_ds._network_config = ds.convert_network_data(
+ cfg_ds._network_config = openstack.convert_net_json(
cfg_ds.network_json, known_macs=KNOWN_MACS)
@@ -435,7 +591,6 @@ def populate_dir(seed_dir, files):
mode = "w"
else:
mode = "wb"
-
with open(path, mode) as fp:
fp.write(content)
diff --git a/tests/unittests/test_datasource/test_gce.py b/tests/unittests/test_datasource/test_gce.py
index 1f7eb99e..6e62a4d2 100644
--- a/tests/unittests/test_datasource/test_gce.py
+++ b/tests/unittests/test_datasource/test_gce.py
@@ -52,7 +52,7 @@ GCE_META_ENCODING = {
HEADERS = {'X-Google-Metadata-Request': 'True'}
MD_URL_RE = re.compile(
- r'http://metadata.google.internal./computeMetadata/v1/.*')
+ r'http://metadata.google.internal/computeMetadata/v1/.*')
def _set_mock_metadata(gce_meta=None):
diff --git a/tests/unittests/test_datasource/test_nocloud.py b/tests/unittests/test_datasource/test_nocloud.py
index 3c528c23..b0fa1130 100644
--- a/tests/unittests/test_datasource/test_nocloud.py
+++ b/tests/unittests/test_datasource/test_nocloud.py
@@ -1,22 +1,13 @@
from cloudinit import helpers
from cloudinit.sources import DataSourceNoCloud
from cloudinit import util
-from ..helpers import TestCase, populate_dir
+from ..helpers import TestCase, populate_dir, mock, ExitStack
import os
import shutil
import tempfile
-import unittest
-import yaml
-try:
- from unittest import mock
-except ImportError:
- import mock
-try:
- from contextlib import ExitStack
-except ImportError:
- from contextlib2 import ExitStack
+import yaml
class TestNoCloudDataSource(TestCase):
@@ -139,7 +130,7 @@ class TestNoCloudDataSource(TestCase):
self.assertTrue(ret)
-class TestParseCommandLineData(unittest.TestCase):
+class TestParseCommandLineData(TestCase):
def test_parse_cmdline_data_valid(self):
ds_id = "ds=nocloud"
diff --git a/tests/unittests/test_datasource/test_smartos.py b/tests/unittests/test_datasource/test_smartos.py
index c809117b..9c6c8768 100644
--- a/tests/unittests/test_datasource/test_smartos.py
+++ b/tests/unittests/test_datasource/test_smartos.py
@@ -34,11 +34,12 @@ import stat
import tempfile
import uuid
-import serial
+from cloudinit import serial
+from cloudinit.sources import DataSourceSmartOS
+
import six
from cloudinit import helpers as c_helpers
-from cloudinit.sources import DataSourceSmartOS
from cloudinit.util import b64e
from ..helpers import mock, FilesystemMockingTestCase, TestCase
@@ -146,7 +147,9 @@ class TestSmartOSDataSource(FilesystemMockingTestCase):
self.addCleanup(shutil.rmtree, self.tmp)
self.paths = c_helpers.Paths({'cloud_dir': self.tmp})
- self.legacy_user_d = tempfile.mkdtemp()
+ self.legacy_user_d = os.path.join(self.tmp, 'legacy_user_tmp')
+ os.mkdir(self.legacy_user_d)
+
self.orig_lud = DataSourceSmartOS.LEGACY_USER_D
DataSourceSmartOS.LEGACY_USER_D = self.legacy_user_d
@@ -378,6 +381,7 @@ class TestJoyentMetadataClient(FilesystemMockingTestCase):
def setUp(self):
super(TestJoyentMetadataClient, self).setUp()
+
self.serial = mock.MagicMock(spec=serial.Serial)
self.request_id = 0xabcdef12
self.metadata_value = 'value'
diff --git a/tests/unittests/test_handler/test_handler_apt_configure_sources_list.py b/tests/unittests/test_handler/test_handler_apt_configure_sources_list.py
new file mode 100644
index 00000000..acde0863
--- /dev/null
+++ b/tests/unittests/test_handler/test_handler_apt_configure_sources_list.py
@@ -0,0 +1,180 @@
+""" test_handler_apt_configure_sources_list
+Test templating of sources list
+"""
+import logging
+import os
+import shutil
+import tempfile
+
+try:
+ from unittest import mock
+except ImportError:
+ import mock
+
+from cloudinit import cloud
+from cloudinit import distros
+from cloudinit import helpers
+from cloudinit import templater
+from cloudinit import util
+
+from cloudinit.config import cc_apt_configure
+from cloudinit.sources import DataSourceNone
+
+from cloudinit.distros.debian import Distro
+
+from .. import helpers as t_help
+
+LOG = logging.getLogger(__name__)
+
+YAML_TEXT_CUSTOM_SL = """
+apt_mirror: http://archive.ubuntu.com/ubuntu/
+apt_custom_sources_list: |
+ ## template:jinja
+ ## Note, this file is written by cloud-init on first boot of an instance
+ ## modifications made here will not survive a re-bundle.
+ ## if you wish to make changes you can:
+ ## a.) add 'apt_preserve_sources_list: true' to /etc/cloud/cloud.cfg
+ ## or do the same in user-data
+ ## b.) add sources in /etc/apt/sources.list.d
+ ## c.) make changes to template file /etc/cloud/templates/sources.list.tmpl
+
+ # See http://help.ubuntu.com/community/UpgradeNotes for how to upgrade to
+ # newer versions of the distribution.
+ deb {{mirror}} {{codename}} main restricted
+ deb-src {{mirror}} {{codename}} main restricted
+ # FIND_SOMETHING_SPECIAL
+"""
+
+EXPECTED_CONVERTED_CONTENT = (
+ """## Note, this file is written by cloud-init on first boot of an instance
+## modifications made here will not survive a re-bundle.
+## if you wish to make changes you can:
+## a.) add 'apt_preserve_sources_list: true' to /etc/cloud/cloud.cfg
+## or do the same in user-data
+## b.) add sources in /etc/apt/sources.list.d
+## c.) make changes to template file /etc/cloud/templates/sources.list.tmpl
+
+# See http://help.ubuntu.com/community/UpgradeNotes for how to upgrade to
+# newer versions of the distribution.
+deb http://archive.ubuntu.com/ubuntu/ fakerelease main restricted
+deb-src http://archive.ubuntu.com/ubuntu/ fakerelease main restricted
+# FIND_SOMETHING_SPECIAL
+""")
+
+
+def load_tfile_or_url(*args, **kwargs):
+ """load_tfile_or_url
+ load file and return content after decoding
+ """
+ return util.decode_binary(util.read_file_or_url(*args, **kwargs).contents)
+
+
+class TestAptSourceConfigSourceList(t_help.FilesystemMockingTestCase):
+ """TestAptSourceConfigSourceList
+ Main Class to test sources list rendering
+ """
+ def setUp(self):
+ super(TestAptSourceConfigSourceList, self).setUp()
+ self.subp = util.subp
+ self.new_root = tempfile.mkdtemp()
+ self.addCleanup(shutil.rmtree, self.new_root)
+
+ def _get_cloud(self, distro, metadata=None):
+ self.patchUtils(self.new_root)
+ paths = helpers.Paths({})
+ cls = distros.fetch(distro)
+ mydist = cls(distro, {}, paths)
+ myds = DataSourceNone.DataSourceNone({}, mydist, paths)
+ if metadata:
+ myds.metadata.update(metadata)
+ return cloud.Cloud(myds, paths, {}, mydist, None)
+
+ def apt_source_list(self, distro, mirror, mirrorcheck=None):
+ """apt_source_list
+ Test rendering of a source.list from template for a given distro
+ """
+ if mirrorcheck is None:
+ mirrorcheck = mirror
+
+ if isinstance(mirror, list):
+ cfg = {'apt_mirror_search': mirror}
+ else:
+ cfg = {'apt_mirror': mirror}
+ mycloud = self._get_cloud(distro)
+
+ with mock.patch.object(templater, 'render_to_file') as mocktmpl:
+ with mock.patch.object(os.path, 'isfile',
+ return_value=True) as mockisfile:
+ with mock.patch.object(util, 'rename'):
+ cc_apt_configure.handle("notimportant", cfg, mycloud,
+ LOG, None)
+
+ mockisfile.assert_any_call(
+ ('/etc/cloud/templates/sources.list.%s.tmpl' % distro))
+ mocktmpl.assert_called_once_with(
+ ('/etc/cloud/templates/sources.list.%s.tmpl' % distro),
+ '/etc/apt/sources.list',
+ {'codename': '', 'primary': mirrorcheck, 'mirror': mirrorcheck})
+
+ def test_apt_source_list_debian(self):
+ """Test rendering of a source.list from template for debian"""
+ self.apt_source_list('debian', 'http://httpredir.debian.org/debian')
+
+ def test_apt_source_list_ubuntu(self):
+ """Test rendering of a source.list from template for ubuntu"""
+ self.apt_source_list('ubuntu', 'http://archive.ubuntu.com/ubuntu/')
+
+ @staticmethod
+ def myresolve(name):
+ """Fake util.is_resolvable for mirrorfail tests"""
+ if name == "does.not.exist":
+ print("Faking FAIL for '%s'" % name)
+ return False
+ else:
+ print("Faking SUCCESS for '%s'" % name)
+ return True
+
+ def test_apt_srcl_debian_mirrorfail(self):
+ """Test rendering of a source.list from template for debian"""
+ with mock.patch.object(util, 'is_resolvable',
+ side_effect=self.myresolve) as mockresolve:
+ self.apt_source_list('debian',
+ ['http://does.not.exist',
+ 'http://httpredir.debian.org/debian'],
+ 'http://httpredir.debian.org/debian')
+ mockresolve.assert_any_call("does.not.exist")
+ mockresolve.assert_any_call("httpredir.debian.org")
+
+ def test_apt_srcl_ubuntu_mirrorfail(self):
+ """Test rendering of a source.list from template for ubuntu"""
+ with mock.patch.object(util, 'is_resolvable',
+ side_effect=self.myresolve) as mockresolve:
+ self.apt_source_list('ubuntu',
+ ['http://does.not.exist',
+ 'http://archive.ubuntu.com/ubuntu/'],
+ 'http://archive.ubuntu.com/ubuntu/')
+ mockresolve.assert_any_call("does.not.exist")
+ mockresolve.assert_any_call("archive.ubuntu.com")
+
+ def test_apt_srcl_custom(self):
+ """Test rendering from a custom source.list template"""
+ cfg = util.load_yaml(YAML_TEXT_CUSTOM_SL)
+ mycloud = self._get_cloud('ubuntu')
+
+ # the second mock restores the original subp
+ with mock.patch.object(util, 'write_file') as mockwrite:
+ with mock.patch.object(util, 'subp', self.subp):
+ with mock.patch.object(cc_apt_configure, 'get_release',
+ return_value='fakerelease'):
+ with mock.patch.object(Distro, 'get_primary_arch',
+ return_value='amd64'):
+ cc_apt_configure.handle("notimportant", cfg, mycloud,
+ LOG, None)
+
+ mockwrite.assert_called_once_with(
+ '/etc/apt/sources.list',
+ EXPECTED_CONVERTED_CONTENT,
+ mode=420)
+
+
+# vi: ts=4 expandtab
diff --git a/tests/unittests/test_handler/test_handler_apt_source.py b/tests/unittests/test_handler/test_handler_apt_source.py
new file mode 100644
index 00000000..99a4d860
--- /dev/null
+++ b/tests/unittests/test_handler/test_handler_apt_source.py
@@ -0,0 +1,516 @@
+""" test_handler_apt_source
+Testing various config variations of the apt_source config
+"""
+import os
+import re
+import shutil
+import tempfile
+
+try:
+ from unittest import mock
+except ImportError:
+ import mock
+from mock import call
+
+from cloudinit.config import cc_apt_configure
+from cloudinit import gpg
+from cloudinit import util
+
+from ..helpers import TestCase
+
+EXPECTEDKEY = """-----BEGIN PGP PUBLIC KEY BLOCK-----
+Version: GnuPG v1
+
+mI0ESuZLUgEEAKkqq3idtFP7g9hzOu1a8+v8ImawQN4TrvlygfScMU1TIS1eC7UQ
+NUA8Qqgr9iUaGnejb0VciqftLrU9D6WYHSKz+EITefgdyJ6SoQxjoJdsCpJ7o9Jy
+8PQnpRttiFm4qHu6BVnKnBNxw/z3ST9YMqW5kbMQpfxbGe+obRox59NpABEBAAG0
+HUxhdW5jaHBhZCBQUEEgZm9yIFNjb3R0IE1vc2VyiLYEEwECACAFAkrmS1ICGwMG
+CwkIBwMCBBUCCAMEFgIDAQIeAQIXgAAKCRAGILvPA2g/d3aEA/9tVjc10HOZwV29
+OatVuTeERjjrIbxflO586GLA8cp0C9RQCwgod/R+cKYdQcHjbqVcP0HqxveLg0RZ
+FJpWLmWKamwkABErwQLGlM/Hwhjfade8VvEQutH5/0JgKHmzRsoqfR+LMO6OS+Sm
+S0ORP6HXET3+jC8BMG4tBWCTK/XEZw==
+=ACB2
+-----END PGP PUBLIC KEY BLOCK-----"""
+
+
+def load_tfile_or_url(*args, **kwargs):
+ """load_tfile_or_url
+ load file and return content after decoding
+ """
+ return util.decode_binary(util.read_file_or_url(*args, **kwargs).contents)
+
+
+class TestAptSourceConfig(TestCase):
+ """TestAptSourceConfig
+ Main Class to test apt_source configs
+ """
+ release = "fantastic"
+
+ def setUp(self):
+ super(TestAptSourceConfig, self).setUp()
+ self.tmp = tempfile.mkdtemp()
+ self.addCleanup(shutil.rmtree, self.tmp)
+ self.aptlistfile = os.path.join(self.tmp, "single-deb.list")
+ self.aptlistfile2 = os.path.join(self.tmp, "single-deb2.list")
+ self.aptlistfile3 = os.path.join(self.tmp, "single-deb3.list")
+ self.join = os.path.join
+ # mock fallback filename into writable tmp dir
+ self.fallbackfn = os.path.join(self.tmp, "etc/apt/sources.list.d/",
+ "cloud_config_sources.list")
+
+ patcher = mock.patch("cloudinit.config.cc_apt_configure.get_release")
+ get_rel = patcher.start()
+ get_rel.return_value = self.release
+ self.addCleanup(patcher.stop)
+
+ @staticmethod
+ def _get_default_params():
+ """get_default_params
+ Get the most basic default mrror and release info to be used in tests
+ """
+ params = {}
+ params['RELEASE'] = cc_apt_configure.get_release()
+ params['MIRROR'] = "http://archive.ubuntu.com/ubuntu"
+ return params
+
+ def myjoin(self, *args, **kwargs):
+ """myjoin - redir into writable tmpdir"""
+ if (args[0] == "/etc/apt/sources.list.d/" and
+ args[1] == "cloud_config_sources.list" and
+ len(args) == 2):
+ return self.join(self.tmp, args[0].lstrip("/"), args[1])
+ else:
+ return self.join(*args, **kwargs)
+
+ def apt_src_basic(self, filename, cfg):
+ """apt_src_basic
+ Test Fix deb source string, has to overwrite mirror conf in params
+ """
+ params = self._get_default_params()
+
+ cc_apt_configure.add_apt_sources(cfg, params)
+
+ self.assertTrue(os.path.isfile(filename))
+
+ contents = load_tfile_or_url(filename)
+ self.assertTrue(re.search(r"%s %s %s %s\n" %
+ ("deb", "http://archive.ubuntu.com/ubuntu",
+ "karmic-backports",
+ "main universe multiverse restricted"),
+ contents, flags=re.IGNORECASE))
+
+ def test_apt_src_basic(self):
+ """Test deb source string, overwrite mirror and filename"""
+ cfg = {'source': ('deb http://archive.ubuntu.com/ubuntu'
+ ' karmic-backports'
+ ' main universe multiverse restricted'),
+ 'filename': self.aptlistfile}
+ self.apt_src_basic(self.aptlistfile, [cfg])
+
+ def test_apt_src_basic_dict(self):
+ """Test deb source string, overwrite mirror and filename (dict)"""
+ cfg = {self.aptlistfile: {'source':
+ ('deb http://archive.ubuntu.com/ubuntu'
+ ' karmic-backports'
+ ' main universe multiverse restricted')}}
+ self.apt_src_basic(self.aptlistfile, cfg)
+
+ def apt_src_basic_tri(self, cfg):
+ """apt_src_basic_tri
+ Test Fix three deb source string, has to overwrite mirror conf in
+ params. Test with filenames provided in config.
+ generic part to check three files with different content
+ """
+ self.apt_src_basic(self.aptlistfile, cfg)
+
+ # extra verify on two extra files of this test
+ contents = load_tfile_or_url(self.aptlistfile2)
+ self.assertTrue(re.search(r"%s %s %s %s\n" %
+ ("deb", "http://archive.ubuntu.com/ubuntu",
+ "precise-backports",
+ "main universe multiverse restricted"),
+ contents, flags=re.IGNORECASE))
+ contents = load_tfile_or_url(self.aptlistfile3)
+ self.assertTrue(re.search(r"%s %s %s %s\n" %
+ ("deb", "http://archive.ubuntu.com/ubuntu",
+ "lucid-backports",
+ "main universe multiverse restricted"),
+ contents, flags=re.IGNORECASE))
+
+ def test_apt_src_basic_tri(self):
+ """Test Fix three deb source string with filenames"""
+ cfg1 = {'source': ('deb http://archive.ubuntu.com/ubuntu'
+ ' karmic-backports'
+ ' main universe multiverse restricted'),
+ 'filename': self.aptlistfile}
+ cfg2 = {'source': ('deb http://archive.ubuntu.com/ubuntu'
+ ' precise-backports'
+ ' main universe multiverse restricted'),
+ 'filename': self.aptlistfile2}
+ cfg3 = {'source': ('deb http://archive.ubuntu.com/ubuntu'
+ ' lucid-backports'
+ ' main universe multiverse restricted'),
+ 'filename': self.aptlistfile3}
+ self.apt_src_basic_tri([cfg1, cfg2, cfg3])
+
+ def test_apt_src_basic_dict_tri(self):
+ """Test Fix three deb source string with filenames (dict)"""
+ cfg = {self.aptlistfile: {'source':
+ ('deb http://archive.ubuntu.com/ubuntu'
+ ' karmic-backports'
+ ' main universe multiverse restricted')},
+ self.aptlistfile2: {'source':
+ ('deb http://archive.ubuntu.com/ubuntu'
+ ' precise-backports'
+ ' main universe multiverse restricted')},
+ self.aptlistfile3: {'source':
+ ('deb http://archive.ubuntu.com/ubuntu'
+ ' lucid-backports'
+ ' main universe multiverse restricted')}}
+ self.apt_src_basic_tri(cfg)
+
+ def test_apt_src_basic_nofn(self):
+ """Test Fix three deb source string without filenames (dict)"""
+ cfg = {'source': ('deb http://archive.ubuntu.com/ubuntu'
+ ' karmic-backports'
+ ' main universe multiverse restricted')}
+ with mock.patch.object(os.path, 'join', side_effect=self.myjoin):
+ self.apt_src_basic(self.fallbackfn, [cfg])
+
+ def apt_src_replacement(self, filename, cfg):
+ """apt_src_replace
+ Test Autoreplacement of MIRROR and RELEASE in source specs
+ """
+ params = self._get_default_params()
+ cc_apt_configure.add_apt_sources(cfg, params)
+
+ self.assertTrue(os.path.isfile(filename))
+
+ contents = load_tfile_or_url(filename)
+ self.assertTrue(re.search(r"%s %s %s %s\n" %
+ ("deb", params['MIRROR'], params['RELEASE'],
+ "multiverse"),
+ contents, flags=re.IGNORECASE))
+
+ def test_apt_src_replace(self):
+ """Test Autoreplacement of MIRROR and RELEASE in source specs"""
+ cfg = {'source': 'deb $MIRROR $RELEASE multiverse',
+ 'filename': self.aptlistfile}
+ self.apt_src_replacement(self.aptlistfile, [cfg])
+
+ def apt_src_replace_tri(self, cfg):
+ """apt_src_replace_tri
+ Test three autoreplacements of MIRROR and RELEASE in source specs with
+ generic part
+ """
+ self.apt_src_replacement(self.aptlistfile, cfg)
+
+ # extra verify on two extra files of this test
+ params = self._get_default_params()
+ contents = load_tfile_or_url(self.aptlistfile2)
+ self.assertTrue(re.search(r"%s %s %s %s\n" %
+ ("deb", params['MIRROR'], params['RELEASE'],
+ "main"),
+ contents, flags=re.IGNORECASE))
+ contents = load_tfile_or_url(self.aptlistfile3)
+ self.assertTrue(re.search(r"%s %s %s %s\n" %
+ ("deb", params['MIRROR'], params['RELEASE'],
+ "universe"),
+ contents, flags=re.IGNORECASE))
+
+ def test_apt_src_replace_tri(self):
+ """Test triple Autoreplacement of MIRROR and RELEASE in source specs"""
+ cfg1 = {'source': 'deb $MIRROR $RELEASE multiverse',
+ 'filename': self.aptlistfile}
+ cfg2 = {'source': 'deb $MIRROR $RELEASE main',
+ 'filename': self.aptlistfile2}
+ cfg3 = {'source': 'deb $MIRROR $RELEASE universe',
+ 'filename': self.aptlistfile3}
+ self.apt_src_replace_tri([cfg1, cfg2, cfg3])
+
+ def test_apt_src_replace_dict_tri(self):
+ """Test triple Autoreplacement in source specs (dict)"""
+ cfg = {self.aptlistfile: {'source': 'deb $MIRROR $RELEASE multiverse'},
+ 'notused': {'source': 'deb $MIRROR $RELEASE main',
+ 'filename': self.aptlistfile2},
+ self.aptlistfile3: {'source': 'deb $MIRROR $RELEASE universe'}}
+ self.apt_src_replace_tri(cfg)
+
+ def test_apt_src_replace_nofn(self):
+ """Test Autoreplacement of MIRROR and RELEASE in source specs nofile"""
+ cfg = {'source': 'deb $MIRROR $RELEASE multiverse'}
+ with mock.patch.object(os.path, 'join', side_effect=self.myjoin):
+ self.apt_src_replacement(self.fallbackfn, [cfg])
+
+ def apt_src_keyid(self, filename, cfg, keynum):
+ """apt_src_keyid
+ Test specification of a source + keyid
+ """
+ params = self._get_default_params()
+
+ with mock.patch.object(util, 'subp',
+ return_value=('fakekey 1234', '')) as mockobj:
+ cc_apt_configure.add_apt_sources(cfg, params)
+
+ # check if it added the right ammount of keys
+ calls = []
+ for _ in range(keynum):
+ calls.append(call(('apt-key', 'add', '-'), 'fakekey 1234'))
+ mockobj.assert_has_calls(calls, any_order=True)
+
+ self.assertTrue(os.path.isfile(filename))
+
+ contents = load_tfile_or_url(filename)
+ self.assertTrue(re.search(r"%s %s %s %s\n" %
+ ("deb",
+ ('http://ppa.launchpad.net/smoser/'
+ 'cloud-init-test/ubuntu'),
+ "xenial", "main"),
+ contents, flags=re.IGNORECASE))
+
+ def test_apt_src_keyid(self):
+ """Test specification of a source + keyid with filename being set"""
+ cfg = {'source': ('deb '
+ 'http://ppa.launchpad.net/'
+ 'smoser/cloud-init-test/ubuntu'
+ ' xenial main'),
+ 'keyid': "03683F77",
+ 'filename': self.aptlistfile}
+ self.apt_src_keyid(self.aptlistfile, [cfg], 1)
+
+ def test_apt_src_keyid_tri(self):
+ """Test 3x specification of a source + keyid with filename being set"""
+ cfg1 = {'source': ('deb '
+ 'http://ppa.launchpad.net/'
+ 'smoser/cloud-init-test/ubuntu'
+ ' xenial main'),
+ 'keyid': "03683F77",
+ 'filename': self.aptlistfile}
+ cfg2 = {'source': ('deb '
+ 'http://ppa.launchpad.net/'
+ 'smoser/cloud-init-test/ubuntu'
+ ' xenial universe'),
+ 'keyid': "03683F77",
+ 'filename': self.aptlistfile2}
+ cfg3 = {'source': ('deb '
+ 'http://ppa.launchpad.net/'
+ 'smoser/cloud-init-test/ubuntu'
+ ' xenial multiverse'),
+ 'keyid': "03683F77",
+ 'filename': self.aptlistfile3}
+
+ self.apt_src_keyid(self.aptlistfile, [cfg1, cfg2, cfg3], 3)
+ contents = load_tfile_or_url(self.aptlistfile2)
+ self.assertTrue(re.search(r"%s %s %s %s\n" %
+ ("deb",
+ ('http://ppa.launchpad.net/smoser/'
+ 'cloud-init-test/ubuntu'),
+ "xenial", "universe"),
+ contents, flags=re.IGNORECASE))
+ contents = load_tfile_or_url(self.aptlistfile3)
+ self.assertTrue(re.search(r"%s %s %s %s\n" %
+ ("deb",
+ ('http://ppa.launchpad.net/smoser/'
+ 'cloud-init-test/ubuntu'),
+ "xenial", "multiverse"),
+ contents, flags=re.IGNORECASE))
+
+ def test_apt_src_keyid_nofn(self):
+ """Test specification of a source + keyid without filename being set"""
+ cfg = {'source': ('deb '
+ 'http://ppa.launchpad.net/'
+ 'smoser/cloud-init-test/ubuntu'
+ ' xenial main'),
+ 'keyid': "03683F77"}
+ with mock.patch.object(os.path, 'join', side_effect=self.myjoin):
+ self.apt_src_keyid(self.fallbackfn, [cfg], 1)
+
+ def apt_src_key(self, filename, cfg):
+ """apt_src_key
+ Test specification of a source + key
+ """
+ params = self._get_default_params()
+
+ with mock.patch.object(util, 'subp') as mockobj:
+ cc_apt_configure.add_apt_sources([cfg], params)
+
+ mockobj.assert_called_with(('apt-key', 'add', '-'), 'fakekey 4321')
+
+ self.assertTrue(os.path.isfile(filename))
+
+ contents = load_tfile_or_url(filename)
+ self.assertTrue(re.search(r"%s %s %s %s\n" %
+ ("deb",
+ ('http://ppa.launchpad.net/smoser/'
+ 'cloud-init-test/ubuntu'),
+ "xenial", "main"),
+ contents, flags=re.IGNORECASE))
+
+ def test_apt_src_key(self):
+ """Test specification of a source + key with filename being set"""
+ cfg = {'source': ('deb '
+ 'http://ppa.launchpad.net/'
+ 'smoser/cloud-init-test/ubuntu'
+ ' xenial main'),
+ 'key': "fakekey 4321",
+ 'filename': self.aptlistfile}
+ self.apt_src_key(self.aptlistfile, cfg)
+
+ def test_apt_src_key_nofn(self):
+ """Test specification of a source + key without filename being set"""
+ cfg = {'source': ('deb '
+ 'http://ppa.launchpad.net/'
+ 'smoser/cloud-init-test/ubuntu'
+ ' xenial main'),
+ 'key': "fakekey 4321"}
+ with mock.patch.object(os.path, 'join', side_effect=self.myjoin):
+ self.apt_src_key(self.fallbackfn, cfg)
+
+ def test_apt_src_keyonly(self):
+ """Test specifying key without source"""
+ params = self._get_default_params()
+ cfg = {'key': "fakekey 4242",
+ 'filename': self.aptlistfile}
+
+ with mock.patch.object(util, 'subp') as mockobj:
+ cc_apt_configure.add_apt_sources([cfg], params)
+
+ mockobj.assert_called_once_with(('apt-key', 'add', '-'),
+ 'fakekey 4242')
+
+ # filename should be ignored on key only
+ self.assertFalse(os.path.isfile(self.aptlistfile))
+
+ def test_apt_src_keyidonly(self):
+ """Test specification of a keyid without source"""
+ params = self._get_default_params()
+ cfg = {'keyid': "03683F77",
+ 'filename': self.aptlistfile}
+
+ with mock.patch.object(util, 'subp',
+ return_value=('fakekey 1212', '')) as mockobj:
+ cc_apt_configure.add_apt_sources([cfg], params)
+
+ mockobj.assert_called_with(('apt-key', 'add', '-'), 'fakekey 1212')
+
+ # filename should be ignored on key only
+ self.assertFalse(os.path.isfile(self.aptlistfile))
+
+ def apt_src_keyid_real(self, cfg, expectedkey):
+ """apt_src_keyid_real
+ Test specification of a keyid without source including
+ up to addition of the key (add_apt_key_raw mocked to keep the
+ environment as is)
+ """
+ params = self._get_default_params()
+
+ with mock.patch.object(cc_apt_configure, 'add_apt_key_raw') as mockkey:
+ with mock.patch.object(gpg, 'get_key_by_id',
+ return_value=expectedkey) as mockgetkey:
+ cc_apt_configure.add_apt_sources([cfg], params)
+
+ mockgetkey.assert_called_with(cfg['keyid'],
+ cfg.get('keyserver',
+ 'keyserver.ubuntu.com'))
+ mockkey.assert_called_with(expectedkey)
+
+ # filename should be ignored on key only
+ self.assertFalse(os.path.isfile(self.aptlistfile))
+
+ def test_apt_src_keyid_real(self):
+ """test_apt_src_keyid_real - Test keyid including key add"""
+ keyid = "03683F77"
+ cfg = {'keyid': keyid,
+ 'filename': self.aptlistfile}
+
+ self.apt_src_keyid_real(cfg, EXPECTEDKEY)
+
+ def test_apt_src_longkeyid_real(self):
+ """test_apt_src_longkeyid_real - Test long keyid including key add"""
+ keyid = "B59D 5F15 97A5 04B7 E230 6DCA 0620 BBCF 0368 3F77"
+ cfg = {'keyid': keyid,
+ 'filename': self.aptlistfile}
+
+ self.apt_src_keyid_real(cfg, EXPECTEDKEY)
+
+ def test_apt_src_longkeyid_ks_real(self):
+ """test_apt_src_longkeyid_ks_real - Test long keyid from other ks"""
+ keyid = "B59D 5F15 97A5 04B7 E230 6DCA 0620 BBCF 0368 3F77"
+ cfg = {'keyid': keyid,
+ 'keyserver': 'keys.gnupg.net',
+ 'filename': self.aptlistfile}
+
+ self.apt_src_keyid_real(cfg, EXPECTEDKEY)
+
+ def test_apt_src_ppa(self):
+ """Test adding a ppa"""
+ params = self._get_default_params()
+ cfg = {'source': 'ppa:smoser/cloud-init-test',
+ 'filename': self.aptlistfile}
+
+ # default matcher needed for ppa
+ matcher = re.compile(r'^[\w-]+:\w').search
+
+ with mock.patch.object(util, 'subp') as mockobj:
+ cc_apt_configure.add_apt_sources([cfg], params,
+ aa_repo_match=matcher)
+ mockobj.assert_called_once_with(['add-apt-repository',
+ 'ppa:smoser/cloud-init-test'])
+
+ # adding ppa should ignore filename (uses add-apt-repository)
+ self.assertFalse(os.path.isfile(self.aptlistfile))
+
+ def test_apt_src_ppa_tri(self):
+ """Test adding three ppa's"""
+ params = self._get_default_params()
+ cfg1 = {'source': 'ppa:smoser/cloud-init-test',
+ 'filename': self.aptlistfile}
+ cfg2 = {'source': 'ppa:smoser/cloud-init-test2',
+ 'filename': self.aptlistfile2}
+ cfg3 = {'source': 'ppa:smoser/cloud-init-test3',
+ 'filename': self.aptlistfile3}
+
+ # default matcher needed for ppa
+ matcher = re.compile(r'^[\w-]+:\w').search
+
+ with mock.patch.object(util, 'subp') as mockobj:
+ cc_apt_configure.add_apt_sources([cfg1, cfg2, cfg3], params,
+ aa_repo_match=matcher)
+ calls = [call(['add-apt-repository', 'ppa:smoser/cloud-init-test']),
+ call(['add-apt-repository', 'ppa:smoser/cloud-init-test2']),
+ call(['add-apt-repository', 'ppa:smoser/cloud-init-test3'])]
+ mockobj.assert_has_calls(calls, any_order=True)
+
+ # adding ppa should ignore all filenames (uses add-apt-repository)
+ self.assertFalse(os.path.isfile(self.aptlistfile))
+ self.assertFalse(os.path.isfile(self.aptlistfile2))
+ self.assertFalse(os.path.isfile(self.aptlistfile3))
+
+ def test_convert_to_new_format(self):
+ """Test the conversion of old to new format"""
+ cfg1 = {'source': 'deb $MIRROR $RELEASE multiverse',
+ 'filename': self.aptlistfile}
+ cfg2 = {'source': 'deb $MIRROR $RELEASE main',
+ 'filename': self.aptlistfile2}
+ cfg3 = {'source': 'deb $MIRROR $RELEASE universe',
+ 'filename': self.aptlistfile3}
+ checkcfg = {self.aptlistfile: {'filename': self.aptlistfile,
+ 'source': 'deb $MIRROR $RELEASE '
+ 'multiverse'},
+ self.aptlistfile2: {'filename': self.aptlistfile2,
+ 'source': 'deb $MIRROR $RELEASE main'},
+ self.aptlistfile3: {'filename': self.aptlistfile3,
+ 'source': 'deb $MIRROR $RELEASE '
+ 'universe'}}
+
+ newcfg = cc_apt_configure.convert_to_new_format([cfg1, cfg2, cfg3])
+ self.assertEqual(newcfg, checkcfg)
+
+ newcfg2 = cc_apt_configure.convert_to_new_format(newcfg)
+ self.assertEqual(newcfg2, checkcfg)
+
+ with self.assertRaises(ValueError):
+ cc_apt_configure.convert_to_new_format(5)
+
+
+# vi: ts=4 expandtab
diff --git a/tests/unittests/test_net.py b/tests/unittests/test_net.py
index 34875f7b..a9268d30 100644
--- a/tests/unittests/test_net.py
+++ b/tests/unittests/test_net.py
@@ -1,7 +1,14 @@
from cloudinit import net
+from cloudinit.net import cmdline
+from cloudinit.net import eni
+from cloudinit.net import network_state
+from cloudinit.net import sysconfig
+from cloudinit.sources.helpers import openstack
from cloudinit import util
+from .helpers import mock
from .helpers import TestCase
+from .helpers import dir2dict
import base64
import copy
@@ -9,9 +16,10 @@ import gzip
import io
import json
import os
+import shutil
+import tempfile
import yaml
-
DHCP_CONTENT_1 = """
DEVICE='eth0'
PROTO='dhcp'
@@ -70,6 +78,71 @@ STATIC_EXPECTED_1 = {
'dns_nameservers': ['10.0.1.1']}],
}
+# Examples (and expected outputs for various renderers).
+OS_SAMPLES = [
+ {
+ 'in_data': {
+ "services": [{"type": "dns", "address": "172.19.0.12"}],
+ "networks": [{
+ "network_id": "dacd568d-5be6-4786-91fe-750c374b78b4",
+ "type": "ipv4", "netmask": "255.255.252.0",
+ "link": "tap1a81968a-79",
+ "routes": [{
+ "netmask": "0.0.0.0",
+ "network": "0.0.0.0",
+ "gateway": "172.19.3.254",
+ }],
+ "ip_address": "172.19.1.34", "id": "network0"
+ }],
+ "links": [
+ {
+ "ethernet_mac_address": "fa:16:3e:ed:9a:59",
+ "mtu": None, "type": "bridge", "id":
+ "tap1a81968a-79",
+ "vif_id": "1a81968a-797a-400f-8a80-567f997eb93f"
+ },
+ ],
+ },
+ 'in_macs': {
+ 'fa:16:3e:ed:9a:59': 'eth0',
+ },
+ 'out_sysconfig': [
+ ('etc/sysconfig/network-scripts/ifcfg-eth0',
+ """
+# Created by cloud-init on instance boot automatically, do not edit.
+#
+BOOTPROTO=static
+DEFROUTE=yes
+DEVICE=eth0
+GATEWAY=172.19.3.254
+HWADDR=fa:16:3e:ed:9a:59
+IPADDR=172.19.1.34
+NETMASK=255.255.252.0
+NM_CONTROLLED=no
+ONBOOT=yes
+TYPE=Ethernet
+USERCTL=no
+""".lstrip()),
+ ('etc/sysconfig/network-scripts/route-eth0',
+ """
+# Created by cloud-init on instance boot automatically, do not edit.
+#
+ADDRESS0=0.0.0.0
+GATEWAY0=172.19.3.254
+NETMASK0=0.0.0.0
+""".lstrip()),
+ ('etc/resolv.conf',
+ """
+; Created by cloud-init on instance boot automatically, do not edit.
+;
+nameserver 172.19.0.12
+""".lstrip()),
+ ('etc/udev/rules.d/70-persistent-net.rules',
+ "".join(['SUBSYSTEM=="net", ACTION=="add", DRIVERS=="?*", ',
+ 'ATTR{address}=="fa:16:3e:ed:9a:59", NAME="eth0"\n']))]
+ }
+]
+
EXAMPLE_ENI = """
auto lo
iface lo inet loopback
@@ -228,21 +301,148 @@ config:
"""
-class TestNetConfigParsing(TestCase):
+def _setup_test(tmp_dir, mock_get_devicelist, mock_sys_netdev_info,
+ mock_sys_dev_path):
+ mock_get_devicelist.return_value = ['eth1000']
+ dev_characteristics = {
+ 'eth1000': {
+ "bridge": False,
+ "carrier": False,
+ "dormant": False,
+ "operstate": "down",
+ "address": "07-1C-C6-75-A4-BE",
+ }
+ }
+
+ def netdev_info(name, field):
+ return dev_characteristics[name][field]
+
+ mock_sys_netdev_info.side_effect = netdev_info
+
+ def sys_dev_path(devname, path=""):
+ return tmp_dir + devname + "/" + path
+
+ for dev in dev_characteristics:
+ os.makedirs(os.path.join(tmp_dir, dev))
+ with open(os.path.join(tmp_dir, dev, 'operstate'), 'w') as fh:
+ fh.write("down")
+
+ mock_sys_dev_path.side_effect = sys_dev_path
+
+
+class TestSysConfigRendering(TestCase):
+
+ @mock.patch("cloudinit.net.sys_dev_path")
+ @mock.patch("cloudinit.net.sys_netdev_info")
+ @mock.patch("cloudinit.net.get_devicelist")
+ def test_default_generation(self, mock_get_devicelist,
+ mock_sys_netdev_info,
+ mock_sys_dev_path):
+ tmp_dir = tempfile.mkdtemp()
+ self.addCleanup(shutil.rmtree, tmp_dir)
+ _setup_test(tmp_dir, mock_get_devicelist,
+ mock_sys_netdev_info, mock_sys_dev_path)
+
+ network_cfg = net.generate_fallback_config()
+ ns = network_state.parse_net_config_data(network_cfg,
+ skip_broken=False)
+
+ render_dir = os.path.join(tmp_dir, "render")
+ os.makedirs(render_dir)
+
+ renderer = sysconfig.Renderer()
+ renderer.render_network_state(render_dir, ns)
+
+ render_file = 'etc/sysconfig/network-scripts/ifcfg-eth1000'
+ with open(os.path.join(render_dir, render_file)) as fh:
+ content = fh.read()
+ expected_content = """
+# Created by cloud-init on instance boot automatically, do not edit.
+#
+BOOTPROTO=dhcp
+DEVICE=eth1000
+HWADDR=07-1C-C6-75-A4-BE
+NM_CONTROLLED=no
+ONBOOT=yes
+TYPE=Ethernet
+USERCTL=no
+""".lstrip()
+ self.assertEqual(expected_content, content)
+
+ def test_openstack_rendering_samples(self):
+ tmp_dir = tempfile.mkdtemp()
+ self.addCleanup(shutil.rmtree, tmp_dir)
+ render_dir = os.path.join(tmp_dir, "render")
+ for os_sample in OS_SAMPLES:
+ ex_input = os_sample['in_data']
+ ex_mac_addrs = os_sample['in_macs']
+ network_cfg = openstack.convert_net_json(
+ ex_input, known_macs=ex_mac_addrs)
+ ns = network_state.parse_net_config_data(network_cfg,
+ skip_broken=False)
+ renderer = sysconfig.Renderer()
+ renderer.render_network_state(render_dir, ns)
+ for fn, expected_content in os_sample.get('out_sysconfig', []):
+ with open(os.path.join(render_dir, fn)) as fh:
+ self.assertEqual(expected_content, fh.read())
+
+
+class TestEniNetRendering(TestCase):
+
+ @mock.patch("cloudinit.net.sys_dev_path")
+ @mock.patch("cloudinit.net.sys_netdev_info")
+ @mock.patch("cloudinit.net.get_devicelist")
+ def test_default_generation(self, mock_get_devicelist,
+ mock_sys_netdev_info,
+ mock_sys_dev_path):
+ tmp_dir = tempfile.mkdtemp()
+ self.addCleanup(shutil.rmtree, tmp_dir)
+ _setup_test(tmp_dir, mock_get_devicelist,
+ mock_sys_netdev_info, mock_sys_dev_path)
+
+ network_cfg = net.generate_fallback_config()
+ ns = network_state.parse_net_config_data(network_cfg,
+ skip_broken=False)
+
+ render_dir = os.path.join(tmp_dir, "render")
+ os.makedirs(render_dir)
+
+ renderer = eni.Renderer(
+ {'links_path_prefix': None,
+ 'eni_path': 'interfaces', 'netrules_path': None,
+ })
+ renderer.render_network_state(render_dir, ns)
+
+ self.assertTrue(os.path.exists(os.path.join(render_dir,
+ 'interfaces')))
+ with open(os.path.join(render_dir, 'interfaces')) as fh:
+ contents = fh.read()
+
+ expected = """
+auto lo
+iface lo inet loopback
+
+auto eth1000
+iface eth1000 inet dhcp
+"""
+ self.assertEqual(expected.lstrip(), contents.lstrip())
+
+
+class TestCmdlineConfigParsing(TestCase):
simple_cfg = {
'config': [{"type": "physical", "name": "eth0",
"mac_address": "c0:d6:9f:2c:e8:80",
"subnets": [{"type": "dhcp"}]}]}
- def test_klibc_convert_dhcp(self):
- found = net._klibc_to_config_entry(DHCP_CONTENT_1)
+ def test_cmdline_convert_dhcp(self):
+ found = cmdline._klibc_to_config_entry(DHCP_CONTENT_1)
self.assertEqual(found, ('eth0', DHCP_EXPECTED_1))
- def test_klibc_convert_static(self):
- found = net._klibc_to_config_entry(STATIC_CONTENT_1)
+ def test_cmdline_convert_static(self):
+ found = cmdline._klibc_to_config_entry(STATIC_CONTENT_1)
self.assertEqual(found, ('eth1', STATIC_EXPECTED_1))
- def test_config_from_klibc_net_cfg(self):
+ def test_config_from_cmdline_net_cfg(self):
files = []
pairs = (('net-eth0.cfg', DHCP_CONTENT_1),
('net-eth1.cfg', STATIC_CONTENT_1))
@@ -263,45 +463,72 @@ class TestNetConfigParsing(TestCase):
files.append(fp)
util.write_file(fp, content)
- found = net.config_from_klibc_net_cfg(files=files, mac_addrs=macs)
+ found = cmdline.config_from_klibc_net_cfg(files=files,
+ mac_addrs=macs)
self.assertEqual(found, expected)
def test_cmdline_with_b64(self):
data = base64.b64encode(json.dumps(self.simple_cfg).encode())
encoded_text = data.decode()
- cmdline = 'ro network-config=' + encoded_text + ' root=foo'
- found = net.read_kernel_cmdline_config(cmdline=cmdline)
+ raw_cmdline = 'ro network-config=' + encoded_text + ' root=foo'
+ found = cmdline.read_kernel_cmdline_config(cmdline=raw_cmdline)
self.assertEqual(found, self.simple_cfg)
def test_cmdline_with_b64_gz(self):
data = _gzip_data(json.dumps(self.simple_cfg).encode())
encoded_text = base64.b64encode(data).decode()
- cmdline = 'ro network-config=' + encoded_text + ' root=foo'
- found = net.read_kernel_cmdline_config(cmdline=cmdline)
+ raw_cmdline = 'ro network-config=' + encoded_text + ' root=foo'
+ found = cmdline.read_kernel_cmdline_config(cmdline=raw_cmdline)
self.assertEqual(found, self.simple_cfg)
class TestEniRoundTrip(TestCase):
+ def setUp(self):
+ super(TestCase, self).setUp()
+ self.tmp_dir = tempfile.mkdtemp()
+ self.addCleanup(shutil.rmtree, self.tmp_dir)
+
+ def _render_and_read(self, network_config=None, state=None, eni_path=None,
+ links_prefix=None, netrules_path=None):
+ if network_config:
+ ns = network_state.parse_net_config_data(network_config)
+ elif state:
+ ns = state
+ else:
+ raise ValueError("Expected data or state, got neither")
+
+ if eni_path is None:
+ eni_path = 'etc/network/interfaces'
+
+ renderer = eni.Renderer(
+ config={'eni_path': eni_path, 'links_path_prefix': links_prefix,
+ 'netrules_path': netrules_path})
+
+ renderer.render_network_state(self.tmp_dir, ns)
+ return dir2dict(self.tmp_dir)
+
def testsimple_convert_and_render(self):
- network_config = net.convert_eni_data(EXAMPLE_ENI)
- ns = net.parse_net_config_data(network_config)
- eni = net.render_interfaces(ns)
- print("Eni looks like:\n%s" % eni)
- raise Exception("FOO")
+ network_config = eni.convert_eni_data(EXAMPLE_ENI)
+ ns = network_state.parse_net_config_data(network_config)
+ eni_path = 'etc/network/interfaces.d/my.interfaces'
+ eni_full_path = os.path.join(self.tmp_dir, eni_path)
+ renderer = eni.Renderer(config={'eni_path': eni_path})
+ renderer.render_network_state(self.tmp_dir, ns)
+ eni_content = util.load_file(eni_full_path)
+ print("Eni looks like: %s" % eni_content)
+ raise Exception("FOO1")
def testsimple_render_all(self):
- network_config = yaml.load(NETWORK_YAML_ALL)
- ns = net.parse_net_config_data(network_config)
- eni = net.render_interfaces(ns)
- print("Eni looks like:\n%s" % eni)
- raise Exception("FOO")
+ files = self._render_and_read(network_config=yaml.load(NETWORK_YAML_ALL))
+ print("files: %s" % files)
+ raise Exception("FOO2")
- def testsimple_render_small(self):
+ def skiptestsimple_render_small(self):
network_config = yaml.load(NETWORK_YAML_SMALL)
- ns = net.parse_net_config_data(network_config)
+ ns = network_state.parse_net_config_data(network_config)
eni = net.render_interfaces(ns)
print("Eni looks like:\n%s" % eni)
- raise Exception("FOO")
+ raise Exception("FOO3")
def _gzip_data(data):
diff --git a/tests/unittests/test_reporting.py b/tests/unittests/test_reporting.py
index 5cad8406..20ca23df 100644
--- a/tests/unittests/test_reporting.py
+++ b/tests/unittests/test_reporting.py
@@ -7,7 +7,9 @@ from cloudinit import reporting
from cloudinit.reporting import events
from cloudinit.reporting import handlers
-from .helpers import (mock, TestCase)
+import mock
+
+from .helpers import TestCase
def _fake_registry():
diff --git a/tests/unittests/test_rh_subscription.py b/tests/unittests/test_rh_subscription.py
index b84c807b..891dbe77 100644
--- a/tests/unittests/test_rh_subscription.py
+++ b/tests/unittests/test_rh_subscription.py
@@ -1,12 +1,24 @@
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 3, as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+import logging
+
from cloudinit.config import cc_rh_subscription
from cloudinit import util
-import logging
-import mock
-import unittest
+from .helpers import TestCase, mock
-class GoodTests(unittest.TestCase):
+class GoodTests(TestCase):
def setUp(self):
super(GoodTests, self).setUp()
self.name = "cc_rh_subscription"
@@ -93,7 +105,7 @@ class GoodTests(unittest.TestCase):
self.assertEqual(self.SM._sub_man_cli.call_count, 9)
-class TestBadInput(unittest.TestCase):
+class TestBadInput(TestCase):
name = "cc_rh_subscription"
cloud_init = None
log = logging.getLogger("bad_tests")
diff --git a/tools/run-pep8 b/tools/run-pep8
index 086400fc..4bd0bbfb 100755
--- a/tools/run-pep8
+++ b/tools/run-pep8
@@ -1,8 +1,7 @@
#!/bin/bash
-pycheck_dirs=( "cloudinit/" "bin/" "tests/" "tools/" )
-# FIXME: cloud-init modifies sys module path, pep8 does not like
-# bin_files=( "bin/cloud-init" )
+pycheck_dirs=( "cloudinit/" "tests/" "tools/" )
+
CR="
"
[ "$1" = "-v" ] && { verbose="$1"; shift; } || verbose=""
diff --git a/tools/run-pyflakes b/tools/run-pyflakes
index 4bea17f4..b3759a94 100755
--- a/tools/run-pyflakes
+++ b/tools/run-pyflakes
@@ -3,7 +3,7 @@
PYTHON_VERSION=${PYTHON_VERSION:-2}
CR="
"
-pycheck_dirs=( "cloudinit/" "bin/" "tests/" "tools/" )
+pycheck_dirs=( "cloudinit/" "tests/" "tools/" )
set -f
if [ $# -eq 0 ]; then
diff --git a/tox.ini b/tox.ini
index dafaaf6d..e7a6f22c 100644
--- a/tox.ini
+++ b/tox.ini
@@ -5,10 +5,9 @@ recreate = True
[testenv]
commands = python -m nose {posargs:tests}
deps = -r{toxinidir}/test-requirements.txt
- -r{toxinidir}/requirements.txt
-
-[testenv:py3]
-basepython = python3
+ -r{toxinidir}/requirements.txt
+setenv =
+ LC_ALL = en_US.utf-8
[testenv:flake8]
basepython = python3
@@ -18,15 +17,11 @@ commands = {envpython} -m flake8 {posargs:cloudinit/ tests/ tools/}
setenv =
LC_ALL = en_US.utf-8
+[testenv:py3]
+basepython = python3
+
[testenv:py26]
commands = nosetests {posargs:tests}
-deps =
- contextlib2
- httpretty>=0.7.1
- mock
- nose
- pep8==1.5.7
- pyflakes
setenv =
LC_ALL = C