From 808edb127507d91ecee1834aaf5cf1f000cd6e28 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Tue, 20 Sep 2016 10:27:54 -0400 Subject: MAAS: improve the main of datasource to look at kernel cmdline config. This just looks in one other maas related path for a config file. The file '91_kernel_cmdline_url' is written by cloud-init when it gets a cloud-config-url parameter. Also now we read the config even if a url is specified to potentially fill in credentials. --- cloudinit/sources/DataSourceMAAS.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) (limited to 'cloudinit/sources') diff --git a/cloudinit/sources/DataSourceMAAS.py b/cloudinit/sources/DataSourceMAAS.py index ab93c0a2..81abcd47 100644 --- a/cloudinit/sources/DataSourceMAAS.py +++ b/cloudinit/sources/DataSourceMAAS.py @@ -310,12 +310,12 @@ if __name__ == "__main__": creds = {'consumer_key': args.ckey, 'token_key': args.tkey, 'token_secret': args.tsec, 'consumer_secret': args.csec} - maaspkg_cfg = "/etc/cloud/cloud.cfg.d/90_dpkg_maas.cfg" - if (args.config is None and args.url is None and - os.path.exists(maaspkg_cfg) and - os.access(maaspkg_cfg, os.R_OK)): - sys.stderr.write("Used config in %s.\n" % maaspkg_cfg) - args.config = maaspkg_cfg + if args.config is None: + for fname in ('91_kernel_cmdline_url', '90_dpkg_maas'): + fpath = "/etc/cloud/cloud.cfg.d/" + fname + ".cfg" + if os.path.exists(fpath) and os.access(fpath, os.R_OK): + sys.stderr.write("Used config in %s.\n" % fpath) + args.config = fpath if args.config: cfg = util.read_conf(args.config) -- cgit v1.2.3 From e8730078df8c99696b1b684e09c803eef7c4926c Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Fri, 30 Sep 2016 15:53:42 -0400 Subject: Fix python2.6 things found running in centos 6. This gets the tests running in centos 6. * ProcessExecutionError: remove setting of .message Nothing in cloud-init seems to use .message anywhere, so it does not seem necessary. The reason to change it is that on 2.6 it spits out: cloudinit/util.py:286: DeprecationWarning: BaseException.message * tox.ini: add a centos6 environment the tox versions listed here replicate a centos6 install with packages from EPEL. You will still need a python2.6 to run this env so we do not enable it by default. --- cloudinit/sources/DataSourceAltCloud.py | 6 ++---- cloudinit/sources/helpers/azure.py | 2 +- cloudinit/util.py | 7 ++----- tests/unittests/test_handler/test_handler_apt_conf_v1.py | 2 +- tests/unittests/test_util.py | 2 +- tox.ini | 16 ++++++++++++++++ 6 files changed, 23 insertions(+), 12 deletions(-) (limited to 'cloudinit/sources') diff --git a/cloudinit/sources/DataSourceAltCloud.py b/cloudinit/sources/DataSourceAltCloud.py index 48136f7c..20345389 100644 --- a/cloudinit/sources/DataSourceAltCloud.py +++ b/cloudinit/sources/DataSourceAltCloud.py @@ -195,8 +195,7 @@ class DataSourceAltCloud(sources.DataSource): (cmd_out, _err) = util.subp(cmd) LOG.debug(('Command: %s\nOutput%s') % (' '.join(cmd), cmd_out)) except ProcessExecutionError as _err: - util.logexc(LOG, 'Failed command: %s\n%s', ' '.join(cmd), - _err.message) + util.logexc(LOG, 'Failed command: %s\n%s', ' '.join(cmd), _err) return False except OSError as _err: util.logexc(LOG, 'Failed command: %s\n%s', ' '.join(cmd), _err) @@ -211,8 +210,7 @@ class DataSourceAltCloud(sources.DataSource): (cmd_out, _err) = util.subp(cmd) LOG.debug(('Command: %s\nOutput%s') % (' '.join(cmd), cmd_out)) except ProcessExecutionError as _err: - util.logexc(LOG, 'Failed command: %s\n%s', ' '.join(cmd), - _err.message) + util.logexc(LOG, 'Failed command: %s\n%s', ' '.join(cmd), _err) return False except OSError as _err: util.logexc(LOG, 'Failed command: %s\n%s', ' '.join(cmd), diff --git a/cloudinit/sources/helpers/azure.py b/cloudinit/sources/helpers/azure.py index 689ed4cc..1b3e9b70 100644 --- a/cloudinit/sources/helpers/azure.py +++ b/cloudinit/sources/helpers/azure.py @@ -232,7 +232,7 @@ class WALinuxAgentShim(object): def _get_value_from_leases_file(fallback_lease_file): leases = [] content = util.load_file(fallback_lease_file) - LOG.debug("content is {}".format(content)) + LOG.debug("content is %s", content) for line in content.splitlines(): if 'unknown-245' in line: # Example line from Ubuntu diff --git a/cloudinit/util.py b/cloudinit/util.py index eb3e5899..4cff83c5 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -199,7 +199,7 @@ def fully_decoded_payload(part): encoding = charset.input_codec else: encoding = 'utf-8' - return cte_payload.decode(encoding, errors='surrogateescape') + return cte_payload.decode(encoding, 'surrogateescape') return cte_payload @@ -282,9 +282,6 @@ class ProcessExecutionError(IOError): 'reason': self.reason, } IOError.__init__(self, message) - # For backward compatibility with Python 2. - if not hasattr(self, 'message'): - self.message = message class SeLinuxGuard(object): @@ -1821,7 +1818,7 @@ def subp(args, data=None, rcs=None, env=None, capture=True, shell=False, def ldecode(data, m='utf-8'): if not isinstance(data, bytes): return data - return data.decode(m, errors=decode) + return data.decode(m, decode) out = ldecode(out) err = ldecode(err) diff --git a/tests/unittests/test_handler/test_handler_apt_conf_v1.py b/tests/unittests/test_handler/test_handler_apt_conf_v1.py index 45714efd..64acc3e0 100644 --- a/tests/unittests/test_handler/test_handler_apt_conf_v1.py +++ b/tests/unittests/test_handler/test_handler_apt_conf_v1.py @@ -118,7 +118,7 @@ class TestConversion(TestCase): def test_convert_with_apt_mirror(self): mirror = 'http://my.mirror/ubuntu' f = cc_apt_configure.convert_to_v3_apt_format({'apt_mirror': mirror}) - self.assertIn(mirror, {m['uri'] for m in f['apt']['primary']}) + self.assertIn(mirror, set(m['uri'] for m in f['apt']['primary'])) def test_no_old_content(self): mirror = 'http://my.mirror/ubuntu' diff --git a/tests/unittests/test_util.py b/tests/unittests/test_util.py index fc6b9d40..881509aa 100644 --- a/tests/unittests/test_util.py +++ b/tests/unittests/test_util.py @@ -553,7 +553,7 @@ class TestSubp(helpers.TestCase): def test_subp_decode_invalid_utf8_replaces(self): (out, _err) = util.subp(self.stdin2out, capture=True, data=self.utf8_invalid) - expected = self.utf8_invalid.decode('utf-8', errors='replace') + expected = self.utf8_invalid.decode('utf-8', 'replace') self.assertEqual(out, expected) def test_subp_decode_strict_raises(self): diff --git a/tox.ini b/tox.ini index 729de2a6..277858ed 100644 --- a/tox.ini +++ b/tox.ini @@ -59,3 +59,19 @@ deps = pyflakes==1.1.0 flake8==2.5.4 hacking==0.10.2 + +[testenv:centos6] +basepython = python2.6 +commands = nosetests {posargs:tests} +deps = + # requirements + argparse==1.2.1 + jinja2==2.2.1 + pyyaml==3.10 + PrettyTable==0.7.2 + oauthlib==0.6.0 + configobj==4.6.0 + requests==2.6.0 + jsonpatch==1.2 + six==1.9.0 + -r{toxinidir}/test-requirements.txt -- cgit v1.2.3 From 9972d246947f1a6ec102b978b99b26acc43133ec Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Thu, 29 Sep 2016 12:45:02 -0400 Subject: OpenNebula: replace 'ip' parsing with cloudinit.net usage. Replace the parsing of 'ip' to get a link and mac address list in OpenNebula's datasource with usage of cloudinit.net. This makes test cases there not depend on 'ip' availability and also uses common code. --- cloudinit/sources/DataSourceOpenNebula.py | 34 ++++++++++------------ tests/unittests/test_datasource/test_opennebula.py | 23 +++++++-------- 2 files changed, 25 insertions(+), 32 deletions(-) (limited to 'cloudinit/sources') diff --git a/cloudinit/sources/DataSourceOpenNebula.py b/cloudinit/sources/DataSourceOpenNebula.py index 635a836c..ba5f3f92 100644 --- a/cloudinit/sources/DataSourceOpenNebula.py +++ b/cloudinit/sources/DataSourceOpenNebula.py @@ -30,6 +30,7 @@ import re import string from cloudinit import log as logging +from cloudinit import net from cloudinit import sources from cloudinit import util @@ -120,17 +121,11 @@ class BrokenContextDiskDir(Exception): class OpenNebulaNetwork(object): - REG_DEV_MAC = re.compile( - r'^\d+: (eth\d+):.*?link\/ether (..:..:..:..:..:..) ?', - re.MULTILINE | re.DOTALL) - - def __init__(self, ip, context): - self.ip = ip + def __init__(self, context, system_nics_by_mac=None): self.context = context - self.ifaces = self.get_ifaces() - - def get_ifaces(self): - return self.REG_DEV_MAC.findall(self.ip) + if system_nics_by_mac is None: + system_nics_by_mac = get_physical_nics_by_mac() + self.ifaces = system_nics_by_mac def mac2ip(self, mac): components = mac.split(':')[2:] @@ -188,9 +183,7 @@ class OpenNebulaNetwork(object): conf.append('iface lo inet loopback') conf.append('') - for i in self.ifaces: - dev = i[0] - mac = i[1] + for mac, dev in self.ifaces.items(): ip_components = self.mac2ip(mac) conf.append('auto ' + dev) @@ -405,16 +398,19 @@ def read_context_disk_dir(source_dir, asuser=None): # generate static /etc/network/interfaces # only if there are any required context variables # http://opennebula.org/documentation:rel3.8:cong#network_configuration - for k in context: - if re.match(r'^ETH\d+_IP$', k): - (out, _) = util.subp(['ip', 'link']) - net = OpenNebulaNetwork(out, context) - results['network-interfaces'] = net.gen_conf() - break + ipaddr_keys = [k for k in context if re.match(r'^ETH\d+_IP$', k)] + if ipaddr_keys: + onet = OpenNebulaNetwork(context) + results['network-interfaces'] = onet.gen_conf() return results +def get_physical_nics_by_mac(): + devs = net.get_interfaces_by_mac() + return dict([(m, n) for m, n in devs.items() if net.is_physical(n)]) + + # Legacy: Must be present in case we load an old pkl object DataSourceOpenNebulaNet = DataSourceOpenNebula diff --git a/tests/unittests/test_datasource/test_opennebula.py b/tests/unittests/test_datasource/test_opennebula.py index d796f030..ce5b5550 100644 --- a/tests/unittests/test_datasource/test_opennebula.py +++ b/tests/unittests/test_datasource/test_opennebula.py @@ -1,7 +1,7 @@ from cloudinit import helpers from cloudinit.sources import DataSourceOpenNebula as ds from cloudinit import util -from ..helpers import TestCase, populate_dir +from ..helpers import mock, populate_dir, TestCase import os import pwd @@ -31,12 +31,7 @@ SSH_KEY = 'ssh-rsa AAAAB3NzaC1....sIkJhq8wdX+4I3A4cYbYP ubuntu@server-460-%i' HOSTNAME = 'foo.example.com' PUBLIC_IP = '10.0.0.3' -CMD_IP_OUT = '''\ -1: lo: mtu 16436 qdisc noqueue state UNKNOWN - link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00 -2: eth0: mtu 1500 qdisc mq state UP qlen 1000 - link/ether 02:00:0a:12:01:01 brd ff:ff:ff:ff:ff:ff -''' +DS_PATH = "cloudinit.sources.DataSourceOpenNebula" class TestOpenNebulaDataSource(TestCase): @@ -233,18 +228,19 @@ class TestOpenNebulaDataSource(TestCase): class TestOpenNebulaNetwork(unittest.TestCase): - def setUp(self): - super(TestOpenNebulaNetwork, self).setUp() + system_nics = {'02:00:0a:12:01:01': 'eth0'} def test_lo(self): - net = ds.OpenNebulaNetwork('', {}) + net = ds.OpenNebulaNetwork(context={}, system_nics_by_mac={}) self.assertEqual(net.gen_conf(), u'''\ auto lo iface lo inet loopback ''') - def test_eth0(self): - net = ds.OpenNebulaNetwork(CMD_IP_OUT, {}) + @mock.patch(DS_PATH + ".get_physical_nics_by_mac") + def test_eth0(self, m_get_phys_by_mac): + m_get_phys_by_mac.return_value = self.system_nics + net = ds.OpenNebulaNetwork({}) self.assertEqual(net.gen_conf(), u'''\ auto lo iface lo inet loopback @@ -267,7 +263,8 @@ iface eth0 inet static 'ETH0_DNS': '1.2.3.6 1.2.3.7' } - net = ds.OpenNebulaNetwork(CMD_IP_OUT, context) + net = ds.OpenNebulaNetwork(context, + system_nics_by_mac=self.system_nics) self.assertEqual(net.gen_conf(), u'''\ auto lo iface lo inet loopback -- cgit v1.2.3