From d1e26fc118cdb641829fbe6b838ef46d4ab1f113 Mon Sep 17 00:00:00 2001 From: Kiril Vladimiroff Date: Wed, 19 Feb 2014 10:45:53 +0200 Subject: Read encoded with base64 user data This allows users of CloudSigma's VM to encode their user data with base64. In order to do that thet have to add the ``cloudinit-user-data`` field to the ``base64_fields``. The latter is a comma-separated field with all the meta fields whit base64 encoded values. --- tests/unittests/test_datasource/test_cloudsigma.py | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) (limited to 'tests/unittests/test_datasource') diff --git a/tests/unittests/test_datasource/test_cloudsigma.py b/tests/unittests/test_datasource/test_cloudsigma.py index 3245aba1..adbb4afb 100644 --- a/tests/unittests/test_datasource/test_cloudsigma.py +++ b/tests/unittests/test_datasource/test_cloudsigma.py @@ -1,4 +1,5 @@ # coding: utf-8 +import copy from unittest import TestCase from cloudinit.cs_utils import Cepko @@ -24,7 +25,8 @@ SERVER_CONTEXT = { class CepkoMock(Cepko): - result = SERVER_CONTEXT + def __init__(self, mocked_context): + self.result = mocked_context def all(self): return self @@ -33,7 +35,7 @@ class CepkoMock(Cepko): class DataSourceCloudSigmaTest(TestCase): def setUp(self): self.datasource = DataSourceCloudSigma.DataSourceCloudSigma("", "", "") - self.datasource.cepko = CepkoMock() + self.datasource.cepko = CepkoMock(SERVER_CONTEXT) self.datasource.get_data() def test_get_hostname(self): @@ -57,3 +59,12 @@ class DataSourceCloudSigmaTest(TestCase): def test_user_data(self): self.assertEqual(self.datasource.userdata_raw, SERVER_CONTEXT['meta']['cloudinit-user-data']) + + def test_encoded_user_data(self): + encoded_context = copy.deepcopy(SERVER_CONTEXT) + encoded_context['meta']['base64_fields'] = 'cloudinit-user-data' + encoded_context['meta']['cloudinit-user-data'] = 'aGkgd29ybGQK' + self.datasource.cepko = CepkoMock(encoded_context) + self.datasource.get_data() + + self.assertEqual(self.datasource.userdata_raw, b'hi world\n') -- cgit v1.2.3 From 9486c1a1abacb9829e5ab172212d57c3735e35e0 Mon Sep 17 00:00:00 2001 From: Enol Fernandez Date: Tue, 25 Mar 2014 16:31:16 +0100 Subject: Added base64 decoding of user data for OpenNebula. --- cloudinit/sources/DataSourceOpenNebula.py | 12 +++++++++++ tests/unittests/test_datasource/test_opennebula.py | 25 ++++++++++++++++++++-- 2 files changed, 35 insertions(+), 2 deletions(-) (limited to 'tests/unittests/test_datasource') diff --git a/cloudinit/sources/DataSourceOpenNebula.py b/cloudinit/sources/DataSourceOpenNebula.py index b0464cbb..d91b80ab 100644 --- a/cloudinit/sources/DataSourceOpenNebula.py +++ b/cloudinit/sources/DataSourceOpenNebula.py @@ -4,11 +4,13 @@ # Copyright (C) 2012 Yahoo! Inc. # Copyright (C) 2012-2013 CERIT Scientific Cloud # Copyright (C) 2012-2013 OpenNebula.org +# Copyright (C) 2014 Consejo Superior de Investigaciones Cientificas # # Author: Scott Moser # Author: Joshua Harlow # Author: Vlastimil Holer # Author: Javier Fontan +# Author: Enol Fernandez # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License version 3, as @@ -22,6 +24,7 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . +import base64 import os import pwd import re @@ -417,6 +420,15 @@ def read_context_disk_dir(source_dir, asuser=None): elif "USERDATA" in context: results['userdata'] = context["USERDATA"] + # b64decode user data if necessary (default) + if 'userdata' in results: + userdata_encoding = context.get('USERDATA_ENCODING', None) + if userdata_encoding in (None, "base64"): + try: + results['userdata'] = base64.b64decode(results['userdata']) + except TypeError: + LOG.warn("Failed base64 decoding of userdata") + # generate static /etc/network/interfaces # only if there are any required context variables # http://opennebula.org/documentation:rel3.8:cong#network_configuration diff --git a/tests/unittests/test_datasource/test_opennebula.py b/tests/unittests/test_datasource/test_opennebula.py index 6fc5b2ac..47e7acbc 100644 --- a/tests/unittests/test_datasource/test_opennebula.py +++ b/tests/unittests/test_datasource/test_opennebula.py @@ -4,6 +4,7 @@ from cloudinit import util from mocker import MockerTestCase from tests.unittests.helpers import populate_dir +from base64 import b64encode import os import pwd @@ -164,10 +165,30 @@ class TestOpenNebulaDataSource(MockerTestCase): public_keys.append(SSH_KEY % (c + 1,)) - def test_user_data(self): + def test_user_data_plain(self): for k in ('USER_DATA', 'USERDATA'): my_d = os.path.join(self.tmp, k) - populate_context_dir(my_d, {k: USER_DATA}) + populate_context_dir(my_d, {k: USER_DATA, + 'USERDATA_ENCODING': ''}) + results = ds.read_context_disk_dir(my_d) + + self.assertTrue('userdata' in results) + self.assertEqual(USER_DATA, results['userdata']) + + def test_user_data_default_encoding(self): + for k in ('USER_DATA', 'USERDATA'): + my_d = os.path.join(self.tmp, k) + populate_context_dir(my_d, {k: b64encode(USER_DATA)}) + results = ds.read_context_disk_dir(my_d) + + self.assertTrue('userdata' in results) + self.assertEqual(USER_DATA, results['userdata']) + + def test_user_data_base64_encoding(self): + for k in ('USER_DATA', 'USERDATA'): + my_d = os.path.join(self.tmp, k) + populate_context_dir(my_d, {k: b64encode(USER_DATA), + 'USERDATA_ENCODING': 'base64'}) results = ds.read_context_disk_dir(my_d) self.assertTrue('userdata' in results) -- cgit v1.2.3 From 2ecefdf51cd93b593bea450b4d751021da91e748 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Thu, 27 Mar 2014 10:03:27 -0400 Subject: change 'default' encoding to be "None" Instead of just trying to see if userdata decodes as the indication that it should be encoded, the user must explicitly set this. The "just try it" will fail in the case where the user had other use of user-data and wanted a blob of data to go through unrecognized by cloud-init. In cases where there can be mistake in automatic behavior, and some users may be relaying on old behavior, its best to just require explicit use. --- cloudinit/sources/DataSourceOpenNebula.py | 5 +++-- tests/unittests/test_datasource/test_opennebula.py | 7 ++++--- 2 files changed, 7 insertions(+), 5 deletions(-) (limited to 'tests/unittests/test_datasource') diff --git a/cloudinit/sources/DataSourceOpenNebula.py b/cloudinit/sources/DataSourceOpenNebula.py index d91b80ab..34557f8b 100644 --- a/cloudinit/sources/DataSourceOpenNebula.py +++ b/cloudinit/sources/DataSourceOpenNebula.py @@ -422,8 +422,9 @@ def read_context_disk_dir(source_dir, asuser=None): # b64decode user data if necessary (default) if 'userdata' in results: - userdata_encoding = context.get('USERDATA_ENCODING', None) - if userdata_encoding in (None, "base64"): + encoding = context.get('USERDATA_ENCODING', + context.get('USER_DATA_ENCODING')) + if encoding == "base64": try: results['userdata'] = base64.b64decode(results['userdata']) except TypeError: diff --git a/tests/unittests/test_datasource/test_opennebula.py b/tests/unittests/test_datasource/test_opennebula.py index 47e7acbc..ec6b752b 100644 --- a/tests/unittests/test_datasource/test_opennebula.py +++ b/tests/unittests/test_datasource/test_opennebula.py @@ -175,14 +175,15 @@ class TestOpenNebulaDataSource(MockerTestCase): self.assertTrue('userdata' in results) self.assertEqual(USER_DATA, results['userdata']) - def test_user_data_default_encoding(self): + def test_user_data_encoding_required_for_decode(self): + b64userdata = b64encode(USER_DATA) for k in ('USER_DATA', 'USERDATA'): my_d = os.path.join(self.tmp, k) - populate_context_dir(my_d, {k: b64encode(USER_DATA)}) + populate_context_dir(my_d, {k: b64userdata}) results = ds.read_context_disk_dir(my_d) self.assertTrue('userdata' in results) - self.assertEqual(USER_DATA, results['userdata']) + self.assertEqual(b64userdata, results['userdata']) def test_user_data_base64_encoding(self): for k in ('USER_DATA', 'USERDATA'): -- cgit v1.2.3 From 5d9726742c22f4c80ce2f386d09c1bbcf4b67164 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Tue, 1 Apr 2014 14:20:57 -0400 Subject: pyflakes cleanups --- cloudinit/config/cc_power_state_change.py | 1 - tests/unittests/test__init__.py | 6 +----- tests/unittests/test_datasource/test_maas.py | 1 - tests/unittests/test_datasource/test_smartos.py | 3 --- tests/unittests/test_handler/test_handler_yum_add_repo.py | 1 - 5 files changed, 1 insertion(+), 11 deletions(-) (limited to 'tests/unittests/test_datasource') diff --git a/cloudinit/config/cc_power_state_change.py b/cloudinit/config/cc_power_state_change.py index 561c5abd..8f99e887 100644 --- a/cloudinit/config/cc_power_state_change.py +++ b/cloudinit/config/cc_power_state_change.py @@ -22,7 +22,6 @@ from cloudinit import util import errno import os import re -import signal import subprocess import time diff --git a/tests/unittests/test__init__.py b/tests/unittests/test__init__.py index 8c41c1ca..03065c8b 100644 --- a/tests/unittests/test__init__.py +++ b/tests/unittests/test__init__.py @@ -1,14 +1,10 @@ -import logging import os -import StringIO -import sys -from mocker import MockerTestCase, ANY, ARGS, KWARGS +from mocker import MockerTestCase, ARGS, KWARGS from cloudinit import handlers from cloudinit import helpers from cloudinit import importer -from cloudinit import log from cloudinit import settings from cloudinit import url_helper from cloudinit import util diff --git a/tests/unittests/test_datasource/test_maas.py b/tests/unittests/test_datasource/test_maas.py index bd5d23fd..73cfadcb 100644 --- a/tests/unittests/test_datasource/test_maas.py +++ b/tests/unittests/test_datasource/test_maas.py @@ -3,7 +3,6 @@ import os from cloudinit.sources import DataSourceMAAS from cloudinit import url_helper -from cloudinit import util from tests.unittests.helpers import populate_dir import mocker diff --git a/tests/unittests/test_datasource/test_smartos.py b/tests/unittests/test_datasource/test_smartos.py index 8f9fa27d..45f1708a 100644 --- a/tests/unittests/test_datasource/test_smartos.py +++ b/tests/unittests/test_datasource/test_smartos.py @@ -24,10 +24,7 @@ import base64 from cloudinit import helpers as c_helpers -from cloudinit import stages -from cloudinit import util from cloudinit.sources import DataSourceSmartOS -from cloudinit.settings import (PER_INSTANCE) from tests.unittests import helpers import os import os.path diff --git a/tests/unittests/test_handler/test_handler_yum_add_repo.py b/tests/unittests/test_handler/test_handler_yum_add_repo.py index 8df592f9..7c6f7c40 100644 --- a/tests/unittests/test_handler/test_handler_yum_add_repo.py +++ b/tests/unittests/test_handler/test_handler_yum_add_repo.py @@ -1,4 +1,3 @@ -from cloudinit import helpers from cloudinit import util from cloudinit.config import cc_yum_add_repo -- cgit v1.2.3 From f7fa9d2aa9abd81b8f8b79b95bdb1fc0c10b5fe9 Mon Sep 17 00:00:00 2001 From: Ben Howard Date: Tue, 27 May 2014 10:17:18 -0600 Subject: Enable vendordata for CloudSigma (LP: #1303986) --- cloudinit/sources/DataSourceCloudSigma.py | 2 ++ tests/unittests/test_datasource/test_cloudsigma.py | 28 +++++++++++++++++++++- 2 files changed, 29 insertions(+), 1 deletion(-) (limited to 'tests/unittests/test_datasource') diff --git a/cloudinit/sources/DataSourceCloudSigma.py b/cloudinit/sources/DataSourceCloudSigma.py index e1c7e566..ad2a044a 100644 --- a/cloudinit/sources/DataSourceCloudSigma.py +++ b/cloudinit/sources/DataSourceCloudSigma.py @@ -66,6 +66,8 @@ class DataSourceCloudSigma(sources.DataSource): self.userdata_raw = server_meta.get('cloudinit-user-data', "") if 'cloudinit-user-data' in base64_fields: self.userdata_raw = b64decode(self.userdata_raw) + if 'cloudinit' in server_context.get('vendor_data', {}): + self.vendordata_raw = server_context["vendor_data"]["cloudinit"] self.metadata = server_context self.ssh_public_key = server_meta['ssh_public_key'] diff --git a/tests/unittests/test_datasource/test_cloudsigma.py b/tests/unittests/test_datasource/test_cloudsigma.py index adbb4afb..a1342a86 100644 --- a/tests/unittests/test_datasource/test_cloudsigma.py +++ b/tests/unittests/test_datasource/test_cloudsigma.py @@ -20,7 +20,11 @@ SERVER_CONTEXT = { "smp": 1, "tags": ["much server", "very performance"], "uuid": "65b2fb23-8c03-4187-a3ba-8b7c919e8890", - "vnc_password": "9e84d6cb49e46379" + "vnc_password": "9e84d6cb49e46379", + "vendor_data": { + "location": "zrh", + "cloudinit": "#cloud-config\n\n...", + } } @@ -68,3 +72,25 @@ class DataSourceCloudSigmaTest(TestCase): self.datasource.get_data() self.assertEqual(self.datasource.userdata_raw, b'hi world\n') + + def test_vendor_data(self): + self.assertEqual(self.datasource.vendordata_raw, + SERVER_CONTEXT['vendor_data']['cloudinit']) + + def test_lack_of_vendor_data(self): + stripped_context = copy.deepcopy(SERVER_CONTEXT) + del stripped_context["vendor_data"] + self.datasource = DataSourceCloudSigma.DataSourceCloudSigma("", "", "") + self.datasource.cepko = CepkoMock(stripped_context) + self.datasource.get_data() + + self.assertIsNone(self.datasource.vendordata_raw) + + def test_lack_of_cloudinit_key_in_vendor_data(self): + stripped_context = copy.deepcopy(SERVER_CONTEXT) + del stripped_context["vendor_data"]["cloudinit"] + self.datasource = DataSourceCloudSigma.DataSourceCloudSigma("", "", "") + self.datasource.cepko = CepkoMock(stripped_context) + self.datasource.get_data() + + self.assertIsNone(self.datasource.vendordata_raw) -- cgit v1.2.3 From 71d817c427f06e9e1f5d547d5db191e541963d31 Mon Sep 17 00:00:00 2001 From: Kiril Vladimiroff Date: Fri, 30 May 2014 14:19:10 +0300 Subject: Use dmidecode to detect if cloud-init runs in CloudSigma's infrastructure --- cloudinit/sources/DataSourceCloudSigma.py | 22 ++++++++++++++++++++++ tests/unittests/test_datasource/test_cloudsigma.py | 1 + 2 files changed, 23 insertions(+) (limited to 'tests/unittests/test_datasource') diff --git a/cloudinit/sources/DataSourceCloudSigma.py b/cloudinit/sources/DataSourceCloudSigma.py index e1c7e566..fffff91e 100644 --- a/cloudinit/sources/DataSourceCloudSigma.py +++ b/cloudinit/sources/DataSourceCloudSigma.py @@ -20,6 +20,7 @@ import re from cloudinit import log as logging from cloudinit import sources +from cloudinit import util from cloudinit.cs_utils import Cepko LOG = logging.getLogger(__name__) @@ -40,12 +41,33 @@ class DataSourceCloudSigma(sources.DataSource): self.ssh_public_key = '' sources.DataSource.__init__(self, sys_cfg, distro, paths) + def is_running_in_cloudsigma(self): + """ + Uses dmidecode to detect if this instance of cloud-init is running + in the CloudSigma's infrastructure. + """ + dmidecode_path = util.which('dmidecode') + if not dmidecode_path: + return False + + LOG.debug("Determining hypervisor product name via dmidecode") + try: + system_product_name, _ = util.subp([dmidecode_path, "-s", "system-product-name"]) + return 'cloudsigma' in system_product_name.lower() + except: + LOG.exception("Failed to get hypervisor product name") + + return False + def get_data(self): """ Metadata is the whole server context and /meta/cloud-config is used as userdata. """ dsmode = None + if not self.is_running_in_cloudsigma(): + return False + try: server_context = self.cepko.all().result server_meta = server_context['meta'] diff --git a/tests/unittests/test_datasource/test_cloudsigma.py b/tests/unittests/test_datasource/test_cloudsigma.py index adbb4afb..25dc12f3 100644 --- a/tests/unittests/test_datasource/test_cloudsigma.py +++ b/tests/unittests/test_datasource/test_cloudsigma.py @@ -35,6 +35,7 @@ class CepkoMock(Cepko): class DataSourceCloudSigmaTest(TestCase): def setUp(self): self.datasource = DataSourceCloudSigma.DataSourceCloudSigma("", "", "") + self.datasource.is_running_in_cloudsigma = lambda: True self.datasource.cepko = CepkoMock(SERVER_CONTEXT) self.datasource.get_data() -- cgit v1.2.3 From 2bb228751a223f21296ff9166b42583c670359a5 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Mon, 2 Jun 2014 16:56:31 -0400 Subject: SmartOS test: do not require existance of /dev/ttyS1. LP: #1316597 --- ChangeLog | 1 + cloudinit/sources/DataSourceSmartOS.py | 10 ++++++++-- tests/unittests/test_datasource/test_smartos.py | 1 + 3 files changed, 10 insertions(+), 2 deletions(-) (limited to 'tests/unittests/test_datasource') diff --git a/ChangeLog b/ChangeLog index 2dee548e..c455f469 100644 --- a/ChangeLog +++ b/ChangeLog @@ -3,6 +3,7 @@ - Enable vendordata on CloudSigma datasource (LP: #1303986) - Poll on /dev/ttyS1 in CloudSigma datasource only if dmidecode says we're running on cloudsigma (LP: #1316475) [Kiril Vladimiroff] + - SmartOS test: do not require existance of /dev/ttyS1. [LP: #1316597] 0.7.5: - open 0.7.5 - Add a debug log message around import failures diff --git a/cloudinit/sources/DataSourceSmartOS.py b/cloudinit/sources/DataSourceSmartOS.py index 7c1eb09a..65ec0339 100644 --- a/cloudinit/sources/DataSourceSmartOS.py +++ b/cloudinit/sources/DataSourceSmartOS.py @@ -170,8 +170,9 @@ class DataSourceSmartOS(sources.DataSource): md = {} ud = "" - if not os.path.exists(self.seed): - LOG.debug("Host does not appear to be on SmartOS") + if not device_exists(self.seed): + LOG.debug("No serial device '%s' found for SmartOS datasource", + self.seed) return False uname_arch = os.uname()[4] @@ -274,6 +275,11 @@ class DataSourceSmartOS(sources.DataSource): b64=b64) +def device_exists(device): + """Symplistic method to determine if the device exists or not""" + return os.path.exists(device) + + def get_serial(seed_device, seed_timeout): """This is replaced in unit testing, allowing us to replace serial.Serial with a mocked class. diff --git a/tests/unittests/test_datasource/test_smartos.py b/tests/unittests/test_datasource/test_smartos.py index 45f1708a..f64aea07 100644 --- a/tests/unittests/test_datasource/test_smartos.py +++ b/tests/unittests/test_datasource/test_smartos.py @@ -171,6 +171,7 @@ class TestSmartOSDataSource(helpers.FilesystemMockingTestCase): self.apply_patches([(mod, 'get_serial', _get_serial)]) self.apply_patches([(mod, 'dmi_data', _dmi_data)]) self.apply_patches([(os, 'uname', _os_uname)]) + self.apply_patches([(mod, 'device_exists', lambda d: True)]) dsrc = mod.DataSourceSmartOS(sys_cfg, distro=None, paths=self.paths) return dsrc -- cgit v1.2.3 From 81525fd93541b41d31b6da13df61a0494cc1e7f6 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Wed, 16 Jul 2014 12:57:24 -0700 Subject: Fix a few tests that have been failing in python 2.6 A few of the current tests have been continually failing in python 2.6 based systems, due to lack of unit test functions that are now added to ensure we can run the unit tests (and not have to ignore those failures) on python 2.6 --- tests/unittests/helpers.py | 24 ++++++++++++++++++++++ tests/unittests/test_datasource/test_cloudsigma.py | 5 +++-- tests/unittests/test_datasource/test_gce.py | 5 +++-- 3 files changed, 30 insertions(+), 4 deletions(-) (limited to 'tests/unittests/test_datasource') diff --git a/tests/unittests/helpers.py b/tests/unittests/helpers.py index 5bed13cc..970eb8cb 100644 --- a/tests/unittests/helpers.py +++ b/tests/unittests/helpers.py @@ -52,6 +52,30 @@ if PY26: standardMsg = standardMsg % (value) self.fail(self._formatMessage(msg, standardMsg)) + def assertDictContainsSubset(self, expected, actual, msg=None): + missing = [] + mismatched = [] + for k, v in expected.iteritems(): + if k not in actual: + missing.append(k) + elif actual[k] != v: + mismatched.append('%r, expected: %r, actual: %r' + % (k, v, actual[k])) + + if len(missing) == 0 and len(mismatched) == 0: + return + + standardMsg = '' + if missing: + standardMsg = 'Missing: %r' % ','.join(m for m in missing) + if mismatched: + if standardMsg: + standardMsg += '; ' + standardMsg += 'Mismatched values: %s' % ','.join(mismatched) + + self.fail(self._formatMessage(msg, standardMsg)) + + else: class TestCase(unittest.TestCase): pass diff --git a/tests/unittests/test_datasource/test_cloudsigma.py b/tests/unittests/test_datasource/test_cloudsigma.py index f92e07b7..eadb3cb7 100644 --- a/tests/unittests/test_datasource/test_cloudsigma.py +++ b/tests/unittests/test_datasource/test_cloudsigma.py @@ -1,10 +1,11 @@ # coding: utf-8 import copy -from unittest import TestCase from cloudinit.cs_utils import Cepko from cloudinit.sources import DataSourceCloudSigma +from tests.unittests import helpers as test_helpers + SERVER_CONTEXT = { "cpu": 1000, @@ -36,7 +37,7 @@ class CepkoMock(Cepko): return self -class DataSourceCloudSigmaTest(TestCase): +class DataSourceCloudSigmaTest(test_helpers.TestCase): def setUp(self): self.datasource = DataSourceCloudSigma.DataSourceCloudSigma("", "", "") self.datasource.is_running_in_cloudsigma = lambda: True diff --git a/tests/unittests/test_datasource/test_gce.py b/tests/unittests/test_datasource/test_gce.py index d91bd531..1979a0de 100644 --- a/tests/unittests/test_datasource/test_gce.py +++ b/tests/unittests/test_datasource/test_gce.py @@ -15,7 +15,6 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -import unittest import httpretty import re @@ -25,6 +24,8 @@ from cloudinit import settings from cloudinit import helpers from cloudinit.sources import DataSourceGCE +from tests.unittests import helpers as test_helpers + GCE_META = { 'instance/id': '123', 'instance/zone': 'foo/bar', @@ -54,7 +55,7 @@ def _request_callback(method, uri, headers): return (404, headers, '') -class TestDataSourceGCE(unittest.TestCase): +class TestDataSourceGCE(test_helpers.TestCase): def setUp(self): self.ds = DataSourceGCE.DataSourceGCE( -- cgit v1.2.3