From f895cb12141281702b34da18f2384deb64c881e7 Mon Sep 17 00:00:00 2001 From: Barry Warsaw Date: Wed, 21 Jan 2015 17:56:53 -0500 Subject: Largely merge lp:~harlowja/cloud-init/py2-3 albeit manually because it seemed to be behind trunk. `tox -e py27` passes full test suite. Now to work on replacing mocker. --- cloudinit/sources/helpers/openstack.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) (limited to 'cloudinit/sources/helpers') diff --git a/cloudinit/sources/helpers/openstack.py b/cloudinit/sources/helpers/openstack.py index b7e19314..88c7a198 100644 --- a/cloudinit/sources/helpers/openstack.py +++ b/cloudinit/sources/helpers/openstack.py @@ -24,6 +24,8 @@ import copy import functools import os +import six + from cloudinit import ec2_utils from cloudinit import log as logging from cloudinit import sources @@ -205,7 +207,7 @@ class BaseReader(object): """ load_json_anytype = functools.partial( - util.load_json, root_types=(dict, basestring, list)) + util.load_json, root_types=(dict, list) + six.string_types) def datafiles(version): files = {} @@ -234,7 +236,7 @@ class BaseReader(object): 'version': 2, } data = datafiles(self._find_working_version()) - for (name, (path, required, translator)) in data.iteritems(): + for (name, (path, required, translator)) in data.items(): path = self._path_join(self.base_path, path) data = None found = False @@ -364,7 +366,7 @@ class ConfigDriveReader(BaseReader): raise NonReadable("%s: no files found" % (self.base_path)) md = {} - for (name, (key, translator, default)) in FILES_V1.iteritems(): + for (name, (key, translator, default)) in FILES_V1.items(): if name in found: path = found[name] try: @@ -478,7 +480,7 @@ def convert_vendordata_json(data, recurse=True): """ if not data: return None - if isinstance(data, (str, unicode, basestring)): + if isinstance(data, six.string_types): return data if isinstance(data, list): return copy.deepcopy(data) -- cgit v1.2.3 From 8cd5d7b143f882d80d45b1c04bdde1949846d4f1 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Wed, 25 Feb 2015 19:40:33 -0500 Subject: move towards user-data being binary UrlResponse: biggest change... make readurl return bytes, making user know what to do with it. util: add load_tfile_or_url for loading text file or url as read_file_or_url now returns bytes ec2_utils: all meta-data is text, remove non-obvious string translations DigitalOcean: adjust for ec2_utils DataSourceGCE, DataSourceMAAS: user-data is binary other fields are text. openstack.py: read paths without decoding to text. This is ok as paths other than user-data are json, and load_json will handle load_file still returns text, and that is what most things use. --- cloudinit/ec2_utils.py | 14 +++++++++++--- cloudinit/sources/DataSourceDigitalOcean.py | 8 ++++++-- cloudinit/sources/DataSourceGCE.py | 21 ++++++++++++--------- cloudinit/sources/DataSourceMAAS.py | 14 +++++++++++--- cloudinit/sources/helpers/openstack.py | 2 +- cloudinit/url_helper.py | 2 +- cloudinit/util.py | 11 ++++++++--- tests/unittests/helpers.py | 5 ++++- tests/unittests/test_datasource/test_configdrive.py | 15 ++++++++++----- tests/unittests/test_datasource/test_gce.py | 2 +- tests/unittests/test_datasource/test_maas.py | 8 ++++---- tests/unittests/test_datasource/test_nocloud.py | 14 +++++++------- tests/unittests/test_datasource/test_openstack.py | 6 +++--- tests/unittests/test_ec2_util.py | 2 +- .../test_handler/test_handler_apt_configure.py | 12 ++++++------ tests/unittests/test_pathprefix2dict.py | 10 +++++----- 16 files changed, 91 insertions(+), 55 deletions(-) (limited to 'cloudinit/sources/helpers') diff --git a/cloudinit/ec2_utils.py b/cloudinit/ec2_utils.py index e1ed4091..7cf99186 100644 --- a/cloudinit/ec2_utils.py +++ b/cloudinit/ec2_utils.py @@ -41,6 +41,10 @@ class MetadataLeafDecoder(object): def __call__(self, field, blob): if not blob: return blob + try: + blob = util.decode_binary(blob) + except UnicodeDecodeError: + return blob if self._maybe_json_object(blob): try: # Assume it's json, unless it fails parsing... @@ -69,6 +73,8 @@ class MetadataMaterializer(object): def _parse(self, blob): leaves = {} children = [] + blob = util.decode_binary(blob) + if not blob: return (leaves, children) @@ -117,12 +123,12 @@ class MetadataMaterializer(object): child_url = url_helper.combine_url(base_url, c) if not child_url.endswith("/"): child_url += "/" - child_blob = str(self._caller(child_url)) + child_blob = self._caller(child_url) child_contents[c] = self._materialize(child_blob, child_url) leaf_contents = {} for (field, resource) in leaves.items(): leaf_url = url_helper.combine_url(base_url, resource) - leaf_blob = self._caller(leaf_url).contents + leaf_blob = self._caller(leaf_url) leaf_contents[field] = self._leaf_decoder(field, leaf_blob) joined = {} joined.update(child_contents) @@ -179,11 +185,13 @@ def get_instance_metadata(api_version='latest', caller = functools.partial(util.read_file_or_url, ssl_details=ssl_details, timeout=timeout, retries=retries) + def mcaller(url): + return caller(url).contents try: response = caller(md_url) materializer = MetadataMaterializer(response.contents, - md_url, caller, + md_url, mcaller, leaf_decoder=leaf_decoder) md = materializer.materialize() if not isinstance(md, (dict)): diff --git a/cloudinit/sources/DataSourceDigitalOcean.py b/cloudinit/sources/DataSourceDigitalOcean.py index 76ddaa9d..5d47564d 100644 --- a/cloudinit/sources/DataSourceDigitalOcean.py +++ b/cloudinit/sources/DataSourceDigitalOcean.py @@ -54,9 +54,13 @@ class DataSourceDigitalOcean(sources.DataSource): def get_data(self): caller = functools.partial(util.read_file_or_url, timeout=self.timeout, retries=self.retries) - md = ec2_utils.MetadataMaterializer(str(caller(self.metadata_address)), + + def mcaller(url): + return caller(url).contents + + md = ec2_utils.MetadataMaterializer(mcaller(self.metadata_address), base_url=self.metadata_address, - caller=caller) + caller=mcaller) self.metadata = md.materialize() diff --git a/cloudinit/sources/DataSourceGCE.py b/cloudinit/sources/DataSourceGCE.py index 6936c74e..608c07f1 100644 --- a/cloudinit/sources/DataSourceGCE.py +++ b/cloudinit/sources/DataSourceGCE.py @@ -53,15 +53,15 @@ class DataSourceGCE(sources.DataSource): # GCE metadata server requires a custom header since v1 headers = {'X-Google-Metadata-Request': True} - # url_map: (our-key, path, required) + # url_map: (our-key, path, required, is_text) url_map = [ - ('instance-id', 'instance/id', True), - ('availability-zone', 'instance/zone', True), - ('local-hostname', 'instance/hostname', True), - ('public-keys', 'project/attributes/sshKeys', False), - ('user-data', 'instance/attributes/user-data', False), + ('instance-id', 'instance/id', True, True), + ('availability-zone', 'instance/zone', True, True), + ('local-hostname', 'instance/hostname', True, True), + ('public-keys', 'project/attributes/sshKeys', False, True), + ('user-data', 'instance/attributes/user-data', False, False), ('user-data-encoding', 'instance/attributes/user-data-encoding', - False), + False, True), ] # if we cannot resolve the metadata server, then no point in trying @@ -71,13 +71,16 @@ class DataSourceGCE(sources.DataSource): # iterate over url_map keys to get metadata items found = False - for (mkey, path, required) in url_map: + for (mkey, path, required, is_text) in url_map: try: resp = url_helper.readurl(url=self.metadata_address + path, headers=headers) if resp.code == 200: found = True - self.metadata[mkey] = resp.contents + if is_text: + self.metadata[mkey] = util.decode_binary(resp.contents) + else: + self.metadata[mkey] = resp.contents else: if required: msg = "required url %s returned code %s. not GCE" diff --git a/cloudinit/sources/DataSourceMAAS.py b/cloudinit/sources/DataSourceMAAS.py index 082cc58f..35c5b5e1 100644 --- a/cloudinit/sources/DataSourceMAAS.py +++ b/cloudinit/sources/DataSourceMAAS.py @@ -36,6 +36,8 @@ from cloudinit import util LOG = logging.getLogger(__name__) MD_VERSION = "2012-03-01" +BINARY_FIELDS = ('user-data',) + class DataSourceMAAS(sources.DataSource): """ @@ -185,7 +187,9 @@ def read_maas_seed_dir(seed_d): md = {} for fname in files: try: - md[fname] = util.load_file(os.path.join(seed_d, fname)) + print("fname: %s / %s" % (fname, fname not in BINARY_FIELDS)) + md[fname] = util.load_file(os.path.join(seed_d, fname), + decode=fname not in BINARY_FIELDS) except IOError as e: if e.errno != errno.ENOENT: raise @@ -218,6 +222,7 @@ def read_maas_seed_url(seed_url, header_cb=None, timeout=None, 'public-keys': "%s/%s" % (base_url, 'meta-data/public-keys'), 'user-data': "%s/%s" % (base_url, 'user-data'), } + md = {} for name in file_order: url = files.get(name) @@ -238,7 +243,10 @@ def read_maas_seed_url(seed_url, header_cb=None, timeout=None, timeout=timeout, ssl_details=ssl_details) if resp.ok(): - md[name] = str(resp) + if name in BINARY_FIELDS: + md[name] = resp.contents + else: + md[name] = util.decode_binary(resp.contents) else: LOG.warn(("Fetching from %s resulted in" " an invalid http code %s"), url, resp.code) @@ -263,7 +271,7 @@ def check_seed_contents(content, seed): if len(missing): raise MAASSeedDirMalformed("%s: missing files %s" % (seed, missing)) - userdata = content.get('user-data', "") + userdata = content.get('user-data', b"") md = {} for (key, val) in content.items(): if key == 'user-data': diff --git a/cloudinit/sources/helpers/openstack.py b/cloudinit/sources/helpers/openstack.py index 88c7a198..bd93d22f 100644 --- a/cloudinit/sources/helpers/openstack.py +++ b/cloudinit/sources/helpers/openstack.py @@ -327,7 +327,7 @@ class ConfigDriveReader(BaseReader): return os.path.join(*components) def _path_read(self, path): - return util.load_file(path) + return util.load_file(path, decode=False) def _fetch_available_versions(self): if self._versions is None: diff --git a/cloudinit/url_helper.py b/cloudinit/url_helper.py index 62001dff..2d81a062 100644 --- a/cloudinit/url_helper.py +++ b/cloudinit/url_helper.py @@ -119,7 +119,7 @@ class UrlResponse(object): @property def contents(self): - return self._response.text + return self._response.content @property def url(self): diff --git a/cloudinit/util.py b/cloudinit/util.py index 4fbdf0a9..efbc3c8d 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -739,6 +739,10 @@ def fetch_ssl_details(paths=None): return ssl_details +def load_tfile_or_url(*args, **kwargs): + return(decode_binary(read_file_or_url(*args, **kwargs).contents)) + + def read_file_or_url(url, timeout=5, retries=10, headers=None, data=None, sec_between=1, ssl_details=None, headers_cb=None, exception_cb=None): @@ -750,7 +754,7 @@ def read_file_or_url(url, timeout=5, retries=10, LOG.warn("Unable to post data to file resource %s", url) file_path = url[len("file://"):] try: - contents = load_file(file_path) + contents = load_file(file_path, decode=False) except IOError as e: code = e.errno if e.errno == errno.ENOENT: @@ -806,7 +810,7 @@ def read_seeded(base="", ext="", timeout=5, retries=10, file_retries=0): ud_url = "%s%s%s" % (base, "user-data", ext) md_url = "%s%s%s" % (base, "meta-data", ext) - md_resp = read_file_or_url(md_url, timeout, retries, file_retries) + md_resp = load_tfile_or_url(md_url, timeout, retries, file_retries) md = None if md_resp.ok(): md = load_yaml(md_resp.contents, default={}) @@ -815,6 +819,7 @@ def read_seeded(base="", ext="", timeout=5, retries=10, file_retries=0): ud = None if ud_resp.ok(): ud = ud_resp.contents + print("returning %s (%s)" % (ud_resp.contents.__class__, ud_resp.contents)) return (md, ud) @@ -2030,7 +2035,7 @@ def pathprefix2dict(base, required=None, optional=None, delim=os.path.sep): ret = {} for f in required + optional: try: - ret[f] = load_file(base + delim + f, quiet=False) + ret[f] = load_file(base + delim + f, quiet=False, decode=False) except IOError as e: if e.errno != errno.ENOENT: raise diff --git a/tests/unittests/helpers.py b/tests/unittests/helpers.py index 7516bd02..24e1e881 100644 --- a/tests/unittests/helpers.py +++ b/tests/unittests/helpers.py @@ -288,7 +288,10 @@ def populate_dir(path, files): os.makedirs(path) for (name, content) in files.items(): with open(os.path.join(path, name), "wb") as fp: - fp.write(content.encode('utf-8')) + if isinstance(content, six.binary_type): + fp.write(content) + else: + fp.write(content.encode('utf-8')) fp.close() diff --git a/tests/unittests/test_datasource/test_configdrive.py b/tests/unittests/test_datasource/test_configdrive.py index e28bdd84..83aca505 100644 --- a/tests/unittests/test_datasource/test_configdrive.py +++ b/tests/unittests/test_datasource/test_configdrive.py @@ -2,6 +2,7 @@ from copy import copy import json import os import shutil +import six import tempfile try: @@ -45,7 +46,7 @@ EC2_META = { 'reservation-id': 'r-iru5qm4m', 'security-groups': ['default'] } -USER_DATA = '#!/bin/sh\necho This is user data\n' +USER_DATA = b'#!/bin/sh\necho This is user data\n' OSTACK_META = { 'availability_zone': 'nova', 'files': [{'content_path': '/content/0000', 'path': '/etc/foo.cfg'}, @@ -56,8 +57,8 @@ OSTACK_META = { 'public_keys': {'mykey': PUBKEY}, 'uuid': 'b0fa911b-69d4-4476-bbe2-1c92bff6535c'} -CONTENT_0 = 'This is contents of /etc/foo.cfg\n' -CONTENT_1 = '# this is /etc/bar/bar.cfg\n' +CONTENT_0 = b'This is contents of /etc/foo.cfg\n' +CONTENT_1 = b'# this is /etc/bar/bar.cfg\n' CFG_DRIVE_FILES_V2 = { 'ec2/2009-04-04/meta-data.json': json.dumps(EC2_META), @@ -346,8 +347,12 @@ def populate_dir(seed_dir, files): dirname = os.path.dirname(path) if not os.path.isdir(dirname): os.makedirs(dirname) - with open(path, "w") as fp: + if isinstance(content, six.text_type): + mode = "w" + else: + mode = "wb" + + with open(path, mode) as fp: fp.write(content) - fp.close() # vi: ts=4 expandtab diff --git a/tests/unittests/test_datasource/test_gce.py b/tests/unittests/test_datasource/test_gce.py index 6dd4b5ed..d28f3b08 100644 --- a/tests/unittests/test_datasource/test_gce.py +++ b/tests/unittests/test_datasource/test_gce.py @@ -32,7 +32,7 @@ GCE_META = { 'instance/zone': 'foo/bar', 'project/attributes/sshKeys': 'user:ssh-rsa AA2..+aRD0fyVw== root@server', 'instance/hostname': 'server.project-foo.local', - 'instance/attributes/user-data': '/bin/echo foo\n', + 'instance/attributes/user-data': b'/bin/echo foo\n', } GCE_META_PARTIAL = { diff --git a/tests/unittests/test_datasource/test_maas.py b/tests/unittests/test_datasource/test_maas.py index d25e1adc..f109bb04 100644 --- a/tests/unittests/test_datasource/test_maas.py +++ b/tests/unittests/test_datasource/test_maas.py @@ -26,7 +26,7 @@ class TestMAASDataSource(TestCase): data = {'instance-id': 'i-valid01', 'local-hostname': 'valid01-hostname', - 'user-data': 'valid01-userdata', + 'user-data': b'valid01-userdata', 'public-keys': 'ssh-rsa AAAAB3Nz...aC1yc2E= keyname'} my_d = os.path.join(self.tmp, "valid") @@ -46,7 +46,7 @@ class TestMAASDataSource(TestCase): data = {'instance-id': 'i-valid-extra', 'local-hostname': 'valid-extra-hostname', - 'user-data': 'valid-extra-userdata', 'foo': 'bar'} + 'user-data': b'valid-extra-userdata', 'foo': 'bar'} my_d = os.path.join(self.tmp, "valid_extra") populate_dir(my_d, data) @@ -103,7 +103,7 @@ class TestMAASDataSource(TestCase): 'meta-data/instance-id': 'i-instanceid', 'meta-data/local-hostname': 'test-hostname', 'meta-data/public-keys': 'test-hostname', - 'user-data': 'foodata', + 'user-data': b'foodata', } valid_order = [ 'meta-data/local-hostname', @@ -143,7 +143,7 @@ class TestMAASDataSource(TestCase): userdata, metadata = DataSourceMAAS.read_maas_seed_url( my_seed, header_cb=my_headers_cb, version=my_ver) - self.assertEqual("foodata", userdata) + self.assertEqual(b"foodata", userdata) self.assertEqual(metadata['instance-id'], valid['meta-data/instance-id']) self.assertEqual(metadata['local-hostname'], diff --git a/tests/unittests/test_datasource/test_nocloud.py b/tests/unittests/test_datasource/test_nocloud.py index 4f967f58..85b4c25a 100644 --- a/tests/unittests/test_datasource/test_nocloud.py +++ b/tests/unittests/test_datasource/test_nocloud.py @@ -37,7 +37,7 @@ class TestNoCloudDataSource(TestCase): def test_nocloud_seed_dir(self): md = {'instance-id': 'IID', 'dsmode': 'local'} - ud = "USER_DATA_HERE" + ud = b"USER_DATA_HERE" populate_dir(os.path.join(self.paths.seed_dir, "nocloud"), {'user-data': ud, 'meta-data': yaml.safe_dump(md)}) @@ -92,20 +92,20 @@ class TestNoCloudDataSource(TestCase): data = { 'fs_label': None, 'meta-data': yaml.safe_dump({'instance-id': 'IID'}), - 'user-data': "USER_DATA_RAW", + 'user-data': b"USER_DATA_RAW", } sys_cfg = {'datasource': {'NoCloud': data}} dsrc = ds(sys_cfg=sys_cfg, distro=None, paths=self.paths) ret = dsrc.get_data() - self.assertEqual(dsrc.userdata_raw, "USER_DATA_RAW") + self.assertEqual(dsrc.userdata_raw, b"USER_DATA_RAW") self.assertEqual(dsrc.metadata.get('instance-id'), 'IID') self.assertTrue(ret) def test_nocloud_seed_with_vendordata(self): md = {'instance-id': 'IID', 'dsmode': 'local'} - ud = "USER_DATA_HERE" - vd = "THIS IS MY VENDOR_DATA" + ud = b"USER_DATA_HERE" + vd = b"THIS IS MY VENDOR_DATA" populate_dir(os.path.join(self.paths.seed_dir, "nocloud"), {'user-data': ud, 'meta-data': yaml.safe_dump(md), @@ -126,7 +126,7 @@ class TestNoCloudDataSource(TestCase): def test_nocloud_no_vendordata(self): populate_dir(os.path.join(self.paths.seed_dir, "nocloud"), - {'user-data': "ud", 'meta-data': "instance-id: IID\n"}) + {'user-data': b"ud", 'meta-data': "instance-id: IID\n"}) sys_cfg = {'datasource': {'NoCloud': {'fs_label': None}}} @@ -134,7 +134,7 @@ class TestNoCloudDataSource(TestCase): dsrc = ds(sys_cfg=sys_cfg, distro=None, paths=self.paths) ret = dsrc.get_data() - self.assertEqual(dsrc.userdata_raw, "ud") + self.assertEqual(dsrc.userdata_raw, b"ud") self.assertFalse(dsrc.vendordata) self.assertTrue(ret) diff --git a/tests/unittests/test_datasource/test_openstack.py b/tests/unittests/test_datasource/test_openstack.py index 81ef1546..81411ced 100644 --- a/tests/unittests/test_datasource/test_openstack.py +++ b/tests/unittests/test_datasource/test_openstack.py @@ -49,7 +49,7 @@ EC2_META = { 'public-ipv4': '0.0.0.1', 'reservation-id': 'r-iru5qm4m', } -USER_DATA = '#!/bin/sh\necho This is user data\n' +USER_DATA = b'#!/bin/sh\necho This is user data\n' VENDOR_DATA = { 'magic': '', } @@ -63,8 +63,8 @@ OSTACK_META = { 'public_keys': {'mykey': PUBKEY}, 'uuid': 'b0fa911b-69d4-4476-bbe2-1c92bff6535c', } -CONTENT_0 = 'This is contents of /etc/foo.cfg\n' -CONTENT_1 = '# this is /etc/bar/bar.cfg\n' +CONTENT_0 = b'This is contents of /etc/foo.cfg\n' +CONTENT_1 = b'# this is /etc/bar/bar.cfg\n' OS_FILES = { 'openstack/latest/meta_data.json': json.dumps(OSTACK_META), 'openstack/latest/user_data': USER_DATA, diff --git a/tests/unittests/test_ec2_util.py b/tests/unittests/test_ec2_util.py index 84aa002e..bd43accf 100644 --- a/tests/unittests/test_ec2_util.py +++ b/tests/unittests/test_ec2_util.py @@ -16,7 +16,7 @@ class TestEc2Util(helpers.HttprettyTestCase): body='stuff', status=200) userdata = eu.get_instance_userdata(self.VERSION) - self.assertEquals('stuff', userdata) + self.assertEquals('stuff', userdata.decode('utf-8')) @hp.activate def test_userdata_fetch_fail_not_found(self): diff --git a/tests/unittests/test_handler/test_handler_apt_configure.py b/tests/unittests/test_handler/test_handler_apt_configure.py index d8fe9a4f..02cad8b2 100644 --- a/tests/unittests/test_handler/test_handler_apt_configure.py +++ b/tests/unittests/test_handler/test_handler_apt_configure.py @@ -30,7 +30,7 @@ class TestAptProxyConfig(TestCase): self.assertTrue(os.path.isfile(self.pfile)) self.assertFalse(os.path.isfile(self.cfile)) - contents = str(util.read_file_or_url(self.pfile)) + contents = util.load_tfile_or_url(self.pfile) self.assertTrue(self._search_apt_config(contents, "http", "myproxy")) def test_apt_http_proxy_written(self): @@ -40,7 +40,7 @@ class TestAptProxyConfig(TestCase): self.assertTrue(os.path.isfile(self.pfile)) self.assertFalse(os.path.isfile(self.cfile)) - contents = str(util.read_file_or_url(self.pfile)) + contents = util.load_tfile_or_url(self.pfile) self.assertTrue(self._search_apt_config(contents, "http", "myproxy")) def test_apt_all_proxy_written(self): @@ -58,7 +58,7 @@ class TestAptProxyConfig(TestCase): self.assertTrue(os.path.isfile(self.pfile)) self.assertFalse(os.path.isfile(self.cfile)) - contents = str(util.read_file_or_url(self.pfile)) + contents = util.load_tfile_or_url(self.pfile) for ptype, pval in values.items(): self.assertTrue(self._search_apt_config(contents, ptype, pval)) @@ -74,7 +74,7 @@ class TestAptProxyConfig(TestCase): cc_apt_configure.apply_apt_config({'apt_proxy': "foo"}, self.pfile, self.cfile) self.assertTrue(os.path.isfile(self.pfile)) - contents = str(util.read_file_or_url(self.pfile)) + contents = util.load_tfile_or_url(self.pfile) self.assertTrue(self._search_apt_config(contents, "http", "foo")) def test_config_written(self): @@ -86,14 +86,14 @@ class TestAptProxyConfig(TestCase): self.assertTrue(os.path.isfile(self.cfile)) self.assertFalse(os.path.isfile(self.pfile)) - self.assertEqual(str(util.read_file_or_url(self.cfile)), payload) + self.assertEqual(util.load_tfile_or_url(self.cfile), payload) def test_config_replaced(self): util.write_file(self.pfile, "content doesnt matter") cc_apt_configure.apply_apt_config({'apt_config': "foo"}, self.pfile, self.cfile) self.assertTrue(os.path.isfile(self.cfile)) - self.assertEqual(str(util.read_file_or_url(self.cfile)), "foo") + self.assertEqual(util.load_tfile_or_url(self.cfile), "foo") def test_config_deleted(self): # if no 'apt_config' is provided, delete any previously written file diff --git a/tests/unittests/test_pathprefix2dict.py b/tests/unittests/test_pathprefix2dict.py index 7089bde6..38fd75b6 100644 --- a/tests/unittests/test_pathprefix2dict.py +++ b/tests/unittests/test_pathprefix2dict.py @@ -14,28 +14,28 @@ class TestPathPrefix2Dict(TestCase): self.addCleanup(shutil.rmtree, self.tmp) def test_required_only(self): - dirdata = {'f1': 'f1content', 'f2': 'f2content'} + dirdata = {'f1': b'f1content', 'f2': b'f2content'} populate_dir(self.tmp, dirdata) ret = util.pathprefix2dict(self.tmp, required=['f1', 'f2']) self.assertEqual(dirdata, ret) def test_required_missing(self): - dirdata = {'f1': 'f1content'} + dirdata = {'f1': b'f1content'} populate_dir(self.tmp, dirdata) kwargs = {'required': ['f1', 'f2']} self.assertRaises(ValueError, util.pathprefix2dict, self.tmp, **kwargs) def test_no_required_and_optional(self): - dirdata = {'f1': 'f1c', 'f2': 'f2c'} + dirdata = {'f1': b'f1c', 'f2': b'f2c'} populate_dir(self.tmp, dirdata) ret = util.pathprefix2dict(self.tmp, required=None, - optional=['f1', 'f2']) + optional=['f1', 'f2']) self.assertEqual(dirdata, ret) def test_required_and_optional(self): - dirdata = {'f1': 'f1c', 'f2': 'f2c'} + dirdata = {'f1': b'f1c', 'f2': b'f2c'} populate_dir(self.tmp, dirdata) ret = util.pathprefix2dict(self.tmp, required=['f1'], optional=['f2']) -- cgit v1.2.3 From b9f26689e8b3bb7a3486771c6362107232a7dcf4 Mon Sep 17 00:00:00 2001 From: Daniel Watkins Date: Fri, 8 May 2015 13:16:42 +0100 Subject: Split WALinuxAgentShim code out to separate file. --- cloudinit/sources/DataSourceAzure.py | 271 +-------------- cloudinit/sources/helpers/azure.py | 273 +++++++++++++++ tests/unittests/test_datasource/test_azure.py | 364 -------------------- .../unittests/test_datasource/test_azure_helper.py | 377 +++++++++++++++++++++ 4 files changed, 653 insertions(+), 632 deletions(-) create mode 100644 cloudinit/sources/helpers/azure.py create mode 100644 tests/unittests/test_datasource/test_azure_helper.py (limited to 'cloudinit/sources/helpers') diff --git a/cloudinit/sources/DataSourceAzure.py b/cloudinit/sources/DataSourceAzure.py index c2dc6b4c..5e147950 100644 --- a/cloudinit/sources/DataSourceAzure.py +++ b/cloudinit/sources/DataSourceAzure.py @@ -17,23 +17,19 @@ # along with this program. If not, see . import base64 +import contextlib import crypt import fnmatch import os import os.path -import re -import socket -import struct -import tempfile -import time -from contextlib import contextmanager from xml.dom import minidom -from xml.etree import ElementTree from cloudinit import log as logging from cloudinit.settings import PER_ALWAYS from cloudinit import sources from cloudinit import util +from cloudinit.sources.helpers.azure import ( + iid_from_shared_config_content, WALinuxAgentShim) LOG = logging.getLogger(__name__) @@ -70,253 +66,6 @@ DS_CFG_PATH = ['datasource', DS_NAME] DEF_EPHEMERAL_LABEL = 'Temporary Storage' - -@contextmanager -def cd(newdir): - prevdir = os.getcwd() - os.chdir(os.path.expanduser(newdir)) - try: - yield - finally: - os.chdir(prevdir) - - -class AzureEndpointHttpClient(object): - - headers = { - 'x-ms-agent-name': 'WALinuxAgent', - 'x-ms-version': '2012-11-30', - } - - def __init__(self, certificate): - self.extra_secure_headers = { - "x-ms-cipher-name": "DES_EDE3_CBC", - "x-ms-guest-agent-public-x509-cert": certificate, - } - - def get(self, url, secure=False): - headers = self.headers - if secure: - headers = self.headers.copy() - headers.update(self.extra_secure_headers) - return util.read_file_or_url(url, headers=headers) - - def post(self, url, data=None, extra_headers=None): - headers = self.headers - if extra_headers is not None: - headers = self.headers.copy() - headers.update(extra_headers) - return util.read_file_or_url(url, data=data, headers=headers) - - -class GoalState(object): - - def __init__(self, xml, http_client): - self.http_client = http_client - self.root = ElementTree.fromstring(xml) - self._certificates_xml = None - - def _text_from_xpath(self, xpath): - element = self.root.find(xpath) - if element is not None: - return element.text - return None - - @property - def container_id(self): - return self._text_from_xpath('./Container/ContainerId') - - @property - def incarnation(self): - return self._text_from_xpath('./Incarnation') - - @property - def instance_id(self): - return self._text_from_xpath( - './Container/RoleInstanceList/RoleInstance/InstanceId') - - @property - def shared_config_xml(self): - url = self._text_from_xpath('./Container/RoleInstanceList/RoleInstance' - '/Configuration/SharedConfig') - return self.http_client.get(url).contents - - @property - def certificates_xml(self): - if self._certificates_xml is None: - url = self._text_from_xpath( - './Container/RoleInstanceList/RoleInstance' - '/Configuration/Certificates') - if url is not None: - self._certificates_xml = self.http_client.get( - url, secure=True).contents - return self._certificates_xml - - -class OpenSSLManager(object): - - certificate_names = { - 'private_key': 'TransportPrivate.pem', - 'certificate': 'TransportCert.pem', - } - - def __init__(self): - self.tmpdir = tempfile.TemporaryDirectory() - self.certificate = None - self.generate_certificate() - - def generate_certificate(self): - LOG.debug('Generating certificate for communication with fabric...') - if self.certificate is not None: - LOG.debug('Certificate already generated.') - return - with cd(self.tmpdir.name): - util.subp([ - 'openssl', 'req', '-x509', '-nodes', '-subj', - '/CN=LinuxTransport', '-days', '32768', '-newkey', 'rsa:2048', - '-keyout', self.certificate_names['private_key'], - '-out', self.certificate_names['certificate'], - ]) - certificate = '' - for line in open(self.certificate_names['certificate']): - if "CERTIFICATE" not in line: - certificate += line.rstrip() - self.certificate = certificate - LOG.debug('New certificate generated.') - - def parse_certificates(self, certificates_xml): - tag = ElementTree.fromstring(certificates_xml).find( - './/Data') - certificates_content = tag.text - lines = [ - b'MIME-Version: 1.0', - b'Content-Disposition: attachment; filename="Certificates.p7m"', - b'Content-Type: application/x-pkcs7-mime; name="Certificates.p7m"', - b'Content-Transfer-Encoding: base64', - b'', - certificates_content.encode('utf-8'), - ] - with cd(self.tmpdir.name): - with open('Certificates.p7m', 'wb') as f: - f.write(b'\n'.join(lines)) - out, _ = util.subp( - 'openssl cms -decrypt -in Certificates.p7m -inkey' - ' {private_key} -recip {certificate} | openssl pkcs12 -nodes' - ' -password pass:'.format(**self.certificate_names), - shell=True) - private_keys, certificates = [], [] - current = [] - for line in out.splitlines(): - current.append(line) - if re.match(r'[-]+END .*?KEY[-]+$', line): - private_keys.append('\n'.join(current)) - current = [] - elif re.match(r'[-]+END .*?CERTIFICATE[-]+$', line): - certificates.append('\n'.join(current)) - current = [] - keys = [] - for certificate in certificates: - with cd(self.tmpdir.name): - public_key, _ = util.subp( - 'openssl x509 -noout -pubkey |' - 'ssh-keygen -i -m PKCS8 -f /dev/stdin', - data=certificate, - shell=True) - keys.append(public_key) - return keys - - -class WALinuxAgentShim(object): - - REPORT_READY_XML_TEMPLATE = '\n'.join([ - '', - '', - ' {incarnation}', - ' ', - ' {container_id}', - ' ', - ' ', - ' {instance_id}', - ' ', - ' Ready', - ' ', - ' ', - ' ', - ' ', - '']) - - def __init__(self): - LOG.debug('WALinuxAgentShim instantiated...') - self.endpoint = self.find_endpoint() - self.openssl_manager = OpenSSLManager() - self.http_client = AzureEndpointHttpClient( - self.openssl_manager.certificate) - self.values = {} - - @staticmethod - def find_endpoint(): - LOG.debug('Finding Azure endpoint...') - content = util.load_file('/var/lib/dhcp/dhclient.eth0.leases') - value = None - for line in content.splitlines(): - if 'unknown-245' in line: - value = line.strip(' ').split(' ', 2)[-1].strip(';\n"') - if value is None: - raise Exception('No endpoint found in DHCP config.') - if ':' in value: - hex_string = '' - for hex_pair in value.split(':'): - if len(hex_pair) == 1: - hex_pair = '0' + hex_pair - hex_string += hex_pair - value = struct.pack('>L', int(hex_string.replace(':', ''), 16)) - else: - value = value.encode('utf-8') - endpoint_ip_address = socket.inet_ntoa(value) - LOG.debug('Azure endpoint found at %s', endpoint_ip_address) - return endpoint_ip_address - - def register_with_azure_and_fetch_data(self): - LOG.info('Registering with Azure...') - for i in range(10): - try: - response = self.http_client.get( - 'http://{}/machine/?comp=goalstate'.format(self.endpoint)) - except Exception: - time.sleep(i + 1) - else: - break - LOG.debug('Successfully fetched GoalState XML.') - goal_state = GoalState(response.contents, self.http_client) - public_keys = [] - if goal_state.certificates_xml is not None: - LOG.debug('Certificate XML found; parsing out public keys.') - public_keys = self.openssl_manager.parse_certificates( - goal_state.certificates_xml) - data = { - 'instance-id': iid_from_shared_config_content( - goal_state.shared_config_xml), - 'public-keys': public_keys, - } - self._report_ready(goal_state) - return data - - def _report_ready(self, goal_state): - LOG.debug('Reporting ready to Azure fabric.') - document = self.REPORT_READY_XML_TEMPLATE.format( - incarnation=goal_state.incarnation, - container_id=goal_state.container_id, - instance_id=goal_state.instance_id, - ) - self.http_client.post( - "http://{}/machine?comp=health".format(self.endpoint), - data=document, - extra_headers={'Content-Type': 'text/xml; charset=utf-8'}, - ) - LOG.info('Reported ready to Azure fabric.') - - def get_hostname(hostname_command='hostname'): return util.subp(hostname_command, capture=True)[0].strip() @@ -690,20 +439,6 @@ def load_azure_ovf_pubkeys(sshnode): return found -def single_node_at_path(node, pathlist): - curnode = node - for tok in pathlist: - results = find_child(curnode, lambda n: n.localName == tok) - if len(results) == 0: - raise ValueError("missing %s token in %s" % (tok, str(pathlist))) - if len(results) > 1: - raise ValueError("found %s nodes of type %s looking for %s" % - (len(results), tok, str(pathlist))) - curnode = results[0] - - return curnode - - def read_azure_ovf(contents): try: dom = minidom.parseString(contents) diff --git a/cloudinit/sources/helpers/azure.py b/cloudinit/sources/helpers/azure.py new file mode 100644 index 00000000..60f116e0 --- /dev/null +++ b/cloudinit/sources/helpers/azure.py @@ -0,0 +1,273 @@ +import logging +import os +import re +import socket +import struct +import tempfile +import time +from contextlib import contextmanager +from xml.etree import ElementTree + +from cloudinit import util + + +LOG = logging.getLogger(__name__) + + +@contextmanager +def cd(newdir): + prevdir = os.getcwd() + os.chdir(os.path.expanduser(newdir)) + try: + yield + finally: + os.chdir(prevdir) + + +class AzureEndpointHttpClient(object): + + headers = { + 'x-ms-agent-name': 'WALinuxAgent', + 'x-ms-version': '2012-11-30', + } + + def __init__(self, certificate): + self.extra_secure_headers = { + "x-ms-cipher-name": "DES_EDE3_CBC", + "x-ms-guest-agent-public-x509-cert": certificate, + } + + def get(self, url, secure=False): + headers = self.headers + if secure: + headers = self.headers.copy() + headers.update(self.extra_secure_headers) + return util.read_file_or_url(url, headers=headers) + + def post(self, url, data=None, extra_headers=None): + headers = self.headers + if extra_headers is not None: + headers = self.headers.copy() + headers.update(extra_headers) + return util.read_file_or_url(url, data=data, headers=headers) + + +class GoalState(object): + + def __init__(self, xml, http_client): + self.http_client = http_client + self.root = ElementTree.fromstring(xml) + self._certificates_xml = None + + def _text_from_xpath(self, xpath): + element = self.root.find(xpath) + if element is not None: + return element.text + return None + + @property + def container_id(self): + return self._text_from_xpath('./Container/ContainerId') + + @property + def incarnation(self): + return self._text_from_xpath('./Incarnation') + + @property + def instance_id(self): + return self._text_from_xpath( + './Container/RoleInstanceList/RoleInstance/InstanceId') + + @property + def shared_config_xml(self): + url = self._text_from_xpath('./Container/RoleInstanceList/RoleInstance' + '/Configuration/SharedConfig') + return self.http_client.get(url).contents + + @property + def certificates_xml(self): + if self._certificates_xml is None: + url = self._text_from_xpath( + './Container/RoleInstanceList/RoleInstance' + '/Configuration/Certificates') + if url is not None: + self._certificates_xml = self.http_client.get( + url, secure=True).contents + return self._certificates_xml + + +class OpenSSLManager(object): + + certificate_names = { + 'private_key': 'TransportPrivate.pem', + 'certificate': 'TransportCert.pem', + } + + def __init__(self): + self.tmpdir = tempfile.TemporaryDirectory() + self.certificate = None + self.generate_certificate() + + def generate_certificate(self): + LOG.debug('Generating certificate for communication with fabric...') + if self.certificate is not None: + LOG.debug('Certificate already generated.') + return + with cd(self.tmpdir.name): + util.subp([ + 'openssl', 'req', '-x509', '-nodes', '-subj', + '/CN=LinuxTransport', '-days', '32768', '-newkey', 'rsa:2048', + '-keyout', self.certificate_names['private_key'], + '-out', self.certificate_names['certificate'], + ]) + certificate = '' + for line in open(self.certificate_names['certificate']): + if "CERTIFICATE" not in line: + certificate += line.rstrip() + self.certificate = certificate + LOG.debug('New certificate generated.') + + def parse_certificates(self, certificates_xml): + tag = ElementTree.fromstring(certificates_xml).find( + './/Data') + certificates_content = tag.text + lines = [ + b'MIME-Version: 1.0', + b'Content-Disposition: attachment; filename="Certificates.p7m"', + b'Content-Type: application/x-pkcs7-mime; name="Certificates.p7m"', + b'Content-Transfer-Encoding: base64', + b'', + certificates_content.encode('utf-8'), + ] + with cd(self.tmpdir.name): + with open('Certificates.p7m', 'wb') as f: + f.write(b'\n'.join(lines)) + out, _ = util.subp( + 'openssl cms -decrypt -in Certificates.p7m -inkey' + ' {private_key} -recip {certificate} | openssl pkcs12 -nodes' + ' -password pass:'.format(**self.certificate_names), + shell=True) + private_keys, certificates = [], [] + current = [] + for line in out.splitlines(): + current.append(line) + if re.match(r'[-]+END .*?KEY[-]+$', line): + private_keys.append('\n'.join(current)) + current = [] + elif re.match(r'[-]+END .*?CERTIFICATE[-]+$', line): + certificates.append('\n'.join(current)) + current = [] + keys = [] + for certificate in certificates: + with cd(self.tmpdir.name): + public_key, _ = util.subp( + 'openssl x509 -noout -pubkey |' + 'ssh-keygen -i -m PKCS8 -f /dev/stdin', + data=certificate, + shell=True) + keys.append(public_key) + return keys + + +def iid_from_shared_config_content(content): + """ + find INSTANCE_ID in: + + + + + """ + root = ElementTree.fromstring(content) + depnode = root.find('Deployment') + return depnode.get('name') + + +class WALinuxAgentShim(object): + + REPORT_READY_XML_TEMPLATE = '\n'.join([ + '', + '', + ' {incarnation}', + ' ', + ' {container_id}', + ' ', + ' ', + ' {instance_id}', + ' ', + ' Ready', + ' ', + ' ', + ' ', + ' ', + '']) + + def __init__(self): + LOG.debug('WALinuxAgentShim instantiated...') + self.endpoint = self.find_endpoint() + self.openssl_manager = OpenSSLManager() + self.http_client = AzureEndpointHttpClient( + self.openssl_manager.certificate) + self.values = {} + + @staticmethod + def find_endpoint(): + LOG.debug('Finding Azure endpoint...') + content = util.load_file('/var/lib/dhcp/dhclient.eth0.leases') + value = None + for line in content.splitlines(): + if 'unknown-245' in line: + value = line.strip(' ').split(' ', 2)[-1].strip(';\n"') + if value is None: + raise Exception('No endpoint found in DHCP config.') + if ':' in value: + hex_string = '' + for hex_pair in value.split(':'): + if len(hex_pair) == 1: + hex_pair = '0' + hex_pair + hex_string += hex_pair + value = struct.pack('>L', int(hex_string.replace(':', ''), 16)) + else: + value = value.encode('utf-8') + endpoint_ip_address = socket.inet_ntoa(value) + LOG.debug('Azure endpoint found at %s', endpoint_ip_address) + return endpoint_ip_address + + def register_with_azure_and_fetch_data(self): + LOG.info('Registering with Azure...') + for i in range(10): + try: + response = self.http_client.get( + 'http://{}/machine/?comp=goalstate'.format(self.endpoint)) + except Exception: + time.sleep(i + 1) + else: + break + LOG.debug('Successfully fetched GoalState XML.') + goal_state = GoalState(response.contents, self.http_client) + public_keys = [] + if goal_state.certificates_xml is not None: + LOG.debug('Certificate XML found; parsing out public keys.') + public_keys = self.openssl_manager.parse_certificates( + goal_state.certificates_xml) + data = { + 'instance-id': iid_from_shared_config_content( + goal_state.shared_config_xml), + 'public-keys': public_keys, + } + self._report_ready(goal_state) + return data + + def _report_ready(self, goal_state): + LOG.debug('Reporting ready to Azure fabric.') + document = self.REPORT_READY_XML_TEMPLATE.format( + incarnation=goal_state.incarnation, + container_id=goal_state.container_id, + instance_id=goal_state.instance_id, + ) + self.http_client.post( + "http://{}/machine?comp=health".format(self.endpoint), + data=document, + extra_headers={'Content-Type': 'text/xml; charset=utf-8'}, + ) + LOG.info('Reported ready to Azure fabric.') diff --git a/tests/unittests/test_datasource/test_azure.py b/tests/unittests/test_datasource/test_azure.py index 28703029..ee7109e1 100644 --- a/tests/unittests/test_datasource/test_azure.py +++ b/tests/unittests/test_datasource/test_azure.py @@ -15,47 +15,9 @@ except ImportError: import crypt import os import stat -import struct import yaml import shutil import tempfile -import unittest - -from cloudinit import url_helper - - -GOAL_STATE_TEMPLATE = """\ - - - 2012-11-30 - {incarnation} - - Started - 300000 - - 16001 - - FALSE - - - {container_id} - - - {instance_id} - Started - - http://100.86.192.70:80/machine/46504ebc-f968-4f23-b9aa-cd2b3e4d470c/68ce47b32ea94952be7b20951c383628.utl%2Dtrusty%2D%2D292258?comp=config&type=hostingEnvironmentConfig&incarnation=1 - {shared_config_url} - http://100.86.192.70:80/machine/46504ebc-f968-4f23-b9aa-cd2b3e4d470c/68ce47b32ea94952be7b20951c383628.utl%2Dtrusty%2D%2D292258?comp=config&type=extensionsConfig&incarnation=1 - http://100.86.192.70:80/machine/46504ebc-f968-4f23-b9aa-cd2b3e4d470c/68ce47b32ea94952be7b20951c383628.utl%2Dtrusty%2D%2D292258?comp=config&type=fullConfig&incarnation=1 - {certificates_url} - 68ce47b32ea94952be7b20951c383628.0.68ce47b32ea94952be7b20951c383628.0.utl-trusty--292258.1.xml - - - - - -""" def construct_valid_ovf_env(data=None, pubkeys=None, userdata=None): @@ -610,329 +572,3 @@ class TestReadAzureOvf(TestCase): for mypk in mypklist: self.assertIn(mypk, cfg['_pubkeys']) - -class TestReadAzureSharedConfig(unittest.TestCase): - def test_valid_content(self): - xml = """ - - - - - - - """ - ret = DataSourceAzure.iid_from_shared_config_content(xml) - self.assertEqual("MY_INSTANCE_ID", ret) - - -class TestFindEndpoint(TestCase): - - def setUp(self): - super(TestFindEndpoint, self).setUp() - patches = ExitStack() - self.addCleanup(patches.close) - - self.load_file = patches.enter_context( - mock.patch.object(DataSourceAzure.util, 'load_file')) - - def test_missing_file(self): - self.load_file.side_effect = IOError - self.assertRaises(IOError, - DataSourceAzure.WALinuxAgentShim.find_endpoint) - - def test_missing_special_azure_line(self): - self.load_file.return_value = '' - self.assertRaises(Exception, - DataSourceAzure.WALinuxAgentShim.find_endpoint) - - def _build_lease_content(self, ip_address, use_hex=True): - ip_address_repr = ':'.join( - [hex(int(part)).replace('0x', '') - for part in ip_address.split('.')]) - if not use_hex: - ip_address_repr = struct.pack( - '>L', int(ip_address_repr.replace(':', ''), 16)) - ip_address_repr = '"{0}"'.format(ip_address_repr.decode('utf-8')) - return '\n'.join([ - 'lease {', - ' interface "eth0";', - ' option unknown-245 {0};'.format(ip_address_repr), - '}']) - - def test_hex_string(self): - ip_address = '98.76.54.32' - file_content = self._build_lease_content(ip_address) - self.load_file.return_value = file_content - self.assertEqual(ip_address, - DataSourceAzure.WALinuxAgentShim.find_endpoint()) - - def test_hex_string_with_single_character_part(self): - ip_address = '4.3.2.1' - file_content = self._build_lease_content(ip_address) - self.load_file.return_value = file_content - self.assertEqual(ip_address, - DataSourceAzure.WALinuxAgentShim.find_endpoint()) - - def test_packed_string(self): - ip_address = '98.76.54.32' - file_content = self._build_lease_content(ip_address, use_hex=False) - self.load_file.return_value = file_content - self.assertEqual(ip_address, - DataSourceAzure.WALinuxAgentShim.find_endpoint()) - - def test_latest_lease_used(self): - ip_addresses = ['4.3.2.1', '98.76.54.32'] - file_content = '\n'.join([self._build_lease_content(ip_address) - for ip_address in ip_addresses]) - self.load_file.return_value = file_content - self.assertEqual(ip_addresses[-1], - DataSourceAzure.WALinuxAgentShim.find_endpoint()) - - -class TestGoalStateParsing(TestCase): - - default_parameters = { - 'incarnation': 1, - 'container_id': 'MyContainerId', - 'instance_id': 'MyInstanceId', - 'shared_config_url': 'MySharedConfigUrl', - 'certificates_url': 'MyCertificatesUrl', - } - - def _get_goal_state(self, http_client=None, **kwargs): - if http_client is None: - http_client = mock.MagicMock() - parameters = self.default_parameters.copy() - parameters.update(kwargs) - xml = GOAL_STATE_TEMPLATE.format(**parameters) - if parameters['certificates_url'] is None: - new_xml_lines = [] - for line in xml.splitlines(): - if 'Certificates' in line: - continue - new_xml_lines.append(line) - xml = '\n'.join(new_xml_lines) - return DataSourceAzure.GoalState(xml, http_client) - - def test_incarnation_parsed_correctly(self): - incarnation = '123' - goal_state = self._get_goal_state(incarnation=incarnation) - self.assertEqual(incarnation, goal_state.incarnation) - - def test_container_id_parsed_correctly(self): - container_id = 'TestContainerId' - goal_state = self._get_goal_state(container_id=container_id) - self.assertEqual(container_id, goal_state.container_id) - - def test_instance_id_parsed_correctly(self): - instance_id = 'TestInstanceId' - goal_state = self._get_goal_state(instance_id=instance_id) - self.assertEqual(instance_id, goal_state.instance_id) - - def test_shared_config_xml_parsed_and_fetched_correctly(self): - http_client = mock.MagicMock() - shared_config_url = 'TestSharedConfigUrl' - goal_state = self._get_goal_state( - http_client=http_client, shared_config_url=shared_config_url) - shared_config_xml = goal_state.shared_config_xml - self.assertEqual(1, http_client.get.call_count) - self.assertEqual(shared_config_url, http_client.get.call_args[0][0]) - self.assertEqual(http_client.get.return_value.contents, - shared_config_xml) - - def test_certificates_xml_parsed_and_fetched_correctly(self): - http_client = mock.MagicMock() - certificates_url = 'TestSharedConfigUrl' - goal_state = self._get_goal_state( - http_client=http_client, certificates_url=certificates_url) - certificates_xml = goal_state.certificates_xml - self.assertEqual(1, http_client.get.call_count) - self.assertEqual(certificates_url, http_client.get.call_args[0][0]) - self.assertTrue(http_client.get.call_args[1].get('secure', False)) - self.assertEqual(http_client.get.return_value.contents, - certificates_xml) - - def test_missing_certificates_skips_http_get(self): - http_client = mock.MagicMock() - goal_state = self._get_goal_state( - http_client=http_client, certificates_url=None) - certificates_xml = goal_state.certificates_xml - self.assertEqual(0, http_client.get.call_count) - self.assertIsNone(certificates_xml) - - -class TestAzureEndpointHttpClient(TestCase): - - regular_headers = { - 'x-ms-agent-name': 'WALinuxAgent', - 'x-ms-version': '2012-11-30', - } - - def setUp(self): - super(TestAzureEndpointHttpClient, self).setUp() - patches = ExitStack() - self.addCleanup(patches.close) - - self.read_file_or_url = patches.enter_context( - mock.patch.object(DataSourceAzure.util, 'read_file_or_url')) - - def test_non_secure_get(self): - client = DataSourceAzure.AzureEndpointHttpClient(mock.MagicMock()) - url = 'MyTestUrl' - response = client.get(url, secure=False) - self.assertEqual(1, self.read_file_or_url.call_count) - self.assertEqual(self.read_file_or_url.return_value, response) - self.assertEqual(mock.call(url, headers=self.regular_headers), - self.read_file_or_url.call_args) - - def test_secure_get(self): - url = 'MyTestUrl' - certificate = mock.MagicMock() - expected_headers = self.regular_headers.copy() - expected_headers.update({ - "x-ms-cipher-name": "DES_EDE3_CBC", - "x-ms-guest-agent-public-x509-cert": certificate, - }) - client = DataSourceAzure.AzureEndpointHttpClient(certificate) - response = client.get(url, secure=True) - self.assertEqual(1, self.read_file_or_url.call_count) - self.assertEqual(self.read_file_or_url.return_value, response) - self.assertEqual(mock.call(url, headers=expected_headers), - self.read_file_or_url.call_args) - - def test_post(self): - data = mock.MagicMock() - url = 'MyTestUrl' - client = DataSourceAzure.AzureEndpointHttpClient(mock.MagicMock()) - response = client.post(url, data=data) - self.assertEqual(1, self.read_file_or_url.call_count) - self.assertEqual(self.read_file_or_url.return_value, response) - self.assertEqual( - mock.call(url, data=data, headers=self.regular_headers), - self.read_file_or_url.call_args) - - def test_post_with_extra_headers(self): - url = 'MyTestUrl' - client = DataSourceAzure.AzureEndpointHttpClient(mock.MagicMock()) - extra_headers = {'test': 'header'} - client.post(url, extra_headers=extra_headers) - self.assertEqual(1, self.read_file_or_url.call_count) - expected_headers = self.regular_headers.copy() - expected_headers.update(extra_headers) - self.assertEqual( - mock.call(mock.ANY, data=mock.ANY, headers=expected_headers), - self.read_file_or_url.call_args) - - -class TestOpenSSLManager(TestCase): - - def setUp(self): - super(TestOpenSSLManager, self).setUp() - patches = ExitStack() - self.addCleanup(patches.close) - - self.subp = patches.enter_context( - mock.patch.object(DataSourceAzure.util, 'subp')) - - @mock.patch.object(DataSourceAzure, 'cd', mock.MagicMock()) - @mock.patch.object(DataSourceAzure.tempfile, 'TemporaryDirectory') - def test_openssl_manager_creates_a_tmpdir(self, TemporaryDirectory): - manager = DataSourceAzure.OpenSSLManager() - self.assertEqual(TemporaryDirectory.return_value, manager.tmpdir) - - @mock.patch('builtins.open') - def test_generate_certificate_uses_tmpdir(self, open): - subp_directory = {} - - def capture_directory(*args, **kwargs): - subp_directory['path'] = os.getcwd() - - self.subp.side_effect = capture_directory - manager = DataSourceAzure.OpenSSLManager() - self.assertEqual(manager.tmpdir.name, subp_directory['path']) - - -class TestWALinuxAgentShim(TestCase): - - def setUp(self): - super(TestWALinuxAgentShim, self).setUp() - patches = ExitStack() - self.addCleanup(patches.close) - - self.AzureEndpointHttpClient = patches.enter_context( - mock.patch.object(DataSourceAzure, 'AzureEndpointHttpClient')) - self.find_endpoint = patches.enter_context( - mock.patch.object( - DataSourceAzure.WALinuxAgentShim, 'find_endpoint')) - self.GoalState = patches.enter_context( - mock.patch.object(DataSourceAzure, 'GoalState')) - self.iid_from_shared_config_content = patches.enter_context( - mock.patch.object(DataSourceAzure, - 'iid_from_shared_config_content')) - self.OpenSSLManager = patches.enter_context( - mock.patch.object(DataSourceAzure, 'OpenSSLManager')) - - def test_http_client_uses_certificate(self): - shim = DataSourceAzure.WALinuxAgentShim() - self.assertEqual( - [mock.call(self.OpenSSLManager.return_value.certificate)], - self.AzureEndpointHttpClient.call_args_list) - self.assertEqual(self.AzureEndpointHttpClient.return_value, - shim.http_client) - - def test_correct_url_used_for_goalstate(self): - self.find_endpoint.return_value = 'test_endpoint' - shim = DataSourceAzure.WALinuxAgentShim() - shim.register_with_azure_and_fetch_data() - get = self.AzureEndpointHttpClient.return_value.get - self.assertEqual( - [mock.call('http://test_endpoint/machine/?comp=goalstate')], - get.call_args_list) - self.assertEqual( - [mock.call(get.return_value.contents, shim.http_client)], - self.GoalState.call_args_list) - - def test_certificates_used_to_determine_public_keys(self): - shim = DataSourceAzure.WALinuxAgentShim() - data = shim.register_with_azure_and_fetch_data() - self.assertEqual( - [mock.call(self.GoalState.return_value.certificates_xml)], - self.OpenSSLManager.return_value.parse_certificates.call_args_list) - self.assertEqual( - self.OpenSSLManager.return_value.parse_certificates.return_value, - data['public-keys']) - - def test_absent_certificates_produces_empty_public_keys(self): - self.GoalState.return_value.certificates_xml = None - shim = DataSourceAzure.WALinuxAgentShim() - data = shim.register_with_azure_and_fetch_data() - self.assertEqual([], data['public-keys']) - - def test_instance_id_returned_in_data(self): - shim = DataSourceAzure.WALinuxAgentShim() - data = shim.register_with_azure_and_fetch_data() - self.assertEqual( - [mock.call(self.GoalState.return_value.shared_config_xml)], - self.iid_from_shared_config_content.call_args_list) - self.assertEqual(self.iid_from_shared_config_content.return_value, - data['instance-id']) - - def test_correct_url_used_for_report_ready(self): - self.find_endpoint.return_value = 'test_endpoint' - shim = DataSourceAzure.WALinuxAgentShim() - shim.register_with_azure_and_fetch_data() - expected_url = 'http://test_endpoint/machine?comp=health' - self.assertEqual( - [mock.call(expected_url, data=mock.ANY, extra_headers=mock.ANY)], - shim.http_client.post.call_args_list) - - def test_goal_state_values_used_for_report_ready(self): - self.GoalState.return_value.incarnation = 'TestIncarnation' - self.GoalState.return_value.container_id = 'TestContainerId' - self.GoalState.return_value.instance_id = 'TestInstanceId' - shim = DataSourceAzure.WALinuxAgentShim() - shim.register_with_azure_and_fetch_data() - posted_document = shim.http_client.post.call_args[1]['data'] - self.assertIn('TestIncarnation', posted_document) - self.assertIn('TestContainerId', posted_document) - self.assertIn('TestInstanceId', posted_document) diff --git a/tests/unittests/test_datasource/test_azure_helper.py b/tests/unittests/test_datasource/test_azure_helper.py new file mode 100644 index 00000000..47b77840 --- /dev/null +++ b/tests/unittests/test_datasource/test_azure_helper.py @@ -0,0 +1,377 @@ +import os +import struct +import unittest + +from cloudinit.sources.helpers import azure as azure_helper +from ..helpers import TestCase + +try: + from unittest import mock +except ImportError: + import mock + +try: + from contextlib import ExitStack +except ImportError: + from contextlib2 import ExitStack + + +GOAL_STATE_TEMPLATE = """\ + + + 2012-11-30 + {incarnation} + + Started + 300000 + + 16001 + + FALSE + + + {container_id} + + + {instance_id} + Started + + http://100.86.192.70:80/machine/46504ebc-f968-4f23-b9aa-cd2b3e4d470c/68ce47b32ea94952be7b20951c383628.utl%2Dtrusty%2D%2D292258?comp=config&type=hostingEnvironmentConfig&incarnation=1 + {shared_config_url} + http://100.86.192.70:80/machine/46504ebc-f968-4f23-b9aa-cd2b3e4d470c/68ce47b32ea94952be7b20951c383628.utl%2Dtrusty%2D%2D292258?comp=config&type=extensionsConfig&incarnation=1 + http://100.86.192.70:80/machine/46504ebc-f968-4f23-b9aa-cd2b3e4d470c/68ce47b32ea94952be7b20951c383628.utl%2Dtrusty%2D%2D292258?comp=config&type=fullConfig&incarnation=1 + {certificates_url} + 68ce47b32ea94952be7b20951c383628.0.68ce47b32ea94952be7b20951c383628.0.utl-trusty--292258.1.xml + + + + + +""" + + +class TestReadAzureSharedConfig(unittest.TestCase): + + def test_valid_content(self): + xml = """ + + + + + + + """ + ret = azure_helper.iid_from_shared_config_content(xml) + self.assertEqual("MY_INSTANCE_ID", ret) + + +class TestFindEndpoint(TestCase): + + def setUp(self): + super(TestFindEndpoint, self).setUp() + patches = ExitStack() + self.addCleanup(patches.close) + + self.load_file = patches.enter_context( + mock.patch.object(azure_helper.util, 'load_file')) + + def test_missing_file(self): + self.load_file.side_effect = IOError + self.assertRaises(IOError, + azure_helper.WALinuxAgentShim.find_endpoint) + + def test_missing_special_azure_line(self): + self.load_file.return_value = '' + self.assertRaises(Exception, + azure_helper.WALinuxAgentShim.find_endpoint) + + def _build_lease_content(self, ip_address, use_hex=True): + ip_address_repr = ':'.join( + [hex(int(part)).replace('0x', '') + for part in ip_address.split('.')]) + if not use_hex: + ip_address_repr = struct.pack( + '>L', int(ip_address_repr.replace(':', ''), 16)) + ip_address_repr = '"{0}"'.format(ip_address_repr.decode('utf-8')) + return '\n'.join([ + 'lease {', + ' interface "eth0";', + ' option unknown-245 {0};'.format(ip_address_repr), + '}']) + + def test_hex_string(self): + ip_address = '98.76.54.32' + file_content = self._build_lease_content(ip_address) + self.load_file.return_value = file_content + self.assertEqual(ip_address, + azure_helper.WALinuxAgentShim.find_endpoint()) + + def test_hex_string_with_single_character_part(self): + ip_address = '4.3.2.1' + file_content = self._build_lease_content(ip_address) + self.load_file.return_value = file_content + self.assertEqual(ip_address, + azure_helper.WALinuxAgentShim.find_endpoint()) + + def test_packed_string(self): + ip_address = '98.76.54.32' + file_content = self._build_lease_content(ip_address, use_hex=False) + self.load_file.return_value = file_content + self.assertEqual(ip_address, + azure_helper.WALinuxAgentShim.find_endpoint()) + + def test_latest_lease_used(self): + ip_addresses = ['4.3.2.1', '98.76.54.32'] + file_content = '\n'.join([self._build_lease_content(ip_address) + for ip_address in ip_addresses]) + self.load_file.return_value = file_content + self.assertEqual(ip_addresses[-1], + azure_helper.WALinuxAgentShim.find_endpoint()) + + +class TestGoalStateParsing(TestCase): + + default_parameters = { + 'incarnation': 1, + 'container_id': 'MyContainerId', + 'instance_id': 'MyInstanceId', + 'shared_config_url': 'MySharedConfigUrl', + 'certificates_url': 'MyCertificatesUrl', + } + + def _get_goal_state(self, http_client=None, **kwargs): + if http_client is None: + http_client = mock.MagicMock() + parameters = self.default_parameters.copy() + parameters.update(kwargs) + xml = GOAL_STATE_TEMPLATE.format(**parameters) + if parameters['certificates_url'] is None: + new_xml_lines = [] + for line in xml.splitlines(): + if 'Certificates' in line: + continue + new_xml_lines.append(line) + xml = '\n'.join(new_xml_lines) + return azure_helper.GoalState(xml, http_client) + + def test_incarnation_parsed_correctly(self): + incarnation = '123' + goal_state = self._get_goal_state(incarnation=incarnation) + self.assertEqual(incarnation, goal_state.incarnation) + + def test_container_id_parsed_correctly(self): + container_id = 'TestContainerId' + goal_state = self._get_goal_state(container_id=container_id) + self.assertEqual(container_id, goal_state.container_id) + + def test_instance_id_parsed_correctly(self): + instance_id = 'TestInstanceId' + goal_state = self._get_goal_state(instance_id=instance_id) + self.assertEqual(instance_id, goal_state.instance_id) + + def test_shared_config_xml_parsed_and_fetched_correctly(self): + http_client = mock.MagicMock() + shared_config_url = 'TestSharedConfigUrl' + goal_state = self._get_goal_state( + http_client=http_client, shared_config_url=shared_config_url) + shared_config_xml = goal_state.shared_config_xml + self.assertEqual(1, http_client.get.call_count) + self.assertEqual(shared_config_url, http_client.get.call_args[0][0]) + self.assertEqual(http_client.get.return_value.contents, + shared_config_xml) + + def test_certificates_xml_parsed_and_fetched_correctly(self): + http_client = mock.MagicMock() + certificates_url = 'TestSharedConfigUrl' + goal_state = self._get_goal_state( + http_client=http_client, certificates_url=certificates_url) + certificates_xml = goal_state.certificates_xml + self.assertEqual(1, http_client.get.call_count) + self.assertEqual(certificates_url, http_client.get.call_args[0][0]) + self.assertTrue(http_client.get.call_args[1].get('secure', False)) + self.assertEqual(http_client.get.return_value.contents, + certificates_xml) + + def test_missing_certificates_skips_http_get(self): + http_client = mock.MagicMock() + goal_state = self._get_goal_state( + http_client=http_client, certificates_url=None) + certificates_xml = goal_state.certificates_xml + self.assertEqual(0, http_client.get.call_count) + self.assertIsNone(certificates_xml) + + +class TestAzureEndpointHttpClient(TestCase): + + regular_headers = { + 'x-ms-agent-name': 'WALinuxAgent', + 'x-ms-version': '2012-11-30', + } + + def setUp(self): + super(TestAzureEndpointHttpClient, self).setUp() + patches = ExitStack() + self.addCleanup(patches.close) + + self.read_file_or_url = patches.enter_context( + mock.patch.object(azure_helper.util, 'read_file_or_url')) + + def test_non_secure_get(self): + client = azure_helper.AzureEndpointHttpClient(mock.MagicMock()) + url = 'MyTestUrl' + response = client.get(url, secure=False) + self.assertEqual(1, self.read_file_or_url.call_count) + self.assertEqual(self.read_file_or_url.return_value, response) + self.assertEqual(mock.call(url, headers=self.regular_headers), + self.read_file_or_url.call_args) + + def test_secure_get(self): + url = 'MyTestUrl' + certificate = mock.MagicMock() + expected_headers = self.regular_headers.copy() + expected_headers.update({ + "x-ms-cipher-name": "DES_EDE3_CBC", + "x-ms-guest-agent-public-x509-cert": certificate, + }) + client = azure_helper.AzureEndpointHttpClient(certificate) + response = client.get(url, secure=True) + self.assertEqual(1, self.read_file_or_url.call_count) + self.assertEqual(self.read_file_or_url.return_value, response) + self.assertEqual(mock.call(url, headers=expected_headers), + self.read_file_or_url.call_args) + + def test_post(self): + data = mock.MagicMock() + url = 'MyTestUrl' + client = azure_helper.AzureEndpointHttpClient(mock.MagicMock()) + response = client.post(url, data=data) + self.assertEqual(1, self.read_file_or_url.call_count) + self.assertEqual(self.read_file_or_url.return_value, response) + self.assertEqual( + mock.call(url, data=data, headers=self.regular_headers), + self.read_file_or_url.call_args) + + def test_post_with_extra_headers(self): + url = 'MyTestUrl' + client = azure_helper.AzureEndpointHttpClient(mock.MagicMock()) + extra_headers = {'test': 'header'} + client.post(url, extra_headers=extra_headers) + self.assertEqual(1, self.read_file_or_url.call_count) + expected_headers = self.regular_headers.copy() + expected_headers.update(extra_headers) + self.assertEqual( + mock.call(mock.ANY, data=mock.ANY, headers=expected_headers), + self.read_file_or_url.call_args) + + +class TestOpenSSLManager(TestCase): + + def setUp(self): + super(TestOpenSSLManager, self).setUp() + patches = ExitStack() + self.addCleanup(patches.close) + + self.subp = patches.enter_context( + mock.patch.object(azure_helper.util, 'subp')) + + @mock.patch.object(azure_helper, 'cd', mock.MagicMock()) + @mock.patch.object(azure_helper.tempfile, 'TemporaryDirectory') + def test_openssl_manager_creates_a_tmpdir(self, TemporaryDirectory): + manager = azure_helper.OpenSSLManager() + self.assertEqual(TemporaryDirectory.return_value, manager.tmpdir) + + @mock.patch('builtins.open') + def test_generate_certificate_uses_tmpdir(self, open): + subp_directory = {} + + def capture_directory(*args, **kwargs): + subp_directory['path'] = os.getcwd() + + self.subp.side_effect = capture_directory + manager = azure_helper.OpenSSLManager() + self.assertEqual(manager.tmpdir.name, subp_directory['path']) + + +class TestWALinuxAgentShim(TestCase): + + def setUp(self): + super(TestWALinuxAgentShim, self).setUp() + patches = ExitStack() + self.addCleanup(patches.close) + + self.AzureEndpointHttpClient = patches.enter_context( + mock.patch.object(azure_helper, 'AzureEndpointHttpClient')) + self.find_endpoint = patches.enter_context( + mock.patch.object( + azure_helper.WALinuxAgentShim, 'find_endpoint')) + self.GoalState = patches.enter_context( + mock.patch.object(azure_helper, 'GoalState')) + self.iid_from_shared_config_content = patches.enter_context( + mock.patch.object(azure_helper, 'iid_from_shared_config_content')) + self.OpenSSLManager = patches.enter_context( + mock.patch.object(azure_helper, 'OpenSSLManager')) + + def test_http_client_uses_certificate(self): + shim = azure_helper.WALinuxAgentShim() + self.assertEqual( + [mock.call(self.OpenSSLManager.return_value.certificate)], + self.AzureEndpointHttpClient.call_args_list) + self.assertEqual(self.AzureEndpointHttpClient.return_value, + shim.http_client) + + def test_correct_url_used_for_goalstate(self): + self.find_endpoint.return_value = 'test_endpoint' + shim = azure_helper.WALinuxAgentShim() + shim.register_with_azure_and_fetch_data() + get = self.AzureEndpointHttpClient.return_value.get + self.assertEqual( + [mock.call('http://test_endpoint/machine/?comp=goalstate')], + get.call_args_list) + self.assertEqual( + [mock.call(get.return_value.contents, shim.http_client)], + self.GoalState.call_args_list) + + def test_certificates_used_to_determine_public_keys(self): + shim = azure_helper.WALinuxAgentShim() + data = shim.register_with_azure_and_fetch_data() + self.assertEqual( + [mock.call(self.GoalState.return_value.certificates_xml)], + self.OpenSSLManager.return_value.parse_certificates.call_args_list) + self.assertEqual( + self.OpenSSLManager.return_value.parse_certificates.return_value, + data['public-keys']) + + def test_absent_certificates_produces_empty_public_keys(self): + self.GoalState.return_value.certificates_xml = None + shim = azure_helper.WALinuxAgentShim() + data = shim.register_with_azure_and_fetch_data() + self.assertEqual([], data['public-keys']) + + def test_instance_id_returned_in_data(self): + shim = azure_helper.WALinuxAgentShim() + data = shim.register_with_azure_and_fetch_data() + self.assertEqual( + [mock.call(self.GoalState.return_value.shared_config_xml)], + self.iid_from_shared_config_content.call_args_list) + self.assertEqual(self.iid_from_shared_config_content.return_value, + data['instance-id']) + + def test_correct_url_used_for_report_ready(self): + self.find_endpoint.return_value = 'test_endpoint' + shim = azure_helper.WALinuxAgentShim() + shim.register_with_azure_and_fetch_data() + expected_url = 'http://test_endpoint/machine?comp=health' + self.assertEqual( + [mock.call(expected_url, data=mock.ANY, extra_headers=mock.ANY)], + shim.http_client.post.call_args_list) + + def test_goal_state_values_used_for_report_ready(self): + self.GoalState.return_value.incarnation = 'TestIncarnation' + self.GoalState.return_value.container_id = 'TestContainerId' + self.GoalState.return_value.instance_id = 'TestInstanceId' + shim = azure_helper.WALinuxAgentShim() + shim.register_with_azure_and_fetch_data() + posted_document = shim.http_client.post.call_args[1]['data'] + self.assertIn('TestIncarnation', posted_document) + self.assertIn('TestContainerId', posted_document) + self.assertIn('TestInstanceId', posted_document) -- cgit v1.2.3 From 9c7643c4a0dee7843963709c361b755baf843a4b Mon Sep 17 00:00:00 2001 From: Daniel Watkins Date: Fri, 8 May 2015 13:16:44 +0100 Subject: Stop using Python 3 only tempfile.TemporaryDirectory (but lose free cleanup). --- cloudinit/sources/helpers/azure.py | 8 ++++---- tests/unittests/test_datasource/test_azure_helper.py | 17 +++++++++++------ 2 files changed, 15 insertions(+), 10 deletions(-) (limited to 'cloudinit/sources/helpers') diff --git a/cloudinit/sources/helpers/azure.py b/cloudinit/sources/helpers/azure.py index 60f116e0..cb13187f 100644 --- a/cloudinit/sources/helpers/azure.py +++ b/cloudinit/sources/helpers/azure.py @@ -104,7 +104,7 @@ class OpenSSLManager(object): } def __init__(self): - self.tmpdir = tempfile.TemporaryDirectory() + self.tmpdir = tempfile.mkdtemp() self.certificate = None self.generate_certificate() @@ -113,7 +113,7 @@ class OpenSSLManager(object): if self.certificate is not None: LOG.debug('Certificate already generated.') return - with cd(self.tmpdir.name): + with cd(self.tmpdir): util.subp([ 'openssl', 'req', '-x509', '-nodes', '-subj', '/CN=LinuxTransport', '-days', '32768', '-newkey', 'rsa:2048', @@ -139,7 +139,7 @@ class OpenSSLManager(object): b'', certificates_content.encode('utf-8'), ] - with cd(self.tmpdir.name): + with cd(self.tmpdir): with open('Certificates.p7m', 'wb') as f: f.write(b'\n'.join(lines)) out, _ = util.subp( @@ -159,7 +159,7 @@ class OpenSSLManager(object): current = [] keys = [] for certificate in certificates: - with cd(self.tmpdir.name): + with cd(self.tmpdir): public_key, _ = util.subp( 'openssl x509 -noout -pubkey |' 'ssh-keygen -i -m PKCS8 -f /dev/stdin', diff --git a/tests/unittests/test_datasource/test_azure_helper.py b/tests/unittests/test_datasource/test_azure_helper.py index 47b77840..398a9007 100644 --- a/tests/unittests/test_datasource/test_azure_helper.py +++ b/tests/unittests/test_datasource/test_azure_helper.py @@ -273,15 +273,20 @@ class TestOpenSSLManager(TestCase): self.subp = patches.enter_context( mock.patch.object(azure_helper.util, 'subp')) + try: + self.open = patches.enter_context( + mock.patch('__builtin__.open')) + except ImportError: + self.open = patches.enter_context( + mock.patch('builtins.open')) @mock.patch.object(azure_helper, 'cd', mock.MagicMock()) - @mock.patch.object(azure_helper.tempfile, 'TemporaryDirectory') - def test_openssl_manager_creates_a_tmpdir(self, TemporaryDirectory): + @mock.patch.object(azure_helper.tempfile, 'mkdtemp') + def test_openssl_manager_creates_a_tmpdir(self, mkdtemp): manager = azure_helper.OpenSSLManager() - self.assertEqual(TemporaryDirectory.return_value, manager.tmpdir) + self.assertEqual(mkdtemp.return_value, manager.tmpdir) - @mock.patch('builtins.open') - def test_generate_certificate_uses_tmpdir(self, open): + def test_generate_certificate_uses_tmpdir(self): subp_directory = {} def capture_directory(*args, **kwargs): @@ -289,7 +294,7 @@ class TestOpenSSLManager(TestCase): self.subp.side_effect = capture_directory manager = azure_helper.OpenSSLManager() - self.assertEqual(manager.tmpdir.name, subp_directory['path']) + self.assertEqual(manager.tmpdir, subp_directory['path']) class TestWALinuxAgentShim(TestCase): -- cgit v1.2.3 From 84868622c404cda5efd2a753e2de30c1afca49a2 Mon Sep 17 00:00:00 2001 From: Daniel Watkins Date: Fri, 8 May 2015 13:18:02 +0100 Subject: Move our walinuxagent implementation to a single function call. --- cloudinit/sources/DataSourceAzure.py | 8 ++-- cloudinit/sources/helpers/azure.py | 31 ++++++++---- tests/unittests/test_datasource/test_azure.py | 19 ++++++-- .../unittests/test_datasource/test_azure_helper.py | 56 ++++++++++++++++++++-- 4 files changed, 92 insertions(+), 22 deletions(-) (limited to 'cloudinit/sources/helpers') diff --git a/cloudinit/sources/DataSourceAzure.py b/cloudinit/sources/DataSourceAzure.py index 5e147950..4053cfa6 100644 --- a/cloudinit/sources/DataSourceAzure.py +++ b/cloudinit/sources/DataSourceAzure.py @@ -29,7 +29,7 @@ from cloudinit.settings import PER_ALWAYS from cloudinit import sources from cloudinit import util from cloudinit.sources.helpers.azure import ( - iid_from_shared_config_content, WALinuxAgentShim) + get_metadata_from_fabric, iid_from_shared_config_content) LOG = logging.getLogger(__name__) @@ -185,15 +185,13 @@ class DataSourceAzureNet(sources.DataSource): write_files(ddir, files, dirmode=0o700) try: - shim = WALinuxAgentShim() - data = shim.register_with_azure_and_fetch_data() + fabric_data = get_metadata_from_fabric() except Exception as exc: LOG.info("Error communicating with Azure fabric; assume we aren't" " on Azure.", exc_info=True) return False - self.metadata['instance-id'] = data['instance-id'] - self.metadata['public-keys'] = data['public-keys'] + self.metadata.update(fabric_data) found_ephemeral = find_ephemeral_disk() if found_ephemeral: diff --git a/cloudinit/sources/helpers/azure.py b/cloudinit/sources/helpers/azure.py index cb13187f..dfdfa7c2 100644 --- a/cloudinit/sources/helpers/azure.py +++ b/cloudinit/sources/helpers/azure.py @@ -108,6 +108,9 @@ class OpenSSLManager(object): self.certificate = None self.generate_certificate() + def clean_up(self): + util.del_dir(self.tmpdir) + def generate_certificate(self): LOG.debug('Generating certificate for communication with fabric...') if self.certificate is not None: @@ -205,11 +208,13 @@ class WALinuxAgentShim(object): def __init__(self): LOG.debug('WALinuxAgentShim instantiated...') self.endpoint = self.find_endpoint() - self.openssl_manager = OpenSSLManager() - self.http_client = AzureEndpointHttpClient( - self.openssl_manager.certificate) + self.openssl_manager = None self.values = {} + def clean_up(self): + if self.openssl_manager is not None: + self.openssl_manager.clean_up() + @staticmethod def find_endpoint(): LOG.debug('Finding Azure endpoint...') @@ -234,17 +239,19 @@ class WALinuxAgentShim(object): return endpoint_ip_address def register_with_azure_and_fetch_data(self): + self.openssl_manager = OpenSSLManager() + http_client = AzureEndpointHttpClient(self.openssl_manager.certificate) LOG.info('Registering with Azure...') for i in range(10): try: - response = self.http_client.get( + response = http_client.get( 'http://{}/machine/?comp=goalstate'.format(self.endpoint)) except Exception: time.sleep(i + 1) else: break LOG.debug('Successfully fetched GoalState XML.') - goal_state = GoalState(response.contents, self.http_client) + goal_state = GoalState(response.contents, http_client) public_keys = [] if goal_state.certificates_xml is not None: LOG.debug('Certificate XML found; parsing out public keys.') @@ -255,19 +262,27 @@ class WALinuxAgentShim(object): goal_state.shared_config_xml), 'public-keys': public_keys, } - self._report_ready(goal_state) + self._report_ready(goal_state, http_client) return data - def _report_ready(self, goal_state): + def _report_ready(self, goal_state, http_client): LOG.debug('Reporting ready to Azure fabric.') document = self.REPORT_READY_XML_TEMPLATE.format( incarnation=goal_state.incarnation, container_id=goal_state.container_id, instance_id=goal_state.instance_id, ) - self.http_client.post( + http_client.post( "http://{}/machine?comp=health".format(self.endpoint), data=document, extra_headers={'Content-Type': 'text/xml; charset=utf-8'}, ) LOG.info('Reported ready to Azure fabric.') + + +def get_metadata_from_fabric(): + shim = WALinuxAgentShim() + try: + return shim.register_with_azure_and_fetch_data() + finally: + shim.clean_up() diff --git a/tests/unittests/test_datasource/test_azure.py b/tests/unittests/test_datasource/test_azure.py index ee7109e1..983be4cd 100644 --- a/tests/unittests/test_datasource/test_azure.py +++ b/tests/unittests/test_datasource/test_azure.py @@ -122,11 +122,10 @@ class TestAzureDataSource(TestCase): mod = DataSourceAzure mod.BUILTIN_DS_CONFIG['data_dir'] = self.waagent_d - fake_shim = mock.MagicMock() - fake_shim().register_with_azure_and_fetch_data.return_value = { + self.get_metadata_from_fabric = mock.MagicMock(return_value={ 'instance-id': 'i-my-azure-id', 'public-keys': [], - } + }) self.apply_patches([ (mod, 'list_possible_azure_ds_devs', dsdevs), @@ -137,7 +136,7 @@ class TestAzureDataSource(TestCase): (mod, 'perform_hostname_bounce', mock.MagicMock()), (mod, 'get_hostname', mock.MagicMock()), (mod, 'set_hostname', mock.MagicMock()), - (mod, 'WALinuxAgentShim', fake_shim), + (mod, 'get_metadata_from_fabric', self.get_metadata_from_fabric), ]) dsrc = mod.DataSourceAzureNet( @@ -388,6 +387,18 @@ class TestAzureDataSource(TestCase): self.assertEqual(new_ovfenv, load_file(os.path.join(self.waagent_d, 'ovf-env.xml'))) + def test_exception_fetching_fabric_data_doesnt_propagate(self): + ds = self._get_ds({'ovfcontent': construct_valid_ovf_env()}) + self.get_metadata_from_fabric.side_effect = Exception + self.assertFalse(ds.get_data()) + + def test_fabric_data_included_in_metadata(self): + ds = self._get_ds({'ovfcontent': construct_valid_ovf_env()}) + self.get_metadata_from_fabric.return_value = {'test': 'value'} + ret = ds.get_data() + self.assertTrue(ret) + self.assertEqual('value', ds.metadata['test']) + class TestAzureBounce(TestCase): diff --git a/tests/unittests/test_datasource/test_azure_helper.py b/tests/unittests/test_datasource/test_azure_helper.py index 398a9007..5fac2ade 100644 --- a/tests/unittests/test_datasource/test_azure_helper.py +++ b/tests/unittests/test_datasource/test_azure_helper.py @@ -296,6 +296,14 @@ class TestOpenSSLManager(TestCase): manager = azure_helper.OpenSSLManager() self.assertEqual(manager.tmpdir, subp_directory['path']) + @mock.patch.object(azure_helper, 'cd', mock.MagicMock()) + @mock.patch.object(azure_helper.tempfile, 'mkdtemp', mock.MagicMock()) + @mock.patch.object(azure_helper.util, 'del_dir') + def test_clean_up(self, del_dir): + manager = azure_helper.OpenSSLManager() + manager.clean_up() + self.assertEqual([mock.call(manager.tmpdir)], del_dir.call_args_list) + class TestWALinuxAgentShim(TestCase): @@ -318,11 +326,10 @@ class TestWALinuxAgentShim(TestCase): def test_http_client_uses_certificate(self): shim = azure_helper.WALinuxAgentShim() + shim.register_with_azure_and_fetch_data() self.assertEqual( [mock.call(self.OpenSSLManager.return_value.certificate)], self.AzureEndpointHttpClient.call_args_list) - self.assertEqual(self.AzureEndpointHttpClient.return_value, - shim.http_client) def test_correct_url_used_for_goalstate(self): self.find_endpoint.return_value = 'test_endpoint' @@ -333,7 +340,8 @@ class TestWALinuxAgentShim(TestCase): [mock.call('http://test_endpoint/machine/?comp=goalstate')], get.call_args_list) self.assertEqual( - [mock.call(get.return_value.contents, shim.http_client)], + [mock.call(get.return_value.contents, + self.AzureEndpointHttpClient.return_value)], self.GoalState.call_args_list) def test_certificates_used_to_determine_public_keys(self): @@ -368,7 +376,7 @@ class TestWALinuxAgentShim(TestCase): expected_url = 'http://test_endpoint/machine?comp=health' self.assertEqual( [mock.call(expected_url, data=mock.ANY, extra_headers=mock.ANY)], - shim.http_client.post.call_args_list) + self.AzureEndpointHttpClient.return_value.post.call_args_list) def test_goal_state_values_used_for_report_ready(self): self.GoalState.return_value.incarnation = 'TestIncarnation' @@ -376,7 +384,45 @@ class TestWALinuxAgentShim(TestCase): self.GoalState.return_value.instance_id = 'TestInstanceId' shim = azure_helper.WALinuxAgentShim() shim.register_with_azure_and_fetch_data() - posted_document = shim.http_client.post.call_args[1]['data'] + posted_document = ( + self.AzureEndpointHttpClient.return_value.post.call_args[1]['data'] + ) self.assertIn('TestIncarnation', posted_document) self.assertIn('TestContainerId', posted_document) self.assertIn('TestInstanceId', posted_document) + + def test_clean_up_can_be_called_at_any_time(self): + shim = azure_helper.WALinuxAgentShim() + shim.clean_up() + + def test_clean_up_will_clean_up_openssl_manager_if_instantiated(self): + shim = azure_helper.WALinuxAgentShim() + shim.register_with_azure_and_fetch_data() + shim.clean_up() + self.assertEqual( + 1, self.OpenSSLManager.return_value.clean_up.call_count) + + +class TestGetMetadataFromFabric(TestCase): + + @mock.patch.object(azure_helper, 'WALinuxAgentShim') + def test_data_from_shim_returned(self, shim): + ret = azure_helper.get_metadata_from_fabric() + self.assertEqual( + shim.return_value.register_with_azure_and_fetch_data.return_value, + ret) + + @mock.patch.object(azure_helper, 'WALinuxAgentShim') + def test_success_calls_clean_up(self, shim): + azure_helper.get_metadata_from_fabric() + self.assertEqual(1, shim.return_value.clean_up.call_count) + + @mock.patch.object(azure_helper, 'WALinuxAgentShim') + def test_failure_in_registration_calls_clean_up(self, shim): + class SentinelException(Exception): + pass + shim.return_value.register_with_azure_and_fetch_data.side_effect = ( + SentinelException) + self.assertRaises(SentinelException, + azure_helper.get_metadata_from_fabric) + self.assertEqual(1, shim.return_value.clean_up.call_count) -- cgit v1.2.3 From 512eb552e0ca740e1d285dc1b66a56579bcf68ec Mon Sep 17 00:00:00 2001 From: Daniel Watkins Date: Fri, 8 May 2015 16:52:49 +0100 Subject: Fix retrying. --- cloudinit/sources/helpers/azure.py | 9 +++++++-- tests/unittests/test_datasource/test_azure_helper.py | 11 +++++++++++ 2 files changed, 18 insertions(+), 2 deletions(-) (limited to 'cloudinit/sources/helpers') diff --git a/cloudinit/sources/helpers/azure.py b/cloudinit/sources/helpers/azure.py index dfdfa7c2..2ce728f5 100644 --- a/cloudinit/sources/helpers/azure.py +++ b/cloudinit/sources/helpers/azure.py @@ -242,14 +242,19 @@ class WALinuxAgentShim(object): self.openssl_manager = OpenSSLManager() http_client = AzureEndpointHttpClient(self.openssl_manager.certificate) LOG.info('Registering with Azure...') - for i in range(10): + attempts = 0 + while True: try: response = http_client.get( 'http://{}/machine/?comp=goalstate'.format(self.endpoint)) except Exception: - time.sleep(i + 1) + if attempts < 10: + time.sleep(attempts + 1) + else: + raise else: break + attempts += 1 LOG.debug('Successfully fetched GoalState XML.') goal_state = GoalState(response.contents, http_client) public_keys = [] diff --git a/tests/unittests/test_datasource/test_azure_helper.py b/tests/unittests/test_datasource/test_azure_helper.py index 5fac2ade..23bc997c 100644 --- a/tests/unittests/test_datasource/test_azure_helper.py +++ b/tests/unittests/test_datasource/test_azure_helper.py @@ -323,6 +323,8 @@ class TestWALinuxAgentShim(TestCase): mock.patch.object(azure_helper, 'iid_from_shared_config_content')) self.OpenSSLManager = patches.enter_context( mock.patch.object(azure_helper, 'OpenSSLManager')) + patches.enter_context( + mock.patch.object(azure_helper.time, 'sleep', mock.MagicMock())) def test_http_client_uses_certificate(self): shim = azure_helper.WALinuxAgentShim() @@ -402,6 +404,15 @@ class TestWALinuxAgentShim(TestCase): self.assertEqual( 1, self.OpenSSLManager.return_value.clean_up.call_count) + def test_failure_to_fetch_goalstate_bubbles_up(self): + class SentinelException(Exception): + pass + self.AzureEndpointHttpClient.return_value.get.side_effect = ( + SentinelException) + shim = azure_helper.WALinuxAgentShim() + self.assertRaises(SentinelException, + shim.register_with_azure_and_fetch_data) + class TestGetMetadataFromFabric(TestCase): -- cgit v1.2.3 From dad01d2cf14a7e0bdca455040fb5a173775cefdc Mon Sep 17 00:00:00 2001 From: Daniel Watkins Date: Fri, 8 May 2015 16:52:58 +0100 Subject: Python 2.6 fixes. --- cloudinit/sources/helpers/azure.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'cloudinit/sources/helpers') diff --git a/cloudinit/sources/helpers/azure.py b/cloudinit/sources/helpers/azure.py index 2ce728f5..281d733e 100644 --- a/cloudinit/sources/helpers/azure.py +++ b/cloudinit/sources/helpers/azure.py @@ -246,7 +246,7 @@ class WALinuxAgentShim(object): while True: try: response = http_client.get( - 'http://{}/machine/?comp=goalstate'.format(self.endpoint)) + 'http://{0}/machine/?comp=goalstate'.format(self.endpoint)) except Exception: if attempts < 10: time.sleep(attempts + 1) @@ -278,7 +278,7 @@ class WALinuxAgentShim(object): instance_id=goal_state.instance_id, ) http_client.post( - "http://{}/machine?comp=health".format(self.endpoint), + "http://{0}/machine?comp=health".format(self.endpoint), data=document, extra_headers={'Content-Type': 'text/xml; charset=utf-8'}, ) -- cgit v1.2.3 From 41900b72f31a1bd0eebe2f58a8598bfab25f0003 Mon Sep 17 00:00:00 2001 From: Daniel Watkins Date: Fri, 9 Oct 2015 14:01:11 +0100 Subject: Handle escaped quotes in WALinuxAgentShim.find_endpoint. This fixes bug 1488891. --- cloudinit/sources/helpers/azure.py | 2 +- tests/unittests/test_datasource/test_azure_helper.py | 10 +++++++++- 2 files changed, 10 insertions(+), 2 deletions(-) (limited to 'cloudinit/sources/helpers') diff --git a/cloudinit/sources/helpers/azure.py b/cloudinit/sources/helpers/azure.py index 281d733e..33003da0 100644 --- a/cloudinit/sources/helpers/azure.py +++ b/cloudinit/sources/helpers/azure.py @@ -233,7 +233,7 @@ class WALinuxAgentShim(object): hex_string += hex_pair value = struct.pack('>L', int(hex_string.replace(':', ''), 16)) else: - value = value.encode('utf-8') + value = value.replace('\\', '').encode('utf-8') endpoint_ip_address = socket.inet_ntoa(value) LOG.debug('Azure endpoint found at %s', endpoint_ip_address) return endpoint_ip_address diff --git a/tests/unittests/test_datasource/test_azure_helper.py b/tests/unittests/test_datasource/test_azure_helper.py index a5228870..68af31cd 100644 --- a/tests/unittests/test_datasource/test_azure_helper.py +++ b/tests/unittests/test_datasource/test_azure_helper.py @@ -97,7 +97,8 @@ class TestFindEndpoint(TestCase): if not use_hex: ip_address_repr = struct.pack( '>L', int(ip_address_repr.replace(':', ''), 16)) - ip_address_repr = '"{0}"'.format(ip_address_repr.decode('utf-8')) + ip_address_repr = '"{0}"'.format( + ip_address_repr.decode('utf-8').replace('"', '\\"')) return '\n'.join([ 'lease {', ' interface "eth0";', @@ -125,6 +126,13 @@ class TestFindEndpoint(TestCase): self.assertEqual(ip_address, azure_helper.WALinuxAgentShim.find_endpoint()) + def test_packed_string_with_escaped_quote(self): + ip_address = '100.72.34.108' + file_content = self._build_lease_content(ip_address, use_hex=False) + self.load_file.return_value = file_content + self.assertEqual(ip_address, + azure_helper.WALinuxAgentShim.find_endpoint()) + def test_latest_lease_used(self): ip_addresses = ['4.3.2.1', '98.76.54.32'] file_content = '\n'.join([self._build_lease_content(ip_address) -- cgit v1.2.3 From 20dc4190e27c7778cfa6c2943961f2ad27e14b48 Mon Sep 17 00:00:00 2001 From: Daniel Watkins Date: Fri, 9 Oct 2015 14:01:11 +0100 Subject: Handle colons in packed strings in WALinuxAgentShim.find_endpoint. This fixes bug 1488896. --- cloudinit/sources/helpers/azure.py | 12 +++++++----- tests/unittests/test_datasource/test_azure_helper.py | 7 +++++++ 2 files changed, 14 insertions(+), 5 deletions(-) (limited to 'cloudinit/sources/helpers') diff --git a/cloudinit/sources/helpers/azure.py b/cloudinit/sources/helpers/azure.py index 33003da0..21b4cd21 100644 --- a/cloudinit/sources/helpers/azure.py +++ b/cloudinit/sources/helpers/azure.py @@ -225,16 +225,18 @@ class WALinuxAgentShim(object): value = line.strip(' ').split(' ', 2)[-1].strip(';\n"') if value is None: raise Exception('No endpoint found in DHCP config.') - if ':' in value: + unescaped_value = value.replace('\\', '') + if len(unescaped_value) > 4: hex_string = '' - for hex_pair in value.split(':'): + for hex_pair in unescaped_value.split(':'): if len(hex_pair) == 1: hex_pair = '0' + hex_pair hex_string += hex_pair - value = struct.pack('>L', int(hex_string.replace(':', ''), 16)) + packed_bytes = struct.pack( + '>L', int(hex_string.replace(':', ''), 16)) else: - value = value.replace('\\', '').encode('utf-8') - endpoint_ip_address = socket.inet_ntoa(value) + packed_bytes = unescaped_value.encode('utf-8') + endpoint_ip_address = socket.inet_ntoa(packed_bytes) LOG.debug('Azure endpoint found at %s', endpoint_ip_address) return endpoint_ip_address diff --git a/tests/unittests/test_datasource/test_azure_helper.py b/tests/unittests/test_datasource/test_azure_helper.py index 68af31cd..5f906837 100644 --- a/tests/unittests/test_datasource/test_azure_helper.py +++ b/tests/unittests/test_datasource/test_azure_helper.py @@ -133,6 +133,13 @@ class TestFindEndpoint(TestCase): self.assertEqual(ip_address, azure_helper.WALinuxAgentShim.find_endpoint()) + def test_packed_string_containing_a_colon(self): + ip_address = '100.72.58.108' + file_content = self._build_lease_content(ip_address, use_hex=False) + self.load_file.return_value = file_content + self.assertEqual(ip_address, + azure_helper.WALinuxAgentShim.find_endpoint()) + def test_latest_lease_used(self): ip_addresses = ['4.3.2.1', '98.76.54.32'] file_content = '\n'.join([self._build_lease_content(ip_address) -- cgit v1.2.3 From d78ea2f8191847242b639f23fe085a5dd8b36014 Mon Sep 17 00:00:00 2001 From: Daniel Watkins Date: Fri, 9 Oct 2015 14:01:11 +0100 Subject: Refactor WALinuxAgentShim.find_endpoint to use a helper method for IP address unpacking. --- cloudinit/sources/helpers/azure.py | 26 +++++++++++++++----------- 1 file changed, 15 insertions(+), 11 deletions(-) (limited to 'cloudinit/sources/helpers') diff --git a/cloudinit/sources/helpers/azure.py b/cloudinit/sources/helpers/azure.py index 21b4cd21..fd08be16 100644 --- a/cloudinit/sources/helpers/azure.py +++ b/cloudinit/sources/helpers/azure.py @@ -216,16 +216,8 @@ class WALinuxAgentShim(object): self.openssl_manager.clean_up() @staticmethod - def find_endpoint(): - LOG.debug('Finding Azure endpoint...') - content = util.load_file('/var/lib/dhcp/dhclient.eth0.leases') - value = None - for line in content.splitlines(): - if 'unknown-245' in line: - value = line.strip(' ').split(' ', 2)[-1].strip(';\n"') - if value is None: - raise Exception('No endpoint found in DHCP config.') - unescaped_value = value.replace('\\', '') + def get_ip_from_lease_value(lease_value): + unescaped_value = lease_value.replace('\\', '') if len(unescaped_value) > 4: hex_string = '' for hex_pair in unescaped_value.split(':'): @@ -236,7 +228,19 @@ class WALinuxAgentShim(object): '>L', int(hex_string.replace(':', ''), 16)) else: packed_bytes = unescaped_value.encode('utf-8') - endpoint_ip_address = socket.inet_ntoa(packed_bytes) + return socket.inet_ntoa(packed_bytes) + + @staticmethod + def find_endpoint(): + LOG.debug('Finding Azure endpoint...') + content = util.load_file('/var/lib/dhcp/dhclient.eth0.leases') + value = None + for line in content.splitlines(): + if 'unknown-245' in line: + value = line.strip(' ').split(' ', 2)[-1].strip(';\n"') + if value is None: + raise Exception('No endpoint found in DHCP config.') + endpoint_ip_address = WALinuxAgentShim.get_ip_from_lease_value(value) LOG.debug('Azure endpoint found at %s', endpoint_ip_address) return endpoint_ip_address -- cgit v1.2.3 From 34b208a05361ae6ab4a51a6a999c9ac4ab77f06a Mon Sep 17 00:00:00 2001 From: Daniel Watkins Date: Fri, 30 Oct 2015 16:26:31 +0000 Subject: Use DMI data to find Azure instance IDs. This replaces the use of SharedConfig.xml in both the walinuxagent case, and the case where we communicate with the Azure fabric ourselves. --- cloudinit/sources/DataSourceAzure.py | 38 +---------- cloudinit/sources/helpers/azure.py | 21 ------ tests/unittests/test_datasource/test_azure.py | 77 +++++----------------- .../unittests/test_datasource/test_azure_helper.py | 42 +----------- 4 files changed, 23 insertions(+), 155 deletions(-) (limited to 'cloudinit/sources/helpers') diff --git a/cloudinit/sources/DataSourceAzure.py b/cloudinit/sources/DataSourceAzure.py index c6228e6c..bd80a8a6 100644 --- a/cloudinit/sources/DataSourceAzure.py +++ b/cloudinit/sources/DataSourceAzure.py @@ -31,8 +31,7 @@ from cloudinit import log as logging from cloudinit.settings import PER_ALWAYS from cloudinit import sources from cloudinit import util -from cloudinit.sources.helpers.azure import ( - get_metadata_from_fabric, iid_from_shared_config_content) +from cloudinit.sources.helpers.azure import get_metadata_from_fabric LOG = logging.getLogger(__name__) @@ -41,7 +40,6 @@ DEFAULT_METADATA = {"instance-id": "iid-AZURE-NODE"} AGENT_START = ['service', 'walinuxagent', 'start'] BOUNCE_COMMAND = ['sh', '-xc', "i=$interface; x=0; ifdown $i || x=$?; ifup $i || x=$?; exit $x"] -DATA_DIR_CLEAN_LIST = ['SharedConfig.xml'] BUILTIN_DS_CONFIG = { 'agent_command': AGENT_START, @@ -144,8 +142,6 @@ class DataSourceAzureNet(sources.DataSource): self.ds_cfg['agent_command']) ddir = self.ds_cfg['data_dir'] - shcfgxml = os.path.join(ddir, "SharedConfig.xml") - wait_for = [shcfgxml] fp_files = [] key_value = None @@ -160,19 +156,11 @@ class DataSourceAzureNet(sources.DataSource): missing = util.log_time(logfunc=LOG.debug, msg="waiting for files", func=wait_for_files, - args=(wait_for + fp_files,)) + args=(fp_files,)) if len(missing): LOG.warn("Did not find files, but going on: %s", missing) metadata = {} - if shcfgxml in missing: - LOG.warn("SharedConfig.xml missing, using static instance-id") - else: - try: - metadata['instance-id'] = iid_from_shared_config(shcfgxml) - except ValueError as e: - LOG.warn("failed to get instance id in %s: %s", shcfgxml, e) - metadata['public-keys'] = key_value or pubkeys_from_crt_files(fp_files) return metadata @@ -229,21 +217,6 @@ class DataSourceAzureNet(sources.DataSource): user_ds_cfg = util.get_cfg_by_path(self.cfg, DS_CFG_PATH, {}) self.ds_cfg = util.mergemanydict([user_ds_cfg, self.ds_cfg]) - if found != ddir: - cached_ovfenv = util.load_file( - os.path.join(ddir, 'ovf-env.xml'), quiet=True, decode=False) - if cached_ovfenv != files['ovf-env.xml']: - # source was not walinux-agent's datadir, so we have to clean - # up so 'wait_for_files' doesn't return early due to stale data - cleaned = [] - for f in [os.path.join(ddir, f) for f in DATA_DIR_CLEAN_LIST]: - if os.path.exists(f): - util.del_file(f) - cleaned.append(f) - if cleaned: - LOG.info("removed stale file(s) in '%s': %s", - ddir, str(cleaned)) - # walinux agent writes files world readable, but expects # the directory to be protected. write_files(ddir, files, dirmode=0o700) @@ -259,6 +232,7 @@ class DataSourceAzureNet(sources.DataSource): " on Azure.", exc_info=True) return False + self.metadata['instance-id'] = util.read_dmi_data('system-uuid') self.metadata.update(fabric_data) found_ephemeral = find_fabric_formatted_ephemeral_disk() @@ -649,12 +623,6 @@ def load_azure_ds_dir(source_dir): return (md, ud, cfg, {'ovf-env.xml': contents}) -def iid_from_shared_config(path): - with open(path, "rb") as fp: - content = fp.read() - return iid_from_shared_config_content(content) - - class BrokenAzureDataSource(Exception): pass diff --git a/cloudinit/sources/helpers/azure.py b/cloudinit/sources/helpers/azure.py index 281d733e..d90c22fd 100644 --- a/cloudinit/sources/helpers/azure.py +++ b/cloudinit/sources/helpers/azure.py @@ -78,12 +78,6 @@ class GoalState(object): return self._text_from_xpath( './Container/RoleInstanceList/RoleInstance/InstanceId') - @property - def shared_config_xml(self): - url = self._text_from_xpath('./Container/RoleInstanceList/RoleInstance' - '/Configuration/SharedConfig') - return self.http_client.get(url).contents - @property def certificates_xml(self): if self._certificates_xml is None: @@ -172,19 +166,6 @@ class OpenSSLManager(object): return keys -def iid_from_shared_config_content(content): - """ - find INSTANCE_ID in: - - - - - """ - root = ElementTree.fromstring(content) - depnode = root.find('Deployment') - return depnode.get('name') - - class WALinuxAgentShim(object): REPORT_READY_XML_TEMPLATE = '\n'.join([ @@ -263,8 +244,6 @@ class WALinuxAgentShim(object): public_keys = self.openssl_manager.parse_certificates( goal_state.certificates_xml) data = { - 'instance-id': iid_from_shared_config_content( - goal_state.shared_config_xml), 'public-keys': public_keys, } self._report_ready(goal_state, http_client) diff --git a/tests/unittests/test_datasource/test_azure.py b/tests/unittests/test_datasource/test_azure.py index ec0435f5..3933794f 100644 --- a/tests/unittests/test_datasource/test_azure.py +++ b/tests/unittests/test_datasource/test_azure.py @@ -115,10 +115,6 @@ class TestAzureDataSource(TestCase): data['pubkey_files'] = flist return ["pubkey_from: %s" % f for f in flist] - def _iid_from_shared_config(path): - data['iid_from_shared_cfg'] = path - return 'i-my-azure-id' - if data.get('ovfcontent') is not None: populate_dir(os.path.join(self.paths.seed_dir, "azure"), {'ovf-env.xml': data['ovfcontent']}) @@ -127,20 +123,22 @@ class TestAzureDataSource(TestCase): mod.BUILTIN_DS_CONFIG['data_dir'] = self.waagent_d self.get_metadata_from_fabric = mock.MagicMock(return_value={ - 'instance-id': 'i-my-azure-id', 'public-keys': [], }) + self.instance_id = 'test-instance-id' + self.apply_patches([ (mod, 'list_possible_azure_ds_devs', dsdevs), (mod, 'invoke_agent', _invoke_agent), (mod, 'wait_for_files', _wait_for_files), (mod, 'pubkeys_from_crt_files', _pubkeys_from_crt_files), - (mod, 'iid_from_shared_config', _iid_from_shared_config), (mod, 'perform_hostname_bounce', mock.MagicMock()), (mod, 'get_hostname', mock.MagicMock()), (mod, 'set_hostname', mock.MagicMock()), (mod, 'get_metadata_from_fabric', self.get_metadata_from_fabric), + (mod.util, 'read_dmi_data', mock.MagicMock( + return_value=self.instance_id)), ]) dsrc = mod.DataSourceAzureNet( @@ -193,7 +191,6 @@ class TestAzureDataSource(TestCase): self.assertEqual(dsrc.metadata['local-hostname'], odata['HostName']) self.assertTrue(os.path.isfile( os.path.join(self.waagent_d, 'ovf-env.xml'))) - self.assertEqual(dsrc.metadata['instance-id'], 'i-my-azure-id') def test_waagent_d_has_0700_perms(self): # we expect /var/lib/waagent to be created 0700 @@ -345,7 +342,6 @@ class TestAzureDataSource(TestCase): for mypk in mypklist: self.assertIn(mypk['value'], dsrc.metadata['public-keys']) - def test_default_ephemeral(self): # make sure the ephemeral device works odata = {} @@ -434,54 +430,6 @@ class TestAzureDataSource(TestCase): dsrc = self._get_ds({'ovfcontent': xml}) dsrc.get_data() - def test_existing_ovf_same(self): - # waagent/SharedConfig left alone if found ovf-env.xml same as cached - odata = {'UserData': b64e("SOMEUSERDATA")} - data = {'ovfcontent': construct_valid_ovf_env(data=odata)} - - populate_dir(self.waagent_d, - {'ovf-env.xml': data['ovfcontent'], - 'otherfile': 'otherfile-content', - 'SharedConfig.xml': 'mysharedconfig'}) - - dsrc = self._get_ds(data) - ret = dsrc.get_data() - self.assertTrue(ret) - self.assertTrue(os.path.exists( - os.path.join(self.waagent_d, 'ovf-env.xml'))) - self.assertTrue(os.path.exists( - os.path.join(self.waagent_d, 'otherfile'))) - self.assertTrue(os.path.exists( - os.path.join(self.waagent_d, 'SharedConfig.xml'))) - - def test_existing_ovf_diff(self): - # waagent/SharedConfig must be removed if ovfenv is found elsewhere - - # 'get_data' should remove SharedConfig.xml in /var/lib/waagent - # if ovf-env.xml differs. - cached_ovfenv = construct_valid_ovf_env( - {'userdata': b64e("FOO_USERDATA")}) - new_ovfenv = construct_valid_ovf_env( - {'userdata': b64e("NEW_USERDATA")}) - - populate_dir(self.waagent_d, - {'ovf-env.xml': cached_ovfenv, - 'SharedConfig.xml': "mysharedconfigxml", - 'otherfile': 'otherfilecontent'}) - - dsrc = self._get_ds({'ovfcontent': new_ovfenv}) - ret = dsrc.get_data() - self.assertTrue(ret) - self.assertEqual(dsrc.userdata_raw, b"NEW_USERDATA") - self.assertTrue(os.path.exists( - os.path.join(self.waagent_d, 'otherfile'))) - self.assertFalse(os.path.exists( - os.path.join(self.waagent_d, 'SharedConfig.xml'))) - self.assertTrue(os.path.exists( - os.path.join(self.waagent_d, 'ovf-env.xml'))) - new_xml = load_file(os.path.join(self.waagent_d, 'ovf-env.xml')) - self.xml_equals(new_ovfenv, new_xml) - def test_exception_fetching_fabric_data_doesnt_propagate(self): ds = self._get_ds({'ovfcontent': construct_valid_ovf_env()}) ds.ds_cfg['agent_command'] = '__builtin__' @@ -496,6 +444,17 @@ class TestAzureDataSource(TestCase): self.assertTrue(ret) self.assertEqual('value', ds.metadata['test']) + def test_instance_id_from_dmidecode_used(self): + ds = self._get_ds({'ovfcontent': construct_valid_ovf_env()}) + ds.get_data() + self.assertEqual(self.instance_id, ds.metadata['instance-id']) + + def test_instance_id_from_dmidecode_used_for_builtin(self): + ds = self._get_ds({'ovfcontent': construct_valid_ovf_env()}) + ds.ds_cfg['agent_command'] = '__builtin__' + ds.get_data() + self.assertEqual(self.instance_id, ds.metadata['instance-id']) + class TestAzureBounce(TestCase): @@ -504,9 +463,6 @@ class TestAzureBounce(TestCase): mock.patch.object(DataSourceAzure, 'invoke_agent')) self.patches.enter_context( mock.patch.object(DataSourceAzure, 'wait_for_files')) - self.patches.enter_context( - mock.patch.object(DataSourceAzure, 'iid_from_shared_config', - mock.MagicMock(return_value='i-my-azure-id'))) self.patches.enter_context( mock.patch.object(DataSourceAzure, 'list_possible_azure_ds_devs', mock.MagicMock(return_value=[]))) @@ -521,6 +477,9 @@ class TestAzureBounce(TestCase): self.patches.enter_context( mock.patch.object(DataSourceAzure, 'get_metadata_from_fabric', mock.MagicMock(return_value={}))) + self.patches.enter_context( + mock.patch.object(DataSourceAzure.util, 'read_dmi_data', + mock.MagicMock(return_value='test-instance-id'))) def setUp(self): super(TestAzureBounce, self).setUp() diff --git a/tests/unittests/test_datasource/test_azure_helper.py b/tests/unittests/test_datasource/test_azure_helper.py index a5228870..0638c974 100644 --- a/tests/unittests/test_datasource/test_azure_helper.py +++ b/tests/unittests/test_datasource/test_azure_helper.py @@ -40,7 +40,7 @@ GOAL_STATE_TEMPLATE = """\ http://100.86.192.70:80/...hostingEnvironmentConfig... - {shared_config_url} + http://100.86.192.70:80/..SharedConfig.. http://100.86.192.70:80/...extensionsConfig... @@ -55,21 +55,6 @@ GOAL_STATE_TEMPLATE = """\ """ -class TestReadAzureSharedConfig(unittest.TestCase): - - def test_valid_content(self): - xml = """ - - - - - - - """ - ret = azure_helper.iid_from_shared_config_content(xml) - self.assertEqual("MY_INSTANCE_ID", ret) - - class TestFindEndpoint(TestCase): def setUp(self): @@ -140,7 +125,6 @@ class TestGoalStateParsing(TestCase): 'incarnation': 1, 'container_id': 'MyContainerId', 'instance_id': 'MyInstanceId', - 'shared_config_url': 'MySharedConfigUrl', 'certificates_url': 'MyCertificatesUrl', } @@ -174,20 +158,9 @@ class TestGoalStateParsing(TestCase): goal_state = self._get_goal_state(instance_id=instance_id) self.assertEqual(instance_id, goal_state.instance_id) - def test_shared_config_xml_parsed_and_fetched_correctly(self): - http_client = mock.MagicMock() - shared_config_url = 'TestSharedConfigUrl' - goal_state = self._get_goal_state( - http_client=http_client, shared_config_url=shared_config_url) - shared_config_xml = goal_state.shared_config_xml - self.assertEqual(1, http_client.get.call_count) - self.assertEqual(shared_config_url, http_client.get.call_args[0][0]) - self.assertEqual(http_client.get.return_value.contents, - shared_config_xml) - def test_certificates_xml_parsed_and_fetched_correctly(self): http_client = mock.MagicMock() - certificates_url = 'TestSharedConfigUrl' + certificates_url = 'TestCertificatesUrl' goal_state = self._get_goal_state( http_client=http_client, certificates_url=certificates_url) certificates_xml = goal_state.certificates_xml @@ -324,8 +297,6 @@ class TestWALinuxAgentShim(TestCase): azure_helper.WALinuxAgentShim, 'find_endpoint')) self.GoalState = patches.enter_context( mock.patch.object(azure_helper, 'GoalState')) - self.iid_from_shared_config_content = patches.enter_context( - mock.patch.object(azure_helper, 'iid_from_shared_config_content')) self.OpenSSLManager = patches.enter_context( mock.patch.object(azure_helper, 'OpenSSLManager')) patches.enter_context( @@ -367,15 +338,6 @@ class TestWALinuxAgentShim(TestCase): data = shim.register_with_azure_and_fetch_data() self.assertEqual([], data['public-keys']) - def test_instance_id_returned_in_data(self): - shim = azure_helper.WALinuxAgentShim() - data = shim.register_with_azure_and_fetch_data() - self.assertEqual( - [mock.call(self.GoalState.return_value.shared_config_xml)], - self.iid_from_shared_config_content.call_args_list) - self.assertEqual(self.iid_from_shared_config_content.return_value, - data['instance-id']) - def test_correct_url_used_for_report_ready(self): self.find_endpoint.return_value = 'test_endpoint' shim = azure_helper.WALinuxAgentShim() -- cgit v1.2.3 From 8844ffb5988bcfbb8cfbe57d9139c3dcb8b429cc Mon Sep 17 00:00:00 2001 From: Sankar Tanguturi Date: Wed, 18 Nov 2015 16:03:15 -0800 Subject: Add Image Customization Parser for VMware vSphere Hypervisor Support. This is the first changeset submitted as a part of project to add cloud-init support for VMware vSphere Hypervisor. This changeset contains _only_ the changes for a simple python parser for a Image Customization Specification file pushed by VMware vSphere hypervisor into the guest VMs. In a later changeset, will be submitting another patch to actually detect the underlying VMware vSphere hypervisor and do the necessary customization. --- cloudinit/sources/helpers/vmware/__init__.py | 13 ++ cloudinit/sources/helpers/vmware/imc/__init__.py | 13 ++ cloudinit/sources/helpers/vmware/imc/boot_proto.py | 11 + cloudinit/sources/helpers/vmware/imc/config.py | 125 ++++++++++++ .../sources/helpers/vmware/imc/config_file.py | 221 +++++++++++++++++++++ .../sources/helpers/vmware/imc/config_namespace.py | 5 + .../sources/helpers/vmware/imc/config_source.py | 2 + cloudinit/sources/helpers/vmware/imc/ipv4_mode.py | 29 +++ cloudinit/sources/helpers/vmware/imc/nic.py | 107 ++++++++++ 9 files changed, 526 insertions(+) create mode 100644 cloudinit/sources/helpers/vmware/__init__.py create mode 100644 cloudinit/sources/helpers/vmware/imc/__init__.py create mode 100644 cloudinit/sources/helpers/vmware/imc/boot_proto.py create mode 100644 cloudinit/sources/helpers/vmware/imc/config.py create mode 100644 cloudinit/sources/helpers/vmware/imc/config_file.py create mode 100644 cloudinit/sources/helpers/vmware/imc/config_namespace.py create mode 100644 cloudinit/sources/helpers/vmware/imc/config_source.py create mode 100644 cloudinit/sources/helpers/vmware/imc/ipv4_mode.py create mode 100644 cloudinit/sources/helpers/vmware/imc/nic.py (limited to 'cloudinit/sources/helpers') diff --git a/cloudinit/sources/helpers/vmware/__init__.py b/cloudinit/sources/helpers/vmware/__init__.py new file mode 100644 index 00000000..386225d5 --- /dev/null +++ b/cloudinit/sources/helpers/vmware/__init__.py @@ -0,0 +1,13 @@ +# vi: ts=4 expandtab +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . diff --git a/cloudinit/sources/helpers/vmware/imc/__init__.py b/cloudinit/sources/helpers/vmware/imc/__init__.py new file mode 100644 index 00000000..386225d5 --- /dev/null +++ b/cloudinit/sources/helpers/vmware/imc/__init__.py @@ -0,0 +1,13 @@ +# vi: ts=4 expandtab +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . diff --git a/cloudinit/sources/helpers/vmware/imc/boot_proto.py b/cloudinit/sources/helpers/vmware/imc/boot_proto.py new file mode 100644 index 00000000..6c3b070a --- /dev/null +++ b/cloudinit/sources/helpers/vmware/imc/boot_proto.py @@ -0,0 +1,11 @@ +# from enum import Enum + +class BootProto: + DHCP = 'dhcp' + STATIC = 'static' + +# def __eq__(self, other): +# return self.name == other.name and self.value == other.value +# +# def __ne__(self, other): +# return not self.__eq__(other) diff --git a/cloudinit/sources/helpers/vmware/imc/config.py b/cloudinit/sources/helpers/vmware/imc/config.py new file mode 100644 index 00000000..ea0873fb --- /dev/null +++ b/cloudinit/sources/helpers/vmware/imc/config.py @@ -0,0 +1,125 @@ +from cloudinit.sources.helpers.vmware.imc.nic import Nic + + +class Config: + DNS = 'DNS|NAMESERVER|' + SUFFIX = 'DNS|SUFFIX|' + PASS = 'PASSWORD|-PASS' + TIMEZONE = 'DATETIME|TIMEZONE' + UTC = 'DATETIME|UTC' + HOSTNAME = 'NETWORK|HOSTNAME' + OMAINNAME = 'NETWORK|DOMAINNAME' + + def __init__(self, configFile): + self._configFile = configFile + + # Retrieves hostname. + # + # Args: + # None + # Results: + # string: hostname + # Throws: + # None + @property + def hostName(self): + return self._configFile.get(Config.HOSTNAME, None) + + # Retrieves domainName. + # + # Args: + # None + # Results: + # string: domainName + # Throws: + # None + @property + def domainName(self): + return self._configFile.get(Config.DOMAINNAME, None) + + # Retrieves timezone. + # + # Args: + # None + # Results: + # string: timezone + # Throws: + # None + @property + def timeZone(self): + return self._configFile.get(Config.TIMEZONE, None) + + # Retrieves whether to set time to UTC or Local. + # + # Args: + # None + # Results: + # boolean: True for yes/YES, True for no/NO, otherwise - None + # Throws: + # None + @property + def utc(self): + return self._configFile.get(Config.UTC, None) + + # Retrieves root password to be set. + # + # Args: + # None + # Results: + # string: base64-encoded root password or None + # Throws: + # None + @property + def adminPassword(self): + return self._configFile.get(Config.PASS, None) + + # Retrieves DNS Servers. + # + # Args: + # None + # Results: + # integer: count or 0 + # Throws: + # None + @property + def nameServers(self): + res = [] + for i in range(1, self._configFile.getCnt(Config.DNS) + 1): + key = Config.DNS + str(i) + res.append(self._configFile[key]) + + return res + + # Retrieves DNS Suffixes. + # + # Args: + # None + # Results: + # integer: count or 0 + # Throws: + # None + @property + def dnsSuffixes(self): + res = [] + for i in range(1, self._configFile.getCnt(Config.SUFFIX) + 1): + key = Config.SUFFIX + str(i) + res.append(self._configFile[key]) + + return res + + # Retrieves NICs. + # + # Args: + # None + # Results: + # integer: count + # Throws: + # None + @property + def nics(self): + res = [] + nics = self._configFile['NIC-CONFIG|NICS'] + for nic in nics.split(','): + res.append(Nic(nic, self._configFile)) + + return res diff --git a/cloudinit/sources/helpers/vmware/imc/config_file.py b/cloudinit/sources/helpers/vmware/imc/config_file.py new file mode 100644 index 00000000..3f9938da --- /dev/null +++ b/cloudinit/sources/helpers/vmware/imc/config_file.py @@ -0,0 +1,221 @@ +import logging +import re + +from cloudinit.sources.helpers.vmware.imc.config_source import ConfigSource + +logger = logging.getLogger(__name__) + + +class ConfigFile(ConfigSource): + def __init__(self): + self._configData = {} + + def __getitem__(self, key): + return self._configData[key] + + def get(self, key, default=None): + return self._configData.get(key, default) + + # Removes all the properties. + # + # Args: + # None + # Results: + # None + # Throws: + # None + def clear(self): + self._configData.clear() + + # Inserts k/v pair. + # + # Does not do any key/cross-key validation. + # + # Args: + # key: string: key + # val: string: value + # Results: + # None + # Throws: + # None + def _insertKey(self, key, val): + # cleaning up on all "input" path + + # remove end char \n (chomp) + key = key.strip() + val = val.strip() + + if key.startswith('-') or '|-' in key: + canLog = 0 + else: + canLog = 1 + + # "sensitive" settings shall not be logged + if canLog: + logger.debug("ADDED KEY-VAL :: '%s' = '%s'" % (key, val)) + else: + logger.debug("ADDED KEY-VAL :: '%s' = '*****************'" % key) + + self._configData[key] = val + + # Determines properties count. + # + # Args: + # None + # Results: + # integer: properties count + # Throws: + # None + def size(self): + return len(self._configData) + + # Parses properties from a .cfg file content. + # + # Any previously available properties will be removed. + # + # Sensitive data will not be logged in case key starts from '-'. + # + # Args: + # content: string: e.g. content of config/cust.cfg + # Results: + # None + # Throws: + # None + def loadConfigContent(self, content): + self.clear() + + # remove end char \n (chomp) + for line in content.split('\n'): + # TODO validate against allowed characters (not done in Perl) + + # spaces at the end are not allowed, things like passwords must be + # at least base64-encoded + line = line.strip() + + # "sensitive" settings shall not be logged + if line.startswith('-'): + canLog = 0 + else: + canLog = 1 + + if canLog: + logger.debug("Processing line: '%s'" % line) + else: + logger.debug("Processing line: '***********************'") + + if not line: + logger.debug("Empty line. Ignored.") + continue + + if line.startswith('#'): + logger.debug("Comment found. Line ignored.") + continue + + matchObj = re.match(r'\[(.+)\]', line) + if matchObj: + category = matchObj.group(1) + logger.debug("FOUND CATEGORY = '%s'" % category) + else: + # POSIX.2 regex doesn't support non-greedy like in (.+?)=(.*) + # key value pair (non-eager '=' for base64) + matchObj = re.match(r'([^=]+)=(.*)', line) + if matchObj: + # cleaning up on all "input" paths + key = category + "|" + matchObj.group(1).strip() + val = matchObj.group(2).strip() + + self._insertKey(key, val) + else: + # TODO document + raise Exception("Unrecognizable line: '%s'" % line) + + self.validate() + + # Parses properties from a .cfg file + # + # Any previously available properties will be removed. + # + # Sensitive data will not be logged in case key starts from '-'. + # + # Args: + # filename: string: full path to a .cfg file + # Results: + # None + # Throws: + # None + def loadConfigFile(self, filename): + logger.info("Opening file name %s." % filename) + # TODO what throws? + with open(filename, "r") as myfile: + self.loadConfigContent(myfile.read()) + + # Determines whether a property with a given key exists. + # + # Args: + # key: string: key + # Results: + # boolean: True if such property exists, otherwise - False. + # Throws: + # None + def hasKey(self, key): + return key in self._configData + + # Determines whether a value for a property must be kept. + # + # If the property is missing, it's treated as it should be not changed by + # the engine. + # + # Args: + # key: string: key + # Results: + # boolean: True if property must be kept, otherwise - False. + # Throws: + # None + def keepCurrentValue(self, key): + # helps to distinguish from "empty" value which is used to indicate + # "removal" + return not self.hasKey(key) + + # Determines whether a value for a property must be removed. + # + # If the property is empty, it's treated as it should be removed by the + # engine. + # + # Args: + # key: string: key + # Results: + # boolean: True if property must be removed, otherwise - False. + # Throws: + # None + def removeCurrentValue(self, key): + # helps to distinguish from "missing" value which is used to indicate + # "keeping unchanged" + if self.hasKey(key): + return not bool(self._configData[key]) + else: + return False + + # TODO + def getCnt(self, prefix): + res = 0 + for key in self._configData.keys(): + if key.startswith(prefix): + res += 1 + + return res + + # TODO + # TODO pass base64 + # Throws: + # Dies in case timezone is present but empty. + # Dies in case password is present but empty. + # Dies in case hostname is present but empty or greater than 63 chars. + # Dies in case UTC is present, but is not yes/YES or no/NO. + # Dies in case NICS is not present. + def validate(self): + # TODO must log all the errors + keyValidators = {'NIC1|IPv6GATEWAY|': None} + crossValidators = {} + + for key in self._configData.keys(): + pass diff --git a/cloudinit/sources/helpers/vmware/imc/config_namespace.py b/cloudinit/sources/helpers/vmware/imc/config_namespace.py new file mode 100644 index 00000000..7f76ac8b --- /dev/null +++ b/cloudinit/sources/helpers/vmware/imc/config_namespace.py @@ -0,0 +1,5 @@ +from cloudinit.sources.helpers.vmware.imc.config_source import ConfigSource + + +class ConfigNamespace(ConfigSource): + pass diff --git a/cloudinit/sources/helpers/vmware/imc/config_source.py b/cloudinit/sources/helpers/vmware/imc/config_source.py new file mode 100644 index 00000000..fad3a389 --- /dev/null +++ b/cloudinit/sources/helpers/vmware/imc/config_source.py @@ -0,0 +1,2 @@ +class ConfigSource: + pass diff --git a/cloudinit/sources/helpers/vmware/imc/ipv4_mode.py b/cloudinit/sources/helpers/vmware/imc/ipv4_mode.py new file mode 100644 index 00000000..66b4fad7 --- /dev/null +++ b/cloudinit/sources/helpers/vmware/imc/ipv4_mode.py @@ -0,0 +1,29 @@ +# from enum import Enum + + +# The IPv4 configuration mode which directly represents the user's goal. +# +# This mode effectively acts as a contract of the inguest customization engine. +# It must be set based on what the user has requested via VMODL/generators API +# and should not be changed by those layers. It's up to the in-guest engine to +# interpret and materialize the user's request. +# +# Also defined in linuxconfiggenerator.h. +class Ipv4Mode: + # The legacy mode which only allows dhcp/static based on whether IPv4 + # addresses list is empty or not + IPV4_MODE_BACKWARDS_COMPATIBLE = 'BACKWARDS_COMPATIBLE' + # IPv4 must use static address. Reserved for future use + IPV4_MODE_STATIC = 'STATIC' + # IPv4 must use DHCPv4. Reserved for future use + IPV4_MODE_DHCP = 'DHCP' + # IPv4 must be disabled + IPV4_MODE_DISABLED = 'DISABLED' + # IPv4 settings should be left untouched. Reserved for future use + IPV4_MODE_AS_IS = 'AS_IS' + + # def __eq__(self, other): + # return self.name == other.name and self.value == other.value + # + # def __ne__(self, other): + # return not self.__eq__(other) diff --git a/cloudinit/sources/helpers/vmware/imc/nic.py b/cloudinit/sources/helpers/vmware/imc/nic.py new file mode 100644 index 00000000..b90a5640 --- /dev/null +++ b/cloudinit/sources/helpers/vmware/imc/nic.py @@ -0,0 +1,107 @@ +from cloudinit.sources.helpers.vmware.imc.boot_proto import BootProto + + +class Nic: + def __init__(self, name, configFile): + self._name = name + self._configFile = configFile + + def _get(self, what): + return self._configFile.get(self.name + what, None) + + def _getCnt(self, prefix): + return self._configFile.getCnt(self.name + prefix) + + @property + def name(self): + return self._name + + @property + def mac(self): + return self._get('|MACADDR').lower() + + @property + def bootProto(self): + return self._get('|BOOTPROTO').lower() + + @property + def ipv4(self): + # TODO implement NONE + if self.bootProto == BootProto.STATIC: + return StaticIpv4Conf(self) + + return DhcpIpv4Conf(self) + + @property + def ipv6(self): + # TODO implement NONE + cnt = self._getCnt("|IPv6ADDR|") + + if cnt != 0: + return StaticIpv6Conf(self) + + return DhcpIpv6Conf(self) + + +class DhcpIpv4Conf: + def __init__(self, nic): + self._nic = nic + + +class StaticIpv4Addr: + def __init__(self, nic): + self._nic = nic + + @property + def ip(self): + return self._nic._get('|IPADDR') + + @property + def netmask(self): + return self._nic._get('|NETMASK') + + @property + def gateway(self): + return self._nic._get('|GATEWAY') + + +class StaticIpv4Conf(DhcpIpv4Conf): + @property + def addrs(self): + return [StaticIpv4Addr(self._nic)] + + +class DhcpIpv6Conf: + def __init__(self, nic): + self._nic = nic + + +class StaticIpv6Addr: + def __init__(self, nic, index): + self._nic = nic + self._index = index + + @property + def ip(self): + return self._nic._get("|IPv6ADDR|" + str(self._index)) + + @property + def prefix(self): + return self._nic._get("|IPv6NETMASK|" + str(self._index)) + + @property + def gateway(self): + return self._nic._get("|IPv6GATEWAY|" + str(self._index)) + + +class StaticIpv6Conf(DhcpIpv6Conf): + @property + def addrs(self): + cnt = self._nic._getCnt("|IPv6ADDR|") + + res = [] + + for i in range(1, cnt + 1): + res.append(StaticIpv6Addr(self._nic, i)) + + return res -- cgit v1.2.3 From 8d9e5bd7fcda8f56a4fe087150db1456af738335 Mon Sep 17 00:00:00 2001 From: Sankar Tanguturi Date: Tue, 5 Jan 2016 12:05:11 -0800 Subject: Fixed all the styling nits. Used proper naming convention for the methods. Added proper documentation. Checked pep8 and flake8 output and no issues were reported. --- cloudinit/sources/helpers/vmware/imc/boot_proto.py | 28 +- cloudinit/sources/helpers/vmware/imc/config.py | 116 +++---- .../sources/helpers/vmware/imc/config_file.py | 372 +++++++++------------ .../sources/helpers/vmware/imc/config_namespace.py | 22 +- .../sources/helpers/vmware/imc/config_source.py | 21 ++ cloudinit/sources/helpers/vmware/imc/ipv4_mode.py | 74 ++-- cloudinit/sources/helpers/vmware/imc/nic.py | 254 ++++++++------ 7 files changed, 448 insertions(+), 439 deletions(-) (limited to 'cloudinit/sources/helpers') diff --git a/cloudinit/sources/helpers/vmware/imc/boot_proto.py b/cloudinit/sources/helpers/vmware/imc/boot_proto.py index 6c3b070a..abfffd75 100644 --- a/cloudinit/sources/helpers/vmware/imc/boot_proto.py +++ b/cloudinit/sources/helpers/vmware/imc/boot_proto.py @@ -1,11 +1,25 @@ -# from enum import Enum +# vi: ts=4 expandtab +# +# Copyright (C) 2015 Canonical Ltd. +# Copyright (C) 2015 VMware Inc. +# +# Author: Sankar Tanguturi +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + class BootProto: + """Specifies the NIC Boot Settings.""" + DHCP = 'dhcp' STATIC = 'static' - -# def __eq__(self, other): -# return self.name == other.name and self.value == other.value -# -# def __ne__(self, other): -# return not self.__eq__(other) diff --git a/cloudinit/sources/helpers/vmware/imc/config.py b/cloudinit/sources/helpers/vmware/imc/config.py index ea0873fb..7eee47a5 100644 --- a/cloudinit/sources/helpers/vmware/imc/config.py +++ b/cloudinit/sources/helpers/vmware/imc/config.py @@ -1,122 +1,90 @@ -from cloudinit.sources.helpers.vmware.imc.nic import Nic +# vi: ts=4 expandtab +# +# Copyright (C) 2015 Canonical Ltd. +# Copyright (C) 2015 VMware Inc. +# +# Author: Sankar Tanguturi +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +from .nic import Nic class Config: + """ + Stores the Contents specified in the Customization + Specification file. + """ + DNS = 'DNS|NAMESERVER|' SUFFIX = 'DNS|SUFFIX|' PASS = 'PASSWORD|-PASS' TIMEZONE = 'DATETIME|TIMEZONE' UTC = 'DATETIME|UTC' HOSTNAME = 'NETWORK|HOSTNAME' - OMAINNAME = 'NETWORK|DOMAINNAME' + DOMAINNAME = 'NETWORK|DOMAINNAME' def __init__(self, configFile): self._configFile = configFile - # Retrieves hostname. - # - # Args: - # None - # Results: - # string: hostname - # Throws: - # None @property - def hostName(self): + def host_name(self): + """Return the hostname.""" return self._configFile.get(Config.HOSTNAME, None) - # Retrieves domainName. - # - # Args: - # None - # Results: - # string: domainName - # Throws: - # None @property - def domainName(self): + def domain_name(self): + """Return the domain name.""" return self._configFile.get(Config.DOMAINNAME, None) - # Retrieves timezone. - # - # Args: - # None - # Results: - # string: timezone - # Throws: - # None @property - def timeZone(self): + def timezone(self): + """Return the timezone.""" return self._configFile.get(Config.TIMEZONE, None) - # Retrieves whether to set time to UTC or Local. - # - # Args: - # None - # Results: - # boolean: True for yes/YES, True for no/NO, otherwise - None - # Throws: - # None @property def utc(self): + """Retrieves whether to set time to UTC or Local.""" return self._configFile.get(Config.UTC, None) - # Retrieves root password to be set. - # - # Args: - # None - # Results: - # string: base64-encoded root password or None - # Throws: - # None @property - def adminPassword(self): + def admin_password(self): + """Return the root password to be set.""" return self._configFile.get(Config.PASS, None) - # Retrieves DNS Servers. - # - # Args: - # None - # Results: - # integer: count or 0 - # Throws: - # None @property - def nameServers(self): + def name_servers(self): + """Return the list of DNS servers.""" res = [] - for i in range(1, self._configFile.getCnt(Config.DNS) + 1): + for i in range(1, self._configFile.get_count(Config.DNS) + 1): key = Config.DNS + str(i) res.append(self._configFile[key]) return res - # Retrieves DNS Suffixes. - # - # Args: - # None - # Results: - # integer: count or 0 - # Throws: - # None @property - def dnsSuffixes(self): + def dns_suffixes(self): + """Return the list of DNS Suffixes.""" res = [] - for i in range(1, self._configFile.getCnt(Config.SUFFIX) + 1): + for i in range(1, self._configFile.get_count(Config.SUFFIX) + 1): key = Config.SUFFIX + str(i) res.append(self._configFile[key]) return res - # Retrieves NICs. - # - # Args: - # None - # Results: - # integer: count - # Throws: - # None @property def nics(self): + """Return the list of associated NICs.""" res = [] nics = self._configFile['NIC-CONFIG|NICS'] for nic in nics.split(','): diff --git a/cloudinit/sources/helpers/vmware/imc/config_file.py b/cloudinit/sources/helpers/vmware/imc/config_file.py index 3f9938da..e08a2a9a 100644 --- a/cloudinit/sources/helpers/vmware/imc/config_file.py +++ b/cloudinit/sources/helpers/vmware/imc/config_file.py @@ -1,221 +1,151 @@ -import logging -import re - -from cloudinit.sources.helpers.vmware.imc.config_source import ConfigSource - -logger = logging.getLogger(__name__) - - -class ConfigFile(ConfigSource): - def __init__(self): - self._configData = {} - - def __getitem__(self, key): - return self._configData[key] - - def get(self, key, default=None): - return self._configData.get(key, default) - - # Removes all the properties. - # - # Args: - # None - # Results: - # None - # Throws: - # None - def clear(self): - self._configData.clear() - - # Inserts k/v pair. - # - # Does not do any key/cross-key validation. - # - # Args: - # key: string: key - # val: string: value - # Results: - # None - # Throws: - # None - def _insertKey(self, key, val): - # cleaning up on all "input" path - - # remove end char \n (chomp) - key = key.strip() - val = val.strip() - - if key.startswith('-') or '|-' in key: - canLog = 0 - else: - canLog = 1 - - # "sensitive" settings shall not be logged - if canLog: - logger.debug("ADDED KEY-VAL :: '%s' = '%s'" % (key, val)) - else: - logger.debug("ADDED KEY-VAL :: '%s' = '*****************'" % key) - - self._configData[key] = val - - # Determines properties count. - # - # Args: - # None - # Results: - # integer: properties count - # Throws: - # None - def size(self): - return len(self._configData) - - # Parses properties from a .cfg file content. - # - # Any previously available properties will be removed. - # - # Sensitive data will not be logged in case key starts from '-'. - # - # Args: - # content: string: e.g. content of config/cust.cfg - # Results: - # None - # Throws: - # None - def loadConfigContent(self, content): - self.clear() - - # remove end char \n (chomp) - for line in content.split('\n'): - # TODO validate against allowed characters (not done in Perl) - - # spaces at the end are not allowed, things like passwords must be - # at least base64-encoded - line = line.strip() - - # "sensitive" settings shall not be logged - if line.startswith('-'): - canLog = 0 - else: - canLog = 1 - - if canLog: - logger.debug("Processing line: '%s'" % line) - else: - logger.debug("Processing line: '***********************'") - - if not line: - logger.debug("Empty line. Ignored.") - continue - - if line.startswith('#'): - logger.debug("Comment found. Line ignored.") - continue - - matchObj = re.match(r'\[(.+)\]', line) - if matchObj: - category = matchObj.group(1) - logger.debug("FOUND CATEGORY = '%s'" % category) - else: - # POSIX.2 regex doesn't support non-greedy like in (.+?)=(.*) - # key value pair (non-eager '=' for base64) - matchObj = re.match(r'([^=]+)=(.*)', line) - if matchObj: - # cleaning up on all "input" paths - key = category + "|" + matchObj.group(1).strip() - val = matchObj.group(2).strip() - - self._insertKey(key, val) - else: - # TODO document - raise Exception("Unrecognizable line: '%s'" % line) - - self.validate() - - # Parses properties from a .cfg file - # - # Any previously available properties will be removed. - # - # Sensitive data will not be logged in case key starts from '-'. - # - # Args: - # filename: string: full path to a .cfg file - # Results: - # None - # Throws: - # None - def loadConfigFile(self, filename): - logger.info("Opening file name %s." % filename) - # TODO what throws? - with open(filename, "r") as myfile: - self.loadConfigContent(myfile.read()) - - # Determines whether a property with a given key exists. - # - # Args: - # key: string: key - # Results: - # boolean: True if such property exists, otherwise - False. - # Throws: - # None - def hasKey(self, key): - return key in self._configData - - # Determines whether a value for a property must be kept. - # - # If the property is missing, it's treated as it should be not changed by - # the engine. - # - # Args: - # key: string: key - # Results: - # boolean: True if property must be kept, otherwise - False. - # Throws: - # None - def keepCurrentValue(self, key): - # helps to distinguish from "empty" value which is used to indicate - # "removal" - return not self.hasKey(key) - - # Determines whether a value for a property must be removed. - # - # If the property is empty, it's treated as it should be removed by the - # engine. - # - # Args: - # key: string: key - # Results: - # boolean: True if property must be removed, otherwise - False. - # Throws: - # None - def removeCurrentValue(self, key): - # helps to distinguish from "missing" value which is used to indicate - # "keeping unchanged" - if self.hasKey(key): - return not bool(self._configData[key]) - else: - return False - - # TODO - def getCnt(self, prefix): - res = 0 - for key in self._configData.keys(): - if key.startswith(prefix): - res += 1 - - return res - - # TODO - # TODO pass base64 - # Throws: - # Dies in case timezone is present but empty. - # Dies in case password is present but empty. - # Dies in case hostname is present but empty or greater than 63 chars. - # Dies in case UTC is present, but is not yes/YES or no/NO. - # Dies in case NICS is not present. - def validate(self): - # TODO must log all the errors - keyValidators = {'NIC1|IPv6GATEWAY|': None} - crossValidators = {} - - for key in self._configData.keys(): - pass +# vi: ts=4 expandtab +# +# Copyright (C) 2015 Canonical Ltd. +# Copyright (C) 2015 VMware Inc. +# +# Author: Sankar Tanguturi +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import logging + +try: + import configparser +except ImportError: + import ConfigParser as configparser + +from .config_source import ConfigSource + +logger = logging.getLogger(__name__) + + +class ConfigFile(ConfigSource, dict): + """ConfigFile module to load the content from a specified source.""" + + def __init__(self): + pass + + def _insertKey(self, key, val): + """ + Inserts a Key Value pair. + + Keyword arguments: + key -- The key to insert + val -- The value to insert for the key + + """ + key = key.strip() + val = val.strip() + + if key.startswith('-') or '|-' in key: + canLog = 0 + else: + canLog = 1 + + # "sensitive" settings shall not be logged + if canLog: + logger.debug("ADDED KEY-VAL :: '%s' = '%s'" % (key, val)) + else: + logger.debug("ADDED KEY-VAL :: '%s' = '*****************'" % key) + + self[key] = val + + def size(self): + """Return the number of properties present.""" + return len(self) + + def loadConfigFile(self, filename): + """ + Parses properties from the specified config file. + + Any previously available properties will be removed. + Sensitive data will not be logged in case the key starts + from '-'. + + Keyword arguments: + filename - The full path to the config file. + """ + logger.info('Parsing the config file %s.' % filename) + + config = configparser.ConfigParser() + config.optionxform = str + config.read(filename) + + self.clear() + + for category in config.sections(): + logger.debug("FOUND CATEGORY = '%s'" % category) + + for (key, value) in config.items(category): + # "sensitive" settings shall not be logged + if key.startswith('-'): + canLog = 0 + else: + canLog = 1 + + if canLog: + logger.debug("Processing key, value: '%s':'%s'" % + (key, value)) + else: + logger.debug("Processing key, value : " + "'*********************'") + + self._insertKey(category + '|' + key, value) + + def keep_current_value(self, key): + """ + Determines whether a value for a property must be kept. + + If the propery is missing, it is treated as it should be not + changed by the engine. + + Keyword arguments: + key -- The key to search for. + """ + # helps to distinguish from "empty" value which is used to indicate + # "removal" + return not key in self + + def remove_current_value(self, key): + """ + Determines whether a value for the property must be removed. + + If the specified key is empty, it is treated as it should be + removed by the engine. + + Return true if the value can be removed, false otherwise. + + Keyword arguments: + key -- The key to search for. + """ + # helps to distinguish from "missing" value which is used to indicate + # "keeping unchanged" + if key in self: + return not bool(self[key]) + else: + return False + + def get_count(self, prefix): + """ + Return the total number of keys that start with the + specified prefix. + + Keyword arguments: + prefix -- prefix of the key + """ + res = 0 + for key in self.keys(): + if key.startswith(prefix): + res += 1 + + return res diff --git a/cloudinit/sources/helpers/vmware/imc/config_namespace.py b/cloudinit/sources/helpers/vmware/imc/config_namespace.py index 7f76ac8b..7266b699 100644 --- a/cloudinit/sources/helpers/vmware/imc/config_namespace.py +++ b/cloudinit/sources/helpers/vmware/imc/config_namespace.py @@ -1,5 +1,25 @@ -from cloudinit.sources.helpers.vmware.imc.config_source import ConfigSource +# vi: ts=4 expandtab +# +# Copyright (C) 2015 Canonical Ltd. +# Copyright (C) 2015 VMware Inc. +# +# Author: Sankar Tanguturi +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +from .config_source import ConfigSource class ConfigNamespace(ConfigSource): + """Specifies the Config Namespace.""" pass diff --git a/cloudinit/sources/helpers/vmware/imc/config_source.py b/cloudinit/sources/helpers/vmware/imc/config_source.py index fad3a389..a367e476 100644 --- a/cloudinit/sources/helpers/vmware/imc/config_source.py +++ b/cloudinit/sources/helpers/vmware/imc/config_source.py @@ -1,2 +1,23 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2015 Canonical Ltd. +# Copyright (C) 2015 VMware Inc. +# +# Author: Sankar Tanguturi +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + + class ConfigSource: + """Specifies a source for the Config Content.""" pass diff --git a/cloudinit/sources/helpers/vmware/imc/ipv4_mode.py b/cloudinit/sources/helpers/vmware/imc/ipv4_mode.py index 66b4fad7..28544e4f 100644 --- a/cloudinit/sources/helpers/vmware/imc/ipv4_mode.py +++ b/cloudinit/sources/helpers/vmware/imc/ipv4_mode.py @@ -1,29 +1,45 @@ -# from enum import Enum - - -# The IPv4 configuration mode which directly represents the user's goal. -# -# This mode effectively acts as a contract of the inguest customization engine. -# It must be set based on what the user has requested via VMODL/generators API -# and should not be changed by those layers. It's up to the in-guest engine to -# interpret and materialize the user's request. -# -# Also defined in linuxconfiggenerator.h. -class Ipv4Mode: - # The legacy mode which only allows dhcp/static based on whether IPv4 - # addresses list is empty or not - IPV4_MODE_BACKWARDS_COMPATIBLE = 'BACKWARDS_COMPATIBLE' - # IPv4 must use static address. Reserved for future use - IPV4_MODE_STATIC = 'STATIC' - # IPv4 must use DHCPv4. Reserved for future use - IPV4_MODE_DHCP = 'DHCP' - # IPv4 must be disabled - IPV4_MODE_DISABLED = 'DISABLED' - # IPv4 settings should be left untouched. Reserved for future use - IPV4_MODE_AS_IS = 'AS_IS' - - # def __eq__(self, other): - # return self.name == other.name and self.value == other.value - # - # def __ne__(self, other): - # return not self.__eq__(other) +# vi: ts=4 expandtab +# +# Copyright (C) 2015 Canonical Ltd. +# Copyright (C) 2015 VMware Inc. +# +# Author: Sankar Tanguturi +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + + +class Ipv4Mode: + """ + The IPv4 configuration mode which directly represents the user's goal. + + This mode effectively acts as a contract of the in-guest customization + engine. It must be set based on what the user has requested and should + not be changed by those layers. It's up to the in-guest engine to + interpret and materialize the user's request. + """ + + # The legacy mode which only allows dhcp/static based on whether IPv4 + # addresses list is empty or not + IPV4_MODE_BACKWARDS_COMPATIBLE = 'BACKWARDS_COMPATIBLE' + + # IPv4 must use static address. Reserved for future use + IPV4_MODE_STATIC = 'STATIC' + + # IPv4 must use DHCPv4. Reserved for future use + IPV4_MODE_DHCP = 'DHCP' + + # IPv4 must be disabled + IPV4_MODE_DISABLED = 'DISABLED' + + # IPv4 settings should be left untouched. Reserved for future use + IPV4_MODE_AS_IS = 'AS_IS' diff --git a/cloudinit/sources/helpers/vmware/imc/nic.py b/cloudinit/sources/helpers/vmware/imc/nic.py index b90a5640..bb45a9e6 100644 --- a/cloudinit/sources/helpers/vmware/imc/nic.py +++ b/cloudinit/sources/helpers/vmware/imc/nic.py @@ -1,107 +1,147 @@ -from cloudinit.sources.helpers.vmware.imc.boot_proto import BootProto - - -class Nic: - def __init__(self, name, configFile): - self._name = name - self._configFile = configFile - - def _get(self, what): - return self._configFile.get(self.name + what, None) - - def _getCnt(self, prefix): - return self._configFile.getCnt(self.name + prefix) - - @property - def name(self): - return self._name - - @property - def mac(self): - return self._get('|MACADDR').lower() - - @property - def bootProto(self): - return self._get('|BOOTPROTO').lower() - - @property - def ipv4(self): - # TODO implement NONE - if self.bootProto == BootProto.STATIC: - return StaticIpv4Conf(self) - - return DhcpIpv4Conf(self) - - @property - def ipv6(self): - # TODO implement NONE - cnt = self._getCnt("|IPv6ADDR|") - - if cnt != 0: - return StaticIpv6Conf(self) - - return DhcpIpv6Conf(self) - - -class DhcpIpv4Conf: - def __init__(self, nic): - self._nic = nic - - -class StaticIpv4Addr: - def __init__(self, nic): - self._nic = nic - - @property - def ip(self): - return self._nic._get('|IPADDR') - - @property - def netmask(self): - return self._nic._get('|NETMASK') - - @property - def gateway(self): - return self._nic._get('|GATEWAY') - - -class StaticIpv4Conf(DhcpIpv4Conf): - @property - def addrs(self): - return [StaticIpv4Addr(self._nic)] - - -class DhcpIpv6Conf: - def __init__(self, nic): - self._nic = nic - - -class StaticIpv6Addr: - def __init__(self, nic, index): - self._nic = nic - self._index = index - - @property - def ip(self): - return self._nic._get("|IPv6ADDR|" + str(self._index)) - - @property - def prefix(self): - return self._nic._get("|IPv6NETMASK|" + str(self._index)) - - @property - def gateway(self): - return self._nic._get("|IPv6GATEWAY|" + str(self._index)) - - -class StaticIpv6Conf(DhcpIpv6Conf): - @property - def addrs(self): - cnt = self._nic._getCnt("|IPv6ADDR|") - - res = [] - - for i in range(1, cnt + 1): - res.append(StaticIpv6Addr(self._nic, i)) - - return res +# vi: ts=4 expandtab +# +# Copyright (C) 2015 Canonical Ltd. +# Copyright (C) 2015 VMware Inc. +# +# Author: Sankar Tanguturi +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +from .boot_proto import BootProto + + +class Nic: + """ + Holds the information about each NIC specified + in the customization specification file + """ + + def __init__(self, name, configFile): + self._name = name + self._configFile = configFile + + def _get(self, what): + return self._configFile.get(self.name + what, None) + + def _get_count(self, prefix): + return self._configFile.get_count(self.name + prefix) + + @property + def name(self): + return self._name + + @property + def mac(self): + return self._get('|MACADDR').lower() + + @property + def bootProto(self): + return self._get('|BOOTPROTO').lower() + + @property + def ipv4(self): + """ + Retrieves the DHCP or Static IPv6 configuration + based on the BOOTPROTO property associated with the NIC + """ + if self.bootProto == BootProto.STATIC: + return StaticIpv4Conf(self) + + return DhcpIpv4Conf(self) + + @property + def ipv6(self): + cnt = self._get_count("|IPv6ADDR|") + + if cnt != 0: + return StaticIpv6Conf(self) + + return DhcpIpv6Conf(self) + + +class DhcpIpv4Conf: + """DHCP Configuration Setting.""" + + def __init__(self, nic): + self._nic = nic + + +class StaticIpv4Addr: + """Static IPV4 Setting.""" + + def __init__(self, nic): + self._nic = nic + + @property + def ip(self): + return self._nic._get('|IPADDR') + + @property + def netmask(self): + return self._nic._get('|NETMASK') + + @property + def gateway(self): + return self._nic._get('|GATEWAY') + + +class StaticIpv4Conf(DhcpIpv4Conf): + """Static IPV4 Configuration.""" + + @property + def addrs(self): + """Return the list of associated IPv4 addresses.""" + return [StaticIpv4Addr(self._nic)] + + +class DhcpIpv6Conf: + """DHCP IPV6 Configuration.""" + + def __init__(self, nic): + self._nic = nic + + +class StaticIpv6Addr: + """Static IPV6 Address.""" + + def __init__(self, nic, index): + self._nic = nic + self._index = index + + @property + def ip(self): + return self._nic._get("|IPv6ADDR|" + str(self._index)) + + @property + def prefix(self): + return self._nic._get("|IPv6NETMASK|" + str(self._index)) + + @property + def gateway(self): + return self._nic._get("|IPv6GATEWAY|" + str(self._index)) + + +class StaticIpv6Conf(DhcpIpv6Conf): + """Static IPV6 Configuration.""" + + @property + def addrs(self): + """Return the list Associated IPV6 addresses.""" + cnt = self._nic._get_count("|IPv6ADDR|") + + res = [] + + for i in range(1, cnt + 1): + res.append(StaticIpv6Addr(self._nic, i)) + + return res -- cgit v1.2.3 From 415c45a2b9b66603e672e8ea54cee8f40a19abd1 Mon Sep 17 00:00:00 2001 From: Sankar Tanguturi Date: Tue, 19 Jan 2016 18:24:54 -0800 Subject: Fixed all the review comments from Daniel. Added a new file i.e. nic_base.py which will be used a base calls for all NIC related configuration. Modified some code in nic.py. --- cloudinit/sources/helpers/vmware/imc/boot_proto.py | 2 +- cloudinit/sources/helpers/vmware/imc/config.py | 6 +- .../sources/helpers/vmware/imc/config_file.py | 40 ++---- cloudinit/sources/helpers/vmware/imc/ipv4_mode.py | 2 +- cloudinit/sources/helpers/vmware/imc/nic.py | 118 +++++++--------- cloudinit/sources/helpers/vmware/imc/nic_base.py | 154 +++++++++++++++++++++ 6 files changed, 222 insertions(+), 100 deletions(-) create mode 100644 cloudinit/sources/helpers/vmware/imc/nic_base.py (limited to 'cloudinit/sources/helpers') diff --git a/cloudinit/sources/helpers/vmware/imc/boot_proto.py b/cloudinit/sources/helpers/vmware/imc/boot_proto.py index abfffd75..faba5887 100644 --- a/cloudinit/sources/helpers/vmware/imc/boot_proto.py +++ b/cloudinit/sources/helpers/vmware/imc/boot_proto.py @@ -18,7 +18,7 @@ # along with this program. If not, see . -class BootProto: +class BootProtoEnum: """Specifies the NIC Boot Settings.""" DHCP = 'dhcp' diff --git a/cloudinit/sources/helpers/vmware/imc/config.py b/cloudinit/sources/helpers/vmware/imc/config.py index 7eee47a5..aebc12a0 100644 --- a/cloudinit/sources/helpers/vmware/imc/config.py +++ b/cloudinit/sources/helpers/vmware/imc/config.py @@ -66,7 +66,8 @@ class Config: def name_servers(self): """Return the list of DNS servers.""" res = [] - for i in range(1, self._configFile.get_count(Config.DNS) + 1): + cnt = self._configFile.get_count_with_prefix(Config.DNS) + for i in range(1, cnt + 1): key = Config.DNS + str(i) res.append(self._configFile[key]) @@ -76,7 +77,8 @@ class Config: def dns_suffixes(self): """Return the list of DNS Suffixes.""" res = [] - for i in range(1, self._configFile.get_count(Config.SUFFIX) + 1): + cnt = self._configFile.get_count_with_prefix(Config.SUFFIX) + for i in range(1, cnt + 1): key = Config.SUFFIX + str(i) res.append(self._configFile[key]) diff --git a/cloudinit/sources/helpers/vmware/imc/config_file.py b/cloudinit/sources/helpers/vmware/imc/config_file.py index e08a2a9a..7c47d14c 100644 --- a/cloudinit/sources/helpers/vmware/imc/config_file.py +++ b/cloudinit/sources/helpers/vmware/imc/config_file.py @@ -32,7 +32,8 @@ logger = logging.getLogger(__name__) class ConfigFile(ConfigSource, dict): """ConfigFile module to load the content from a specified source.""" - def __init__(self): + def __init__(self, filename): + self._loadConfigFile(filename) pass def _insertKey(self, key, val): @@ -48,9 +49,9 @@ class ConfigFile(ConfigSource, dict): val = val.strip() if key.startswith('-') or '|-' in key: - canLog = 0 + canLog = False else: - canLog = 1 + canLog = True # "sensitive" settings shall not be logged if canLog: @@ -64,7 +65,7 @@ class ConfigFile(ConfigSource, dict): """Return the number of properties present.""" return len(self) - def loadConfigFile(self, filename): + def _loadConfigFile(self, filename): """ Parses properties from the specified config file. @@ -87,22 +88,9 @@ class ConfigFile(ConfigSource, dict): logger.debug("FOUND CATEGORY = '%s'" % category) for (key, value) in config.items(category): - # "sensitive" settings shall not be logged - if key.startswith('-'): - canLog = 0 - else: - canLog = 1 - - if canLog: - logger.debug("Processing key, value: '%s':'%s'" % - (key, value)) - else: - logger.debug("Processing key, value : " - "'*********************'") - self._insertKey(category + '|' + key, value) - def keep_current_value(self, key): + def should_keep_current_value(self, key): """ Determines whether a value for a property must be kept. @@ -114,9 +102,9 @@ class ConfigFile(ConfigSource, dict): """ # helps to distinguish from "empty" value which is used to indicate # "removal" - return not key in self + return key not in self - def remove_current_value(self, key): + def should_remove_current_value(self, key): """ Determines whether a value for the property must be removed. @@ -135,17 +123,11 @@ class ConfigFile(ConfigSource, dict): else: return False - def get_count(self, prefix): + def get_count_with_prefix(self, prefix): """ - Return the total number of keys that start with the - specified prefix. + Return the total count of keys that start with the specified prefix. Keyword arguments: prefix -- prefix of the key """ - res = 0 - for key in self.keys(): - if key.startswith(prefix): - res += 1 - - return res + return len([key for key in self if key.startswith(prefix)]) diff --git a/cloudinit/sources/helpers/vmware/imc/ipv4_mode.py b/cloudinit/sources/helpers/vmware/imc/ipv4_mode.py index 28544e4f..33f88726 100644 --- a/cloudinit/sources/helpers/vmware/imc/ipv4_mode.py +++ b/cloudinit/sources/helpers/vmware/imc/ipv4_mode.py @@ -18,7 +18,7 @@ # along with this program. If not, see . -class Ipv4Mode: +class Ipv4ModeEnum: """ The IPv4 configuration mode which directly represents the user's goal. diff --git a/cloudinit/sources/helpers/vmware/imc/nic.py b/cloudinit/sources/helpers/vmware/imc/nic.py index bb45a9e6..a7594874 100644 --- a/cloudinit/sources/helpers/vmware/imc/nic.py +++ b/cloudinit/sources/helpers/vmware/imc/nic.py @@ -17,10 +17,11 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -from .boot_proto import BootProto +from .boot_proto import BootProtoEnum +from .nic_base import NicBase, StaticIpv4Base, StaticIpv6Base -class Nic: +class Nic(NicBase): """ Holds the information about each NIC specified in the customization specification file @@ -31,10 +32,10 @@ class Nic: self._configFile = configFile def _get(self, what): - return self._configFile.get(self.name + what, None) + return self._configFile.get(self.name + '|' + what, None) - def _get_count(self, prefix): - return self._configFile.get_count(self.name + prefix) + def _get_count_with_prefix(self, prefix): + return self._configFile.get_count_with_prefix(self.name + prefix) @property def name(self): @@ -42,41 +43,52 @@ class Nic: @property def mac(self): - return self._get('|MACADDR').lower() + return self._get('MACADDR').lower() @property - def bootProto(self): - return self._get('|BOOTPROTO').lower() + def primary(self): + value = self._get('PRIMARY').lower() + return value == 'yes' or value == 'true' @property - def ipv4(self): - """ - Retrieves the DHCP or Static IPv6 configuration - based on the BOOTPROTO property associated with the NIC - """ - if self.bootProto == BootProto.STATIC: - return StaticIpv4Conf(self) + def onboot(self): + value = self._get('ONBOOT').lower() + return value == 'yes' or value == 'true' - return DhcpIpv4Conf(self) + @property + def bootProto(self): + return self._get('BOOTPROTO').lower() @property - def ipv6(self): - cnt = self._get_count("|IPv6ADDR|") + def ipv4_mode(self): + return self._get('IPv4_MODE').lower() - if cnt != 0: - return StaticIpv6Conf(self) + @property + def staticIpv4(self): + """ + Checks the BOOTPROTO property and returns StaticIPv4Addr + configuration object if STATIC configuration is set. + """ + if self.bootProto == BootProtoEnum.STATIC: + return [StaticIpv4Addr(self)] + else: + return None - return DhcpIpv6Conf(self) + @property + def staticIpv6(self): + cnt = self._get_count_with_prefix('|IPv6ADDR|') + if not cnt: + return None -class DhcpIpv4Conf: - """DHCP Configuration Setting.""" + result = [] + for index in range(1, cnt + 1): + result.append(StaticIpv6Addr(self, index)) - def __init__(self, nic): - self._nic = nic + return result -class StaticIpv4Addr: +class StaticIpv4Addr(StaticIpv4Base): """Static IPV4 Setting.""" def __init__(self, nic): @@ -84,34 +96,22 @@ class StaticIpv4Addr: @property def ip(self): - return self._nic._get('|IPADDR') + return self._nic._get('IPADDR') @property def netmask(self): - return self._nic._get('|NETMASK') + return self._nic._get('NETMASK') @property - def gateway(self): - return self._nic._get('|GATEWAY') + def gateways(self): + value = self._nic._get('GATEWAY') + if value: + return [x.strip() for x in value.split(',')] + else: + return None -class StaticIpv4Conf(DhcpIpv4Conf): - """Static IPV4 Configuration.""" - - @property - def addrs(self): - """Return the list of associated IPv4 addresses.""" - return [StaticIpv4Addr(self._nic)] - - -class DhcpIpv6Conf: - """DHCP IPV6 Configuration.""" - - def __init__(self, nic): - self._nic = nic - - -class StaticIpv6Addr: +class StaticIpv6Addr(StaticIpv6Base): """Static IPV6 Address.""" def __init__(self, nic, index): @@ -120,28 +120,12 @@ class StaticIpv6Addr: @property def ip(self): - return self._nic._get("|IPv6ADDR|" + str(self._index)) + return self._nic._get('IPv6ADDR|' + str(self._index)) @property - def prefix(self): - return self._nic._get("|IPv6NETMASK|" + str(self._index)) + def netmask(self): + return self._nic._get('IPv6NETMASK|' + str(self._index)) @property def gateway(self): - return self._nic._get("|IPv6GATEWAY|" + str(self._index)) - - -class StaticIpv6Conf(DhcpIpv6Conf): - """Static IPV6 Configuration.""" - - @property - def addrs(self): - """Return the list Associated IPV6 addresses.""" - cnt = self._nic._get_count("|IPv6ADDR|") - - res = [] - - for i in range(1, cnt + 1): - res.append(StaticIpv6Addr(self._nic, i)) - - return res + return self._nic._get('IPv6GATEWAY|' + str(self._index)) diff --git a/cloudinit/sources/helpers/vmware/imc/nic_base.py b/cloudinit/sources/helpers/vmware/imc/nic_base.py new file mode 100644 index 00000000..030ba311 --- /dev/null +++ b/cloudinit/sources/helpers/vmware/imc/nic_base.py @@ -0,0 +1,154 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2015 Canonical Ltd. +# Copyright (C) 2015 VMware Inc. +# +# Author: Sankar Tanguturi +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + + +class NicBase: + """ + Define what are expected of each nic. + The following properties should be provided in an implementation class. + """ + + @property + def mac(self): + """ + Retrieves the mac address of the nic + @return (str) : the MACADDR setting + """ + raise NotImplementedError('MACADDR') + + @property + def primary(self): + """ + Retrieves whether the nic is the primary nic + Indicates whether NIC will be used to define the default gateway. + If none of the NICs is configured to be primary, default gateway won't + be set. + @return (bool): the PRIMARY setting + """ + raise NotImplementedError('PRIMARY') + + @property + def onboot(self): + """ + Retrieves whether the nic should be up at the boot time + @return (bool) : the ONBOOT setting + """ + raise NotImplementedError('ONBOOT') + + @property + def bootProto(self): + """ + Retrieves the boot protocol of the nic + @return (str): the BOOTPROTO setting, valid values: dhcp and static. + """ + raise NotImplementedError('BOOTPROTO') + + @property + def ipv4_mode(self): + """ + Retrieves the IPv4_MODE + @return (str): the IPv4_MODE setting, valid values: + backwards_compatible, static, dhcp, disabled, as_is + """ + raise NotImplementedError('IPv4_MODE') + + @property + def staticIpv4(self): + """ + Retrieves the static IPv4 configuration of the nic + @return (StaticIpv4Base list): the static ipv4 setting + """ + raise NotImplementedError('Static IPv4') + + @property + def staticIpv6(self): + """ + Retrieves the IPv6 configuration of the nic + @return (StaticIpv6Base list): the static ipv6 setting + """ + raise NotImplementedError('Static Ipv6') + + def validate(self): + """ + Validate the object + For example, the staticIpv4 property is required and should not be + empty when ipv4Mode is STATIC + """ + raise NotImplementedError('Check constraints on properties') + + +class StaticIpv4Base: + """ + Define what are expected of a static IPv4 setting + The following properties should be provided in an implementation class. + """ + + @property + def ip(self): + """ + Retrieves the Ipv4 address + @return (str): the IPADDR setting + """ + raise NotImplementedError('Ipv4 Address') + + @property + def netmask(self): + """ + Retrieves the Ipv4 NETMASK setting + @return (str): the NETMASK setting + """ + raise NotImplementedError('Ipv4 NETMASK') + + @property + def gateways(self): + """ + Retrieves the gateways on this Ipv4 subnet + @return (str list): the GATEWAY setting + """ + raise NotImplementedError('Ipv4 GATEWAY') + + +class StaticIpv6Base: + """Define what are expected of a static IPv6 setting + The following properties should be provided in an implementation class. + """ + + @property + def ip(self): + """ + Retrieves the Ipv6 address + @return (str): the IPv6ADDR setting + """ + raise NotImplementedError('Ipv6 Address') + + @property + def netmask(self): + """ + Retrieves the Ipv6 NETMASK setting + @return (str): the IPv6NETMASK setting + """ + raise NotImplementedError('Ipv6 NETMASK') + + @property + def gateway(self): + """ + Retrieves the Ipv6 GATEWAY setting + @return (str): the IPv6GATEWAY setting + """ + raise NotImplementedError('Ipv6 GATEWAY') -- cgit v1.2.3 From 39f668e5db8d09c46eee3a5df73a69f8d85ba489 Mon Sep 17 00:00:00 2001 From: Sankar Tanguturi Date: Tue, 9 Feb 2016 17:54:07 -0800 Subject: - Added the code to configure the NICs. - Added the code to detect VMware Virtual Platform and apply the customization based on the 'Customization Specification File' Pushed into the guest VM. --- cloudinit/sources/DataSourceOVF.py | 107 ++++++++- cloudinit/sources/helpers/vmware/imc/config_nic.py | 246 +++++++++++++++++++++ cloudinit/sources/helpers/vmware/imc/nic.py | 28 ++- 3 files changed, 372 insertions(+), 9 deletions(-) create mode 100644 cloudinit/sources/helpers/vmware/imc/config_nic.py (limited to 'cloudinit/sources/helpers') diff --git a/cloudinit/sources/DataSourceOVF.py b/cloudinit/sources/DataSourceOVF.py index 58a4b2a2..add7d243 100644 --- a/cloudinit/sources/DataSourceOVF.py +++ b/cloudinit/sources/DataSourceOVF.py @@ -24,11 +24,16 @@ from xml.dom import minidom import base64 import os +import shutil import re +import time from cloudinit import log as logging from cloudinit import sources from cloudinit import util +from cloudinit.sources.helpers.vmware.imc.config import Config +from cloudinit.sources.helpers.vmware.imc.config_file import ConfigFile +from cloudinit.sources.helpers.vmware.imc.config_nic import NicConfigurator LOG = logging.getLogger(__name__) @@ -50,13 +55,51 @@ class DataSourceOVF(sources.DataSource): found = [] md = {} ud = "" + vmwarePlatformFound = False + vmwareImcConfigFilePath = '' defaults = { "instance-id": "iid-dsovf", } (seedfile, contents) = get_ovf_env(self.paths.seed_dir) - if seedfile: + dmi_info = dmi_data() + system_uuid = "" + system_type = "" + + if dmi_info is False: + LOG.debug("No dmidata utility found") + else: + system_uuid, system_type = tuple(dmi_info) + + if 'vmware' in system_type.lower(): + LOG.debug("VMware Virtual Platform found") + deployPkgPluginPath = search_file("/usr/lib/vmware-tools", "libdeployPkgPlugin.so") + if deployPkgPluginPath: + vmwareImcConfigFilePath = util.log_time(logfunc=LOG.debug, + msg="waiting for configuration file", + func=wait_for_imc_cfg_file, + args=("/tmp", "cust.cfg")) + + if vmwareImcConfigFilePath: + LOG.debug("Found VMware DeployPkg Config File Path at %s" % vmwareImcConfigFilePath) + else: + LOG.debug("Didn't find VMware DeployPkg Config File Path") + + if vmwareImcConfigFilePath: + try: + cf = ConfigFile(vmwareImcConfigFilePath) + conf = Config(cf) + (md, ud, cfg) = read_vmware_imc(conf) + nicConfigurator = NicConfigurator(conf.nics) + nicConfigurator.configure() + vmwarePlatformFound = True + except Exception as inst: + LOG.debug("Error while parsing the Customization Config File") + finally: + dirPath = os.path.dirname(vmwareImcConfigFilePath) + shutil.rmtree(dirPath) + elif seedfile: # Found a seed dir seed = os.path.join(self.paths.seed_dir, seedfile) (md, ud, cfg) = read_ovf_environment(contents) @@ -76,7 +119,7 @@ class DataSourceOVF(sources.DataSource): found.append(name) # There was no OVF transports found - if len(found) == 0: + if len(found) == 0 and not vmwarePlatformFound: return False if 'seedfrom' in md and md['seedfrom']: @@ -108,7 +151,7 @@ class DataSourceOVF(sources.DataSource): def get_public_ssh_keys(self): if 'public-keys' not in self.metadata: - return [] + return [] pks = self.metadata['public-keys'] if isinstance(pks, (list)): return pks @@ -129,6 +172,31 @@ class DataSourceOVFNet(DataSourceOVF): self.supported_seed_starts = ("http://", "https://", "ftp://") +def wait_for_imc_cfg_file(directoryPath, filename, maxwait=180, naplen=5): + waited = 0 + + while waited < maxwait: + fileFullPath = search_file(directoryPath, filename) + if fileFullPath: + return fileFullPath + time.sleep(naplen) + waited += naplen + return None + +# This will return a dict with some content +# meta-data, user-data, some config +def read_vmware_imc(config): + md = {} + cfg = {} + ud = "" + if config.host_name: + if config.domain_name: + md['local-hostname'] = config.host_name + "." + config.domain_name + else: + md['local-hostname'] = config.host_name + + return (md, ud, cfg) + # This will return a dict with some content # meta-data, user-data, some config def read_ovf_environment(contents): @@ -280,6 +348,39 @@ def get_properties(contents): return props +def dmi_data(): + sys_uuid = util.read_dmi_data("system-uuid") + sys_type = util.read_dmi_data("system-product-name") + + if not sys_uuid or not sys_type: + return None + + return (sys_uuid.lower(), sys_type) + +def search_file(directoryPath, filename): + if not directoryPath or not filename: + return None + + dirs = [] + + if os.path.isdir(directoryPath): + dirs.append(directoryPath) + + while dirs: + dir = dirs.pop() + children = [] + try: + children.extend(os.listdir(dir)) + except: + LOG.debug("Ignoring the error while searching the directory %s" % dir) + for child in children: + childFullPath = os.path.join(dir, child) + if os.path.isdir(childFullPath): + dirs.append(childFullPath) + elif child == filename: + return childFullPath + + return None class XmlError(Exception): pass diff --git a/cloudinit/sources/helpers/vmware/imc/config_nic.py b/cloudinit/sources/helpers/vmware/imc/config_nic.py new file mode 100644 index 00000000..8e2fc5d3 --- /dev/null +++ b/cloudinit/sources/helpers/vmware/imc/config_nic.py @@ -0,0 +1,246 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2015 Canonical Ltd. +# Copyright (C) 2016 VMware INC. +# +# Author: Sankar Tanguturi +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import logging +import os +import subprocess +import re + +logger = logging.getLogger(__name__) + + +class NicConfigurator: + def __init__(self, nics): + """ + Initialize the Nic Configurator + @param nics (list) an array of nics to configure + """ + self.nics = nics + self.mac2Name = {} + self.ipv4PrimaryGateway = None + self.ipv6PrimaryGateway = None + self.find_devices() + self._primaryNic = self.get_primary_nic() + + def get_primary_nic(self): + """ + Retrieve the primary nic if it exists + @return (NicBase): the primary nic if exists, None otherwise + """ + primaryNic = None + + for nic in self.nics: + if nic.primary: + if primaryNic: + raise Exception('There can only be one primary nic', + primaryNic.mac, nic.mac) + primaryNic = nic + + return primaryNic + + def find_devices(self): + """ + Create the mac2Name dictionary + The mac address(es) are in the lower case + """ + cmd = 'ip addr show' + outText = subprocess.check_output(cmd, shell=True).decode() + sections = re.split(r'\n\d+: ', '\n' + outText)[1:] + + macPat = r'link/ether (([0-9A-Fa-f]{2}[:]){5}([0-9A-Fa-f]{2}))' + for section in sections: + matcher = re.search(macPat, section) + if not matcher: # Only keep info about nics + continue + mac = matcher.group(1).lower() + name = section.split(':', 1)[0] + self.mac2Name[mac] = name + + def gen_one_nic(self, nic): + """ + Return the lines needed to configure a nic + @return (str list): the string list to configure the nic + @param nic (NicBase): the nic to configure + """ + lines = [] + name = self.mac2Name.get(nic.mac.lower()) + if not name: + raise ValueError('No known device has MACADDR: %s' % nic.mac) + + if nic.onboot: + lines.append('auto %s' % name) + + # Customize IPv4 + lines.extend(self.gen_ipv4(name, nic)) + + # Customize IPv6 + lines.extend(self.gen_ipv6(name, nic)) + + lines.append('') + + return lines + + def gen_ipv4(self, name, nic): + """ + Return the lines needed to configure the IPv4 setting of a nic + @return (str list): the string list to configure the gateways + @param name (str): name of the nic + @param nic (NicBase): the nic to configure + """ + lines = [] + + bootproto = nic.bootProto.lower() + if nic.ipv4_mode.lower() == 'disabled': + bootproto = 'manual' + lines.append('iface %s inet %s' % (name, bootproto)) + + if bootproto != 'static': + return lines + + # Static Ipv4 + v4 = nic.staticIpv4 + if v4.ip: + lines.append(' address %s' % v4.ip) + if v4.netmask: + lines.append(' netmask %s' % v4.netmask) + + # Add the primary gateway + if nic.primary and v4.gateways: + self.ipv4PrimaryGateway = v4.gateways[0] + lines.append(' gateway %s metric 0' % self.ipv4PrimaryGateway) + return lines + + # Add routes if there is no primary nic + if not self._primaryNic: + lines.extend(self.gen_ipv4_route(nic, v4.gateways)) + + return lines + + def gen_ipv4_route(self, nic, gateways): + """ + Return the lines needed to configure additional Ipv4 route + @return (str list): the string list to configure the gateways + @param nic (NicBase): the nic to configure + @param gateways (str list): the list of gateways + """ + lines = [] + + for gateway in gateways: + lines.append(' up route add default gw %s metric 10000' % gateway) + + return lines + + def gen_ipv6(self, name, nic): + """ + Return the lines needed to configure the gateways for a nic + @return (str list): the string list to configure the gateways + @param name (str): name of the nic + @param nic (NicBase): the nic to configure + """ + lines = [] + + if not nic.staticIpv6: + return lines + + # Static Ipv6 + addrs = nic.staticIpv6 + lines.append('iface %s inet6 static' % name) + lines.append(' address %s' % addrs[0].ip) + lines.append(' netmask %s' % addrs[0].netmask) + + for addr in addrs[1:]: + lines.append(' up ifconfig %s inet6 add %s/%s' % (name, addr.ip, + addr.netmask)) + # Add the primary gateway + if nic.primary: + for addr in addrs: + if addr.gateway: + self.ipv6PrimaryGateway = addr.gateway + lines.append(' gateway %s' % self.ipv6PrimaryGateway) + return lines + + # Add routes if there is no primary nic + if not self._primaryNic: + lines.extend(self._genIpv6Route(name, nic, addrs)) + + return lines + + def _genIpv6Route(self, name, nic, addrs): + lines = [] + + for addr in addrs: + lines.append(' up route -A inet6 add default gw %s metric 10000' % + addr.gateway) + + return lines + + def generate(self): + """Return the lines that is needed to configure the nics""" + lines = [] + lines.append('iface lo inet loopback') + lines.append('auto lo') + lines.append('') + + for nic in self.nics: + lines.extend(self.gen_one_nic(nic)) + + return lines + + def clear_dhcp(self): + logger.info('Clearing DHCP leases') + + subprocess.call('pkill dhclient', shell=True) + subprocess.check_call('rm -f /var/lib/dhcp/*', shell=True) + + def if_down_up(self): + names = [] + for nic in self.nics: + name = self.mac2Name.get(nic.mac.lower()) + names.append(name) + + for name in names: + logger.info('Bring down interface %s' % name) + subprocess.check_call('ifdown %s' % name, shell=True) + + self.clear_dhcp() + + for name in names: + logger.info('Bring up interface %s' % name) + subprocess.check_call('ifup %s' % name, shell=True) + + def configure(self): + """ + Configure the /etc/network/intefaces + Make a back up of the original + """ + containingDir = '/etc/network' + + interfaceFile = os.path.join(containingDir, 'interfaces') + originalFile = os.path.join(containingDir, + 'interfaces.before_vmware_customization') + + if not os.path.exists(originalFile) and os.path.exists(interfaceFile): + os.rename(interfaceFile, originalFile) + + lines = self.generate() + with open(interfaceFile, 'w') as fp: + for line in lines: + fp.write('%s\n' % line) + + self.if_down_up() diff --git a/cloudinit/sources/helpers/vmware/imc/nic.py b/cloudinit/sources/helpers/vmware/imc/nic.py index a7594874..6628a3ec 100644 --- a/cloudinit/sources/helpers/vmware/imc/nic.py +++ b/cloudinit/sources/helpers/vmware/imc/nic.py @@ -47,21 +47,37 @@ class Nic(NicBase): @property def primary(self): - value = self._get('PRIMARY').lower() - return value == 'yes' or value == 'true' + value = self._get('PRIMARY') + if value: + value = value.lower() + return value == 'yes' or value == 'true' + else: + return False @property def onboot(self): - value = self._get('ONBOOT').lower() - return value == 'yes' or value == 'true' + value = self._get('ONBOOT') + if value: + value = value.lower() + return value == 'yes' or value == 'true' + else: + return False @property def bootProto(self): - return self._get('BOOTPROTO').lower() + value = self._get('BOOTPROTO') + if value: + return value.lower() + else: + return "" @property def ipv4_mode(self): - return self._get('IPv4_MODE').lower() + value = self._get('IPv4_MODE') + if value: + return value.lower() + else: + return "" @property def staticIpv4(self): -- cgit v1.2.3 From 0ce71cb8975e19677eea415101e15da5f4095cd5 Mon Sep 17 00:00:00 2001 From: Sankar Tanguturi Date: Tue, 16 Feb 2016 17:34:24 -0800 Subject: - Used proper 4 space indentations for config_nic.py and nic.py - Implemented the 'search_file' function using 'os.walk()' - Fixed few variable names. - Removed size() function in config_file.py - Updated the test_config_file.py to use len() instead of .size() --- cloudinit/sources/DataSourceOVF.py | 34 +- .../sources/helpers/vmware/imc/config_file.py | 4 - cloudinit/sources/helpers/vmware/imc/config_nic.py | 433 +++++++++++---------- cloudinit/sources/helpers/vmware/imc/nic.py | 20 +- tests/unittests/test_vmware_config_file.py | 4 +- 5 files changed, 238 insertions(+), 257 deletions(-) (limited to 'cloudinit/sources/helpers') diff --git a/cloudinit/sources/DataSourceOVF.py b/cloudinit/sources/DataSourceOVF.py index add7d243..6d3bf7bb 100644 --- a/cloudinit/sources/DataSourceOVF.py +++ b/cloudinit/sources/DataSourceOVF.py @@ -64,13 +64,12 @@ class DataSourceOVF(sources.DataSource): (seedfile, contents) = get_ovf_env(self.paths.seed_dir) dmi_info = dmi_data() - system_uuid = "" system_type = "" - if dmi_info is False: + if dmi_info is None: LOG.debug("No dmidata utility found") else: - system_uuid, system_type = tuple(dmi_info) + (_, system_type) = dmi_info if 'vmware' in system_type.lower(): LOG.debug("VMware Virtual Platform found") @@ -172,11 +171,11 @@ class DataSourceOVFNet(DataSourceOVF): self.supported_seed_starts = ("http://", "https://", "ftp://") -def wait_for_imc_cfg_file(directoryPath, filename, maxwait=180, naplen=5): +def wait_for_imc_cfg_file(dirpath, filename, maxwait=180, naplen=5): waited = 0 while waited < maxwait: - fileFullPath = search_file(directoryPath, filename) + fileFullPath = search_file(dirpath, filename) if fileFullPath: return fileFullPath time.sleep(naplen) @@ -357,28 +356,13 @@ def dmi_data(): return (sys_uuid.lower(), sys_type) -def search_file(directoryPath, filename): - if not directoryPath or not filename: +def search_file(dirpath, filename): + if not dirpath or not filename: return None - dirs = [] - - if os.path.isdir(directoryPath): - dirs.append(directoryPath) - - while dirs: - dir = dirs.pop() - children = [] - try: - children.extend(os.listdir(dir)) - except: - LOG.debug("Ignoring the error while searching the directory %s" % dir) - for child in children: - childFullPath = os.path.join(dir, child) - if os.path.isdir(childFullPath): - dirs.append(childFullPath) - elif child == filename: - return childFullPath + for root, dirs, files in os.walk(dirpath): + if filename in files: + return os.path.join(root, filename) return None diff --git a/cloudinit/sources/helpers/vmware/imc/config_file.py b/cloudinit/sources/helpers/vmware/imc/config_file.py index 7c47d14c..bb9fb7dc 100644 --- a/cloudinit/sources/helpers/vmware/imc/config_file.py +++ b/cloudinit/sources/helpers/vmware/imc/config_file.py @@ -61,10 +61,6 @@ class ConfigFile(ConfigSource, dict): self[key] = val - def size(self): - """Return the number of properties present.""" - return len(self) - def _loadConfigFile(self, filename): """ Parses properties from the specified config file. diff --git a/cloudinit/sources/helpers/vmware/imc/config_nic.py b/cloudinit/sources/helpers/vmware/imc/config_nic.py index 8e2fc5d3..d79e6936 100644 --- a/cloudinit/sources/helpers/vmware/imc/config_nic.py +++ b/cloudinit/sources/helpers/vmware/imc/config_nic.py @@ -26,221 +26,222 @@ logger = logging.getLogger(__name__) class NicConfigurator: - def __init__(self, nics): - """ - Initialize the Nic Configurator - @param nics (list) an array of nics to configure - """ - self.nics = nics - self.mac2Name = {} - self.ipv4PrimaryGateway = None - self.ipv6PrimaryGateway = None - self.find_devices() - self._primaryNic = self.get_primary_nic() - - def get_primary_nic(self): - """ - Retrieve the primary nic if it exists - @return (NicBase): the primary nic if exists, None otherwise - """ - primaryNic = None - - for nic in self.nics: - if nic.primary: - if primaryNic: - raise Exception('There can only be one primary nic', - primaryNic.mac, nic.mac) + def __init__(self, nics): + """ + Initialize the Nic Configurator + @param nics (list) an array of nics to configure + """ + self.nics = nics + self.mac2Name = {} + self.ipv4PrimaryGateway = None + self.ipv6PrimaryGateway = None + self.find_devices() + self._primaryNic = self.get_primary_nic() + + def get_primary_nic(self): + """ + Retrieve the primary nic if it exists + @return (NicBase): the primary nic if exists, None otherwise + """ + primaryNic = None + + for nic in self.nics: + if nic.primary: + if primaryNic: + raise Exception('There can only be one primary nic', + primaryNic.mac, nic.mac) primaryNic = nic - return primaryNic - - def find_devices(self): - """ - Create the mac2Name dictionary - The mac address(es) are in the lower case - """ - cmd = 'ip addr show' - outText = subprocess.check_output(cmd, shell=True).decode() - sections = re.split(r'\n\d+: ', '\n' + outText)[1:] - - macPat = r'link/ether (([0-9A-Fa-f]{2}[:]){5}([0-9A-Fa-f]{2}))' - for section in sections: - matcher = re.search(macPat, section) - if not matcher: # Only keep info about nics - continue - mac = matcher.group(1).lower() - name = section.split(':', 1)[0] - self.mac2Name[mac] = name - - def gen_one_nic(self, nic): - """ - Return the lines needed to configure a nic - @return (str list): the string list to configure the nic - @param nic (NicBase): the nic to configure - """ - lines = [] - name = self.mac2Name.get(nic.mac.lower()) - if not name: - raise ValueError('No known device has MACADDR: %s' % nic.mac) - - if nic.onboot: - lines.append('auto %s' % name) - - # Customize IPv4 - lines.extend(self.gen_ipv4(name, nic)) - - # Customize IPv6 - lines.extend(self.gen_ipv6(name, nic)) - - lines.append('') - - return lines - - def gen_ipv4(self, name, nic): - """ - Return the lines needed to configure the IPv4 setting of a nic - @return (str list): the string list to configure the gateways - @param name (str): name of the nic - @param nic (NicBase): the nic to configure - """ - lines = [] - - bootproto = nic.bootProto.lower() - if nic.ipv4_mode.lower() == 'disabled': - bootproto = 'manual' - lines.append('iface %s inet %s' % (name, bootproto)) - - if bootproto != 'static': - return lines - - # Static Ipv4 - v4 = nic.staticIpv4 - if v4.ip: - lines.append(' address %s' % v4.ip) - if v4.netmask: - lines.append(' netmask %s' % v4.netmask) - - # Add the primary gateway - if nic.primary and v4.gateways: - self.ipv4PrimaryGateway = v4.gateways[0] - lines.append(' gateway %s metric 0' % self.ipv4PrimaryGateway) - return lines - - # Add routes if there is no primary nic - if not self._primaryNic: - lines.extend(self.gen_ipv4_route(nic, v4.gateways)) - - return lines - - def gen_ipv4_route(self, nic, gateways): - """ - Return the lines needed to configure additional Ipv4 route - @return (str list): the string list to configure the gateways - @param nic (NicBase): the nic to configure - @param gateways (str list): the list of gateways - """ - lines = [] - - for gateway in gateways: - lines.append(' up route add default gw %s metric 10000' % gateway) - - return lines - - def gen_ipv6(self, name, nic): - """ - Return the lines needed to configure the gateways for a nic - @return (str list): the string list to configure the gateways - @param name (str): name of the nic - @param nic (NicBase): the nic to configure - """ - lines = [] - - if not nic.staticIpv6: - return lines - - # Static Ipv6 - addrs = nic.staticIpv6 - lines.append('iface %s inet6 static' % name) - lines.append(' address %s' % addrs[0].ip) - lines.append(' netmask %s' % addrs[0].netmask) - - for addr in addrs[1:]: - lines.append(' up ifconfig %s inet6 add %s/%s' % (name, addr.ip, - addr.netmask)) - # Add the primary gateway - if nic.primary: - for addr in addrs: - if addr.gateway: - self.ipv6PrimaryGateway = addr.gateway - lines.append(' gateway %s' % self.ipv6PrimaryGateway) - return lines - - # Add routes if there is no primary nic - if not self._primaryNic: - lines.extend(self._genIpv6Route(name, nic, addrs)) - - return lines - - def _genIpv6Route(self, name, nic, addrs): - lines = [] - - for addr in addrs: - lines.append(' up route -A inet6 add default gw %s metric 10000' % - addr.gateway) - - return lines - - def generate(self): - """Return the lines that is needed to configure the nics""" - lines = [] - lines.append('iface lo inet loopback') - lines.append('auto lo') - lines.append('') - - for nic in self.nics: - lines.extend(self.gen_one_nic(nic)) - - return lines - - def clear_dhcp(self): - logger.info('Clearing DHCP leases') - - subprocess.call('pkill dhclient', shell=True) - subprocess.check_call('rm -f /var/lib/dhcp/*', shell=True) - - def if_down_up(self): - names = [] - for nic in self.nics: - name = self.mac2Name.get(nic.mac.lower()) - names.append(name) - - for name in names: - logger.info('Bring down interface %s' % name) - subprocess.check_call('ifdown %s' % name, shell=True) - - self.clear_dhcp() - - for name in names: - logger.info('Bring up interface %s' % name) - subprocess.check_call('ifup %s' % name, shell=True) - - def configure(self): - """ - Configure the /etc/network/intefaces - Make a back up of the original - """ - containingDir = '/etc/network' - - interfaceFile = os.path.join(containingDir, 'interfaces') - originalFile = os.path.join(containingDir, - 'interfaces.before_vmware_customization') - - if not os.path.exists(originalFile) and os.path.exists(interfaceFile): - os.rename(interfaceFile, originalFile) - - lines = self.generate() - with open(interfaceFile, 'w') as fp: - for line in lines: - fp.write('%s\n' % line) - - self.if_down_up() + return primaryNic + + def find_devices(self): + """ + Create the mac2Name dictionary + The mac address(es) are in the lower case + """ + cmd = 'ip addr show' + outText = subprocess.check_output(cmd, shell=True).decode() + sections = re.split(r'\n\d+: ', '\n' + outText)[1:] + + macPat = r'link/ether (([0-9A-Fa-f]{2}[:]){5}([0-9A-Fa-f]{2}))' + for section in sections: + matcher = re.search(macPat, section) + if not matcher: # Only keep info about nics + continue + mac = matcher.group(1).lower() + name = section.split(':', 1)[0] + self.mac2Name[mac] = name + + def gen_one_nic(self, nic): + """ + Return the lines needed to configure a nic + @return (str list): the string list to configure the nic + @param nic (NicBase): the nic to configure + """ + lines = [] + name = self.mac2Name.get(nic.mac.lower()) + if not name: + raise ValueError('No known device has MACADDR: %s' % nic.mac) + + if nic.onboot: + lines.append('auto %s' % name) + + # Customize IPv4 + lines.extend(self.gen_ipv4(name, nic)) + + # Customize IPv6 + lines.extend(self.gen_ipv6(name, nic)) + + lines.append('') + + return lines + + def gen_ipv4(self, name, nic): + """ + Return the lines needed to configure the IPv4 setting of a nic + @return (str list): the string list to configure the gateways + @param name (str): name of the nic + @param nic (NicBase): the nic to configure + """ + lines = [] + + bootproto = nic.bootProto.lower() + if nic.ipv4_mode.lower() == 'disabled': + bootproto = 'manual' + lines.append('iface %s inet %s' % (name, bootproto)) + + if bootproto != 'static': + return lines + + # Static Ipv4 + v4 = nic.staticIpv4 + if v4.ip: + lines.append(' address %s' % v4.ip) + if v4.netmask: + lines.append(' netmask %s' % v4.netmask) + + # Add the primary gateway + if nic.primary and v4.gateways: + self.ipv4PrimaryGateway = v4.gateways[0] + lines.append(' gateway %s metric 0' % self.ipv4PrimaryGateway) + return lines + + # Add routes if there is no primary nic + if not self._primaryNic: + lines.extend(self.gen_ipv4_route(nic, v4.gateways)) + + return lines + + def gen_ipv4_route(self, nic, gateways): + """ + Return the lines needed to configure additional Ipv4 route + @return (str list): the string list to configure the gateways + @param nic (NicBase): the nic to configure + @param gateways (str list): the list of gateways + """ + lines = [] + + for gateway in gateways: + lines.append(' up route add default gw %s metric 10000' % + gateway) + + return lines + + def gen_ipv6(self, name, nic): + """ + Return the lines needed to configure the gateways for a nic + @return (str list): the string list to configure the gateways + @param name (str): name of the nic + @param nic (NicBase): the nic to configure + """ + lines = [] + + if not nic.staticIpv6: + return lines + + # Static Ipv6 + addrs = nic.staticIpv6 + lines.append('iface %s inet6 static' % name) + lines.append(' address %s' % addrs[0].ip) + lines.append(' netmask %s' % addrs[0].netmask) + + for addr in addrs[1:]: + lines.append(' up ifconfig %s inet6 add %s/%s' % (name, addr.ip, + addr.netmask)) + # Add the primary gateway + if nic.primary: + for addr in addrs: + if addr.gateway: + self.ipv6PrimaryGateway = addr.gateway + lines.append(' gateway %s' % self.ipv6PrimaryGateway) + return lines + + # Add routes if there is no primary nic + if not self._primaryNic: + lines.extend(self._genIpv6Route(name, nic, addrs)) + + return lines + + def _genIpv6Route(self, name, nic, addrs): + lines = [] + + for addr in addrs: + lines.append(' up route -A inet6 add default gw %s metric 10000' % + addr.gateway) + + return lines + + def generate(self): + """Return the lines that is needed to configure the nics""" + lines = [] + lines.append('iface lo inet loopback') + lines.append('auto lo') + lines.append('') + + for nic in self.nics: + lines.extend(self.gen_one_nic(nic)) + + return lines + + def clear_dhcp(self): + logger.info('Clearing DHCP leases') + + subprocess.call('pkill dhclient', shell=True) + subprocess.check_call('rm -f /var/lib/dhcp/*', shell=True) + + def if_down_up(self): + names = [] + for nic in self.nics: + name = self.mac2Name.get(nic.mac.lower()) + names.append(name) + + for name in names: + logger.info('Bring down interface %s' % name) + subprocess.check_call('ifdown %s' % name, shell=True) + + self.clear_dhcp() + + for name in names: + logger.info('Bring up interface %s' % name) + subprocess.check_call('ifup %s' % name, shell=True) + + def configure(self): + """ + Configure the /etc/network/intefaces + Make a back up of the original + """ + containingDir = '/etc/network' + + interfaceFile = os.path.join(containingDir, 'interfaces') + originalFile = os.path.join(containingDir, + 'interfaces.before_vmware_customization') + + if not os.path.exists(originalFile) and os.path.exists(interfaceFile): + os.rename(interfaceFile, originalFile) + + lines = self.generate() + with open(interfaceFile, 'w') as fp: + for line in lines: + fp.write('%s\n' % line) + + self.if_down_up() diff --git a/cloudinit/sources/helpers/vmware/imc/nic.py b/cloudinit/sources/helpers/vmware/imc/nic.py index 6628a3ec..b5d704ea 100644 --- a/cloudinit/sources/helpers/vmware/imc/nic.py +++ b/cloudinit/sources/helpers/vmware/imc/nic.py @@ -49,35 +49,35 @@ class Nic(NicBase): def primary(self): value = self._get('PRIMARY') if value: - value = value.lower() - return value == 'yes' or value == 'true' + value = value.lower() + return value == 'yes' or value == 'true' else: - return False + return False @property def onboot(self): value = self._get('ONBOOT') if value: - value = value.lower() - return value == 'yes' or value == 'true' + value = value.lower() + return value == 'yes' or value == 'true' else: - return False + return False @property def bootProto(self): value = self._get('BOOTPROTO') if value: - return value.lower() + return value.lower() else: - return "" + return "" @property def ipv4_mode(self): value = self._get('IPv4_MODE') if value: - return value.lower() + return value.lower() else: - return "" + return "" @property def staticIpv4(self): diff --git a/tests/unittests/test_vmware_config_file.py b/tests/unittests/test_vmware_config_file.py index 51166dd7..d5c7367b 100644 --- a/tests/unittests/test_vmware_config_file.py +++ b/tests/unittests/test_vmware_config_file.py @@ -36,12 +36,12 @@ class TestVmwareConfigFile(unittest.TestCase): cf.clear() - self.assertEqual(0, cf.size(), "clear size") + self.assertEqual(0, len(cf), "clear size") cf._insertKey(" PASSWORD|-PASS ", " foo ") cf._insertKey("BAR", " ") - self.assertEqual(2, cf.size(), "insert size") + self.assertEqual(2, len(cf), "insert size") self.assertEqual('foo', cf["PASSWORD|-PASS"], "password") self.assertTrue("PASSWORD|-PASS" in cf, "hasPassword") self.assertFalse(cf.should_keep_current_value("PASSWORD|-PASS"), -- cgit v1.2.3 From c5d2f79a982258d86181368b25ce6bc6638ef645 Mon Sep 17 00:00:00 2001 From: Sankar Tanguturi Date: Thu, 18 Feb 2016 18:31:07 -0800 Subject: - Removed dmi_data function. - Fixed few variable names. - Used util.subp methods for process related manipulations. --- cloudinit/sources/DataSourceOVF.py | 20 +++-------- cloudinit/sources/helpers/vmware/imc/config_nic.py | 40 +++++++++++----------- 2 files changed, 24 insertions(+), 36 deletions(-) (limited to 'cloudinit/sources/helpers') diff --git a/cloudinit/sources/DataSourceOVF.py b/cloudinit/sources/DataSourceOVF.py index 6d3bf7bb..72ba5aba 100644 --- a/cloudinit/sources/DataSourceOVF.py +++ b/cloudinit/sources/DataSourceOVF.py @@ -63,15 +63,11 @@ class DataSourceOVF(sources.DataSource): } (seedfile, contents) = get_ovf_env(self.paths.seed_dir) - dmi_info = dmi_data() - system_type = "" - if dmi_info is None: - LOG.debug("No dmidata utility found") - else: - (_, system_type) = dmi_info - - if 'vmware' in system_type.lower(): + system_type = util.read_dmi_data("system-product-name") + if system_type is None: + LOG.debug("No system-product-name found") + elif 'vmware' in system_type.lower(): LOG.debug("VMware Virtual Platform found") deployPkgPluginPath = search_file("/usr/lib/vmware-tools", "libdeployPkgPlugin.so") if deployPkgPluginPath: @@ -347,14 +343,6 @@ def get_properties(contents): return props -def dmi_data(): - sys_uuid = util.read_dmi_data("system-uuid") - sys_type = util.read_dmi_data("system-product-name") - - if not sys_uuid or not sys_type: - return None - - return (sys_uuid.lower(), sys_type) def search_file(dirpath, filename): if not dirpath or not filename: diff --git a/cloudinit/sources/helpers/vmware/imc/config_nic.py b/cloudinit/sources/helpers/vmware/imc/config_nic.py index d79e6936..172a1649 100644 --- a/cloudinit/sources/helpers/vmware/imc/config_nic.py +++ b/cloudinit/sources/helpers/vmware/imc/config_nic.py @@ -22,6 +22,8 @@ import os import subprocess import re +from cloudinit import util + logger = logging.getLogger(__name__) @@ -43,32 +45,30 @@ class NicConfigurator: Retrieve the primary nic if it exists @return (NicBase): the primary nic if exists, None otherwise """ - primaryNic = None - - for nic in self.nics: - if nic.primary: - if primaryNic: - raise Exception('There can only be one primary nic', - primaryNic.mac, nic.mac) - primaryNic = nic - - return primaryNic + primary_nics = [nic for nic in self.nics if nic.primary] + if not primary_nics: + return None + elif len(primary_nics) > 1: + raise Exception('There can only be one primary nic', + [nic.mac for nic in primary_nics]) + else: + return primary_nics[0] def find_devices(self): """ Create the mac2Name dictionary The mac address(es) are in the lower case """ - cmd = 'ip addr show' - outText = subprocess.check_output(cmd, shell=True).decode() - sections = re.split(r'\n\d+: ', '\n' + outText)[1:] + cmd = ['ip', 'addr', 'show'] + (output, err) = util.subp(cmd) + sections = re.split(r'\n\d+: ', '\n' + output)[1:] macPat = r'link/ether (([0-9A-Fa-f]{2}[:]){5}([0-9A-Fa-f]{2}))' for section in sections: - matcher = re.search(macPat, section) - if not matcher: # Only keep info about nics + match = re.search(macPat, section) + if not match: # Only keep info about nics continue - mac = matcher.group(1).lower() + mac = match.group(1).lower() name = section.split(':', 1)[0] self.mac2Name[mac] = name @@ -206,8 +206,8 @@ class NicConfigurator: def clear_dhcp(self): logger.info('Clearing DHCP leases') - subprocess.call('pkill dhclient', shell=True) - subprocess.check_call('rm -f /var/lib/dhcp/*', shell=True) + util.subp(["pkill", "dhclient"]) + util.subp(["rm", "-f", "/var/lib/dhcp/*"]) def if_down_up(self): names = [] @@ -217,13 +217,13 @@ class NicConfigurator: for name in names: logger.info('Bring down interface %s' % name) - subprocess.check_call('ifdown %s' % name, shell=True) + util.subp(["ifdown", "%s" % name]) self.clear_dhcp() for name in names: logger.info('Bring up interface %s' % name) - subprocess.check_call('ifup %s' % name, shell=True) + util.subp(["ifup", "%s" % name]) def configure(self): """ -- cgit v1.2.3 From c496b6a11d504ef62371cb5e03ac80b4ceb37540 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Thu, 3 Mar 2016 12:20:48 -0500 Subject: run pyflakes in more places, fix fallout this makes 'make' run pyflakes, so failures there will stop a build. also adds it to tox. --- Makefile | 6 ++++-- cloudinit/sources/DataSourceOVF.py | 3 ++- cloudinit/sources/helpers/vmware/imc/config_nic.py | 1 - cloudinit/util.py | 2 +- tests/unittests/test_datasource/test_azure_helper.py | 2 -- tests/unittests/test_datasource/test_smartos.py | 1 - tests/unittests/test_handler/test_handler_power_state.py | 2 +- tox.ini | 6 +++++- 8 files changed, 13 insertions(+), 10 deletions(-) (limited to 'cloudinit/sources/helpers') diff --git a/Makefile b/Makefile index bb0c5253..8987d51c 100644 --- a/Makefile +++ b/Makefile @@ -14,13 +14,15 @@ ifeq ($(distro),) distro = redhat endif -all: test check_version +all: check + +check: test check_version pyflakes pep8: @$(CWD)/tools/run-pep8 $(PY_FILES) pyflakes: - @$(CWD)/tools/tox-venv py34 pyflakes $(PY_FILES) + @pyflakes $(PY_FILES) pip-requirements: @echo "Installing cloud-init dependencies..." diff --git a/cloudinit/sources/DataSourceOVF.py b/cloudinit/sources/DataSourceOVF.py index 72ba5aba..d12601a4 100644 --- a/cloudinit/sources/DataSourceOVF.py +++ b/cloudinit/sources/DataSourceOVF.py @@ -90,7 +90,8 @@ class DataSourceOVF(sources.DataSource): nicConfigurator.configure() vmwarePlatformFound = True except Exception as inst: - LOG.debug("Error while parsing the Customization Config File") + LOG.debug("Error while parsing the Customization " + "Config File: %s", inst) finally: dirPath = os.path.dirname(vmwareImcConfigFilePath) shutil.rmtree(dirPath) diff --git a/cloudinit/sources/helpers/vmware/imc/config_nic.py b/cloudinit/sources/helpers/vmware/imc/config_nic.py index 172a1649..6d721134 100644 --- a/cloudinit/sources/helpers/vmware/imc/config_nic.py +++ b/cloudinit/sources/helpers/vmware/imc/config_nic.py @@ -19,7 +19,6 @@ import logging import os -import subprocess import re from cloudinit import util diff --git a/cloudinit/util.py b/cloudinit/util.py index 45d49e66..0a639bb9 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -2147,7 +2147,7 @@ def _read_dmi_syspath(key): LOG.debug("dmi data %s returned %s", dmi_key_path, key_data) return key_data.strip() - except Exception as e: + except Exception: logexc(LOG, "failed read of %s", dmi_key_path) return None diff --git a/tests/unittests/test_datasource/test_azure_helper.py b/tests/unittests/test_datasource/test_azure_helper.py index 8dbdfb0b..1134199b 100644 --- a/tests/unittests/test_datasource/test_azure_helper.py +++ b/tests/unittests/test_datasource/test_azure_helper.py @@ -1,6 +1,4 @@ import os -import struct -import unittest from cloudinit.sources.helpers import azure as azure_helper from ..helpers import TestCase diff --git a/tests/unittests/test_datasource/test_smartos.py b/tests/unittests/test_datasource/test_smartos.py index 1235436d..ccb9f080 100644 --- a/tests/unittests/test_datasource/test_smartos.py +++ b/tests/unittests/test_datasource/test_smartos.py @@ -31,7 +31,6 @@ import shutil import stat import tempfile import uuid -import unittest from binascii import crc32 import serial diff --git a/tests/unittests/test_handler/test_handler_power_state.py b/tests/unittests/test_handler/test_handler_power_state.py index 5687b10d..cd376e9c 100644 --- a/tests/unittests/test_handler/test_handler_power_state.py +++ b/tests/unittests/test_handler/test_handler_power_state.py @@ -107,7 +107,7 @@ def check_lps_ret(psc_return, mode=None): if 'shutdown' not in psc_return[0][0]: errs.append("string 'shutdown' not in cmd") - if 'condition' is None: + if condition is None: errs.append("condition was not returned") if mode is not None: diff --git a/tox.ini b/tox.ini index b72df0c9..fd65f6ef 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py27,py3 +envlist = py27,py3,pyflakes recreate = True [testenv] @@ -10,6 +10,10 @@ deps = -r{toxinidir}/test-requirements.txt [testenv:py3] basepython = python3 +[testenv:pyflakes] +basepython = python3 +commands = {envpython} -m pyflakes {posargs:cloudinit/ tests/ tools/} + # https://github.com/gabrielfalcao/HTTPretty/issues/223 setenv = LC_ALL = en_US.utf-8 -- cgit v1.2.3 From ef7368ef61c47fbb0bc03e6e7a5bc4571d492baf Mon Sep 17 00:00:00 2001 From: Sankar Tanguturi Date: Tue, 8 Mar 2016 12:41:08 -0800 Subject: - Ignored return code 1 for 'pkill' command in config_nic.py - Added few utility functions to report events to the underlying VMware Virtualization platform - Re-factored code little bit. - Executed ./tools/run-pep8 and no pep8 errors were reported. --- cloudinit/sources/DataSourceOVF.py | 40 +++++++++-- cloudinit/sources/helpers/vmware/imc/config_nic.py | 14 ++-- .../sources/helpers/vmware/imc/guestcust_error.py | 24 +++++++ .../sources/helpers/vmware/imc/guestcust_event.py | 27 ++++++++ .../sources/helpers/vmware/imc/guestcust_state.py | 25 +++++++ .../sources/helpers/vmware/imc/guestcust_util.py | 79 ++++++++++++++++++++++ 6 files changed, 198 insertions(+), 11 deletions(-) create mode 100644 cloudinit/sources/helpers/vmware/imc/guestcust_error.py create mode 100644 cloudinit/sources/helpers/vmware/imc/guestcust_event.py create mode 100644 cloudinit/sources/helpers/vmware/imc/guestcust_state.py create mode 100644 cloudinit/sources/helpers/vmware/imc/guestcust_util.py (limited to 'cloudinit/sources/helpers') diff --git a/cloudinit/sources/DataSourceOVF.py b/cloudinit/sources/DataSourceOVF.py index d07f6219..0fbdf0b8 100644 --- a/cloudinit/sources/DataSourceOVF.py +++ b/cloudinit/sources/DataSourceOVF.py @@ -34,6 +34,14 @@ from cloudinit import util from cloudinit.sources.helpers.vmware.imc.config import Config from cloudinit.sources.helpers.vmware.imc.config_file import ConfigFile from cloudinit.sources.helpers.vmware.imc.config_nic import NicConfigurator +from cloudinit.sources.helpers.vmware.imc.guestcust_event import \ + GuestCustEventEnum +from cloudinit.sources.helpers.vmware.imc.guestcust_state import \ + GuestCustStateEnum +from cloudinit.sources.helpers.vmware.imc.guestcust_error import \ + GuestCustErrorEnum +from cloudinit.sources.helpers.vmware.imc.guestcust_util import \ + set_customization_status LOG = logging.getLogger(__name__) @@ -74,6 +82,9 @@ class DataSourceOVF(sources.DataSource): True): deployPkgPluginPath = search_file("/usr/lib/vmware-tools", "libdeployPkgPlugin.so") + if not deployPkgPluginPath: + deployPkgPluginPath = search_file("/usr/lib/open-vm-tools", + "libdeployPkgPlugin.so") if deployPkgPluginPath: vmwareImcConfigFilePath = util.log_time(logfunc=LOG.debug, msg="waiting for configuration file", @@ -93,14 +104,33 @@ class DataSourceOVF(sources.DataSource): cf = ConfigFile(vmwareImcConfigFilePath) conf = Config(cf) (md, ud, cfg) = read_vmware_imc(conf) - nicConfigurator = NicConfigurator(conf.nics) - nicConfigurator.configure() - vmwarePlatformFound = True - except Exception as inst: - LOG.debug("Error while parsing the Customization Config File") + except Exception as e: + LOG.debug("Error parsing the customization Config File") + LOG.exception(e) + set_customization_status( + GuestCustStateEnum.GUESTCUST_STATE_RUNNING, + GuestCustEventEnum.GUESTCUST_EVENT_CUSTOMIZE_FAILED) + return False finally: dirPath = os.path.dirname(vmwareImcConfigFilePath) shutil.rmtree(dirPath) + + try: + LOG.debug("Applying the Network customization") + nicConfigurator = NicConfigurator(conf.nics) + nicConfigurator.configure() + except Exception as e: + LOG.debug("Error applying the Network Configuration") + LOG.exception(e) + set_customization_status( + GuestCustStateEnum.GUESTCUST_STATE_RUNNING, + GuestCustEventEnum.GUESTCUST_EVENT_NETWORK_SETUP_FAILED) + return False + + vmwarePlatformFound = True + set_customization_status( + GuestCustStateEnum.GUESTCUST_STATE_DONE, + GuestCustErrorEnum.GUESTCUST_ERROR_SUCCESS) elif seedfile: # Found a seed dir seed = os.path.join(self.paths.seed_dir, seedfile) diff --git a/cloudinit/sources/helpers/vmware/imc/config_nic.py b/cloudinit/sources/helpers/vmware/imc/config_nic.py index 172a1649..42fbcc7e 100644 --- a/cloudinit/sources/helpers/vmware/imc/config_nic.py +++ b/cloudinit/sources/helpers/vmware/imc/config_nic.py @@ -47,12 +47,12 @@ class NicConfigurator: """ primary_nics = [nic for nic in self.nics if nic.primary] if not primary_nics: - return None + return None elif len(primary_nics) > 1: - raise Exception('There can only be one primary nic', + raise Exception('There can only be one primary nic', [nic.mac for nic in primary_nics]) else: - return primary_nics[0] + return primary_nics[0] def find_devices(self): """ @@ -186,8 +186,9 @@ class NicConfigurator: lines = [] for addr in addrs: - lines.append(' up route -A inet6 add default gw %s metric 10000' % - addr.gateway) + lines.append( + ' up route -A inet6 add default gw %s metric 10000' % + addr.gateway) return lines @@ -206,7 +207,8 @@ class NicConfigurator: def clear_dhcp(self): logger.info('Clearing DHCP leases') - util.subp(["pkill", "dhclient"]) + # Ignore the return code 1. + util.subp(["pkill", "dhclient"], rcs=[0, 1]) util.subp(["rm", "-f", "/var/lib/dhcp/*"]) def if_down_up(self): diff --git a/cloudinit/sources/helpers/vmware/imc/guestcust_error.py b/cloudinit/sources/helpers/vmware/imc/guestcust_error.py new file mode 100644 index 00000000..1b04161f --- /dev/null +++ b/cloudinit/sources/helpers/vmware/imc/guestcust_error.py @@ -0,0 +1,24 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2016 Canonical Ltd. +# Copyright (C) 2016 VMware Inc. +# +# Author: Sankar Tanguturi +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + + +class GuestCustErrorEnum: + """Specifies different errors of Guest Customization engine""" + + GUESTCUST_ERROR_SUCCESS = 0 diff --git a/cloudinit/sources/helpers/vmware/imc/guestcust_event.py b/cloudinit/sources/helpers/vmware/imc/guestcust_event.py new file mode 100644 index 00000000..fc22568f --- /dev/null +++ b/cloudinit/sources/helpers/vmware/imc/guestcust_event.py @@ -0,0 +1,27 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2016 Canonical Ltd. +# Copyright (C) 2016 VMware Inc. +# +# Author: Sankar Tanguturi +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + + +class GuestCustEventEnum: + """Specifies different types of Guest Customization Events""" + + GUESTCUST_EVENT_CUSTOMIZE_FAILED = 100 + GUESTCUST_EVENT_NETWORK_SETUP_FAILED = 101 + GUESTCUST_EVENT_ENABLE_NICS = 103 + GUESTCUST_EVENT_QUERY_NICS = 104 diff --git a/cloudinit/sources/helpers/vmware/imc/guestcust_state.py b/cloudinit/sources/helpers/vmware/imc/guestcust_state.py new file mode 100644 index 00000000..f255be5f --- /dev/null +++ b/cloudinit/sources/helpers/vmware/imc/guestcust_state.py @@ -0,0 +1,25 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2016 Canonical Ltd. +# Copyright (C) 2016 VMware Inc. +# +# Author: Sankar Tanguturi +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + + +class GuestCustStateEnum: + """Specifies different states of Guest Customization engine""" + + GUESTCUST_STATE_RUNNING = 4 + GUESTCUST_STATE_DONE = 5 diff --git a/cloudinit/sources/helpers/vmware/imc/guestcust_util.py b/cloudinit/sources/helpers/vmware/imc/guestcust_util.py new file mode 100644 index 00000000..2466a47e --- /dev/null +++ b/cloudinit/sources/helpers/vmware/imc/guestcust_util.py @@ -0,0 +1,79 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2016 Canonical Ltd. +# Copyright (C) 2016 VMware Inc. +# +# Author: Sankar Tanguturi +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import logging +import os + +from cloudinit import util + + +logger = logging.getLogger(__name__) + + +CLOUDINIT_LOG_FILE = "/var/log/cloud-init.log" + + +# This will send a RPC command to the underlying +# VMware Virtualization Platform. +def send_rpc(rpc): + if not rpc: + return None + + rc = 1 + output = "Error sending the RPC command" + + try: + logger.debug("Sending RPC command: %s", rpc) + (rc, output) = util.subp(["vmware-rpctool", rpc], rcs=[0]) + except Exception as e: + logger.debug("Failed to send RPC command") + logger.exception(e) + + return (rc, output) + + +# This will send the customization status to the +# underlying VMware Virtualization Platform. +def set_customization_status(custstate, custerror, errormessage=None): + message = "" + + if errormessage: + message = CLOUDINIT_LOG_FILE + "@" + errormessage + else: + message = CLOUDINIT_LOG_FILE + + rpc = "deployPkg.update.state %d %d %s" % (custstate, custerror, message) + (rc, output) = send_rpc(rpc) + + +# This will read the file nics.txt in the specified directory +# and return the content +def get_nics_to_enable(dirpath): + if not dirpath: + return None + + NICS_SIZE = 1024 + nicsfilepath = os.path.join(dirpath, "nics.txt") + if not os.path.exists(nicsfilepath): + return None + + with open(nicsfilepath, 'r') as fp: + nics = fp.read(NICS_SIZE) + + return nics -- cgit v1.2.3 From a6e0922a4d34ede6df000dd8fc4bb3531218d69f Mon Sep 17 00:00:00 2001 From: Sankar Tanguturi Date: Wed, 9 Mar 2016 16:02:34 -0800 Subject: - Fixed few issues with return values form util.subp() - Added a new utility method to send a RPC for enabling NICS - Modified DataSourceOVF.py to enable nics. - Executed ./tools/run-pep8 and no issues were reported. --- cloudinit/sources/DataSourceOVF.py | 11 +++- .../sources/helpers/vmware/imc/guestcust_util.py | 60 ++++++++++++++++++++-- 2 files changed, 64 insertions(+), 7 deletions(-) (limited to 'cloudinit/sources/helpers') diff --git a/cloudinit/sources/DataSourceOVF.py b/cloudinit/sources/DataSourceOVF.py index 0fbdf0b8..bc13b71a 100644 --- a/cloudinit/sources/DataSourceOVF.py +++ b/cloudinit/sources/DataSourceOVF.py @@ -40,8 +40,11 @@ from cloudinit.sources.helpers.vmware.imc.guestcust_state import \ GuestCustStateEnum from cloudinit.sources.helpers.vmware.imc.guestcust_error import \ GuestCustErrorEnum -from cloudinit.sources.helpers.vmware.imc.guestcust_util import \ - set_customization_status +from cloudinit.sources.helpers.vmware.imc.guestcust_util import ( + set_customization_status, + get_nics_to_enable, + enable_nics +) LOG = logging.getLogger(__name__) @@ -100,10 +103,13 @@ class DataSourceOVF(sources.DataSource): LOG.debug("Customization for VMware platform is disabled.") if vmwareImcConfigFilePath: + nics = "" try: cf = ConfigFile(vmwareImcConfigFilePath) conf = Config(cf) (md, ud, cfg) = read_vmware_imc(conf) + dirpath = os.path.dirname(vmwareImcConfigFilePath) + nics = get_nics_to_enable(dirpath) except Exception as e: LOG.debug("Error parsing the customization Config File") LOG.exception(e) @@ -128,6 +134,7 @@ class DataSourceOVF(sources.DataSource): return False vmwarePlatformFound = True + enable_nics(nics) set_customization_status( GuestCustStateEnum.GUESTCUST_STATE_DONE, GuestCustErrorEnum.GUESTCUST_ERROR_SUCCESS) diff --git a/cloudinit/sources/helpers/vmware/imc/guestcust_util.py b/cloudinit/sources/helpers/vmware/imc/guestcust_util.py index 2466a47e..b8c58f1e 100644 --- a/cloudinit/sources/helpers/vmware/imc/guestcust_util.py +++ b/cloudinit/sources/helpers/vmware/imc/guestcust_util.py @@ -19,14 +19,20 @@ import logging import os +import time from cloudinit import util +from .guestcust_state import GuestCustStateEnum +from .guestcust_error import GuestCustErrorEnum +from .guestcust_event import GuestCustEventEnum logger = logging.getLogger(__name__) CLOUDINIT_LOG_FILE = "/var/log/cloud-init.log" +QUERY_NICS_SUPPORTED = "queryNicsSupported" +NICS_STATUS_CONNECTED = "connected" # This will send a RPC command to the underlying @@ -35,17 +41,20 @@ def send_rpc(rpc): if not rpc: return None - rc = 1 - output = "Error sending the RPC command" + out = "" + err = "Error sending the RPC command" try: logger.debug("Sending RPC command: %s", rpc) - (rc, output) = util.subp(["vmware-rpctool", rpc], rcs=[0]) + (out, err) = util.subp(["vmware-rpctool", rpc], rcs=[0]) + # Remove the trailing newline in the output. + if out: + out = out.rstrip() except Exception as e: logger.debug("Failed to send RPC command") logger.exception(e) - return (rc, output) + return (out, err) # This will send the customization status to the @@ -59,7 +68,8 @@ def set_customization_status(custstate, custerror, errormessage=None): message = CLOUDINIT_LOG_FILE rpc = "deployPkg.update.state %d %d %s" % (custstate, custerror, message) - (rc, output) = send_rpc(rpc) + (out, err) = send_rpc(rpc) + return (out, err) # This will read the file nics.txt in the specified directory @@ -77,3 +87,43 @@ def get_nics_to_enable(dirpath): nics = fp.read(NICS_SIZE) return nics + + +# This will send a RPC command to the underlying VMware Virtualization platform +# and enable nics. +def enable_nics(nics): + if not nics: + logger.warning("No Nics found") + return + + enableNicsWaitRetries = 5 + enableNicsWaitCount = 5 + enableNicsWaitSeconds = 1 + + for attempt in range(0, enableNicsWaitRetries): + logger.debug("Trying to connect interfaces, attempt %d", attempt) + (out, err) = set_customization_status( + GuestCustStateEnum.GUESTCUST_STATE_RUNNING, + GuestCustEventEnum.GUESTCUST_EVENT_ENABLE_NICS, + nics) + if not out: + time.sleep(enableNicsWaitCount * enableNicsWaitSeconds) + continue + + if out != QUERY_NICS_SUPPORTED: + logger.warning("NICS connection status query is not supported") + return + + for count in range(0, enableNicsWaitCount): + (out, err) = set_customization_status( + GuestCustStateEnum.GUESTCUST_STATE_RUNNING, + GuestCustEventEnum.GUESTCUST_EVENT_QUERY_NICS, + nics) + if out and out == NICS_STATUS_CONNECTED: + logger.info("NICS are connected on %d second", count) + return + + time.sleep(enableNicsWaitSeconds) + + logger.warning("Can't connect network interfaces after %d attempts", + enableNicsWaitRetries) -- cgit v1.2.3 From 03998cd336b3906dc1eb675fff1ddeb1272668d3 Mon Sep 17 00:00:00 2001 From: Sankar Tanguturi Date: Fri, 11 Mar 2016 13:29:28 -0800 Subject: - Fixed few pep8 and flake8 issues. - Changed the really long 'from ... import ...' statements. --- cloudinit/sources/DataSourceOVF.py | 21 +++++++++------------ .../sources/helpers/vmware/imc/guestcust_util.py | 13 ++++++------- 2 files changed, 15 insertions(+), 19 deletions(-) (limited to 'cloudinit/sources/helpers') diff --git a/cloudinit/sources/DataSourceOVF.py b/cloudinit/sources/DataSourceOVF.py index fec13b93..65cefc48 100644 --- a/cloudinit/sources/DataSourceOVF.py +++ b/cloudinit/sources/DataSourceOVF.py @@ -31,16 +31,13 @@ import time from cloudinit import log as logging from cloudinit import sources from cloudinit import util -from cloudinit.sources.helpers.vmware.imc.config import Config -from cloudinit.sources.helpers.vmware.imc.config_file import ConfigFile -from cloudinit.sources.helpers.vmware.imc.config_nic import NicConfigurator -from cloudinit.sources.helpers.vmware.imc.guestcust_event import \ - GuestCustEventEnum -from cloudinit.sources.helpers.vmware.imc.guestcust_state import \ - GuestCustStateEnum -from cloudinit.sources.helpers.vmware.imc.guestcust_error import \ - GuestCustErrorEnum -from cloudinit.sources.helpers.vmware.imc.guestcust_util import ( +from .helpers.vmware.imc.config import Config +from .helpers.vmware.imc.config_file import ConfigFile +from .helpers.vmware.imc.config_nic import NicConfigurator +from .helpers.vmware.imc.guestcust_event import GuestCustEventEnum +from .helpers.vmware.imc.guestcust_state import GuestCustStateEnum +from .helpers.vmware.imc.guestcust_error import GuestCustErrorEnum +from .helpers.vmware.imc.guestcust_util import ( set_customization_status, get_nics_to_enable, enable_nics @@ -135,8 +132,8 @@ class DataSourceOVF(sources.DataSource): vmwarePlatformFound = True enable_nics(nics) set_customization_status( - GuestCustStateEnum.GUESTCUST_STATE_DONE, - GuestCustErrorEnum.GUESTCUST_ERROR_SUCCESS) + GuestCustStateEnum.GUESTCUST_STATE_DONE, + GuestCustErrorEnum.GUESTCUST_ERROR_SUCCESS) elif seedfile: # Found a seed dir seed = os.path.join(self.paths.seed_dir, seedfile) diff --git a/cloudinit/sources/helpers/vmware/imc/guestcust_util.py b/cloudinit/sources/helpers/vmware/imc/guestcust_util.py index b8c58f1e..d39f0a65 100644 --- a/cloudinit/sources/helpers/vmware/imc/guestcust_util.py +++ b/cloudinit/sources/helpers/vmware/imc/guestcust_util.py @@ -24,7 +24,6 @@ import time from cloudinit import util from .guestcust_state import GuestCustStateEnum -from .guestcust_error import GuestCustErrorEnum from .guestcust_event import GuestCustEventEnum logger = logging.getLogger(__name__) @@ -103,9 +102,9 @@ def enable_nics(nics): for attempt in range(0, enableNicsWaitRetries): logger.debug("Trying to connect interfaces, attempt %d", attempt) (out, err) = set_customization_status( - GuestCustStateEnum.GUESTCUST_STATE_RUNNING, - GuestCustEventEnum.GUESTCUST_EVENT_ENABLE_NICS, - nics) + GuestCustStateEnum.GUESTCUST_STATE_RUNNING, + GuestCustEventEnum.GUESTCUST_EVENT_ENABLE_NICS, + nics) if not out: time.sleep(enableNicsWaitCount * enableNicsWaitSeconds) continue @@ -116,9 +115,9 @@ def enable_nics(nics): for count in range(0, enableNicsWaitCount): (out, err) = set_customization_status( - GuestCustStateEnum.GUESTCUST_STATE_RUNNING, - GuestCustEventEnum.GUESTCUST_EVENT_QUERY_NICS, - nics) + GuestCustStateEnum.GUESTCUST_STATE_RUNNING, + GuestCustEventEnum.GUESTCUST_EVENT_QUERY_NICS, + nics) if out and out == NICS_STATUS_CONNECTED: logger.info("NICS are connected on %d second", count) return -- cgit v1.2.3 From 2b85dabb802766e0b3b1949d744c8860c0cb838a Mon Sep 17 00:00:00 2001 From: Ryan Harper Date: Wed, 23 Mar 2016 11:05:22 -0500 Subject: configdata: parse and convert openstack network_data json to network_config --- cloudinit/net/__init__.py | 34 +++-- cloudinit/net/network_state.py | 45 ++++++- cloudinit/sources/DataSourceConfigDrive.py | 137 +++++++++++++++++++++ cloudinit/sources/helpers/openstack.py | 34 ++++- .../unittests/test_datasource/test_configdrive.py | 50 +++++++- 5 files changed, 289 insertions(+), 11 deletions(-) (limited to 'cloudinit/sources/helpers') diff --git a/cloudinit/net/__init__.py b/cloudinit/net/__init__.py index ae7b1c04..76cd4e8b 100644 --- a/cloudinit/net/__init__.py +++ b/cloudinit/net/__init__.py @@ -336,7 +336,7 @@ def iface_add_attrs(iface): 'index', 'subnets', ] - if iface['type'] not in ['bond', 'bridge']: + if iface['type'] not in ['bond', 'bridge', 'vlan']: ignore_map.append('mac_address') for key, value in iface.items(): @@ -348,19 +348,34 @@ def iface_add_attrs(iface): return content -def render_route(route): - content = "up route add" +def render_route(route, indent=""): + content = "" + up = indent + "post-up route add" + down = indent + "pre-down route del" + eol = " || true\n" mapping = { 'network': '-net', 'netmask': 'netmask', 'gateway': 'gw', 'metric': 'metric', } - for k in ['network', 'netmask', 'gateway', 'metric']: - if k in route: - content += " %s %s" % (mapping[k], route[k]) + if route['network'] == '0.0.0.0' and route['netmask'] == '0.0.0.0': + default_gw = " default gw %s" % route['gateway'] + content += up + default_gw + eol + content += down + default_gw + eol + elif route['network'] == '::' and route['netmask'] == 0: + # ipv6! + default_gw = " -A inet6 default gw %s" % route['gateway'] + content += up + default_gw + eol + content += down + default_gw + eol + else: + route_line = "" + for k in ['network', 'netmask', 'gateway', 'metric']: + if k in route: + route_line += " %s %s" % (mapping[k], route[k]) + content += up + route_line + eol + content += down + route_line + eol - content += '\n' return content @@ -384,6 +399,7 @@ def render_interfaces(network_state): if len(value): content += " dns-{} {}\n".format(dnskey, " ".join(value)) + content += "\n" for iface in sorted(interfaces.values(), key=lambda k: (order[k['type']], k['name'])): content += "auto {name}\n".format(**iface) @@ -409,6 +425,8 @@ def render_interfaces(network_state): content += iface_add_subnet(iface, subnet) content += iface_add_attrs(iface) + for route in subnet.get('routes', []): + content += render_route(route, indent=" ") content += "\n" else: content += "iface {name} {inet} {mode}\n".format(**iface) @@ -419,7 +437,7 @@ def render_interfaces(network_state): content += render_route(route) # global replacements until v2 format - content = content.replace('mac_address', 'hwaddress') + content = content.replace('mac_address', 'hwaddress ether') return content diff --git a/cloudinit/net/network_state.py b/cloudinit/net/network_state.py index df04c526..e32d2cdf 100644 --- a/cloudinit/net/network_state.py +++ b/cloudinit/net/network_state.py @@ -124,6 +124,17 @@ class NetworkState: iface = interfaces.get(command['name'], {}) for param, val in command.get('params', {}).items(): iface.update({param: val}) + + # convert subnet ipv6 netmask to cidr as needed + subnets = command.get('subnets') + if subnets: + for subnet in subnets: + if subnet['type'] == 'static': + if 'netmask' in subnet and ':' in subnet['address']: + subnet['netmask'] = mask2cidr(subnet['netmask']) + for route in subnet.get('routes', []): + if 'netmask' in route: + route['netmask'] = mask2cidr(route['netmask']) iface.update({ 'name': command.get('name'), 'type': command.get('type'), @@ -133,7 +144,7 @@ class NetworkState: 'mtu': command.get('mtu'), 'address': None, 'gateway': None, - 'subnets': command.get('subnets'), + 'subnets': subnets, }) self.network_state['interfaces'].update({command.get('name'): iface}) self.dump_network_state() @@ -144,6 +155,7 @@ class NetworkState: iface eth0.222 inet static address 10.10.10.1 netmask 255.255.255.0 + hwaddress ether BC:76:4E:06:96:B3 vlan-raw-device eth0 ''' required_keys = [ @@ -335,6 +347,37 @@ def cidr2mask(cidr): return ".".join([str(x) for x in mask]) +def ipv4mask2cidr(mask): + if '.' not in mask: + return mask + return sum([bin(int(x)).count('1') for x in mask.split('.')]) + + +def ipv6mask2cidr(mask): + if ':' not in mask: + return mask + + bitCount = [0, 0x8000, 0xc000, 0xe000, 0xf000, 0xf800, 0xfc00, 0xfe00, + 0xff00, 0xff80, 0xffc0, 0xffe0, 0xfff0, 0xfff8, 0xfffc, + 0xfffe, 0xffff] + cidr = 0 + for word in mask.split(':'): + if not word or int(word, 16) == 0: + break + cidr += bitCount.index(int(word, 16)) + + return cidr + + +def mask2cidr(mask): + if ':' in mask: + return ipv6mask2cidr(mask) + elif '.' in mask: + return ipv4mask2cidr(mask) + else: + return mask + + if __name__ == '__main__': import sys import random diff --git a/cloudinit/sources/DataSourceConfigDrive.py b/cloudinit/sources/DataSourceConfigDrive.py index 6fc9e05b..d84fab54 100644 --- a/cloudinit/sources/DataSourceConfigDrive.py +++ b/cloudinit/sources/DataSourceConfigDrive.py @@ -18,6 +18,7 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . +import copy import os from cloudinit import log as logging @@ -50,6 +51,8 @@ class DataSourceConfigDrive(openstack.SourceMixin, sources.DataSource): self.seed_dir = os.path.join(paths.seed_dir, 'config_drive') self.version = None self.ec2_metadata = None + self._network_config = None + self.network_json = None self.files = {} def __str__(self): @@ -144,12 +147,27 @@ class DataSourceConfigDrive(openstack.SourceMixin, sources.DataSource): LOG.warn("Invalid content in vendor-data: %s", e) self.vendordata_raw = None + nd = results.get('networkdata') + self.networkdata_pure = nd + try: + self.network_json = openstack.convert_networkdata_json(nd) + except ValueError as e: + LOG.warn("Invalid content in network-data: %s", e) + self.network_json = None + + if self.network_json: + self._network_config = convert_network_data(self.network_json) + return True def check_instance_id(self): # quickly (local check only) if self.instance_id is still valid return sources.instance_id_matches_system_uuid(self.get_instance_id()) + @property + def network_config(self): + return self._network_config + class DataSourceConfigDriveNet(DataSourceConfigDrive): def __init__(self, sys_cfg, distro, paths): @@ -287,3 +305,122 @@ datasources = [ # Return a list of data sources that match this set of dependencies def get_datasource_list(depends): return sources.list_from_depends(depends, datasources) + + +# Convert OpenStack ConfigDrive NetworkData json to network_config yaml +def convert_network_data(network_json=None): + """Return a dictionary of network_config by parsing provided + OpenStack ConfigDrive NetworkData json format + + OpenStack network_data.json provides a 3 element dictionary + - "links" (links are network devices, physical or virtual) + - "networks" (networks are ip network configurations for one or more + links) + - services (non-ip services, like dns) + + networks and links are combined via network items referencing specific + links via a 'link_id' which maps to a links 'id' field. + + To convert this format to network_config yaml, we first iterate over the + links and then walk the network list to determine if any of the networks + utilize the current link; if so we generate a subnet entry for the device + + We also need to map network_data.json fields to network_config fields. For + example, the network_data links 'id' field is equivalent to network_config + 'name' field for devices. We apply more of this mapping to the various + link types that we encounter. + + There are additional fields that are populated in the network_data.json + from OpenStack that are not relevant to network_config yaml, so we + enumerate a dictionary of valid keys for network_yaml and apply filtering + to drop these superflous keys from the network_config yaml. + """ + if network_json is None: + return None + + # dict of network_config key for filtering network_json + valid_keys = { + 'physical': [ + 'name', + 'type', + 'mac_address', + 'subnets', + 'params', + ], + 'subnet': [ + 'type', + 'address', + 'netmask', + 'broadcast', + 'metric', + 'gateway', + 'pointopoint', + 'mtu', + 'scope', + 'dns_nameservers', + 'dns_search', + 'routes', + ], + } + + links = network_json.get('links', []) + networks = network_json.get('networks', []) + services = network_json.get('services', []) + + config = [] + for link in links: + subnets = [] + cfg = {k: v for k, v in link.items() + if k in valid_keys['physical']} + cfg.update({'name': link['id']}) + for network in [net for net in networks + if net['link'] == link['id']]: + subnet = {k: v for k, v in network.items() + if k in valid_keys['subnet']} + if 'dhcp' in network['type']: + t = 'dhcp6' if network['type'].startswith('ipv6') else 'dhcp4' + subnet.update({ + 'type': t, + }) + else: + subnet.update({ + 'type': 'static', + 'address': network.get('ip_address'), + }) + subnets.append(subnet) + cfg.update({'subnets': subnets}) + if link['type'] in ['ethernet', 'vif', 'ovs']: + cfg.update({ + 'type': 'physical', + 'mac_address': link['ethernet_mac_address']}) + elif link['type'] in ['bond']: + params = {} + for k, v in link.items(): + if k == 'bond_links': + continue + elif k.startswith('bond'): + params.update({k: v}) + cfg.update({ + 'bond_interfaces': copy.deepcopy(link['bond_links']), + 'params': params, + }) + elif link['type'] in ['vlan']: + cfg.update({ + 'name': "%s.%s" % (link['vlan_link'], + link['vlan_id']), + 'vlan_link': link['vlan_link'], + 'vlan_id': link['vlan_id'], + 'mac_address': link['vlan_mac_address'], + }) + else: + raise ValueError( + 'Unknown network_data link type: %s' % link['type']) + + config.append(cfg) + + for service in services: + cfg = service + cfg.update({'type': 'nameserver'}) + config.append(cfg) + + return {'version': 1, 'config': config} diff --git a/cloudinit/sources/helpers/openstack.py b/cloudinit/sources/helpers/openstack.py index bd93d22f..eb50a7be 100644 --- a/cloudinit/sources/helpers/openstack.py +++ b/cloudinit/sources/helpers/openstack.py @@ -51,11 +51,13 @@ OS_LATEST = 'latest' OS_FOLSOM = '2012-08-10' OS_GRIZZLY = '2013-04-04' OS_HAVANA = '2013-10-17' +OS_KILO = '2015-10-15' # keep this in chronological order. new supported versions go at the end. OS_VERSIONS = ( OS_FOLSOM, OS_GRIZZLY, OS_HAVANA, + OS_KILO, ) @@ -229,6 +231,11 @@ class BaseReader(object): False, load_json_anytype, ) + files['networkdata'] = ( + self._path_join("openstack", version, 'network_data.json'), + False, + load_json_anytype, + ) return files results = { @@ -334,7 +341,7 @@ class ConfigDriveReader(BaseReader): path = self._path_join(self.base_path, 'openstack') found = [d for d in os.listdir(path) if os.path.isdir(os.path.join(path))] - self._versions = found + self._versions = sorted(found) return self._versions def _read_ec2_metadata(self): @@ -490,3 +497,28 @@ def convert_vendordata_json(data, recurse=True): recurse=False) raise ValueError("vendordata['cloud-init'] cannot be dict") raise ValueError("Unknown data type for vendordata: %s" % type(data)) + + +def convert_networkdata_json(data, recurse=True): + """ data: a loaded json *object* (strings, arrays, dicts). + return something suitable for cloudinit networkdata_raw. + + if data is: + None: return None + string: return string + list: return data + the list is then processed in UserDataProcessor + dict: return convert_networkdata_json(data.get('cloud-init')) + """ + if not data: + return None + if isinstance(data, six.string_types): + return data + if isinstance(data, list): + return copy.deepcopy(data) + if isinstance(data, dict): + if recurse is True: + return convert_networkdata_json(data.get('cloud-init'), + recurse=False) + raise ValueError("networkdata['cloud-init'] cannot be dict") + raise ValueError("Unknown data type for networkdata: %s" % type(data)) diff --git a/tests/unittests/test_datasource/test_configdrive.py b/tests/unittests/test_datasource/test_configdrive.py index bfd787d1..01f8c5ce 100644 --- a/tests/unittests/test_datasource/test_configdrive.py +++ b/tests/unittests/test_datasource/test_configdrive.py @@ -59,6 +59,34 @@ OSTACK_META = { CONTENT_0 = b'This is contents of /etc/foo.cfg\n' CONTENT_1 = b'# this is /etc/bar/bar.cfg\n' +NETWORK_DATA = { + 'services': [ + {'type': 'dns', 'address': '199.204.44.24'}, + {'type': 'dns', 'address': '199.204.47.54'} + ], + 'links': [ + {'vif_id': '2ecc7709-b3f7-4448-9580-e1ec32d75bbd', + 'ethernet_mac_address': 'fa:16:3e:69:b0:58', + 'type': 'ovs', 'mtu': None, 'id': 'tap2ecc7709-b3'}, + {'vif_id': '2f88d109-5b57-40e6-af32-2472df09dc33', + 'ethernet_mac_address': 'fa:16:3e:d4:57:ad', + 'type': 'ovs', 'mtu': None, 'id': 'tap2f88d109-5b'}, + {'vif_id': '1a5382f8-04c5-4d75-ab98-d666c1ef52cc', + 'ethernet_mac_address': 'fa:16:3e:05:30:fe', + 'type': 'ovs', 'mtu': None, 'id': 'tap1a5382f8-04'} + ], + 'networks': [ + {'link': 'tap2ecc7709-b3', 'type': 'ipv4_dhcp', + 'network_id': '6d6357ac-0f70-4afa-8bd7-c274cc4ea235', + 'id': 'network0'}, + {'link': 'tap2f88d109-5b', 'type': 'ipv4_dhcp', + 'network_id': 'd227a9b3-6960-4d94-8976-ee5788b44f54', + 'id': 'network1'}, + {'link': 'tap1a5382f8-04', 'type': 'ipv4_dhcp', + 'network_id': 'dab2ba57-cae2-4311-a5ed-010b263891f5', + 'id': 'network2'} + ] +} CFG_DRIVE_FILES_V2 = { 'ec2/2009-04-04/meta-data.json': json.dumps(EC2_META), @@ -70,7 +98,11 @@ CFG_DRIVE_FILES_V2 = { 'openstack/content/0000': CONTENT_0, 'openstack/content/0001': CONTENT_1, 'openstack/latest/meta_data.json': json.dumps(OSTACK_META), - 'openstack/latest/user_data': USER_DATA} + 'openstack/latest/user_data': USER_DATA, + 'openstack/latest/network_data.json': json.dumps(NETWORK_DATA), + 'openstack/2015-10-15/meta_data.json': json.dumps(OSTACK_META), + 'openstack/2015-10-15/user_data': USER_DATA, + 'openstack/2015-10-15/network_data.json': json.dumps(NETWORK_DATA)} class TestConfigDriveDataSource(TestCase): @@ -225,6 +257,7 @@ class TestConfigDriveDataSource(TestCase): self.assertEqual(USER_DATA, found['userdata']) self.assertEqual(expected_md, found['metadata']) + self.assertEqual(NETWORK_DATA, found['networkdata']) self.assertEqual(found['files']['/etc/foo.cfg'], CONTENT_0) self.assertEqual(found['files']['/etc/bar/bar.cfg'], CONTENT_1) @@ -321,6 +354,19 @@ class TestConfigDriveDataSource(TestCase): self.assertEqual(myds.get_public_ssh_keys(), [OSTACK_META['public_keys']['mykey']]) + def test_network_data_is_found(self): + """Verify that network_data is present in ds in config-drive-v2.""" + populate_dir(self.tmp, CFG_DRIVE_FILES_V2) + myds = cfg_ds_from_dir(self.tmp) + self.assertEqual(myds.network_json, NETWORK_DATA) + + def test_network_config_is_converted(self): + """Verify that network_data is converted and present on ds object.""" + populate_dir(self.tmp, CFG_DRIVE_FILES_V2) + myds = cfg_ds_from_dir(self.tmp) + network_config = ds.convert_network_data(NETWORK_DATA) + self.assertEqual(myds.network_config, network_config) + def cfg_ds_from_dir(seed_d): found = ds.read_config_drive(seed_d) @@ -339,6 +385,8 @@ def populate_ds_from_read_config(cfg_ds, source, results): cfg_ds.ec2_metadata = results.get('ec2-metadata') cfg_ds.userdata_raw = results.get('userdata') cfg_ds.version = results.get('version') + cfg_ds.network_json = results.get('networkdata') + cfg_ds._network_config = ds.convert_network_data(cfg_ds.network_json) def populate_dir(seed_dir, files): -- cgit v1.2.3 From 6b79e2c6f9a7342163691be9e785cef1aa642541 Mon Sep 17 00:00:00 2001 From: Ryan Harper Date: Wed, 23 Mar 2016 14:17:10 -0500 Subject: fix openstack versions s/KILO/LIBERY drop networkdata read helper --- cloudinit/sources/DataSourceConfigDrive.py | 2 +- cloudinit/sources/helpers/openstack.py | 29 ++--------------------------- 2 files changed, 3 insertions(+), 28 deletions(-) (limited to 'cloudinit/sources/helpers') diff --git a/cloudinit/sources/DataSourceConfigDrive.py b/cloudinit/sources/DataSourceConfigDrive.py index d84fab54..15dddefe 100644 --- a/cloudinit/sources/DataSourceConfigDrive.py +++ b/cloudinit/sources/DataSourceConfigDrive.py @@ -150,7 +150,7 @@ class DataSourceConfigDrive(openstack.SourceMixin, sources.DataSource): nd = results.get('networkdata') self.networkdata_pure = nd try: - self.network_json = openstack.convert_networkdata_json(nd) + self.network_json = util.load_json(nd) except ValueError as e: LOG.warn("Invalid content in network-data: %s", e) self.network_json = None diff --git a/cloudinit/sources/helpers/openstack.py b/cloudinit/sources/helpers/openstack.py index eb50a7be..1aa6bbae 100644 --- a/cloudinit/sources/helpers/openstack.py +++ b/cloudinit/sources/helpers/openstack.py @@ -51,13 +51,13 @@ OS_LATEST = 'latest' OS_FOLSOM = '2012-08-10' OS_GRIZZLY = '2013-04-04' OS_HAVANA = '2013-10-17' -OS_KILO = '2015-10-15' +OS_LIBERTY = '2015-10-15' # keep this in chronological order. new supported versions go at the end. OS_VERSIONS = ( OS_FOLSOM, OS_GRIZZLY, OS_HAVANA, - OS_KILO, + OS_LIBERTY, ) @@ -497,28 +497,3 @@ def convert_vendordata_json(data, recurse=True): recurse=False) raise ValueError("vendordata['cloud-init'] cannot be dict") raise ValueError("Unknown data type for vendordata: %s" % type(data)) - - -def convert_networkdata_json(data, recurse=True): - """ data: a loaded json *object* (strings, arrays, dicts). - return something suitable for cloudinit networkdata_raw. - - if data is: - None: return None - string: return string - list: return data - the list is then processed in UserDataProcessor - dict: return convert_networkdata_json(data.get('cloud-init')) - """ - if not data: - return None - if isinstance(data, six.string_types): - return data - if isinstance(data, list): - return copy.deepcopy(data) - if isinstance(data, dict): - if recurse is True: - return convert_networkdata_json(data.get('cloud-init'), - recurse=False) - raise ValueError("networkdata['cloud-init'] cannot be dict") - raise ValueError("Unknown data type for networkdata: %s" % type(data)) -- cgit v1.2.3