From 10aeda45b32645542d03cd42bd830558a6354495 Mon Sep 17 00:00:00 2001 From: Daniel Watkins Date: Tue, 17 Feb 2015 16:33:23 +0000 Subject: Clean up imports in DataSourceCloudStack.py. --- cloudinit/sources/DataSourceCloudStack.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) (limited to 'cloudinit/sources/DataSourceCloudStack.py') diff --git a/cloudinit/sources/DataSourceCloudStack.py b/cloudinit/sources/DataSourceCloudStack.py index 1bbeca59..b8974dc1 100644 --- a/cloudinit/sources/DataSourceCloudStack.py +++ b/cloudinit/sources/DataSourceCloudStack.py @@ -26,14 +26,13 @@ import os import time +from socket import inet_ntoa +from struct import pack from cloudinit import ec2_utils as ec2 from cloudinit import log as logging -from cloudinit import sources from cloudinit import url_helper as uhelp -from cloudinit import util -from socket import inet_ntoa -from struct import pack +from cloudinit import sources, util LOG = logging.getLogger(__name__) -- cgit v1.2.3 From e626359a6ea47880f0c17add03502513ee3a6792 Mon Sep 17 00:00:00 2001 From: Daniel Watkins Date: Tue, 17 Feb 2015 16:33:23 +0000 Subject: Fetch and use passwords from CloudStack virtual router. --- cloudinit/sources/DataSourceCloudStack.py | 36 ++++++++++++++++++++++++++++--- 1 file changed, 33 insertions(+), 3 deletions(-) (limited to 'cloudinit/sources/DataSourceCloudStack.py') diff --git a/cloudinit/sources/DataSourceCloudStack.py b/cloudinit/sources/DataSourceCloudStack.py index b8974dc1..0377d940 100644 --- a/cloudinit/sources/DataSourceCloudStack.py +++ b/cloudinit/sources/DataSourceCloudStack.py @@ -29,6 +29,8 @@ import time from socket import inet_ntoa from struct import pack +from six.moves import http_client + from cloudinit import ec2_utils as ec2 from cloudinit import log as logging from cloudinit import url_helper as uhelp @@ -44,10 +46,11 @@ class DataSourceCloudStack(sources.DataSource): # Cloudstack has its metadata/userdata URLs located at # http:///latest/ self.api_ver = 'latest' - vr_addr = get_vr_address() - if not vr_addr: + self.vr_addr = get_vr_address() + if not self.vr_addr: raise RuntimeError("No virtual router found!") - self.metadata_address = "http://%s/" % (vr_addr) + self.metadata_address = "http://%s/" % (self.vr_addr,) + self.cfg = {} def _get_url_settings(self): mcfg = self.ds_cfg @@ -92,6 +95,9 @@ class DataSourceCloudStack(sources.DataSource): return bool(url) + def get_config_obj(self): + return self.cfg + def get_data(self): seed_ret = {} if util.read_optional_seed(seed_ret, base=(self.seed_dir + "/")): @@ -109,12 +115,36 @@ class DataSourceCloudStack(sources.DataSource): self.metadata_address) LOG.debug("Crawl of metadata service took %s seconds", int(time.time() - start_time)) + set_password = self.get_password() + if set_password: + self.cfg = { + 'ssh_pwauth': True, + 'password': set_password, + 'chpasswd': { + 'expire': False, + }, + } return True except Exception: util.logexc(LOG, 'Failed fetching from metadata service %s', self.metadata_address) return False + def get_password(self): + def _do_request(req_string): + conn = http_client.HTTPConnection(self.vr_addr, 8080) + conn.request('GET', '', headers={'DomU_Request': req_string}) + output = conn.sock.recv(1024).decode('utf-8').strip() + conn.close() + return output + password = _do_request('send_my_password') + if password in ['', 'saved_password']: + return None + if password == 'bad_request': + raise RuntimeError('Error when attempting to fetch root password.') + _do_request('saved_password') + return password + def get_instance_id(self): return self.metadata['instance-id'] -- cgit v1.2.3 From e01795dac74cd31bd6054e3185c2dba6203690ca Mon Sep 17 00:00:00 2001 From: Daniel Watkins Date: Tue, 17 Feb 2015 16:33:23 +0000 Subject: Add explanatory comment. --- cloudinit/sources/DataSourceCloudStack.py | 3 +++ 1 file changed, 3 insertions(+) (limited to 'cloudinit/sources/DataSourceCloudStack.py') diff --git a/cloudinit/sources/DataSourceCloudStack.py b/cloudinit/sources/DataSourceCloudStack.py index 0377d940..5eda10a5 100644 --- a/cloudinit/sources/DataSourceCloudStack.py +++ b/cloudinit/sources/DataSourceCloudStack.py @@ -132,6 +132,9 @@ class DataSourceCloudStack(sources.DataSource): def get_password(self): def _do_request(req_string): + # We have to provide a valid HTTP request, but a valid HTTP + # response is not returned. This means that getresponse() chokes, + # so we use the socket directly to read off the password. conn = http_client.HTTPConnection(self.vr_addr, 8080) conn.request('GET', '', headers={'DomU_Request': req_string}) output = conn.sock.recv(1024).decode('utf-8').strip() -- cgit v1.2.3 From 5e864eb373ead67d2bc29a19d970f9d3d94c53df Mon Sep 17 00:00:00 2001 From: Daniel Watkins Date: Wed, 18 Feb 2015 18:09:34 +0000 Subject: Failing to fetch a CloudStack password should never fail the whole DS. There might be some CloudStack deployments without the :8080 password server, and there's no reason the rest of the data source can't be used for them. --- cloudinit/sources/DataSourceCloudStack.py | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) (limited to 'cloudinit/sources/DataSourceCloudStack.py') diff --git a/cloudinit/sources/DataSourceCloudStack.py b/cloudinit/sources/DataSourceCloudStack.py index 5eda10a5..a8f8daec 100644 --- a/cloudinit/sources/DataSourceCloudStack.py +++ b/cloudinit/sources/DataSourceCloudStack.py @@ -115,15 +115,21 @@ class DataSourceCloudStack(sources.DataSource): self.metadata_address) LOG.debug("Crawl of metadata service took %s seconds", int(time.time() - start_time)) - set_password = self.get_password() - if set_password: - self.cfg = { - 'ssh_pwauth': True, - 'password': set_password, - 'chpasswd': { - 'expire': False, - }, - } + try: + set_password = self.get_password() + except Exception: + util.logexc(LOG, + 'Failed to fetch password from virtual router %s', + self.vr_addr) + else: + if set_password: + self.cfg = { + 'ssh_pwauth': True, + 'password': set_password, + 'chpasswd': { + 'expire': False, + }, + } return True except Exception: util.logexc(LOG, 'Failed fetching from metadata service %s', -- cgit v1.2.3 From d3d44a3efaf22c91d342f2cb81470745b7be0658 Mon Sep 17 00:00:00 2001 From: Daniel Watkins Date: Wed, 18 Feb 2015 18:10:15 +0000 Subject: Set an explicit timeout when fetching CloudStack passwords. --- cloudinit/sources/DataSourceCloudStack.py | 1 + 1 file changed, 1 insertion(+) (limited to 'cloudinit/sources/DataSourceCloudStack.py') diff --git a/cloudinit/sources/DataSourceCloudStack.py b/cloudinit/sources/DataSourceCloudStack.py index a8f8daec..89f58e1e 100644 --- a/cloudinit/sources/DataSourceCloudStack.py +++ b/cloudinit/sources/DataSourceCloudStack.py @@ -143,6 +143,7 @@ class DataSourceCloudStack(sources.DataSource): # so we use the socket directly to read off the password. conn = http_client.HTTPConnection(self.vr_addr, 8080) conn.request('GET', '', headers={'DomU_Request': req_string}) + conn.sock.settimeout(30) output = conn.sock.recv(1024).decode('utf-8').strip() conn.close() return output -- cgit v1.2.3 From b57c6a109491f344fa6e6fc2593ab2e60ca65249 Mon Sep 17 00:00:00 2001 From: Daniel Watkins Date: Fri, 20 Feb 2015 10:57:06 +0000 Subject: Minor formatting clean-up in CloudStack DS. --- cloudinit/sources/DataSourceCloudStack.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) (limited to 'cloudinit/sources/DataSourceCloudStack.py') diff --git a/cloudinit/sources/DataSourceCloudStack.py b/cloudinit/sources/DataSourceCloudStack.py index 89f58e1e..85f20c23 100644 --- a/cloudinit/sources/DataSourceCloudStack.py +++ b/cloudinit/sources/DataSourceCloudStack.py @@ -84,14 +84,14 @@ class DataSourceCloudStack(sources.DataSource): 'latest/meta-data/instance-id')] start_time = time.time() url = uhelp.wait_for_url(urls=urls, max_wait=max_wait, - timeout=timeout, status_cb=LOG.warn) + timeout=timeout, status_cb=LOG.warn) if url: LOG.debug("Using metadata source: '%s'", url) else: LOG.critical(("Giving up on waiting for the metadata from %s" " after %s seconds"), - urls, int(time.time() - start_time)) + urls, int(time.time() - start_time)) return bool(url) @@ -109,8 +109,8 @@ class DataSourceCloudStack(sources.DataSource): if not self.wait_for_metadata_service(): return False start_time = time.time() - self.userdata_raw = ec2.get_instance_userdata(self.api_ver, - self.metadata_address) + self.userdata_raw = ec2.get_instance_userdata( + self.api_ver, self.metadata_address) self.metadata = ec2.get_instance_metadata(self.api_ver, self.metadata_address) LOG.debug("Crawl of metadata service took %s seconds", @@ -231,7 +231,7 @@ def get_vr_address(): # Used to match classes to dependencies datasources = [ - (DataSourceCloudStack, (sources.DEP_FILESYSTEM, sources.DEP_NETWORK)), + (DataSourceCloudStack, (sources.DEP_FILESYSTEM, sources.DEP_NETWORK)), ] -- cgit v1.2.3 From f8d9ebbe3743bcada75bc1a980b49f493e2da2f1 Mon Sep 17 00:00:00 2001 From: Daniel Watkins Date: Fri, 20 Feb 2015 10:57:18 +0000 Subject: Split CloudStack password handling out to separate class. --- cloudinit/sources/DataSourceCloudStack.py | 65 +++++++++++++++++++++---------- 1 file changed, 45 insertions(+), 20 deletions(-) (limited to 'cloudinit/sources/DataSourceCloudStack.py') diff --git a/cloudinit/sources/DataSourceCloudStack.py b/cloudinit/sources/DataSourceCloudStack.py index 85f20c23..0c3c51c0 100644 --- a/cloudinit/sources/DataSourceCloudStack.py +++ b/cloudinit/sources/DataSourceCloudStack.py @@ -39,6 +39,49 @@ from cloudinit import sources, util LOG = logging.getLogger(__name__) +class CloudStackPasswordServerClient(object): + """ + Implements password fetching from the CloudStack password server. + + http://cloudstack-administration.readthedocs.org/en/latest/templates.html#adding-password-management-to-your-templates + has documentation about the system. This implementation is following that + found at + https://github.com/shankerbalan/cloudstack-scripts/blob/master/cloud-set-guest-password-debian + + The CloudStack password server is, essentially, a broken HTTP + server. It requires us to provide a valid HTTP request (including a + DomU_Request header, which is the meat of the request), but just + writes the text of its response on to the socket, without a status + line or any HTTP headers. This makes HTTP libraries sad, which + explains the screwiness of the implementation of this class. + """ + + def __init__(self, virtual_router_address): + self.virtual_router_address = virtual_router_address + + def _do_request(self, domu_request): + # We have to provide a valid HTTP request, but a valid HTTP + # response is not returned. This means that getresponse() chokes, + # so we use the socket directly to read off the response. + # Because we're reading off the socket directly, we can't re-use the + # connection. + conn = http_client.HTTPConnection(self.virtual_router_address, 8080) + conn.request('GET', '', headers={'DomU_Request': domu_request}) + conn.sock.settimeout(30) + output = conn.sock.recv(1024).decode('utf-8').strip() + conn.close() + return output + + def get_password(self): + password = self._do_request('send_my_password') + if password in ['', 'saved_password']: + return None + if password == 'bad_request': + raise RuntimeError('Error when attempting to fetch root password.') + self._do_request('saved_password') + return password + + class DataSourceCloudStack(sources.DataSource): def __init__(self, sys_cfg, distro, paths): sources.DataSource.__init__(self, sys_cfg, distro, paths) @@ -115,8 +158,9 @@ class DataSourceCloudStack(sources.DataSource): self.metadata_address) LOG.debug("Crawl of metadata service took %s seconds", int(time.time() - start_time)) + password_client = CloudStackPasswordServerClient(self.vr_addr) try: - set_password = self.get_password() + set_password = password_client.get_password() except Exception: util.logexc(LOG, 'Failed to fetch password from virtual router %s', @@ -136,25 +180,6 @@ class DataSourceCloudStack(sources.DataSource): self.metadata_address) return False - def get_password(self): - def _do_request(req_string): - # We have to provide a valid HTTP request, but a valid HTTP - # response is not returned. This means that getresponse() chokes, - # so we use the socket directly to read off the password. - conn = http_client.HTTPConnection(self.vr_addr, 8080) - conn.request('GET', '', headers={'DomU_Request': req_string}) - conn.sock.settimeout(30) - output = conn.sock.recv(1024).decode('utf-8').strip() - conn.close() - return output - password = _do_request('send_my_password') - if password in ['', 'saved_password']: - return None - if password == 'bad_request': - raise RuntimeError('Error when attempting to fetch root password.') - _do_request('saved_password') - return password - def get_instance_id(self): return self.metadata['instance-id'] -- cgit v1.2.3 From ef84bd214a1d5e0b922c0dd38096f694f8ff406e Mon Sep 17 00:00:00 2001 From: Daniel Watkins Date: Mon, 23 Feb 2015 09:22:50 +0000 Subject: Always close the password server connection, even on failure. --- cloudinit/sources/DataSourceCloudStack.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) (limited to 'cloudinit/sources/DataSourceCloudStack.py') diff --git a/cloudinit/sources/DataSourceCloudStack.py b/cloudinit/sources/DataSourceCloudStack.py index 0c3c51c0..996076b1 100644 --- a/cloudinit/sources/DataSourceCloudStack.py +++ b/cloudinit/sources/DataSourceCloudStack.py @@ -66,10 +66,12 @@ class CloudStackPasswordServerClient(object): # Because we're reading off the socket directly, we can't re-use the # connection. conn = http_client.HTTPConnection(self.virtual_router_address, 8080) - conn.request('GET', '', headers={'DomU_Request': domu_request}) - conn.sock.settimeout(30) - output = conn.sock.recv(1024).decode('utf-8').strip() - conn.close() + try: + conn.request('GET', '', headers={'DomU_Request': domu_request}) + conn.sock.settimeout(30) + output = conn.sock.recv(1024).decode('utf-8').strip() + finally: + conn.close() return output def get_password(self): -- cgit v1.2.3 From 9ab6bbab42ffb5cadbe0afb36aa6967ed94459c3 Mon Sep 17 00:00:00 2001 From: Daniel Watkins Date: Mon, 23 Feb 2015 09:36:36 +0000 Subject: Add documentation about upstream CloudStack HTTP fix. --- cloudinit/sources/DataSourceCloudStack.py | 3 +++ 1 file changed, 3 insertions(+) (limited to 'cloudinit/sources/DataSourceCloudStack.py') diff --git a/cloudinit/sources/DataSourceCloudStack.py b/cloudinit/sources/DataSourceCloudStack.py index 996076b1..7b32e1fa 100644 --- a/cloudinit/sources/DataSourceCloudStack.py +++ b/cloudinit/sources/DataSourceCloudStack.py @@ -54,6 +54,9 @@ class CloudStackPasswordServerClient(object): writes the text of its response on to the socket, without a status line or any HTTP headers. This makes HTTP libraries sad, which explains the screwiness of the implementation of this class. + + This should be fixed in CloudStack by commit + a72f14ea9cb832faaac946b3cf9f56856b50142a in December 2014. """ def __init__(self, virtual_router_address): -- cgit v1.2.3 From ba7fc871f2e73e0adbf883ef8253180f41cdcfe8 Mon Sep 17 00:00:00 2001 From: Daniel Watkins Date: Tue, 16 Jun 2015 17:35:03 +0100 Subject: Use wget to fetch CloudStack passwords. Different versions of the CloudStack password server respond differently; wget handles these nicely for us, so it's easier to just use wget. LP: #1440263, #1464253 --- cloudinit/sources/DataSourceCloudStack.py | 35 +++++++--------------- tests/unittests/test_datasource/test_cloudstack.py | 30 +++++++++---------- 2 files changed, 25 insertions(+), 40 deletions(-) (limited to 'cloudinit/sources/DataSourceCloudStack.py') diff --git a/cloudinit/sources/DataSourceCloudStack.py b/cloudinit/sources/DataSourceCloudStack.py index 7b32e1fa..d0cac5bb 100644 --- a/cloudinit/sources/DataSourceCloudStack.py +++ b/cloudinit/sources/DataSourceCloudStack.py @@ -29,8 +29,6 @@ import time from socket import inet_ntoa from struct import pack -from six.moves import http_client - from cloudinit import ec2_utils as ec2 from cloudinit import log as logging from cloudinit import url_helper as uhelp @@ -47,35 +45,22 @@ class CloudStackPasswordServerClient(object): has documentation about the system. This implementation is following that found at https://github.com/shankerbalan/cloudstack-scripts/blob/master/cloud-set-guest-password-debian - - The CloudStack password server is, essentially, a broken HTTP - server. It requires us to provide a valid HTTP request (including a - DomU_Request header, which is the meat of the request), but just - writes the text of its response on to the socket, without a status - line or any HTTP headers. This makes HTTP libraries sad, which - explains the screwiness of the implementation of this class. - - This should be fixed in CloudStack by commit - a72f14ea9cb832faaac946b3cf9f56856b50142a in December 2014. """ def __init__(self, virtual_router_address): self.virtual_router_address = virtual_router_address def _do_request(self, domu_request): - # We have to provide a valid HTTP request, but a valid HTTP - # response is not returned. This means that getresponse() chokes, - # so we use the socket directly to read off the response. - # Because we're reading off the socket directly, we can't re-use the - # connection. - conn = http_client.HTTPConnection(self.virtual_router_address, 8080) - try: - conn.request('GET', '', headers={'DomU_Request': domu_request}) - conn.sock.settimeout(30) - output = conn.sock.recv(1024).decode('utf-8').strip() - finally: - conn.close() - return output + # The password server was in the past, a broken HTTP server, but is now + # fixed. wget handles this seamlessly, so it's easier to shell out to + # that rather than write our own handling code. + output, _ = util.subp([ + 'wget', '--quiet', '--tries', '3', '--timeout', '20', + '--output-document', '-', '--header', + 'DomU_Request: {0}'.format(domu_request), + '{0}:8080'.format(self.virtual_router_address) + ]) + return output.strip() def get_password(self): password = self._do_request('send_my_password') diff --git a/tests/unittests/test_datasource/test_cloudstack.py b/tests/unittests/test_datasource/test_cloudstack.py index 959d78ae..656d80d1 100644 --- a/tests/unittests/test_datasource/test_cloudstack.py +++ b/tests/unittests/test_datasource/test_cloudstack.py @@ -23,13 +23,11 @@ class TestCloudStackPasswordFetching(TestCase): self.patches.enter_context(mock.patch('{0}.uhelp'.format(mod_name))) def _set_password_server_response(self, response_string): - http_client = mock.MagicMock() - http_client.HTTPConnection.return_value.sock.recv.return_value = \ - response_string.encode('utf-8') + subp = mock.MagicMock(return_value=(response_string, '')) self.patches.enter_context( - mock.patch('cloudinit.sources.DataSourceCloudStack.http_client', - http_client)) - return http_client + mock.patch('cloudinit.sources.DataSourceCloudStack.util.subp', + subp)) + return subp def test_empty_password_doesnt_create_config(self): self._set_password_server_response('') @@ -55,26 +53,28 @@ class TestCloudStackPasswordFetching(TestCase): ds = DataSourceCloudStack({}, None, helpers.Paths({})) self.assertTrue(ds.get_data()) - def assertRequestTypesSent(self, http_client, expected_request_types): - request_types = [ - kwargs['headers']['DomU_Request'] - for _, kwargs - in http_client.HTTPConnection.return_value.request.call_args_list] + def assertRequestTypesSent(self, subp, expected_request_types): + request_types = [] + for call in subp.call_args_list: + args = call[0][0] + for arg in args: + if arg.startswith('DomU_Request'): + request_types.append(arg.split()[1]) self.assertEqual(expected_request_types, request_types) def test_valid_response_means_password_marked_as_saved(self): password = 'SekritSquirrel' - http_client = self._set_password_server_response(password) + subp = self._set_password_server_response(password) ds = DataSourceCloudStack({}, None, helpers.Paths({})) ds.get_data() - self.assertRequestTypesSent(http_client, + self.assertRequestTypesSent(subp, ['send_my_password', 'saved_password']) def _check_password_not_saved_for(self, response_string): - http_client = self._set_password_server_response(response_string) + subp = self._set_password_server_response(response_string) ds = DataSourceCloudStack({}, None, helpers.Paths({})) ds.get_data() - self.assertRequestTypesSent(http_client, ['send_my_password']) + self.assertRequestTypesSent(subp, ['send_my_password']) def test_password_not_saved_if_empty(self): self._check_password_not_saved_for('') -- cgit v1.2.3 From 328cc7fbaf4d60b51193fb8c14e52d8c6f3273f2 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Tue, 4 Aug 2015 21:57:57 -0500 Subject: pep8 fixes --- cloudinit/config/cc_rh_subscription.py | 6 +++--- cloudinit/config/cc_rsyslog.py | 1 + cloudinit/sources/DataSourceCloudStack.py | 6 ++++-- 3 files changed, 8 insertions(+), 5 deletions(-) (limited to 'cloudinit/sources/DataSourceCloudStack.py') diff --git a/cloudinit/config/cc_rh_subscription.py b/cloudinit/config/cc_rh_subscription.py index 6da26d25..3b30c47e 100644 --- a/cloudinit/config/cc_rh_subscription.py +++ b/cloudinit/config/cc_rh_subscription.py @@ -130,9 +130,9 @@ class SubscriptionManager(object): ((not self.auto_attach) or (util.is_false(str(self.auto_attach)))): - no_auto = "The service-level key must be used in conjunction with "\ - "the auto-attach key. Please re-run with auto-attach: "\ - "True" + no_auto = ("The service-level key must be used in conjunction " + "with the auto-attach key. Please re-run with " + "auto-attach: True") return False, no_auto return True, None diff --git a/cloudinit/config/cc_rsyslog.py b/cloudinit/config/cc_rsyslog.py index a0132d28..b8642d65 100644 --- a/cloudinit/config/cc_rsyslog.py +++ b/cloudinit/config/cc_rsyslog.py @@ -130,6 +130,7 @@ HOST_PORT_RE = re.compile( '(([[](?P[^\]]*)[\]])|(?P[^:]*))' '([:](?P[0-9]+))?$') + def reload_syslog(command=DEF_RELOAD, systemd=False): service = 'rsyslog' if command == DEF_RELOAD: diff --git a/cloudinit/sources/DataSourceCloudStack.py b/cloudinit/sources/DataSourceCloudStack.py index d0cac5bb..64595020 100644 --- a/cloudinit/sources/DataSourceCloudStack.py +++ b/cloudinit/sources/DataSourceCloudStack.py @@ -41,10 +41,12 @@ class CloudStackPasswordServerClient(object): """ Implements password fetching from the CloudStack password server. - http://cloudstack-administration.readthedocs.org/en/latest/templates.html#adding-password-management-to-your-templates + http://cloudstack-administration.readthedocs.org/ + en/latest/templates.html#adding-password-management-to-your-templates has documentation about the system. This implementation is following that found at - https://github.com/shankerbalan/cloudstack-scripts/blob/master/cloud-set-guest-password-debian + https://github.com/shankerbalan/cloudstack-scripts/ + blob/master/cloud-set-guest-password-debian """ def __init__(self, virtual_router_address): -- cgit v1.2.3