summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--ChangeLog7
-rw-r--r--cloudinit/distros/__init__.py16
-rw-r--r--cloudinit/distros/rhel.py17
-rw-r--r--cloudinit/sources/DataSourceAzure.py169
-rw-r--r--cloudinit/sources/helpers/azure.py293
-rw-r--r--cloudinit/util.py8
-rw-r--r--doc/rtd/topics/datasources.rst6
-rw-r--r--doc/sources/openstack/README.rst24
-rwxr-xr-xpackages/brpm4
-rw-r--r--packages/debian/control.in2
-rw-r--r--tests/unittests/helpers.py10
-rw-r--r--tests/unittests/test_cli.py10
-rw-r--r--tests/unittests/test_datasource/test_azure.py113
-rw-r--r--tests/unittests/test_datasource/test_azure_helper.py444
-rw-r--r--tests/unittests/test_distros/test_generic.py23
-rw-r--r--tests/unittests/test_handler/test_handler_apt_configure.py16
-rw-r--r--tests/unittests/test_util.py17
17 files changed, 1045 insertions, 134 deletions
diff --git a/ChangeLog b/ChangeLog
index ff5f2aac..30885329 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -39,6 +39,13 @@
[Surojit Pathak]
- Azure: do not re-set hostname if user has changed it (LP: #1375252)
- Fix exception when running with no arguments on Python 3. [Daniel Watkins]
+ - Centos: detect/expect use of systemd on centos 7. [Brian Rak]
+ - Azure: remove dependency on walinux-agent [Daniel Watkins]
+ - EC2: know about eu-central-1 availability-zone (LP: #1456684)
+ - Azure: remove password from on-disk ovf-env.xml (LP: #1443311) [Ben Howard]
+ - Doc: include information on user-data in OpenStack [Daniel Watkins]
+ - Systemd: check for systemd using sd_booted symantics (LP: #1461201)
+ [Lars Kellogg-Stedman]
0.7.6:
- open 0.7.6
- Enable vendordata on CloudSigma datasource (LP: #1303986)
diff --git a/cloudinit/distros/__init__.py b/cloudinit/distros/__init__.py
index 05721922..8a947867 100644
--- a/cloudinit/distros/__init__.py
+++ b/cloudinit/distros/__init__.py
@@ -27,6 +27,7 @@ from six import StringIO
import abc
import os
import re
+import stat
from cloudinit import importer
from cloudinit import log as logging
@@ -89,6 +90,13 @@ class Distro(object):
self._write_hostname(writeable_hostname, self.hostname_conf_fn)
self._apply_hostname(writeable_hostname)
+ def uses_systemd(self):
+ try:
+ res = os.lstat('/run/systemd/system')
+ return stat.S_ISDIR(res.st_mode)
+ except:
+ return False
+
@abc.abstractmethod
def package_command(self, cmd, args=None, pkgs=None):
raise NotImplementedError()
@@ -556,8 +564,12 @@ def _get_package_mirror_info(mirror_info, availability_zone=None,
if not mirror_info:
mirror_info = {}
- ec2_az_re = ("^[a-z][a-z]-(%s)-[1-9][0-9]*[a-z]$" %
- "north|northeast|east|southeast|south|southwest|west|northwest")
+ # ec2 availability zones are named cc-direction-[0-9][a-d] (us-east-1b)
+ # the region is us-east-1. so region = az[0:-1]
+ directions_re = '|'.join([
+ 'central', 'east', 'north', 'northeast', 'northwest',
+ 'south', 'southeast', 'southwest', 'west'])
+ ec2_az_re = ("^[a-z][a-z]-(%s)-[1-9][0-9]*[a-z]$" % directions_re)
subst = {}
if availability_zone:
diff --git a/cloudinit/distros/rhel.py b/cloudinit/distros/rhel.py
index 7408989c..812e7002 100644
--- a/cloudinit/distros/rhel.py
+++ b/cloudinit/distros/rhel.py
@@ -111,13 +111,6 @@ class Distro(distros.Distro):
rhel_util.update_sysconfig_file(self.network_conf_fn, net_cfg)
return dev_names
- def uses_systemd(self):
- # Fedora 18 and RHEL 7 were the first adopters in their series
- (dist, vers) = util.system_info()['dist'][:2]
- major = (int)(vers.split('.')[0])
- return ((dist.startswith('Red Hat Enterprise Linux') and major >= 7)
- or (dist.startswith('Fedora') and major >= 18))
-
def apply_locale(self, locale, out_fn=None):
if self.uses_systemd():
if not out_fn:
@@ -132,7 +125,11 @@ class Distro(distros.Distro):
rhel_util.update_sysconfig_file(out_fn, locale_cfg)
def _write_hostname(self, hostname, out_fn):
- if self.uses_systemd():
+ # systemd will never update previous-hostname for us, so
+ # we need to do it ourselves
+ if self.uses_systemd() and out_fn.endswith('/previous-hostname'):
+ util.write_file(out_fn, hostname)
+ elif self.uses_systemd():
util.subp(['hostnamectl', 'set-hostname', str(hostname)])
else:
host_cfg = {
@@ -155,7 +152,9 @@ class Distro(distros.Distro):
return (host_fn, self._read_hostname(host_fn))
def _read_hostname(self, filename, default=None):
- if self.uses_systemd():
+ if self.uses_systemd() and filename.endswith('/previous-hostname'):
+ return util.load_file(filename).strip()
+ elif self.uses_systemd():
(out, _err) = util.subp(['hostname'])
if len(out):
return out
diff --git a/cloudinit/sources/DataSourceAzure.py b/cloudinit/sources/DataSourceAzure.py
index a19d9ca2..d0a882ca 100644
--- a/cloudinit/sources/DataSourceAzure.py
+++ b/cloudinit/sources/DataSourceAzure.py
@@ -23,12 +23,16 @@ import fnmatch
import os
import os.path
import time
+import xml.etree.ElementTree as ET
+
from xml.dom import minidom
from cloudinit import log as logging
from cloudinit.settings import PER_ALWAYS
from cloudinit import sources
from cloudinit import util
+from cloudinit.sources.helpers.azure import (
+ get_metadata_from_fabric, iid_from_shared_config_content)
LOG = logging.getLogger(__name__)
@@ -66,6 +70,10 @@ BUILTIN_CLOUD_CONFIG = {
DS_CFG_PATH = ['datasource', DS_NAME]
DEF_EPHEMERAL_LABEL = 'Temporary Storage'
+# The redacted password fails to meet password complexity requirements
+# so we can safely use this to mask/redact the password in the ovf-env.xml
+DEF_PASSWD_REDACTION = 'REDACTED'
+
def get_hostname(hostname_command='hostname'):
return util.subp(hostname_command, capture=True)[0].strip()
@@ -111,6 +119,56 @@ class DataSourceAzureNet(sources.DataSource):
root = sources.DataSource.__str__(self)
return "%s [seed=%s]" % (root, self.seed)
+ def get_metadata_from_agent(self):
+ temp_hostname = self.metadata.get('local-hostname')
+ hostname_command = self.ds_cfg['hostname_bounce']['hostname_command']
+ with temporary_hostname(temp_hostname, self.ds_cfg,
+ hostname_command=hostname_command) \
+ as previous_hostname:
+ if (previous_hostname is not None
+ and util.is_true(self.ds_cfg.get('set_hostname'))):
+ cfg = self.ds_cfg['hostname_bounce']
+ try:
+ perform_hostname_bounce(hostname=temp_hostname,
+ cfg=cfg,
+ prev_hostname=previous_hostname)
+ except Exception as e:
+ LOG.warn("Failed publishing hostname: %s", e)
+ util.logexc(LOG, "handling set_hostname failed")
+
+ try:
+ invoke_agent(self.ds_cfg['agent_command'])
+ except util.ProcessExecutionError:
+ # claim the datasource even if the command failed
+ util.logexc(LOG, "agent command '%s' failed.",
+ self.ds_cfg['agent_command'])
+
+ ddir = self.ds_cfg['data_dir']
+ shcfgxml = os.path.join(ddir, "SharedConfig.xml")
+ wait_for = [shcfgxml]
+
+ fp_files = []
+ for pk in self.cfg.get('_pubkeys', []):
+ bname = str(pk['fingerprint'] + ".crt")
+ fp_files += [os.path.join(ddir, bname)]
+
+ missing = util.log_time(logfunc=LOG.debug, msg="waiting for files",
+ func=wait_for_files,
+ args=(wait_for + fp_files,))
+ if len(missing):
+ LOG.warn("Did not find files, but going on: %s", missing)
+
+ metadata = {}
+ if shcfgxml in missing:
+ LOG.warn("SharedConfig.xml missing, using static instance-id")
+ else:
+ try:
+ metadata['instance-id'] = iid_from_shared_config(shcfgxml)
+ except ValueError as e:
+ LOG.warn("failed to get instance id in %s: %s", shcfgxml, e)
+ metadata['public-keys'] = pubkeys_from_crt_files(fp_files)
+ return metadata
+
def get_data(self):
# azure removes/ejects the cdrom containing the ovf-env.xml
# file on reboot. So, in order to successfully reboot we
@@ -163,8 +221,6 @@ class DataSourceAzureNet(sources.DataSource):
# now update ds_cfg to reflect contents pass in config
user_ds_cfg = util.get_cfg_by_path(self.cfg, DS_CFG_PATH, {})
self.ds_cfg = util.mergemanydict([user_ds_cfg, self.ds_cfg])
- mycfg = self.ds_cfg
- ddir = mycfg['data_dir']
if found != ddir:
cached_ovfenv = util.load_file(
@@ -185,53 +241,18 @@ class DataSourceAzureNet(sources.DataSource):
# the directory to be protected.
write_files(ddir, files, dirmode=0o700)
- temp_hostname = self.metadata.get('local-hostname')
- hostname_command = mycfg['hostname_bounce']['hostname_command']
- with temporary_hostname(temp_hostname, mycfg,
- hostname_command=hostname_command) \
- as previous_hostname:
- if (previous_hostname is not None
- and util.is_true(mycfg.get('set_hostname'))):
- cfg = mycfg['hostname_bounce']
- try:
- perform_hostname_bounce(hostname=temp_hostname,
- cfg=cfg,
- prev_hostname=previous_hostname)
- except Exception as e:
- LOG.warn("Failed publishing hostname: %s", e)
- util.logexc(LOG, "handling set_hostname failed")
-
- try:
- invoke_agent(mycfg['agent_command'])
- except util.ProcessExecutionError:
- # claim the datasource even if the command failed
- util.logexc(LOG, "agent command '%s' failed.",
- mycfg['agent_command'])
-
- shcfgxml = os.path.join(ddir, "SharedConfig.xml")
- wait_for = [shcfgxml]
-
- fp_files = []
- for pk in self.cfg.get('_pubkeys', []):
- bname = str(pk['fingerprint'] + ".crt")
- fp_files += [os.path.join(ddir, bname)]
-
- missing = util.log_time(logfunc=LOG.debug, msg="waiting for files",
- func=wait_for_files,
- args=(wait_for + fp_files,))
- if len(missing):
- LOG.warn("Did not find files, but going on: %s", missing)
-
- if shcfgxml in missing:
- LOG.warn("SharedConfig.xml missing, using static instance-id")
+ if self.ds_cfg['agent_command'] == '__builtin__':
+ metadata_func = get_metadata_from_fabric
else:
- try:
- self.metadata['instance-id'] = iid_from_shared_config(shcfgxml)
- except ValueError as e:
- LOG.warn("failed to get instance id in %s: %s", shcfgxml, e)
+ metadata_func = self.get_metadata_from_agent
+ try:
+ fabric_data = metadata_func()
+ except Exception as exc:
+ LOG.info("Error communicating with Azure fabric; assume we aren't"
+ " on Azure.", exc_info=True)
+ return False
- pubkeys = pubkeys_from_crt_files(fp_files)
- self.metadata['public-keys'] = pubkeys
+ self.metadata.update(fabric_data)
found_ephemeral = find_ephemeral_disk()
if found_ephemeral:
@@ -363,10 +384,11 @@ def perform_hostname_bounce(hostname, cfg, prev_hostname):
'env': env})
-def crtfile_to_pubkey(fname):
+def crtfile_to_pubkey(fname, data=None):
pipeline = ('openssl x509 -noout -pubkey < "$0" |'
'ssh-keygen -i -m PKCS8 -f /dev/stdin')
- (out, _err) = util.subp(['sh', '-c', pipeline, fname], capture=True)
+ (out, _err) = util.subp(['sh', '-c', pipeline, fname],
+ capture=True, data=data)
return out.rstrip()
@@ -398,14 +420,30 @@ def wait_for_files(flist, maxwait=60, naplen=.5):
def write_files(datadir, files, dirmode=None):
+
+ def _redact_password(cnt, fname):
+ """Azure provides the UserPassword in plain text. So we redact it"""
+ try:
+ root = ET.fromstring(cnt)
+ for elem in root.iter():
+ if ('UserPassword' in elem.tag and
+ elem.text != DEF_PASSWD_REDACTION):
+ elem.text = DEF_PASSWD_REDACTION
+ return ET.tostring(root)
+ except Exception as e:
+ LOG.critical("failed to redact userpassword in {}".format(fname))
+ return cnt
+
if not datadir:
return
if not files:
files = {}
util.ensure_dir(datadir, dirmode)
for (name, content) in files.items():
- util.write_file(filename=os.path.join(datadir, name),
- content=content, mode=0o600)
+ fname = os.path.join(datadir, name)
+ if 'ovf-env.xml' in name:
+ content = _redact_password(content, fname)
+ util.write_file(filename=fname, content=content, mode=0o600)
def invoke_agent(cmd):
@@ -476,20 +514,6 @@ def load_azure_ovf_pubkeys(sshnode):
return found
-def single_node_at_path(node, pathlist):
- curnode = node
- for tok in pathlist:
- results = find_child(curnode, lambda n: n.localName == tok)
- if len(results) == 0:
- raise ValueError("missing %s token in %s" % (tok, str(pathlist)))
- if len(results) > 1:
- raise ValueError("found %s nodes of type %s looking for %s" %
- (len(results), tok, str(pathlist)))
- curnode = results[0]
-
- return curnode
-
-
def read_azure_ovf(contents):
try:
dom = minidom.parseString(contents)
@@ -574,7 +598,7 @@ def read_azure_ovf(contents):
defuser = {}
if username:
defuser['name'] = username
- if password:
+ if password and DEF_PASSWD_REDACTION != password:
defuser['passwd'] = encrypt_pass(password)
defuser['lock_passwd'] = False
@@ -620,19 +644,6 @@ def iid_from_shared_config(path):
return iid_from_shared_config_content(content)
-def iid_from_shared_config_content(content):
- """
- find INSTANCE_ID in:
- <?xml version="1.0" encoding="utf-8"?>
- <SharedConfig version="1.0.0.0" goalStateIncarnation="1">
- <Deployment name="INSTANCE_ID" guid="{...}" incarnation="0">
- <Service name="..." guid="{00000000-0000-0000-0000-000000000000}" />
- """
- dom = minidom.parseString(content)
- depnode = single_node_at_path(dom, ["SharedConfig", "Deployment"])
- return depnode.attributes.get('name').value
-
-
class BrokenAzureDataSource(Exception):
pass
diff --git a/cloudinit/sources/helpers/azure.py b/cloudinit/sources/helpers/azure.py
new file mode 100644
index 00000000..281d733e
--- /dev/null
+++ b/cloudinit/sources/helpers/azure.py
@@ -0,0 +1,293 @@
+import logging
+import os
+import re
+import socket
+import struct
+import tempfile
+import time
+from contextlib import contextmanager
+from xml.etree import ElementTree
+
+from cloudinit import util
+
+
+LOG = logging.getLogger(__name__)
+
+
+@contextmanager
+def cd(newdir):
+ prevdir = os.getcwd()
+ os.chdir(os.path.expanduser(newdir))
+ try:
+ yield
+ finally:
+ os.chdir(prevdir)
+
+
+class AzureEndpointHttpClient(object):
+
+ headers = {
+ 'x-ms-agent-name': 'WALinuxAgent',
+ 'x-ms-version': '2012-11-30',
+ }
+
+ def __init__(self, certificate):
+ self.extra_secure_headers = {
+ "x-ms-cipher-name": "DES_EDE3_CBC",
+ "x-ms-guest-agent-public-x509-cert": certificate,
+ }
+
+ def get(self, url, secure=False):
+ headers = self.headers
+ if secure:
+ headers = self.headers.copy()
+ headers.update(self.extra_secure_headers)
+ return util.read_file_or_url(url, headers=headers)
+
+ def post(self, url, data=None, extra_headers=None):
+ headers = self.headers
+ if extra_headers is not None:
+ headers = self.headers.copy()
+ headers.update(extra_headers)
+ return util.read_file_or_url(url, data=data, headers=headers)
+
+
+class GoalState(object):
+
+ def __init__(self, xml, http_client):
+ self.http_client = http_client
+ self.root = ElementTree.fromstring(xml)
+ self._certificates_xml = None
+
+ def _text_from_xpath(self, xpath):
+ element = self.root.find(xpath)
+ if element is not None:
+ return element.text
+ return None
+
+ @property
+ def container_id(self):
+ return self._text_from_xpath('./Container/ContainerId')
+
+ @property
+ def incarnation(self):
+ return self._text_from_xpath('./Incarnation')
+
+ @property
+ def instance_id(self):
+ return self._text_from_xpath(
+ './Container/RoleInstanceList/RoleInstance/InstanceId')
+
+ @property
+ def shared_config_xml(self):
+ url = self._text_from_xpath('./Container/RoleInstanceList/RoleInstance'
+ '/Configuration/SharedConfig')
+ return self.http_client.get(url).contents
+
+ @property
+ def certificates_xml(self):
+ if self._certificates_xml is None:
+ url = self._text_from_xpath(
+ './Container/RoleInstanceList/RoleInstance'
+ '/Configuration/Certificates')
+ if url is not None:
+ self._certificates_xml = self.http_client.get(
+ url, secure=True).contents
+ return self._certificates_xml
+
+
+class OpenSSLManager(object):
+
+ certificate_names = {
+ 'private_key': 'TransportPrivate.pem',
+ 'certificate': 'TransportCert.pem',
+ }
+
+ def __init__(self):
+ self.tmpdir = tempfile.mkdtemp()
+ self.certificate = None
+ self.generate_certificate()
+
+ def clean_up(self):
+ util.del_dir(self.tmpdir)
+
+ def generate_certificate(self):
+ LOG.debug('Generating certificate for communication with fabric...')
+ if self.certificate is not None:
+ LOG.debug('Certificate already generated.')
+ return
+ with cd(self.tmpdir):
+ util.subp([
+ 'openssl', 'req', '-x509', '-nodes', '-subj',
+ '/CN=LinuxTransport', '-days', '32768', '-newkey', 'rsa:2048',
+ '-keyout', self.certificate_names['private_key'],
+ '-out', self.certificate_names['certificate'],
+ ])
+ certificate = ''
+ for line in open(self.certificate_names['certificate']):
+ if "CERTIFICATE" not in line:
+ certificate += line.rstrip()
+ self.certificate = certificate
+ LOG.debug('New certificate generated.')
+
+ def parse_certificates(self, certificates_xml):
+ tag = ElementTree.fromstring(certificates_xml).find(
+ './/Data')
+ certificates_content = tag.text
+ lines = [
+ b'MIME-Version: 1.0',
+ b'Content-Disposition: attachment; filename="Certificates.p7m"',
+ b'Content-Type: application/x-pkcs7-mime; name="Certificates.p7m"',
+ b'Content-Transfer-Encoding: base64',
+ b'',
+ certificates_content.encode('utf-8'),
+ ]
+ with cd(self.tmpdir):
+ with open('Certificates.p7m', 'wb') as f:
+ f.write(b'\n'.join(lines))
+ out, _ = util.subp(
+ 'openssl cms -decrypt -in Certificates.p7m -inkey'
+ ' {private_key} -recip {certificate} | openssl pkcs12 -nodes'
+ ' -password pass:'.format(**self.certificate_names),
+ shell=True)
+ private_keys, certificates = [], []
+ current = []
+ for line in out.splitlines():
+ current.append(line)
+ if re.match(r'[-]+END .*?KEY[-]+$', line):
+ private_keys.append('\n'.join(current))
+ current = []
+ elif re.match(r'[-]+END .*?CERTIFICATE[-]+$', line):
+ certificates.append('\n'.join(current))
+ current = []
+ keys = []
+ for certificate in certificates:
+ with cd(self.tmpdir):
+ public_key, _ = util.subp(
+ 'openssl x509 -noout -pubkey |'
+ 'ssh-keygen -i -m PKCS8 -f /dev/stdin',
+ data=certificate,
+ shell=True)
+ keys.append(public_key)
+ return keys
+
+
+def iid_from_shared_config_content(content):
+ """
+ find INSTANCE_ID in:
+ <?xml version="1.0" encoding="utf-8"?>
+ <SharedConfig version="1.0.0.0" goalStateIncarnation="1">
+ <Deployment name="INSTANCE_ID" guid="{...}" incarnation="0">
+ <Service name="..." guid="{00000000-0000-0000-0000-000000000000}"/>
+ """
+ root = ElementTree.fromstring(content)
+ depnode = root.find('Deployment')
+ return depnode.get('name')
+
+
+class WALinuxAgentShim(object):
+
+ REPORT_READY_XML_TEMPLATE = '\n'.join([
+ '<?xml version="1.0" encoding="utf-8"?>',
+ '<Health xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"'
+ ' xmlns:xsd="http://www.w3.org/2001/XMLSchema">',
+ ' <GoalStateIncarnation>{incarnation}</GoalStateIncarnation>',
+ ' <Container>',
+ ' <ContainerId>{container_id}</ContainerId>',
+ ' <RoleInstanceList>',
+ ' <Role>',
+ ' <InstanceId>{instance_id}</InstanceId>',
+ ' <Health>',
+ ' <State>Ready</State>',
+ ' </Health>',
+ ' </Role>',
+ ' </RoleInstanceList>',
+ ' </Container>',
+ '</Health>'])
+
+ def __init__(self):
+ LOG.debug('WALinuxAgentShim instantiated...')
+ self.endpoint = self.find_endpoint()
+ self.openssl_manager = None
+ self.values = {}
+
+ def clean_up(self):
+ if self.openssl_manager is not None:
+ self.openssl_manager.clean_up()
+
+ @staticmethod
+ def find_endpoint():
+ LOG.debug('Finding Azure endpoint...')
+ content = util.load_file('/var/lib/dhcp/dhclient.eth0.leases')
+ value = None
+ for line in content.splitlines():
+ if 'unknown-245' in line:
+ value = line.strip(' ').split(' ', 2)[-1].strip(';\n"')
+ if value is None:
+ raise Exception('No endpoint found in DHCP config.')
+ if ':' in value:
+ hex_string = ''
+ for hex_pair in value.split(':'):
+ if len(hex_pair) == 1:
+ hex_pair = '0' + hex_pair
+ hex_string += hex_pair
+ value = struct.pack('>L', int(hex_string.replace(':', ''), 16))
+ else:
+ value = value.encode('utf-8')
+ endpoint_ip_address = socket.inet_ntoa(value)
+ LOG.debug('Azure endpoint found at %s', endpoint_ip_address)
+ return endpoint_ip_address
+
+ def register_with_azure_and_fetch_data(self):
+ self.openssl_manager = OpenSSLManager()
+ http_client = AzureEndpointHttpClient(self.openssl_manager.certificate)
+ LOG.info('Registering with Azure...')
+ attempts = 0
+ while True:
+ try:
+ response = http_client.get(
+ 'http://{0}/machine/?comp=goalstate'.format(self.endpoint))
+ except Exception:
+ if attempts < 10:
+ time.sleep(attempts + 1)
+ else:
+ raise
+ else:
+ break
+ attempts += 1
+ LOG.debug('Successfully fetched GoalState XML.')
+ goal_state = GoalState(response.contents, http_client)
+ public_keys = []
+ if goal_state.certificates_xml is not None:
+ LOG.debug('Certificate XML found; parsing out public keys.')
+ public_keys = self.openssl_manager.parse_certificates(
+ goal_state.certificates_xml)
+ data = {
+ 'instance-id': iid_from_shared_config_content(
+ goal_state.shared_config_xml),
+ 'public-keys': public_keys,
+ }
+ self._report_ready(goal_state, http_client)
+ return data
+
+ def _report_ready(self, goal_state, http_client):
+ LOG.debug('Reporting ready to Azure fabric.')
+ document = self.REPORT_READY_XML_TEMPLATE.format(
+ incarnation=goal_state.incarnation,
+ container_id=goal_state.container_id,
+ instance_id=goal_state.instance_id,
+ )
+ http_client.post(
+ "http://{0}/machine?comp=health".format(self.endpoint),
+ data=document,
+ extra_headers={'Content-Type': 'text/xml; charset=utf-8'},
+ )
+ LOG.info('Reported ready to Azure fabric.')
+
+
+def get_metadata_from_fabric():
+ shim = WALinuxAgentShim()
+ try:
+ return shim.register_with_azure_and_fetch_data()
+ finally:
+ shim.clean_up()
diff --git a/cloudinit/util.py b/cloudinit/util.py
index cae57770..db4e02b8 100644
--- a/cloudinit/util.py
+++ b/cloudinit/util.py
@@ -766,10 +766,6 @@ def fetch_ssl_details(paths=None):
return ssl_details
-def load_tfile_or_url(*args, **kwargs):
- return(decode_binary(read_file_or_url(*args, **kwargs).contents))
-
-
def read_file_or_url(url, timeout=5, retries=10,
headers=None, data=None, sec_between=1, ssl_details=None,
headers_cb=None, exception_cb=None):
@@ -837,10 +833,10 @@ def read_seeded(base="", ext="", timeout=5, retries=10, file_retries=0):
ud_url = "%s%s%s" % (base, "user-data", ext)
md_url = "%s%s%s" % (base, "meta-data", ext)
- md_resp = load_tfile_or_url(md_url, timeout, retries, file_retries)
+ md_resp = read_file_or_url(md_url, timeout, retries, file_retries)
md = None
if md_resp.ok():
- md = load_yaml(md_resp.contents, default={})
+ md = load_yaml(decode_binary(md_resp.contents), default={})
ud_resp = read_file_or_url(ud_url, timeout, retries, file_retries)
ud = None
diff --git a/doc/rtd/topics/datasources.rst b/doc/rtd/topics/datasources.rst
index a2024bdc..0d7d4aca 100644
--- a/doc/rtd/topics/datasources.rst
+++ b/doc/rtd/topics/datasources.rst
@@ -177,6 +177,12 @@ OVF
For now see: https://bazaar.launchpad.net/~cloud-init-dev/cloud-init/trunk/files/head:/doc/sources/ovf/
---------------------------
+OpenStack
+---------------------------
+
+.. include:: ../../sources/openstack/README.rst
+
+---------------------------
Fallback/None
---------------------------
diff --git a/doc/sources/openstack/README.rst b/doc/sources/openstack/README.rst
new file mode 100644
index 00000000..8102597e
--- /dev/null
+++ b/doc/sources/openstack/README.rst
@@ -0,0 +1,24 @@
+*TODO*
+
+Vendor Data
+~~~~~~~~~~~
+
+The OpenStack metadata server can be configured to serve up vendor data
+which is available to all instances for consumption. OpenStack vendor
+data is, generally, a JSON object.
+
+cloud-init will look for configuration in the ``cloud-init`` attribute
+of the vendor data JSON object. cloud-init processes this configuration
+using the same handlers as user data, so any formats that work for user
+data should work for vendor data.
+
+For example, configuring the following as vendor data in OpenStack would
+upgrade packages and install ``htop`` on all instances:
+
+.. sourcecode:: json
+
+ {"cloud-init": "#cloud-config\npackage_upgrade: True\npackages:\n - htop"}
+
+For more general information about how cloud-init handles vendor data,
+including how it can be disabled by users on instances, see
+https://bazaar.launchpad.net/~cloud-init-dev/cloud-init/trunk/view/head:/doc/vendordata.txt
diff --git a/packages/brpm b/packages/brpm
index c6d79e75..b41b675f 100755
--- a/packages/brpm
+++ b/packages/brpm
@@ -40,7 +40,7 @@ PKG_MP = {
'jinja2': 'python-jinja2',
'configobj': 'python-configobj',
'jsonpatch': 'python-jsonpatch',
- 'oauthlib': 'python-oauth',
+ 'oauthlib': 'python-oauthlib',
'prettytable': 'python-prettytable',
'pyserial': 'pyserial',
'pyyaml': 'PyYAML',
@@ -52,7 +52,7 @@ PKG_MP = {
'cheetah': 'python-cheetah',
'configobj': 'python-configobj',
'jsonpatch': 'python-jsonpatch',
- 'oauth': 'python-oauth',
+ 'oauthlib': 'python-oauthlib',
'prettytable': 'python-prettytable',
'pyserial': 'python-pyserial',
'pyyaml': 'python-yaml',
diff --git a/packages/debian/control.in b/packages/debian/control.in
index 6de09f23..401a542f 100644
--- a/packages/debian/control.in
+++ b/packages/debian/control.in
@@ -19,7 +19,7 @@ Depends: procps,
${python},
${requires},
${misc:Depends},
-Recommends: eatmydata, sudo, software-properties-common
+Recommends: eatmydata, sudo, software-properties-common, gdisk
XB-Python-Version: ${python:Versions}
Description: Init scripts for cloud instances
Cloud instances need special scripts to run during initialisation
diff --git a/tests/unittests/helpers.py b/tests/unittests/helpers.py
index 61a1f6ff..7f4b8784 100644
--- a/tests/unittests/helpers.py
+++ b/tests/unittests/helpers.py
@@ -248,13 +248,15 @@ class FilesystemMockingTestCase(ResourceUsingTestCase):
def patchOS(self, new_root):
patch_funcs = {
- os.path: ['isfile', 'exists', 'islink', 'isdir'],
- os: ['listdir'],
+ os.path: [('isfile', 1), ('exists', 1),
+ ('islink', 1), ('isdir', 1)],
+ os: [('listdir', 1), ('mkdir', 1),
+ ('lstat', 1), ('symlink', 2)],
}
for (mod, funcs) in patch_funcs.items():
- for f in funcs:
+ for f, nargs in funcs:
func = getattr(mod, f)
- trap_func = retarget_many_wrapper(new_root, 1, func)
+ trap_func = retarget_many_wrapper(new_root, nargs, func)
self.patched_funcs.enter_context(
mock.patch.object(mod, f, trap_func))
diff --git a/tests/unittests/test_cli.py b/tests/unittests/test_cli.py
index ba447f87..ed863399 100644
--- a/tests/unittests/test_cli.py
+++ b/tests/unittests/test_cli.py
@@ -1,6 +1,6 @@
import imp
+import os
import sys
-
import six
from . import helpers as test_helpers
@@ -11,6 +11,9 @@ except ImportError:
import mock
+BIN_CLOUDINIT = "bin/cloud-init"
+
+
class TestCLI(test_helpers.FilesystemMockingTestCase):
def setUp(self):
@@ -25,16 +28,18 @@ class TestCLI(test_helpers.FilesystemMockingTestCase):
self.patched_funcs.enter_context(
mock.patch.object(sys, 'argv', ['cloud-init']))
cli = imp.load_module(
- 'cli', open('bin/cloud-init'), '', ('', 'r', imp.PY_SOURCE))
+ 'cli', open(BIN_CLOUDINIT), '', ('', 'r', imp.PY_SOURCE))
try:
return cli.main()
except:
pass
+ @test_helpers.skipIf(not os.path.isfile(BIN_CLOUDINIT), "no bin/cloudinit")
def test_no_arguments_shows_usage(self):
self._call_main()
self.assertIn('usage: cloud-init', self.stderr.getvalue())
+ @test_helpers.skipIf(not os.path.isfile(BIN_CLOUDINIT), "no bin/cloudinit")
def test_no_arguments_exits_2(self):
exit_code = self._call_main()
if self.sys_exit.call_count:
@@ -42,6 +47,7 @@ class TestCLI(test_helpers.FilesystemMockingTestCase):
else:
self.assertEqual(2, exit_code)
+ @test_helpers.skipIf(not os.path.isfile(BIN_CLOUDINIT), "no bin/cloudinit")
def test_no_arguments_shows_error_message(self):
self._call_main()
self.assertIn('cloud-init: error: too few arguments',
diff --git a/tests/unittests/test_datasource/test_azure.py b/tests/unittests/test_datasource/test_azure.py
index 7e789853..33b971f6 100644
--- a/tests/unittests/test_datasource/test_azure.py
+++ b/tests/unittests/test_datasource/test_azure.py
@@ -18,7 +18,7 @@ import stat
import yaml
import shutil
import tempfile
-import unittest
+import xml.etree.ElementTree as ET
def construct_valid_ovf_env(data=None, pubkeys=None, userdata=None):
@@ -123,6 +123,11 @@ class TestAzureDataSource(TestCase):
mod = DataSourceAzure
mod.BUILTIN_DS_CONFIG['data_dir'] = self.waagent_d
+ self.get_metadata_from_fabric = mock.MagicMock(return_value={
+ 'instance-id': 'i-my-azure-id',
+ 'public-keys': [],
+ })
+
self.apply_patches([
(mod, 'list_possible_azure_ds_devs', dsdevs),
(mod, 'invoke_agent', _invoke_agent),
@@ -132,13 +137,47 @@ class TestAzureDataSource(TestCase):
(mod, 'perform_hostname_bounce', mock.MagicMock()),
(mod, 'get_hostname', mock.MagicMock()),
(mod, 'set_hostname', mock.MagicMock()),
- ])
+ (mod, 'get_metadata_from_fabric', self.get_metadata_from_fabric),
+ ])
dsrc = mod.DataSourceAzureNet(
data.get('sys_cfg', {}), distro=None, paths=self.paths)
return dsrc
+ def xml_equals(self, oxml, nxml):
+ """Compare two sets of XML to make sure they are equal"""
+
+ def create_tag_index(xml):
+ et = ET.fromstring(xml)
+ ret = {}
+ for x in et.iter():
+ ret[x.tag] = x
+ return ret
+
+ def tags_exists(x, y):
+ for tag in x.keys():
+ self.assertIn(tag, y)
+ for tag in y.keys():
+ self.assertIn(tag, x)
+
+ def tags_equal(x, y):
+ for x_tag, x_val in x.items():
+ y_val = y.get(x_val.tag)
+ self.assertEquals(x_val.text, y_val.text)
+
+ old_cnt = create_tag_index(oxml)
+ new_cnt = create_tag_index(nxml)
+ tags_exists(old_cnt, new_cnt)
+ tags_equal(old_cnt, new_cnt)
+
+ def xml_notequals(self, oxml, nxml):
+ try:
+ self.xml_equals(oxml, nxml)
+ except AssertionError as e:
+ return
+ raise AssertionError("XML is the same")
+
def test_basic_seed_dir(self):
odata = {'HostName': "myhost", 'UserName': "myuser"}
data = {'ovfcontent': construct_valid_ovf_env(data=odata),
@@ -317,6 +356,31 @@ class TestAzureDataSource(TestCase):
self.assertEqual(userdata.encode('us-ascii'), dsrc.userdata_raw)
+ def test_password_redacted_in_ovf(self):
+ odata = {'HostName': "myhost", 'UserName': "myuser",
+ 'UserPassword': "mypass"}
+ data = {'ovfcontent': construct_valid_ovf_env(data=odata)}
+ dsrc = self._get_ds(data)
+ ret = dsrc.get_data()
+
+ self.assertTrue(ret)
+ ovf_env_path = os.path.join(self.waagent_d, 'ovf-env.xml')
+
+ # The XML should not be same since the user password is redacted
+ on_disk_ovf = load_file(ovf_env_path)
+ self.xml_notequals(data['ovfcontent'], on_disk_ovf)
+
+ # Make sure that the redacted password on disk is not used by CI
+ self.assertNotEquals(dsrc.cfg.get('password'),
+ DataSourceAzure.DEF_PASSWD_REDACTION)
+
+ # Make sure that the password was really encrypted
+ et = ET.fromstring(on_disk_ovf)
+ for elem in et.iter():
+ if 'UserPassword' in elem.tag:
+ self.assertEquals(DataSourceAzure.DEF_PASSWD_REDACTION,
+ elem.text)
+
def test_ovf_env_arrives_in_waagent_dir(self):
xml = construct_valid_ovf_env(data={}, userdata="FOODATA")
dsrc = self._get_ds({'ovfcontent': xml})
@@ -326,7 +390,7 @@ class TestAzureDataSource(TestCase):
# we expect that the ovf-env.xml file is copied there.
ovf_env_path = os.path.join(self.waagent_d, 'ovf-env.xml')
self.assertTrue(os.path.exists(ovf_env_path))
- self.assertEqual(xml, load_file(ovf_env_path))
+ self.xml_equals(xml, load_file(ovf_env_path))
def test_ovf_can_include_unicode(self):
xml = construct_valid_ovf_env(data={})
@@ -375,12 +439,26 @@ class TestAzureDataSource(TestCase):
self.assertEqual(dsrc.userdata_raw, b"NEW_USERDATA")
self.assertTrue(os.path.exists(
os.path.join(self.waagent_d, 'otherfile')))
- self.assertFalse(
- os.path.exists(os.path.join(self.waagent_d, 'SharedConfig.xml')))
- self.assertTrue(
- os.path.exists(os.path.join(self.waagent_d, 'ovf-env.xml')))
- self.assertEqual(new_ovfenv,
- load_file(os.path.join(self.waagent_d, 'ovf-env.xml')))
+ self.assertFalse(os.path.exists(
+ os.path.join(self.waagent_d, 'SharedConfig.xml')))
+ self.assertTrue(os.path.exists(
+ os.path.join(self.waagent_d, 'ovf-env.xml')))
+ new_xml = load_file(os.path.join(self.waagent_d, 'ovf-env.xml'))
+ self.xml_equals(new_ovfenv, new_xml)
+
+ def test_exception_fetching_fabric_data_doesnt_propagate(self):
+ ds = self._get_ds({'ovfcontent': construct_valid_ovf_env()})
+ ds.ds_cfg['agent_command'] = '__builtin__'
+ self.get_metadata_from_fabric.side_effect = Exception
+ self.assertFalse(ds.get_data())
+
+ def test_fabric_data_included_in_metadata(self):
+ ds = self._get_ds({'ovfcontent': construct_valid_ovf_env()})
+ ds.ds_cfg['agent_command'] = '__builtin__'
+ self.get_metadata_from_fabric.return_value = {'test': 'value'}
+ ret = ds.get_data()
+ self.assertTrue(ret)
+ self.assertEqual('value', ds.metadata['test'])
class TestAzureBounce(TestCase):
@@ -402,6 +480,9 @@ class TestAzureBounce(TestCase):
self.patches.enter_context(
mock.patch.object(DataSourceAzure, 'find_ephemeral_part',
mock.MagicMock(return_value=None)))
+ self.patches.enter_context(
+ mock.patch.object(DataSourceAzure, 'get_metadata_from_fabric',
+ mock.MagicMock(return_value={})))
def setUp(self):
super(TestAzureBounce, self).setUp()
@@ -565,17 +646,3 @@ class TestReadAzureOvf(TestCase):
(_md, _ud, cfg) = DataSourceAzure.read_azure_ovf(content)
for mypk in mypklist:
self.assertIn(mypk, cfg['_pubkeys'])
-
-
-class TestReadAzureSharedConfig(unittest.TestCase):
- def test_valid_content(self):
- xml = """<?xml version="1.0" encoding="utf-8"?>
- <SharedConfig>
- <Deployment name="MY_INSTANCE_ID">
- <Service name="myservice"/>
- <ServiceInstance name="INSTANCE_ID.0" guid="{abcd-uuid}" />
- </Deployment>
- <Incarnation number="1"/>
- </SharedConfig>"""
- ret = DataSourceAzure.iid_from_shared_config_content(xml)
- self.assertEqual("MY_INSTANCE_ID", ret)
diff --git a/tests/unittests/test_datasource/test_azure_helper.py b/tests/unittests/test_datasource/test_azure_helper.py
new file mode 100644
index 00000000..a5228870
--- /dev/null
+++ b/tests/unittests/test_datasource/test_azure_helper.py
@@ -0,0 +1,444 @@
+import os
+import struct
+import unittest
+
+from cloudinit.sources.helpers import azure as azure_helper
+from ..helpers import TestCase
+
+try:
+ from unittest import mock
+except ImportError:
+ import mock
+
+try:
+ from contextlib import ExitStack
+except ImportError:
+ from contextlib2 import ExitStack
+
+
+GOAL_STATE_TEMPLATE = """\
+<?xml version="1.0" encoding="utf-8"?>
+<GoalState xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:noNamespaceSchemaLocation="goalstate10.xsd">
+ <Version>2012-11-30</Version>
+ <Incarnation>{incarnation}</Incarnation>
+ <Machine>
+ <ExpectedState>Started</ExpectedState>
+ <StopRolesDeadlineHint>300000</StopRolesDeadlineHint>
+ <LBProbePorts>
+ <Port>16001</Port>
+ </LBProbePorts>
+ <ExpectHealthReport>FALSE</ExpectHealthReport>
+ </Machine>
+ <Container>
+ <ContainerId>{container_id}</ContainerId>
+ <RoleInstanceList>
+ <RoleInstance>
+ <InstanceId>{instance_id}</InstanceId>
+ <State>Started</State>
+ <Configuration>
+ <HostingEnvironmentConfig>
+ http://100.86.192.70:80/...hostingEnvironmentConfig...
+ </HostingEnvironmentConfig>
+ <SharedConfig>{shared_config_url}</SharedConfig>
+ <ExtensionsConfig>
+ http://100.86.192.70:80/...extensionsConfig...
+ </ExtensionsConfig>
+ <FullConfig>http://100.86.192.70:80/...fullConfig...</FullConfig>
+ <Certificates>{certificates_url}</Certificates>
+ <ConfigName>68ce47.0.68ce47.0.utl-trusty--292258.1.xml</ConfigName>
+ </Configuration>
+ </RoleInstance>
+ </RoleInstanceList>
+ </Container>
+</GoalState>
+"""
+
+
+class TestReadAzureSharedConfig(unittest.TestCase):
+
+ def test_valid_content(self):
+ xml = """<?xml version="1.0" encoding="utf-8"?>
+ <SharedConfig>
+ <Deployment name="MY_INSTANCE_ID">
+ <Service name="myservice"/>
+ <ServiceInstance name="INSTANCE_ID.0" guid="{abcd-uuid}" />
+ </Deployment>
+ <Incarnation number="1"/>
+ </SharedConfig>"""
+ ret = azure_helper.iid_from_shared_config_content(xml)
+ self.assertEqual("MY_INSTANCE_ID", ret)
+
+
+class TestFindEndpoint(TestCase):
+
+ def setUp(self):
+ super(TestFindEndpoint, self).setUp()
+ patches = ExitStack()
+ self.addCleanup(patches.close)
+
+ self.load_file = patches.enter_context(
+ mock.patch.object(azure_helper.util, 'load_file'))
+
+ def test_missing_file(self):
+ self.load_file.side_effect = IOError
+ self.assertRaises(IOError,
+ azure_helper.WALinuxAgentShim.find_endpoint)
+
+ def test_missing_special_azure_line(self):
+ self.load_file.return_value = ''
+ self.assertRaises(Exception,
+ azure_helper.WALinuxAgentShim.find_endpoint)
+
+ def _build_lease_content(self, ip_address, use_hex=True):
+ ip_address_repr = ':'.join(
+ [hex(int(part)).replace('0x', '')
+ for part in ip_address.split('.')])
+ if not use_hex:
+ ip_address_repr = struct.pack(
+ '>L', int(ip_address_repr.replace(':', ''), 16))
+ ip_address_repr = '"{0}"'.format(ip_address_repr.decode('utf-8'))
+ return '\n'.join([
+ 'lease {',
+ ' interface "eth0";',
+ ' option unknown-245 {0};'.format(ip_address_repr),
+ '}'])
+
+ def test_hex_string(self):
+ ip_address = '98.76.54.32'
+ file_content = self._build_lease_content(ip_address)
+ self.load_file.return_value = file_content
+ self.assertEqual(ip_address,
+ azure_helper.WALinuxAgentShim.find_endpoint())
+
+ def test_hex_string_with_single_character_part(self):
+ ip_address = '4.3.2.1'
+ file_content = self._build_lease_content(ip_address)
+ self.load_file.return_value = file_content
+ self.assertEqual(ip_address,
+ azure_helper.WALinuxAgentShim.find_endpoint())
+
+ def test_packed_string(self):
+ ip_address = '98.76.54.32'
+ file_content = self._build_lease_content(ip_address, use_hex=False)
+ self.load_file.return_value = file_content
+ self.assertEqual(ip_address,
+ azure_helper.WALinuxAgentShim.find_endpoint())
+
+ def test_latest_lease_used(self):
+ ip_addresses = ['4.3.2.1', '98.76.54.32']
+ file_content = '\n'.join([self._build_lease_content(ip_address)
+ for ip_address in ip_addresses])
+ self.load_file.return_value = file_content
+ self.assertEqual(ip_addresses[-1],
+ azure_helper.WALinuxAgentShim.find_endpoint())
+
+
+class TestGoalStateParsing(TestCase):
+
+ default_parameters = {
+ 'incarnation': 1,
+ 'container_id': 'MyContainerId',
+ 'instance_id': 'MyInstanceId',
+ 'shared_config_url': 'MySharedConfigUrl',
+ 'certificates_url': 'MyCertificatesUrl',
+ }
+
+ def _get_goal_state(self, http_client=None, **kwargs):
+ if http_client is None:
+ http_client = mock.MagicMock()
+ parameters = self.default_parameters.copy()
+ parameters.update(kwargs)
+ xml = GOAL_STATE_TEMPLATE.format(**parameters)
+ if parameters['certificates_url'] is None:
+ new_xml_lines = []
+ for line in xml.splitlines():
+ if 'Certificates' in line:
+ continue
+ new_xml_lines.append(line)
+ xml = '\n'.join(new_xml_lines)
+ return azure_helper.GoalState(xml, http_client)
+
+ def test_incarnation_parsed_correctly(self):
+ incarnation = '123'
+ goal_state = self._get_goal_state(incarnation=incarnation)
+ self.assertEqual(incarnation, goal_state.incarnation)
+
+ def test_container_id_parsed_correctly(self):
+ container_id = 'TestContainerId'
+ goal_state = self._get_goal_state(container_id=container_id)
+ self.assertEqual(container_id, goal_state.container_id)
+
+ def test_instance_id_parsed_correctly(self):
+ instance_id = 'TestInstanceId'
+ goal_state = self._get_goal_state(instance_id=instance_id)
+ self.assertEqual(instance_id, goal_state.instance_id)
+
+ def test_shared_config_xml_parsed_and_fetched_correctly(self):
+ http_client = mock.MagicMock()
+ shared_config_url = 'TestSharedConfigUrl'
+ goal_state = self._get_goal_state(
+ http_client=http_client, shared_config_url=shared_config_url)
+ shared_config_xml = goal_state.shared_config_xml
+ self.assertEqual(1, http_client.get.call_count)
+ self.assertEqual(shared_config_url, http_client.get.call_args[0][0])
+ self.assertEqual(http_client.get.return_value.contents,
+ shared_config_xml)
+
+ def test_certificates_xml_parsed_and_fetched_correctly(self):
+ http_client = mock.MagicMock()
+ certificates_url = 'TestSharedConfigUrl'
+ goal_state = self._get_goal_state(
+ http_client=http_client, certificates_url=certificates_url)
+ certificates_xml = goal_state.certificates_xml
+ self.assertEqual(1, http_client.get.call_count)
+ self.assertEqual(certificates_url, http_client.get.call_args[0][0])
+ self.assertTrue(http_client.get.call_args[1].get('secure', False))
+ self.assertEqual(http_client.get.return_value.contents,
+ certificates_xml)
+
+ def test_missing_certificates_skips_http_get(self):
+ http_client = mock.MagicMock()
+ goal_state = self._get_goal_state(
+ http_client=http_client, certificates_url=None)
+ certificates_xml = goal_state.certificates_xml
+ self.assertEqual(0, http_client.get.call_count)
+ self.assertIsNone(certificates_xml)
+
+
+class TestAzureEndpointHttpClient(TestCase):
+
+ regular_headers = {
+ 'x-ms-agent-name': 'WALinuxAgent',
+ 'x-ms-version': '2012-11-30',
+ }
+
+ def setUp(self):
+ super(TestAzureEndpointHttpClient, self).setUp()
+ patches = ExitStack()
+ self.addCleanup(patches.close)
+
+ self.read_file_or_url = patches.enter_context(
+ mock.patch.object(azure_helper.util, 'read_file_or_url'))
+
+ def test_non_secure_get(self):
+ client = azure_helper.AzureEndpointHttpClient(mock.MagicMock())
+ url = 'MyTestUrl'
+ response = client.get(url, secure=False)
+ self.assertEqual(1, self.read_file_or_url.call_count)
+ self.assertEqual(self.read_file_or_url.return_value, response)
+ self.assertEqual(mock.call(url, headers=self.regular_headers),
+ self.read_file_or_url.call_args)
+
+ def test_secure_get(self):
+ url = 'MyTestUrl'
+ certificate = mock.MagicMock()
+ expected_headers = self.regular_headers.copy()
+ expected_headers.update({
+ "x-ms-cipher-name": "DES_EDE3_CBC",
+ "x-ms-guest-agent-public-x509-cert": certificate,
+ })
+ client = azure_helper.AzureEndpointHttpClient(certificate)
+ response = client.get(url, secure=True)
+ self.assertEqual(1, self.read_file_or_url.call_count)
+ self.assertEqual(self.read_file_or_url.return_value, response)
+ self.assertEqual(mock.call(url, headers=expected_headers),
+ self.read_file_or_url.call_args)
+
+ def test_post(self):
+ data = mock.MagicMock()
+ url = 'MyTestUrl'
+ client = azure_helper.AzureEndpointHttpClient(mock.MagicMock())
+ response = client.post(url, data=data)
+ self.assertEqual(1, self.read_file_or_url.call_count)
+ self.assertEqual(self.read_file_or_url.return_value, response)
+ self.assertEqual(
+ mock.call(url, data=data, headers=self.regular_headers),
+ self.read_file_or_url.call_args)
+
+ def test_post_with_extra_headers(self):
+ url = 'MyTestUrl'
+ client = azure_helper.AzureEndpointHttpClient(mock.MagicMock())
+ extra_headers = {'test': 'header'}
+ client.post(url, extra_headers=extra_headers)
+ self.assertEqual(1, self.read_file_or_url.call_count)
+ expected_headers = self.regular_headers.copy()
+ expected_headers.update(extra_headers)
+ self.assertEqual(
+ mock.call(mock.ANY, data=mock.ANY, headers=expected_headers),
+ self.read_file_or_url.call_args)
+
+
+class TestOpenSSLManager(TestCase):
+
+ def setUp(self):
+ super(TestOpenSSLManager, self).setUp()
+ patches = ExitStack()
+ self.addCleanup(patches.close)
+
+ self.subp = patches.enter_context(
+ mock.patch.object(azure_helper.util, 'subp'))
+ try:
+ self.open = patches.enter_context(
+ mock.patch('__builtin__.open'))
+ except ImportError:
+ self.open = patches.enter_context(
+ mock.patch('builtins.open'))
+
+ @mock.patch.object(azure_helper, 'cd', mock.MagicMock())
+ @mock.patch.object(azure_helper.tempfile, 'mkdtemp')
+ def test_openssl_manager_creates_a_tmpdir(self, mkdtemp):
+ manager = azure_helper.OpenSSLManager()
+ self.assertEqual(mkdtemp.return_value, manager.tmpdir)
+
+ def test_generate_certificate_uses_tmpdir(self):
+ subp_directory = {}
+
+ def capture_directory(*args, **kwargs):
+ subp_directory['path'] = os.getcwd()
+
+ self.subp.side_effect = capture_directory
+ manager = azure_helper.OpenSSLManager()
+ self.assertEqual(manager.tmpdir, subp_directory['path'])
+
+ @mock.patch.object(azure_helper, 'cd', mock.MagicMock())
+ @mock.patch.object(azure_helper.tempfile, 'mkdtemp', mock.MagicMock())
+ @mock.patch.object(azure_helper.util, 'del_dir')
+ def test_clean_up(self, del_dir):
+ manager = azure_helper.OpenSSLManager()
+ manager.clean_up()
+ self.assertEqual([mock.call(manager.tmpdir)], del_dir.call_args_list)
+
+
+class TestWALinuxAgentShim(TestCase):
+
+ def setUp(self):
+ super(TestWALinuxAgentShim, self).setUp()
+ patches = ExitStack()
+ self.addCleanup(patches.close)
+
+ self.AzureEndpointHttpClient = patches.enter_context(
+ mock.patch.object(azure_helper, 'AzureEndpointHttpClient'))
+ self.find_endpoint = patches.enter_context(
+ mock.patch.object(
+ azure_helper.WALinuxAgentShim, 'find_endpoint'))
+ self.GoalState = patches.enter_context(
+ mock.patch.object(azure_helper, 'GoalState'))
+ self.iid_from_shared_config_content = patches.enter_context(
+ mock.patch.object(azure_helper, 'iid_from_shared_config_content'))
+ self.OpenSSLManager = patches.enter_context(
+ mock.patch.object(azure_helper, 'OpenSSLManager'))
+ patches.enter_context(
+ mock.patch.object(azure_helper.time, 'sleep', mock.MagicMock()))
+
+ def test_http_client_uses_certificate(self):
+ shim = azure_helper.WALinuxAgentShim()
+ shim.register_with_azure_and_fetch_data()
+ self.assertEqual(
+ [mock.call(self.OpenSSLManager.return_value.certificate)],
+ self.AzureEndpointHttpClient.call_args_list)
+
+ def test_correct_url_used_for_goalstate(self):
+ self.find_endpoint.return_value = 'test_endpoint'
+ shim = azure_helper.WALinuxAgentShim()
+ shim.register_with_azure_and_fetch_data()
+ get = self.AzureEndpointHttpClient.return_value.get
+ self.assertEqual(
+ [mock.call('http://test_endpoint/machine/?comp=goalstate')],
+ get.call_args_list)
+ self.assertEqual(
+ [mock.call(get.return_value.contents,
+ self.AzureEndpointHttpClient.return_value)],
+ self.GoalState.call_args_list)
+
+ def test_certificates_used_to_determine_public_keys(self):
+ shim = azure_helper.WALinuxAgentShim()
+ data = shim.register_with_azure_and_fetch_data()
+ self.assertEqual(
+ [mock.call(self.GoalState.return_value.certificates_xml)],
+ self.OpenSSLManager.return_value.parse_certificates.call_args_list)
+ self.assertEqual(
+ self.OpenSSLManager.return_value.parse_certificates.return_value,
+ data['public-keys'])
+
+ def test_absent_certificates_produces_empty_public_keys(self):
+ self.GoalState.return_value.certificates_xml = None
+ shim = azure_helper.WALinuxAgentShim()
+ data = shim.register_with_azure_and_fetch_data()
+ self.assertEqual([], data['public-keys'])
+
+ def test_instance_id_returned_in_data(self):
+ shim = azure_helper.WALinuxAgentShim()
+ data = shim.register_with_azure_and_fetch_data()
+ self.assertEqual(
+ [mock.call(self.GoalState.return_value.shared_config_xml)],
+ self.iid_from_shared_config_content.call_args_list)
+ self.assertEqual(self.iid_from_shared_config_content.return_value,
+ data['instance-id'])
+
+ def test_correct_url_used_for_report_ready(self):
+ self.find_endpoint.return_value = 'test_endpoint'
+ shim = azure_helper.WALinuxAgentShim()
+ shim.register_with_azure_and_fetch_data()
+ expected_url = 'http://test_endpoint/machine?comp=health'
+ self.assertEqual(
+ [mock.call(expected_url, data=mock.ANY, extra_headers=mock.ANY)],
+ self.AzureEndpointHttpClient.return_value.post.call_args_list)
+
+ def test_goal_state_values_used_for_report_ready(self):
+ self.GoalState.return_value.incarnation = 'TestIncarnation'
+ self.GoalState.return_value.container_id = 'TestContainerId'
+ self.GoalState.return_value.instance_id = 'TestInstanceId'
+ shim = azure_helper.WALinuxAgentShim()
+ shim.register_with_azure_and_fetch_data()
+ posted_document = (
+ self.AzureEndpointHttpClient.return_value.post.call_args[1]['data']
+ )
+ self.assertIn('TestIncarnation', posted_document)
+ self.assertIn('TestContainerId', posted_document)
+ self.assertIn('TestInstanceId', posted_document)
+
+ def test_clean_up_can_be_called_at_any_time(self):
+ shim = azure_helper.WALinuxAgentShim()
+ shim.clean_up()
+
+ def test_clean_up_will_clean_up_openssl_manager_if_instantiated(self):
+ shim = azure_helper.WALinuxAgentShim()
+ shim.register_with_azure_and_fetch_data()
+ shim.clean_up()
+ self.assertEqual(
+ 1, self.OpenSSLManager.return_value.clean_up.call_count)
+
+ def test_failure_to_fetch_goalstate_bubbles_up(self):
+ class SentinelException(Exception):
+ pass
+ self.AzureEndpointHttpClient.return_value.get.side_effect = (
+ SentinelException)
+ shim = azure_helper.WALinuxAgentShim()
+ self.assertRaises(SentinelException,
+ shim.register_with_azure_and_fetch_data)
+
+
+class TestGetMetadataFromFabric(TestCase):
+
+ @mock.patch.object(azure_helper, 'WALinuxAgentShim')
+ def test_data_from_shim_returned(self, shim):
+ ret = azure_helper.get_metadata_from_fabric()
+ self.assertEqual(
+ shim.return_value.register_with_azure_and_fetch_data.return_value,
+ ret)
+
+ @mock.patch.object(azure_helper, 'WALinuxAgentShim')
+ def test_success_calls_clean_up(self, shim):
+ azure_helper.get_metadata_from_fabric()
+ self.assertEqual(1, shim.return_value.clean_up.call_count)
+
+ @mock.patch.object(azure_helper, 'WALinuxAgentShim')
+ def test_failure_in_registration_calls_clean_up(self, shim):
+ class SentinelException(Exception):
+ pass
+ shim.return_value.register_with_azure_and_fetch_data.side_effect = (
+ SentinelException)
+ self.assertRaises(SentinelException,
+ azure_helper.get_metadata_from_fabric)
+ self.assertEqual(1, shim.return_value.clean_up.call_count)
diff --git a/tests/unittests/test_distros/test_generic.py b/tests/unittests/test_distros/test_generic.py
index 35153f0d..8e3bd78a 100644
--- a/tests/unittests/test_distros/test_generic.py
+++ b/tests/unittests/test_distros/test_generic.py
@@ -194,6 +194,29 @@ class TestGenericDistro(helpers.FilesystemMockingTestCase):
{'primary': 'http://fs-primary-intel',
'security': 'http://security-mirror2-intel'})
+ def test_systemd_in_use(self):
+ cls = distros.fetch("ubuntu")
+ d = cls("ubuntu", {}, None)
+ self.patchOS(self.tmp)
+ self.patchUtils(self.tmp)
+ os.makedirs('/run/systemd/system')
+ self.assertTrue(d.uses_systemd())
+
+ def test_systemd_not_in_use(self):
+ cls = distros.fetch("ubuntu")
+ d = cls("ubuntu", {}, None)
+ self.patchOS(self.tmp)
+ self.patchUtils(self.tmp)
+ self.assertFalse(d.uses_systemd())
+
+ def test_systemd_symlink(self):
+ cls = distros.fetch("ubuntu")
+ d = cls("ubuntu", {}, None)
+ self.patchOS(self.tmp)
+ self.patchUtils(self.tmp)
+ os.makedirs('/run/systemd')
+ os.symlink('/', '/run/systemd/system')
+ self.assertFalse(d.uses_systemd())
# def _get_package_mirror_info(mirror_info, availability_zone=None,
# mirror_filter=util.search_for_mirror):
diff --git a/tests/unittests/test_handler/test_handler_apt_configure.py b/tests/unittests/test_handler/test_handler_apt_configure.py
index 895728b3..1ed185ca 100644
--- a/tests/unittests/test_handler/test_handler_apt_configure.py
+++ b/tests/unittests/test_handler/test_handler_apt_configure.py
@@ -9,6 +9,10 @@ import shutil
import tempfile
+def load_tfile_or_url(*args, **kwargs):
+ return(util.decode_binary(util.read_file_or_url(*args, **kwargs).contents))
+
+
class TestAptProxyConfig(TestCase):
def setUp(self):
super(TestAptProxyConfig, self).setUp()
@@ -29,7 +33,7 @@ class TestAptProxyConfig(TestCase):
self.assertTrue(os.path.isfile(self.pfile))
self.assertFalse(os.path.isfile(self.cfile))
- contents = util.load_tfile_or_url(self.pfile)
+ contents = load_tfile_or_url(self.pfile)
self.assertTrue(self._search_apt_config(contents, "http", "myproxy"))
def test_apt_http_proxy_written(self):
@@ -39,7 +43,7 @@ class TestAptProxyConfig(TestCase):
self.assertTrue(os.path.isfile(self.pfile))
self.assertFalse(os.path.isfile(self.cfile))
- contents = util.load_tfile_or_url(self.pfile)
+ contents = load_tfile_or_url(self.pfile)
self.assertTrue(self._search_apt_config(contents, "http", "myproxy"))
def test_apt_all_proxy_written(self):
@@ -57,7 +61,7 @@ class TestAptProxyConfig(TestCase):
self.assertTrue(os.path.isfile(self.pfile))
self.assertFalse(os.path.isfile(self.cfile))
- contents = util.load_tfile_or_url(self.pfile)
+ contents = load_tfile_or_url(self.pfile)
for ptype, pval in values.items():
self.assertTrue(self._search_apt_config(contents, ptype, pval))
@@ -73,7 +77,7 @@ class TestAptProxyConfig(TestCase):
cc_apt_configure.apply_apt_config({'apt_proxy': "foo"},
self.pfile, self.cfile)
self.assertTrue(os.path.isfile(self.pfile))
- contents = util.load_tfile_or_url(self.pfile)
+ contents = load_tfile_or_url(self.pfile)
self.assertTrue(self._search_apt_config(contents, "http", "foo"))
def test_config_written(self):
@@ -85,14 +89,14 @@ class TestAptProxyConfig(TestCase):
self.assertTrue(os.path.isfile(self.cfile))
self.assertFalse(os.path.isfile(self.pfile))
- self.assertEqual(util.load_tfile_or_url(self.cfile), payload)
+ self.assertEqual(load_tfile_or_url(self.cfile), payload)
def test_config_replaced(self):
util.write_file(self.pfile, "content doesnt matter")
cc_apt_configure.apply_apt_config({'apt_config': "foo"},
self.pfile, self.cfile)
self.assertTrue(os.path.isfile(self.cfile))
- self.assertEqual(util.load_tfile_or_url(self.cfile), "foo")
+ self.assertEqual(load_tfile_or_url(self.cfile), "foo")
def test_config_deleted(self):
# if no 'apt_config' is provided, delete any previously written file
diff --git a/tests/unittests/test_util.py b/tests/unittests/test_util.py
index 1619b5d2..95990165 100644
--- a/tests/unittests/test_util.py
+++ b/tests/unittests/test_util.py
@@ -459,4 +459,21 @@ class TestMessageFromString(helpers.TestCase):
roundtripped = util.message_from_string(u'\n').as_string()
self.assertNotIn('\x00', roundtripped)
+
+class TestReadSeeded(helpers.TestCase):
+ def setUp(self):
+ super(TestReadSeeded, self).setUp()
+ self.tmp = tempfile.mkdtemp()
+ self.addCleanup(shutil.rmtree, self.tmp)
+
+ def test_unicode_not_messed_up(self):
+ ud = b"userdatablob"
+ helpers.populate_dir(
+ self.tmp, {'meta-data': "key1: val1", 'user-data': ud})
+ sdir = self.tmp + os.path.sep
+ (found_md, found_ud) = util.read_seeded(sdir)
+
+ self.assertEqual(found_md, {'key1': 'val1'})
+ self.assertEqual(found_ud, ud)
+
# vi: ts=4 expandtab
generated by cgit v1.2.3 (git 2.39.1) at 2025-05-10 10:58:15 +0000