From 626f49e427cdbe91285dc337d134bfe2011fd268 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Tue, 5 Mar 2013 18:49:03 +0000 Subject: add doc --- doc/examples/cloud-config-growpart.txt | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 doc/examples/cloud-config-growpart.txt (limited to 'doc/examples') diff --git a/doc/examples/cloud-config-growpart.txt b/doc/examples/cloud-config-growpart.txt new file mode 100644 index 00000000..705f02c2 --- /dev/null +++ b/doc/examples/cloud-config-growpart.txt @@ -0,0 +1,24 @@ +#cloud-config +# +# growpart entry is a dict, if it is not present at all +# in config, then the default is used ({'mode': 'auto', 'devices': ['/']}) +# +# mode: +# values: +# * auto: use any option possible (growpart or parted) +# if none are available, do not warn, but debug. +# * growpart: use growpart to grow partitions +# if growpart is not available, this is an error. +# * parted: use parted (parted resizepart) to resize partitions +# if parted is not available, this is an error. +# * off, false +# +# devices: +# a list of things to resize. +# items can be filesystem paths or devices (in /dev) +# examples: +# devices: [/, /dev/vdb1] +# +growpart: + mode: auto + devices: ['/'] -- cgit v1.2.3 From 05c22e4f202332332de051e6849bbf5210aa19f6 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Wed, 3 Apr 2013 16:06:43 -0500 Subject: invoke 'dist-upgrade' instead of 'upgrade' on for upgrades. In general, dist-upgrade is the correct behavior here. It will get a new kernel, though, which could be annoying. So, allow a way to turn it off (by setting 'apt_get_upgrade_subcommand: upgrade'). LP: #1164147 --- cloudinit/distros/debian.py | 8 +++++++- doc/examples/cloud-config.txt | 13 +++++++++++++ 2 files changed, 20 insertions(+), 1 deletion(-) (limited to 'doc/examples') diff --git a/cloudinit/distros/debian.py b/cloudinit/distros/debian.py index 4b779d57..a1e28ad5 100644 --- a/cloudinit/distros/debian.py +++ b/cloudinit/distros/debian.py @@ -161,7 +161,13 @@ class Distro(distros.Distro): elif args and isinstance(args, list): cmd.extend(args) - cmd.append(command) + + subcmd = command + if command == "upgrade": + subcmd = self.get_option("apt_get_upgrade_subcommand", + "dist-upgrade") + + cmd.append(subcmd) pkglist = util.expand_package_list('%s=%s', pkgs) cmd.extend(pkglist) diff --git a/doc/examples/cloud-config.txt b/doc/examples/cloud-config.txt index 09298655..b8abb67a 100644 --- a/doc/examples/cloud-config.txt +++ b/doc/examples/cloud-config.txt @@ -125,6 +125,19 @@ apt_sources: =Y2oI -----END PGP PUBLIC KEY BLOCK----- +# apt_get_command: [command, argument, argument] +# Specify a different 'apt-get' command. must be a list. subcommands are +# appended to it. default is: +# ['apt-get', '--option=Dpkg::Options::=--force-confold', +# '--option=Dpkg::options::=--force-unsafe-io', '--assume-yes', '--quiet'] +# +# apt_get_upgrade_subcommand: +# Specify a different 'apt-get upgrade' subcommand. when 'apt_upgrade' or +# package_upgrade is set to true above, then this subcommand will be invoked. +# default is 'dist-upgrade'. For example, you could set this to 'upgrade'. +apt_get_upgrade_subcommand: dist-upgrade + + # Install additional packages on first boot # # Default: none -- cgit v1.2.3 From 691fe6d4ef3dad5d77e1b250d05bb0858234afee Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Mon, 8 Jul 2013 17:04:53 -0400 Subject: commit dev state --- cloudinit/sources/DataSourceAzure.py | 205 ++++++++++++++++++++++++++++++ doc/examples/cloud-config-datasources.txt | 4 + 2 files changed, 209 insertions(+) create mode 100644 cloudinit/sources/DataSourceAzure.py (limited to 'doc/examples') diff --git a/cloudinit/sources/DataSourceAzure.py b/cloudinit/sources/DataSourceAzure.py new file mode 100644 index 00000000..83c4603c --- /dev/null +++ b/cloudinit/sources/DataSourceAzure.py @@ -0,0 +1,205 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2013 Canonical Ltd. +# +# Author: Scott Moser +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import base64 +import os +from xml.dom import minidom + +from cloudinit import log as logging +from cloudinit import sources +from cloudinit import util + +LOG = logging.getLogger(__name__) + +DS_NAME = 'Azure' +DEFAULT_METADATA = {"instance-id": "iid-AZURE-NODE"} +AGENT_START = ['service', 'walinuxagent', 'start'] +DEFAULT_DS_CONFIG = {'datasource': {DS_NAME: {'agent_command': AGENT_START}}} + + +class DataSourceAzureNet(sources.DataSource): + def __init__(self, sys_cfg, distro, paths): + sources.DataSource.__init__(self, sys_cfg, distro, paths) + self.seed_dir = os.path.join(paths.seed_dir, 'azure') + self.cfg = {} + self.seed = None + + def __str__(self): + root = sources.DataSource.__str__(self) + return "%s [seed=%s]" % (root, self.seed) + + def get_data(self): + candidates = [self.seed_dir] + candidates.extend(list_possible_azure_ds_devs()) + found = None + + for cdev in candidates: + try: + if cdev.startswith("/dev/"): + ret = util.mount_cb(cdev, load_azure_ds_dir) + else: + ret = load_azure_ds_dir(cdev) + + except NonAzureDataSource: + pass + except BrokenAzureDataSource as exc: + raise exc + except util.MountFailedError: + LOG.warn("%s was not mountable" % cdev) + + (md, self.userdata_raw, cfg) = ret + self.seed = cdev + self.metadata = util.mergemanydict([md, DEFAULT_METADATA]) + self.cfg = util.mergemanydict([cfg, DEFAULT_DS_CONFIG]) + found = cdev + + LOG.debug("found datasource in %s", cdev) + break + + if not found: + return False + + path = ['datasource', DS_NAME, 'agent_command'] + cmd = None + for cfg in (self.cfg, self.sys_cfg): + cmd = util.get_cfg_by_path(cfg, keyp=path) + if cmd is not None: + break + invoke_agent(cmd) + + def get_config_obj(self): + return self.cfg + + +def invoke_agent(cmd): + if cmd: + LOG.debug("invoking agent: %s" % cmd) + util.subp(cmd, shell=(not isinstance(cmd, list))) + else: + LOG.debug("not invoking agent") + + +def find_child(node, filter_func): + ret = [] + if not node.hasChildNodes(): + return ret + for child in node.childNodes: + if filter_func(child): + ret.append(child) + return ret + + +def read_azure_ovf(contents): + dom = minidom.parseString(contents) + results = find_child(dom.documentElement, + lambda n: n.localName == "ProvisioningSection") + + if len(results) == 0: + raise NonAzureDataSource("No ProvisioningSection") + if len(results) > 1: + raise BrokenAzureDataSource("found '%d' ProvisioningSection items" % + len(results)) + provSection = results[0] + + lpcs_nodes = find_child(provSection, + lambda n: n.localName == "LinuxProvisioningConfigurationSet") + + if len(results) == 0: + raise NonAzureDataSource("No LinuxProvisioningConfigurationSet") + if len(results) > 1: + raise BrokenAzureDataSource("found '%d' %ss" % + ("LinuxProvisioningConfigurationSet", + len(results))) + lpcs = lpcs_nodes[0] + + if not lpcs.hasChildNodes(): + raise BrokenAzureDataSource("no child nodes of configuration set") + + md_props = 'seedfrom' + md = {'azure_data': {}} + cfg = {} + ud = "" + + for child in lpcs.childNodes: + if child.nodeType == dom.TEXT_NODE or not child.localName: + continue + + name = child.localName.lower() + + simple = False + if (len(child.childNodes) == 1 and + child.childNodes[0].nodeType == dom.TEXT_NODE): + simple = True + value = child.childNodes[0].wholeText + + if name == "userdata": + ud = base64.b64decode(''.join(value.split())) + elif name == "username": + cfg['system_info'] = {'default_user': {'name': value}} + elif name == "hostname": + md['local-hostname'] = value + elif name == "dscfg": + cfg['datasource'] = {DS_NAME: util.load_yaml(value, default={})} + elif simple: + if name in md_props: + md[name] = value + else: + md['azure_data'][name] = value + + return (md, ud, cfg) + + +def list_possible_azure_ds_devs(): + # return a sorted list of devices that might have a azure datasource + devlist = [] + for fstype in ("iso9660", "udf"): + devlist.extend(util.find_devs_with("TYPE=%s" % fstype)) + + devlist.sort(reverse=True) + return devlist + + +def load_azure_ds_dir(source_dir): + ovf_file = os.path.join(source_dir, "ovf-env.xml") + + if not os.path.isfile(ovf_file): + raise NonAzureDataSource("No ovf-env file found") + + with open(ovf_file, "r") as fp: + contents = fp.read() + + return read_azure_ovf(contents) + + +class BrokenAzureDataSource(Exception): + pass + + +class NonAzureDataSource(Exception): + pass + + +# Used to match classes to dependencies +datasources = [ + (DataSourceAzureNet, (sources.DEP_FILESYSTEM, sources.DEP_NETWORK)), +] + + +# Return a list of data sources that match this set of dependencies +def get_datasource_list(depends): + return sources.list_from_depends(depends, datasources) diff --git a/doc/examples/cloud-config-datasources.txt b/doc/examples/cloud-config-datasources.txt index fc8c22d4..fbabcad9 100644 --- a/doc/examples/cloud-config-datasources.txt +++ b/doc/examples/cloud-config-datasources.txt @@ -42,3 +42,7 @@ datasource: meta-data: instance-id: i-87018aed local-hostname: myhost.internal + + Azure: + agent_command: [service, walinuxagent, start] + -- cgit v1.2.3 From 0be043f1e677f533f64d0191f02b5fe956844157 Mon Sep 17 00:00:00 2001 From: Ben Howard Date: Tue, 23 Jul 2013 16:44:54 -0600 Subject: Added blurb to documentation about the datasource --- doc/examples/cloud-config-datasources.txt | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) (limited to 'doc/examples') diff --git a/doc/examples/cloud-config-datasources.txt b/doc/examples/cloud-config-datasources.txt index fbabcad9..9f0ac386 100644 --- a/doc/examples/cloud-config-datasources.txt +++ b/doc/examples/cloud-config-datasources.txt @@ -45,4 +45,16 @@ datasource: Azure: agent_command: [service, walinuxagent, start] - + + SmartOS: + Smart OS provisions via a serial console actings a server. By default, + the second serial console is the device. SmartOS also uses a serial + timeout of 60 seconds, although that should never be hit. + + serial device: /dev/ttyS1 + serial timeout: 60 + + To change the defaults, you can define it via a cloud-config by creating + a .cfg file in /etc/cloud/cloud.cfg.d with the following: + serial_device: /dev/ttyS1 + serial_timeout: 30 -- cgit v1.2.3 From c1f3fa86b45ba8b7ca6e0f6971de171ac5fccf15 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Wed, 24 Jul 2013 10:26:01 -0400 Subject: shorten comments in example cloud-config --- doc/examples/cloud-config-datasources.txt | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) (limited to 'doc/examples') diff --git a/doc/examples/cloud-config-datasources.txt b/doc/examples/cloud-config-datasources.txt index 9f0ac386..a19353fc 100644 --- a/doc/examples/cloud-config-datasources.txt +++ b/doc/examples/cloud-config-datasources.txt @@ -47,14 +47,8 @@ datasource: agent_command: [service, walinuxagent, start] SmartOS: - Smart OS provisions via a serial console actings a server. By default, - the second serial console is the device. SmartOS also uses a serial - timeout of 60 seconds, although that should never be hit. - + # Smart OS datasource works over a serial console interacting with + # a server on the other end. By default, the second serial console is the + # device. SmartOS also uses a serial timeout of 60 seconds. serial device: /dev/ttyS1 serial timeout: 60 - - To change the defaults, you can define it via a cloud-config by creating - a .cfg file in /etc/cloud/cloud.cfg.d with the following: - serial_device: /dev/ttyS1 - serial_timeout: 30 -- cgit v1.2.3 From 3d10b8d080a874be022f9e25063ba77f0293c5e8 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Thu, 25 Jul 2013 14:37:10 -0400 Subject: azure: support bouncing interfaces to publish new hostname See the added doc/sources/azure/README.rst for why this is necessary. Essentially, we now are doing the following in the get_data() method of azure datasource to publish this NewHostname: hostname NewHostName ifdown eth0; ifup eth0 LP: #1202758 --- cloudinit/sources/DataSourceAzure.py | 114 +++++++++++++++++----- doc/examples/cloud-config-datasources.txt | 5 + doc/sources/azure/README.rst | 134 ++++++++++++++++++++++++++ tests/unittests/test_datasource/test_azure.py | 77 ++++++++++++++- 4 files changed, 300 insertions(+), 30 deletions(-) create mode 100644 doc/sources/azure/README.rst (limited to 'doc/examples') diff --git a/cloudinit/sources/DataSourceAzure.py b/cloudinit/sources/DataSourceAzure.py index 0a5caebe..30b06fef 100644 --- a/cloudinit/sources/DataSourceAzure.py +++ b/cloudinit/sources/DataSourceAzure.py @@ -31,9 +31,20 @@ LOG = logging.getLogger(__name__) DS_NAME = 'Azure' DEFAULT_METADATA = {"instance-id": "iid-AZURE-NODE"} AGENT_START = ['service', 'walinuxagent', 'start'] -BUILTIN_DS_CONFIG = {'datasource': {DS_NAME: { - 'agent_command': AGENT_START, - 'data_dir': "/var/lib/waagent"}}} +BOUNCE_COMMAND = ("i=$interface; x=0; ifdown $i || x=$?; " + "ifup $i || x=$?; exit $x") +BUILTIN_DS_CONFIG = { + 'agent_command': AGENT_START, + 'data_dir': "/var/lib/waagent", + 'set_hostname': True, + 'hostname_bounce': { + 'interface': 'eth0', + 'policy': True, + 'command': BOUNCE_COMMAND, + 'hostname_command': 'hostname', + } +} +DS_CFG_PATH = ['datasource', DS_NAME] class DataSourceAzureNet(sources.DataSource): @@ -42,19 +53,19 @@ class DataSourceAzureNet(sources.DataSource): self.seed_dir = os.path.join(paths.seed_dir, 'azure') self.cfg = {} self.seed = None + self.ds_cfg = util.mergemanydict([ + util.get_cfg_by_path(sys_cfg, DS_CFG_PATH), + BUILTIN_DS_CONFIG]) def __str__(self): root = sources.DataSource.__str__(self) return "%s [seed=%s]" % (root, self.seed) def get_data(self): - ddir_cfgpath = ['datasource', DS_NAME, 'data_dir'] # azure removes/ejects the cdrom containing the ovf-env.xml # file on reboot. So, in order to successfully reboot we # need to look in the datadir and consider that valid - ddir = util.get_cfg_by_path(self.sys_cfg, ddir_cfgpath) - if ddir is None: - ddir = util.get_cfg_by_path(BUILTIN_DS_CONFIG, ddir_cfgpath) + ddir = self.ds_cfg['data_dir'] candidates = [self.seed_dir] candidates.extend(list_possible_azure_ds_devs()) @@ -91,36 +102,40 @@ class DataSourceAzureNet(sources.DataSource): return False if found == ddir: - LOG.debug("using cached datasource in %s", ddir) - - fields = [('cmd', ['datasource', DS_NAME, 'agent_command']), - ('datadir', ddir_cfgpath)] - mycfg = {} - for cfg in (self.cfg, self.sys_cfg, BUILTIN_DS_CONFIG): - for name, path in fields: - if name in mycfg: - continue - value = util.get_cfg_by_path(cfg, keyp=path) - if value is not None: - mycfg[name] = value + LOG.debug("using files cached in %s", ddir) + + # now update ds_cfg to reflect contents pass in config + usercfg = util.get_cfg_by_path(self.cfg, DS_CFG_PATH, {}) + self.ds_cfg = util.mergemanydict([usercfg, self.ds_cfg]) + mycfg = self.ds_cfg # walinux agent writes files world readable, but expects # the directory to be protected. - write_files(mycfg['datadir'], files, dirmode=0700) + write_files(mycfg['data_dir'], files, dirmode=0700) + + # handle the hostname 'publishing' + try: + handle_set_hostname(mycfg.get('set_hostname'), + self.metadata.get('local-hostname'), + mycfg['hostname_bounce']) + except Exception as e: + LOG.warn("Failed publishing hostname: %s" % e) + util.logexc(LOG, "handling set_hostname failed") try: - invoke_agent(mycfg['cmd']) + invoke_agent(mycfg['agent_command']) except util.ProcessExecutionError: # claim the datasource even if the command failed - util.logexc(LOG, "agent command '%s' failed.", mycfg['cmd']) + util.logexc(LOG, "agent command '%s' failed.", + mycfg['agent_command']) - shcfgxml = os.path.join(mycfg['datadir'], "SharedConfig.xml") + shcfgxml = os.path.join(mycfg['data_dir'], "SharedConfig.xml") wait_for = [shcfgxml] fp_files = [] for pk in self.cfg.get('_pubkeys', []): bname = pk['fingerprint'] + ".crt" - fp_files += [os.path.join(mycfg['datadir'], bname)] + fp_files += [os.path.join(mycfg['data_dir'], bname)] start = time.time() missing = wait_for_files(wait_for + fp_files) @@ -148,6 +163,43 @@ class DataSourceAzureNet(sources.DataSource): return self.cfg +def handle_set_hostname(enabled, hostname, cfg): + if not util.is_true(enabled): + return + + if not hostname: + LOG.warn("set_hostname was true but no local-hostname") + return + + apply_hostname_bounce(hostname=hostname, policy=cfg['policy'], + interface=cfg['interface'], + command=cfg['command'], + hostname_command=cfg['hostname_command']) + + +def apply_hostname_bounce(hostname, policy, interface, command, + hostname_command="hostname"): + # set the hostname to 'hostname' if it is not already set to that. + # then, if policy is not off, bounce the interface using command + prev_hostname = util.subp(hostname_command, capture=True)[0].strip() + + util.subp([hostname_command, hostname]) + + if util.is_false(policy): + return + + if prev_hostname == hostname and policy != "force": + return + + env = os.environ.copy() + env['interface'] = interface + + if command == "builtin": + command = BOUNCE_COMMAND + + util.subp(command, shell=(not isinstance(command, list)), capture=True) + + def crtfile_to_pubkey(fname): pipeline = ('openssl x509 -noout -pubkey < "$0" |' 'ssh-keygen -i -m PKCS8 -f /dev/stdin') @@ -319,15 +371,21 @@ def read_azure_ovf(contents): name = child.localName.lower() simple = False + value = "" if (len(child.childNodes) == 1 and child.childNodes[0].nodeType == dom.TEXT_NODE): simple = True value = child.childNodes[0].wholeText + attrs = {k: v for k, v in child.attributes.items()} + # we accept either UserData or CustomData. If both are present # then behavior is undefined. if (name == "userdata" or name == "customdata"): - ud = base64.b64decode(''.join(value.split())) + if attrs.get('encoding') in (None, "base64"): + ud = base64.b64decode(''.join(value.split())) + else: + ud = value elif name == "username": username = value elif name == "userpassword": @@ -335,7 +393,11 @@ def read_azure_ovf(contents): elif name == "hostname": md['local-hostname'] = value elif name == "dscfg": - cfg['datasource'] = {DS_NAME: util.load_yaml(value, default={})} + if attrs.get('encoding') in (None, "base64"): + dscfg = base64.b64decode(''.join(value.split())) + else: + dscfg = value + cfg['datasource'] = {DS_NAME: util.load_yaml(dscfg, default={})} elif name == "ssh": cfg['_pubkeys'] = load_azure_ovf_pubkeys(child) elif name == "disablesshpasswordauthentication": diff --git a/doc/examples/cloud-config-datasources.txt b/doc/examples/cloud-config-datasources.txt index a19353fc..6544448e 100644 --- a/doc/examples/cloud-config-datasources.txt +++ b/doc/examples/cloud-config-datasources.txt @@ -45,6 +45,11 @@ datasource: Azure: agent_command: [service, walinuxagent, start] + set_hostname: True + hostname_bounce: + interface: eth0 + policy: on # [can be 'on', 'off' or 'force'] + } SmartOS: # Smart OS datasource works over a serial console interacting with diff --git a/doc/sources/azure/README.rst b/doc/sources/azure/README.rst new file mode 100644 index 00000000..8239d1fa --- /dev/null +++ b/doc/sources/azure/README.rst @@ -0,0 +1,134 @@ +================ +Azure Datasource +================ + +This datasource finds metadata and user-data from the Azure cloud platform. + +Azure Platform +-------------- +The azure cloud-platform provides initial data to an instance via an attached +CD formated in UDF. That CD contains a 'ovf-env.xml' file that provides some +information. Additional information is obtained via interaction with the +"endpoint". The ip address of the endpoint is advertised to the instance +inside of dhcp option 245. On ubuntu, that can be seen in +/var/lib/dhcp/dhclient.eth0.leases as a colon delimited hex value (example: +``option unknown-245 64:41:60:82;`` is 100.65.96.130) + +walinuxagent +------------ +In order to operate correctly, cloud-init needs walinuxagent to provide much +of the interaction with azure. In addition to "provisioning" code, walinux +does the following on the agent is a long running daemon that handles the +following things: +- generate a x509 certificate and send that to the endpoint + +waagent.conf config +~~~~~~~~~~~~~~~~~~~ +in order to use waagent.conf with cloud-init, the following settings are recommended. Other values can be changed or set to the defaults. + + :: + + # disabling provisioning turns off all 'Provisioning.*' function + Provisioning.Enabled=n + # this is currently not handled by cloud-init, so let walinuxagent do it. + ResourceDisk.Format=y + ResourceDisk.MountPoint=/mnt + + +Userdata +-------- +Userdata is provided to cloud-init inside the ovf-env.xml file. Cloud-init +expects that user-data will be provided as base64 encoded value inside the +text child of a element named ``UserData`` or ``CustomData`` which is a direct +child of the ``LinuxProvisioningConfigurationSet`` (a sibling to ``UserName``) +If both ``UserData`` and ``CustomData`` are provided behavior is undefined on +which will be selected. + +In the example below, user-data provided is 'this is my userdata', and the +datasource config provided is ``{"agent_command": ["start", "walinuxagent"]}``. +That agent command will take affect as if it were specified in system config. + +Example: + +.. code:: + + + 1.0 + + LinuxProvisioningConfiguration + myHost + myuser + + dGhpcyBpcyBteSB1c2VyZGF0YQ=== + eyJhZ2VudF9jb21tYW5kIjogWyJzdGFydCIsICJ3YWxpbnV4YWdlbnQiXX0= + true + + + + 6BE7A7C3C8A8F4B123CCA5D0C2F1BE4CA7B63ED7 + this-value-unused + + + + + + +Configuration +------------- +Configuration for the datasource can be read from the system config's or set +via the `dscfg` entry in the `LinuxProvisioningConfigurationSet`. Content in +dscfg node is expected to be base64 encoded yaml content, and it will be +merged into the 'datasource: Azure' entry. + +The '``hostname_bounce: command``' entry can be either the literal string +'builtin' or a command to execute. The command will be invoked after the +hostname is set, and will have the 'interface' in its environment. If +``set_hostname`` is not true, then ``hostname_bounce`` will be ignored. + +An example might be: + command: ["sh", "-c", "killall dhclient; dhclient $interface"] + +.. code:: + + datasource: + agent_command + Azure: + agent_command: [service, walinuxagent, start] + set_hostname: True + hostname_bounce: + # the name of the interface to bounce + interface: eth0 + # policy can be 'on', 'off' or 'force' + policy: on + # the method 'bounce' command. + command: "builtin" + hostname_command: "hostname" + } + +hostname +-------- +When the user launches an instance, they provide a hostname for that instance. +The hostname is provided to the instance in the ovf-env.xml file as +``HostName``. + +Whatever value the instance provides in its dhcp request will resolve in the +domain returned in the 'search' request. + +The interesting issue is that a generic image will already have a hostname +configured. The ubuntu cloud images have 'ubuntu' as the hostname of the +system, and the initial dhcp request on eth0 is not guaranteed to occur after +the datasource code has been run. So, on first boot, that initial value will +be sent in the dhcp request and *that* value will resolve. + +In order to make the ``HostName`` provided in the ovf-env.xml resolve, a +dhcp request must be made with the new value. Walinuxagent (in its current +version) handles this by polling the state of hostname and bouncing ('``ifdown +eth0; ifup eth0``' the network interface if it sees that a change has been +made. + +cloud-init handles this by setting the hostname in the DataSource's 'get_data' +method via '``hostname $HostName``', and then bouncing the interface. This +behavior can be configured or disabled in the datasource config. See +'Configuration' above. diff --git a/tests/unittests/test_datasource/test_azure.py b/tests/unittests/test_datasource/test_azure.py index 2e8583f9..c944cb13 100644 --- a/tests/unittests/test_datasource/test_azure.py +++ b/tests/unittests/test_datasource/test_azure.py @@ -26,8 +26,15 @@ def construct_valid_ovf_env(data=None, pubkeys=None, userdata=None): xmlns:i="http://www.w3.org/2001/XMLSchema-instance"> LinuxProvisioningConfiguration """ - for key, val in data.items(): - content += "<%s>%s\n" % (key, val, key) + for key, dval in data.items(): + if isinstance(dval, dict): + val = dval.get('text') + attrs = ' ' + ' '.join(["%s='%s'" % (k, v) for k, v in dval.items() + if k != 'text']) + else: + val = dval + attrs = "" + content += "<%s%s>%s\n" % (key, attrs, val, key) if userdata: content += "%s\n" % (base64.b64encode(userdata)) @@ -103,6 +110,9 @@ class TestAzureDataSource(MockerTestCase): data['iid_from_shared_cfg'] = path return 'i-my-azure-id' + def _apply_hostname_bounce(**kwargs): + data['apply_hostname_bounce'] = kwargs + if data.get('ovfcontent') is not None: populate_dir(os.path.join(self.paths.seed_dir, "azure"), {'ovf-env.xml': data['ovfcontent']}) @@ -118,7 +128,9 @@ class TestAzureDataSource(MockerTestCase): (mod, 'pubkeys_from_crt_files', _pubkeys_from_crt_files), (mod, 'iid_from_shared_config', - _iid_from_shared_config), ]) + _iid_from_shared_config), + (mod, 'apply_hostname_bounce', + _apply_hostname_bounce), ]) dsrc = mod.DataSourceAzureNet( data.get('sys_cfg', {}), distro=None, paths=self.paths) @@ -139,11 +151,26 @@ class TestAzureDataSource(MockerTestCase): self.assertEqual(0700, data['datadir_mode']) self.assertEqual(dsrc.metadata['instance-id'], 'i-my-azure-id') + def test_user_cfg_set_agent_command_plain(self): + # set dscfg in via plaintext + cfg = {'agent_command': "my_command"} + odata = {'HostName': "myhost", 'UserName': "myuser", + 'dscfg': {'text': yaml.dump(cfg), 'encoding': 'plain'}} + data = {'ovfcontent': construct_valid_ovf_env(data=odata)} + + dsrc = self._get_ds(data) + ret = dsrc.get_data() + self.assertTrue(ret) + self.assertEqual(data['agent_invoked'], cfg['agent_command']) + def test_user_cfg_set_agent_command(self): + # set dscfg in via base64 encoded yaml cfg = {'agent_command': "my_command"} odata = {'HostName': "myhost", 'UserName': "myuser", - 'dscfg': yaml.dump(cfg)} + 'dscfg': {'text': base64.b64encode(yaml.dump(cfg)), + 'encoding': 'base64'}} data = {'ovfcontent': construct_valid_ovf_env(data=odata)} + print data dsrc = self._get_ds(data) ret = dsrc.get_data() @@ -218,6 +245,48 @@ class TestAzureDataSource(MockerTestCase): for mypk in mypklist: self.assertIn(mypk, dsrc.cfg['_pubkeys']) + def test_disabled_bounce(self): + pass + + def test_apply_bounce_call_1(self): + # hostname needs to get through to apply_hostname_bounce + mydata = "FOOBAR" + odata = {'HostName': 'my-random-hostname'} + data = {'ovfcontent': construct_valid_ovf_env(data=odata)} + + self._get_ds(data).get_data() + self.assertIn('hostname', data['apply_hostname_bounce']) + self.assertEqual(data['apply_hostname_bounce']['hostname'], + odata['HostName']) + + def test_apply_bounce_call_configurable(self): + # hostname_bounce should be configurable in datasource cfg + cfg = {'hostname_bounce': {'interface': 'eth1', 'policy': 'off', + 'command': 'my-bounce-command', + 'hostname_command': 'my-hostname-command'}} + odata = {'HostName': "xhost", + 'dscfg': {'text': base64.b64encode(yaml.dump(cfg)), + 'encoding': 'base64'}} + data = {'ovfcontent': construct_valid_ovf_env(data=odata)} + self._get_ds(data).get_data() + + for k in cfg['hostname_bounce']: + self.assertIn(k, data['apply_hostname_bounce']) + + for k, v in cfg['hostname_bounce'].items(): + self.assertEqual(data['apply_hostname_bounce'][k], v) + + def test_set_hostname_disabled(self): + # config specifying set_hostname off should not bounce + cfg = {'set_hostname': False} + odata = {'HostName': "xhost", + 'dscfg': {'text': base64.b64encode(yaml.dump(cfg)), + 'encoding': 'base64'}} + data = {'ovfcontent': construct_valid_ovf_env(data=odata)} + self._get_ds(data).get_data() + + self.assertEqual(data.get('apply_hostname_bounce', "N/A"), "N/A") + class TestReadAzureOvf(MockerTestCase): def test_invalid_xml_raises_non_azure_ds(self): -- cgit v1.2.3 From 1e84d5dad7dcd21a521d0977963450424ec35c9c Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Tue, 6 Aug 2013 07:15:36 +0100 Subject: initially add support for apt_ftp_proxy and apt_https_proxy. tests to come. LP: #1057195 --- cloudinit/config/cc_apt_configure.py | 41 ++++++++++++++++++++++++------------ doc/examples/cloud-config-TODO.txt | 20 ------------------ doc/examples/cloud-config.txt | 3 +++ 3 files changed, 31 insertions(+), 33 deletions(-) delete mode 100644 doc/examples/cloud-config-TODO.txt (limited to 'doc/examples') diff --git a/cloudinit/config/cc_apt_configure.py b/cloudinit/config/cc_apt_configure.py index 3ce3b351..0c26050a 100644 --- a/cloudinit/config/cc_apt_configure.py +++ b/cloudinit/config/cc_apt_configure.py @@ -21,13 +21,17 @@ import glob import os +from cloudinit import log as logging from cloudinit import templater from cloudinit import util +LOG = logging.getLogger(__name__) + distros = ['ubuntu', 'debian'] PROXY_TPL = "Acquire::HTTP::Proxy \"%s\";\n" -PROXY_FN = "/etc/apt/apt.conf.d/95cloud-init-proxy" +APT_CONFIG_FN = "/etc/apt/apt.conf.d/94cloud-init-config" +APT_PROXY_FN = "/etc/apt/apt.conf.d/95cloud-init-proxy" # A temporary shell program to get a given gpg key # from a given keyserver @@ -67,18 +71,10 @@ def handle(name, cfg, cloud, log, _args): "security": "security.ubuntu.com/ubuntu"}) rename_apt_lists(old_mirrors, mirrors) - # Set up any apt proxy - proxy = cfg.get("apt_proxy", None) - proxy_filename = PROXY_FN - if proxy: - try: - # See man 'apt.conf' - contents = PROXY_TPL % (proxy) - util.write_file(proxy_filename, contents) - except Exception as e: - util.logexc(log, "Failed to write proxy to %s", proxy_filename) - elif os.path.isfile(proxy_filename): - util.del_file(proxy_filename) + try: + apply_apt_config(cfg, APT_PROXY_FN, APT_CONFIG_FN) + except Exception as e: + log.warn("failed to proxy or apt config info: %s", e) # Process 'apt_sources' if 'apt_sources' in cfg: @@ -256,3 +252,22 @@ def find_apt_mirror_info(cloud, cfg): mirror_info.update({'primary': mirror}) return mirror_info + + +def apply_apt_config(cfg, proxy_fname, config_fname): + # Set up any apt proxy + cfgs = (('apt_proxy', 'Acquire::HTTP::Proxy "%s";'), + ('apt_http_proxy', 'Acquire::HTTP::Proxy "%s";'), + ('apt_ftp_proxy', 'Acquire::FTP::Proxy "%s";'), + ('apt_https_proxy', 'Acquire::HTTPS::Proxy "%s";')) + + proxies = [fmt % cfg.get(name) for (name, fmt) in cfgs if cfg.get(name)] + if len(proxies): + util.write_file(proxy_fname, '\n'.join(proxies) + '\n') + elif os.path.isfile(proxy_fname): + util.del_file(proxy_fname) + + if cfg.get('apt_config', None): + util.write_file(config_fname, cfg.get('apt_config')) + elif os.path.isfile(config_fname): + util.del_file(config_fname) diff --git a/doc/examples/cloud-config-TODO.txt b/doc/examples/cloud-config-TODO.txt deleted file mode 100644 index c7ed54ab..00000000 --- a/doc/examples/cloud-config-TODO.txt +++ /dev/null @@ -1,20 +0,0 @@ -# Add apt configuration files -# Add an apt.conf.d/ file with the relevant content -# -# See apt.conf man page for more information. -# -# Defaults: -# + filename: 00-boot-conf -# -apt_conf: - - # Creates an apt proxy configuration in /etc/apt/apt.conf.d/01-proxy - - filename: "01-proxy" - content: | - Acquire::http::Proxy "http://proxy.example.org:3142/ubuntu"; - - # Add the following line to /etc/apt/apt.conf.d/00-boot-conf - # (run debconf at a critical priority) - - content: | - DPkg::Pre-Install-Pkgs:: "/usr/sbin/dpkg-preconfigure --apt -p critical|| true"; - diff --git a/doc/examples/cloud-config.txt b/doc/examples/cloud-config.txt index 24b4b36c..bcfd7917 100644 --- a/doc/examples/cloud-config.txt +++ b/doc/examples/cloud-config.txt @@ -53,6 +53,9 @@ apt_mirror_search: apt_mirror_search_dns: False # apt_proxy (configure Acquire::HTTP::Proxy) +# 'apt_http_proxy' is an alias for 'apt_proxy'. +# Also, available are 'apt_ftp_proxy' and 'apt_https_proxy'. +# These affect Acquire::FTP::Proxy and Acquire::HTTPS::Proxy respectively apt_proxy: http://my.apt.proxy:3128 # apt_pipelining (configure Acquire::http::Pipeline-Depth) -- cgit v1.2.3 From e683ab2baa3e67614edcd409122bd1aec99737e0 Mon Sep 17 00:00:00 2001 From: Ben Howard Date: Tue, 20 Aug 2013 09:56:25 -0600 Subject: Fixed no_base64_decode settings --- cloudinit/sources/DataSourceSmartOS.py | 26 ++++++++++++++++++++----- doc/examples/cloud-config-datasources.txt | 2 +- doc/sources/smartos/README.rst | 12 ++++++------ tests/unittests/test_datasource/test_smartos.py | 8 ++++++++ 4 files changed, 36 insertions(+), 12 deletions(-) (limited to 'doc/examples') diff --git a/cloudinit/sources/DataSourceSmartOS.py b/cloudinit/sources/DataSourceSmartOS.py index e0bb871c..1cf9e4f0 100644 --- a/cloudinit/sources/DataSourceSmartOS.py +++ b/cloudinit/sources/DataSourceSmartOS.py @@ -35,7 +35,7 @@ import os import os.path import serial - +DS_NAME = 'SmartOS' DEF_TTY_LOC = '/dev/ttyS1' DEF_TTY_TIMEOUT = 60 LOG = logging.getLogger(__name__) @@ -60,13 +60,14 @@ class DataSourceSmartOS(sources.DataSource): sources.DataSource.__init__(self, sys_cfg, distro, paths) self.seed_dir = os.path.join(paths.seed_dir, 'sdc') self.is_smartdc = None + self.base_64_encoded = [] self.seed = self.sys_cfg.get("serial_device", DEF_TTY_LOC) self.all_base64 = self.sys_cfg.get("decode_base64", False) - self.base_64_encoded = [] - self.smartos_no_base64 = self.sys_cfg.get("no_base64_decode", - SMARTOS_NO_BASE64) self.seed_timeout = self.sys_cfg.get("serial_timeout", DEF_TTY_TIMEOUT) + self.smartos_no_base64 = SMARTOS_NO_BASE64 + if 'no_base64_decode' in self.ds_cfg: + self.smartos_no_base64 = self.ds_cfg['no_base64_decode'] def __str__(self): root = sources.DataSource.__str__(self) @@ -137,10 +138,25 @@ class DataSourceSmartOS(sources.DataSource): def get_instance_id(self): return self.metadata['instance-id'] + def not_b64_var(self, var): + """Return true if value is read as b64.""" + if var in self.smartos_no_base64 or \ + not self.all_base64: + return True + return False + + def is_b64_var(self, var): + """Return true if value is read as b64.""" + if self.all_base64 or ( + var not in self.smartos_no_base64 and + var in self.base_64_encoded): + return True + return False + def get_serial(seed_device, seed_timeout): """This is replaced in unit testing, allowing us to replace - serial.Serial with a mocked class + serial.Serial with a mocked class. The timeout value of 60 seconds should never be hit. The value is taken from SmartOS own provisioning tools. Since we are reading diff --git a/doc/examples/cloud-config-datasources.txt b/doc/examples/cloud-config-datasources.txt index 6544448e..6ec0d57e 100644 --- a/doc/examples/cloud-config-datasources.txt +++ b/doc/examples/cloud-config-datasources.txt @@ -55,5 +55,5 @@ datasource: # Smart OS datasource works over a serial console interacting with # a server on the other end. By default, the second serial console is the # device. SmartOS also uses a serial timeout of 60 seconds. - serial device: /dev/ttyS1 + serial_device: /dev/ttyS1 serial timeout: 60 diff --git a/doc/sources/smartos/README.rst b/doc/sources/smartos/README.rst index 96310857..ba90e7af 100644 --- a/doc/sources/smartos/README.rst +++ b/doc/sources/smartos/README.rst @@ -12,8 +12,7 @@ serial console. On Linux, this is /dev/ttyS1. The data is a provided via a simple protocol, where something queries for the userdata, where the console responds with the status and if "SUCCESS" returns until a single ".\n". -The format is lossy. As such, new versions of the SmartOS tooling will include -support for base64 encoded data. +New versions of the SmartOS tooling will include support for base64 encoded data. Userdata -------- @@ -48,15 +47,16 @@ always supercede any user-script data. This is for consistency. base64 ------ -In order to provide a lossy format, all data except for: +The following are excempt from base64 encoding, owing to the fact that they +are provided by SmartOS: * root_authorized_keys * enable_motd_sys_info * iptables_disable This means that user-script and user-data as well as other values can be -base64 encoded to provide a lossy format. Since Cloud-init can only guess -as to whether or not something is truly base64 encoded, the following -meta-data keys are hints as to whether or not to base64 decode something: +base64 encoded. Since Cloud-init can only guess as to whether or not something +is truly base64 encoded, the following meta-data keys are hints as to whether +or not to base64 decode something: * decode_base64: Except for excluded keys, attempt to base64 decode the values. If the value fails to decode properly, it will be returned in its text diff --git a/tests/unittests/test_datasource/test_smartos.py b/tests/unittests/test_datasource/test_smartos.py index ae621433..b9b3a479 100644 --- a/tests/unittests/test_datasource/test_smartos.py +++ b/tests/unittests/test_datasource/test_smartos.py @@ -36,6 +36,7 @@ mock_returns = { 'enable_motd_sys_info': None, 'system_uuid': str(uuid.uuid4()), 'smartdc': 'smartdc', + 'test-var1': 'some data', 'user-data': """ #!/bin/sh /bin/true @@ -156,6 +157,13 @@ class TestSmartOSDataSource(MockerTestCase): self.assertTrue(ret) self.assertTrue(dsrc.is_smartdc) + def test_no_base64(self): + sys_cfg = {'no_base64_decode': ['test_var1'], 'all_base': True} + dsrc = self._get_ds(sys_cfg=sys_cfg) + ret = dsrc.get_data() + self.assertTrue(ret) + self.assertTrue(dsrc.not_b64_var('test-var')) + def test_uuid(self): dsrc = self._get_ds() ret = dsrc.get_data() -- cgit v1.2.3 From 10c8ec1e5c1b16572a38afd08ee794d28c450054 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Sat, 24 Aug 2013 00:07:35 -0400 Subject: changes to behavior on specifying keys. The most likely end user operation (or at least a valid one) for base64 encoding would be to encode the user-data, but leave all other values as plaintext. In order to facilitate that, the user can simply add: b64-user-data=true to indicate that user-data is base64 encoded. Other changes here are to change the cloud-config and metadata keynames that are used. base64_all = boolean(True) base64_keys = [list, of, keys] Fixed up tests to accomodate. --- cloudinit/sources/DataSourceSmartOS.py | 94 ++++++--------- doc/examples/cloud-config-datasources.txt | 10 +- doc/sources/smartos/README.rst | 16 ++- tests/unittests/test_datasource/test_smartos.py | 153 ++++++++++++++---------- 4 files changed, 149 insertions(+), 124 deletions(-) (limited to 'doc/examples') diff --git a/cloudinit/sources/DataSourceSmartOS.py b/cloudinit/sources/DataSourceSmartOS.py index 45f03a7e..d348d20b 100644 --- a/cloudinit/sources/DataSourceSmartOS.py +++ b/cloudinit/sources/DataSourceSmartOS.py @@ -35,7 +35,6 @@ import os import os.path import serial -DS_NAME = 'SmartOS' DEF_TTY_LOC = '/dev/ttyS1' DEF_TTY_TIMEOUT = 60 LOG = logging.getLogger(__name__) @@ -51,6 +50,7 @@ SMARTOS_ATTRIB_MAP = { } # These are values which will never be base64 encoded. +# They come from the cloud platform, not user SMARTOS_NO_BASE64 = ['root_authorized_keys', 'motd_sys_info', 'iptables_disable'] @@ -60,17 +60,13 @@ class DataSourceSmartOS(sources.DataSource): sources.DataSource.__init__(self, sys_cfg, distro, paths) self.seed_dir = os.path.join(paths.seed_dir, 'sdc') self.is_smartdc = None - self.base_64_encoded = [] - self.seed = self.sys_cfg.get("serial_device", DEF_TTY_LOC) - self.seed_timeout = self.sys_cfg.get("serial_timeout", - DEF_TTY_TIMEOUT) - self.all_base64 = False - if 'decode_base64' in self.ds_cfg: - self.all_base64 = self.ds_cfg['decode_base64'] - - self.smartos_no_base64 = SMARTOS_NO_BASE64 - if 'no_base64_decode' in self.ds_cfg: - self.smartos_no_base64 = self.ds_cfg['no_base64_decode'] + + self.seed = self.ds_cfg.get("serial_device", DEF_TTY_LOC) + self.seed_timeout = self.ds_cfg.get("serial_timeout", DEF_TTY_TIMEOUT) + self.smartos_no_base64 = self.ds_cfg.get('no_base64_decode', + SMARTOS_NO_BASE64) + self.b64_keys = self.ds_cfg.get('base64_keys', []) + self.b64_all = self.ds_cfg.get('base64_all', False) def __str__(self): root = sources.DataSource.__str__(self) @@ -92,38 +88,22 @@ class DataSourceSmartOS(sources.DataSource): system_uuid, system_type = dmi_info if 'smartdc' not in system_type.lower(): - LOG.debug("Host is not on SmartOS") + LOG.debug("Host is not on SmartOS. system_type=%s", system_type) return False self.is_smartdc = True md['instance-id'] = system_uuid - self.base_64_encoded = query_data('base_64_enocded', - self.seed, - self.seed_timeout, - strip=True) - if self.base_64_encoded: - self.base_64_encoded = str(self.base_64_encoded).split(',') - else: - self.base_64_encoded = [] + b64_keys = self.query('base64_keys', strip=True, b64=False) + if b64_keys is not None: + self.b64_keys = [k.strip() for k in str(b64_keys).split(',')] - if not self.all_base64: - self.all_base64 = util.is_true(query_data('meta_encoded_base64', - self.seed, - self.seed_timeout, - strip=True)) + b64_all = self.query('base64_all', strip=True, b64=False) + if b64_all is not None: + self.b64_all = util.is_true(b64_all) for ci_noun, attribute in SMARTOS_ATTRIB_MAP.iteritems(): smartos_noun, strip = attribute - - b64encoded = False - if self.all_base64 and \ - (smartos_noun not in self.smartos_no_base64 and \ - ci_noun not in self.smartos_no_base64): - b64encoded = True - - md[ci_noun] = query_data(smartos_noun, self.seed, - self.seed_timeout, strip=strip, - b64encoded=b64encoded) + md[ci_noun] = self.query(smartos_noun, strip=strip) if not md['local-hostname']: md['local-hostname'] = system_uuid @@ -141,20 +121,16 @@ class DataSourceSmartOS(sources.DataSource): def get_instance_id(self): return self.metadata['instance-id'] - def not_b64_var(self, var): - """Return true if value is read as b64.""" - if var in self.smartos_no_base64 or \ - not self.all_base64: - return True - return False + def query(self, noun, strip=False, default=None, b64=None): + if b64 is None: + if noun in self.smartos_no_base64: + b64 = False + elif self.b64_all or noun in self.b64_keys: + b64 = True - def is_b64_var(self, var): - """Return true if value is read as b64.""" - if self.all_base64 or ( - var not in self.smartos_no_base64 and - var in self.base_64_encoded): - return True - return False + return query_data(noun=noun, strip=strip, seed_device=self.seed, + seed_timeout=self.seed_timeout, default=default, + b64=b64) def get_serial(seed_device, seed_timeout): @@ -176,7 +152,8 @@ def get_serial(seed_device, seed_timeout): return ser -def query_data(noun, seed_device, seed_timeout, strip=False, b64encoded=False): +def query_data(noun, seed_device, seed_timeout, strip=False, default=None, + b64=None): """Makes a request to via the serial console via "GET " In the response, the first line is the status, while subsequent lines @@ -200,7 +177,7 @@ def query_data(noun, seed_device, seed_timeout, strip=False, b64encoded=False): if 'SUCCESS' not in status: ser.close() - return None + return default while not eom_found: m = ser.readline() @@ -211,18 +188,23 @@ def query_data(noun, seed_device, seed_timeout, strip=False, b64encoded=False): ser.close() + if b64 is None: + b64 = query_data('b64-%s' % noun, seed_device=seed_device, + seed_timeout=seed_timeout, b64=False, + default=False, strip=True) + b64 = util.is_true(b64) + resp = None - if not strip: - resp = "".join(response) - elif b64encoded: + if b64 or strip: resp = "".join(response).rstrip() else: - resp = "".join(response).rstrip() + resp = "".join(response) - if b64encoded: + if b64: try: return base64.b64decode(resp) except TypeError: + LOG.warn("Failed base64 decoding key '%s'", noun) return resp return resp diff --git a/doc/examples/cloud-config-datasources.txt b/doc/examples/cloud-config-datasources.txt index 6ec0d57e..65a3cdf5 100644 --- a/doc/examples/cloud-config-datasources.txt +++ b/doc/examples/cloud-config-datasources.txt @@ -56,4 +56,12 @@ datasource: # a server on the other end. By default, the second serial console is the # device. SmartOS also uses a serial timeout of 60 seconds. serial_device: /dev/ttyS1 - serial timeout: 60 + serial_timeout: 60 + + # a list of keys that will not be base64 decoded even if base64_all + no_base64_decode: ['root_authorized_keys', 'motd_sys_info', + 'iptables_disable'] + # a plaintext, comma delimited list of keys whose values are b64 encoded + base64_keys: [] + # a boolean indicating that all keys not in 'no_base64_decode' are encoded + base64_all: False diff --git a/doc/sources/smartos/README.rst b/doc/sources/smartos/README.rst index 8f72fa0f..fd4e496d 100644 --- a/doc/sources/smartos/README.rst +++ b/doc/sources/smartos/README.rst @@ -53,14 +53,20 @@ are provided by SmartOS: * enable_motd_sys_info * iptables_disable +This list can be changed through system config of variable 'no_base64_decode'. + This means that user-script and user-data as well as other values can be base64 encoded. Since Cloud-init can only guess as to whether or not something is truly base64 encoded, the following meta-data keys are hints as to whether or not to base64 decode something: - * decode_base64: Except for excluded keys, attempt to base64 decode + * base64_all: Except for excluded keys, attempt to base64 decode the values. If the value fails to decode properly, it will be returned in its text - * base_64_encoded: A comma deliminated list of which values are base64 - encoded. - * no_base64_decode: This is a configuration setting (i.e. /etc/cloud/cloud.cfg.d) - that sets which values should not be base64 decoded. + * base64_keys: A comma deliminated list of which keys are base64 encoded. + * b64-: + for any key, if there exists an entry in the metadata for 'b64-' + Then 'b64-' is expected to be a plaintext boolean indicating whether + or not its value is encoded. + * no_base64_decode: This is a configuration setting + (i.e. /etc/cloud/cloud.cfg.d) that sets which values should not be + base64 decoded. diff --git a/tests/unittests/test_datasource/test_smartos.py b/tests/unittests/test_datasource/test_smartos.py index b9b3a479..f53715b0 100644 --- a/tests/unittests/test_datasource/test_smartos.py +++ b/tests/unittests/test_datasource/test_smartos.py @@ -29,20 +29,17 @@ from cloudinit.sources import DataSourceSmartOS from mocker import MockerTestCase import uuid -mock_returns = { +MOCK_RETURNS = { 'hostname': 'test-host', 'root_authorized_keys': 'ssh-rsa AAAAB3Nz...aC1yc2E= keyname', 'disable_iptables_flag': None, 'enable_motd_sys_info': None, - 'system_uuid': str(uuid.uuid4()), - 'smartdc': 'smartdc', 'test-var1': 'some data', - 'user-data': """ -#!/bin/sh -/bin/true -""", + 'user-data': '\n'.join(['#!/bin/sh', '/bin/true', '']), } +DMI_DATA_RETURN = (str(uuid.uuid4()), 'smartdc') + class MockSerial(object): """Fake a serial terminal for testing the code that @@ -50,14 +47,13 @@ class MockSerial(object): port = None - def __init__(self, b64encode=False): + def __init__(self, mockdata): self.last = None self.last = None self.new = True self.count = 0 self.mocked_out = [] - self.b64encode = b64encode - self.b64excluded = DataSourceSmartOS.SMARTOS_NO_BASE64 + self.mockdata = mockdata def open(self): return True @@ -75,12 +71,12 @@ class MockSerial(object): def readline(self): if self.new: self.new = False - if self.last in mock_returns: + if self.last in self.mockdata: return 'SUCCESS\n' else: return 'NOTFOUND %s\n' % self.last - if self.last in mock_returns: + if self.last in self.mockdata: if not self.mocked_out: self.mocked_out = [x for x in self._format_out()] print self.mocked_out @@ -90,21 +86,16 @@ class MockSerial(object): return self.mocked_out[self.count - 1] def _format_out(self): - if self.last in mock_returns: - _mret = mock_returns[self.last] - if self.b64encode and \ - self.last not in self.b64excluded: - yield base64.b64encode(_mret) - - else: - try: - for l in _mret.splitlines(): - yield "%s\n" % l.rstrip() - except: - yield "%s\n" % _mret.rstrip() + if self.last in self.mockdata: + _mret = self.mockdata[self.last] + try: + for l in _mret.splitlines(): + yield "%s\n" % l.rstrip() + except: + yield "%s\n" % _mret.rstrip() - yield '\n' yield '.' + yield '\n' class TestSmartOSDataSource(MockerTestCase): @@ -126,26 +117,36 @@ class TestSmartOSDataSource(MockerTestCase): ret = apply_patches(patches) self.unapply += ret - def _get_ds(self, b64encode=False, sys_cfg=None): + def _get_ds(self, sys_cfg=None, ds_cfg=None, mockdata=None, dmi_data=None): mod = DataSourceSmartOS + if mockdata is None: + mockdata = MOCK_RETURNS + + if dmi_data is None: + dmi_data = DMI_DATA_RETURN + def _get_serial(*_): - return MockSerial(b64encode=b64encode) + return MockSerial(mockdata) def _dmi_data(): - return mock_returns['system_uuid'], 'smartdc' + return dmi_data - if not sys_cfg: + if sys_cfg is None: sys_cfg = {} - data = {'sys_cfg': sys_cfg} + if ds_cfg is not None: + sys_cfg['datasource'] = sys_cfg.get('datasource', {}) + sys_cfg['datasource']['SmartOS'] = ds_cfg + self.apply_patches([(mod, 'get_serial', _get_serial)]) self.apply_patches([(mod, 'dmi_data', _dmi_data)]) - dsrc = mod.DataSourceSmartOS( - data.get('sys_cfg', {}), distro=None, paths=self.paths) + dsrc = mod.DataSourceSmartOS(sys_cfg, distro=None, + paths=self.paths) return dsrc def test_seed(self): + # default seed should be /dev/ttyS1 dsrc = self._get_ds() ret = dsrc.get_data() self.assertTrue(ret) @@ -158,78 +159,106 @@ class TestSmartOSDataSource(MockerTestCase): self.assertTrue(dsrc.is_smartdc) def test_no_base64(self): - sys_cfg = {'no_base64_decode': ['test_var1'], 'all_base': True} - dsrc = self._get_ds(sys_cfg=sys_cfg) + ds_cfg = {'no_base64_decode': ['test_var1'], 'all_base': True} + dsrc = self._get_ds(ds_cfg=ds_cfg) ret = dsrc.get_data() self.assertTrue(ret) - self.assertTrue(dsrc.not_b64_var('test-var')) def test_uuid(self): - dsrc = self._get_ds() + dsrc = self._get_ds(mockdata=MOCK_RETURNS) ret = dsrc.get_data() self.assertTrue(ret) - self.assertEquals(mock_returns['system_uuid'], - dsrc.metadata['instance-id']) + self.assertEquals(DMI_DATA_RETURN[0], dsrc.metadata['instance-id']) def test_root_keys(self): - dsrc = self._get_ds() + dsrc = self._get_ds(mockdata=MOCK_RETURNS) ret = dsrc.get_data() self.assertTrue(ret) - self.assertEquals(mock_returns['root_authorized_keys'], + self.assertEquals(MOCK_RETURNS['root_authorized_keys'], dsrc.metadata['public-keys']) def test_hostname_b64(self): - dsrc = self._get_ds(b64encode=True) + dsrc = self._get_ds(mockdata=MOCK_RETURNS) ret = dsrc.get_data() self.assertTrue(ret) - self.assertEquals(base64.b64encode(mock_returns['hostname']), + self.assertEquals(MOCK_RETURNS['hostname'], dsrc.metadata['local-hostname']) def test_hostname(self): - dsrc = self._get_ds() + dsrc = self._get_ds(mockdata=MOCK_RETURNS) ret = dsrc.get_data() self.assertTrue(ret) - self.assertEquals(mock_returns['hostname'], + self.assertEquals(MOCK_RETURNS['hostname'], dsrc.metadata['local-hostname']) - def test_base64(self): - """This tests to make sure that SmartOS system key/value pairs - are not interpetted as being base64 encoded, while making - sure that the others are when 'decode_base64' is set""" - dsrc = self._get_ds(sys_cfg={'decode_base64': True}, - b64encode=True) + def test_base64_all(self): + # metadata provided base64_all of true + my_returns = MOCK_RETURNS.copy() + my_returns['base64_all'] = "true" + for k in ('hostname', 'user-data'): + my_returns[k] = base64.b64encode(my_returns[k]) + + dsrc = self._get_ds(mockdata=my_returns) ret = dsrc.get_data() self.assertTrue(ret) - self.assertEquals(mock_returns['hostname'], + self.assertEquals(MOCK_RETURNS['hostname'], dsrc.metadata['local-hostname']) - self.assertEquals("%s" % mock_returns['user-data'], + self.assertEquals(MOCK_RETURNS['user-data'], dsrc.userdata_raw) - self.assertEquals(mock_returns['root_authorized_keys'], + self.assertEquals(MOCK_RETURNS['root_authorized_keys'], dsrc.metadata['public-keys']) - self.assertEquals(mock_returns['disable_iptables_flag'], + self.assertEquals(MOCK_RETURNS['disable_iptables_flag'], dsrc.metadata['iptables_disable']) - self.assertEquals(mock_returns['enable_motd_sys_info'], + self.assertEquals(MOCK_RETURNS['enable_motd_sys_info'], dsrc.metadata['motd_sys_info']) + def test_b64_userdata(self): + my_returns = MOCK_RETURNS.copy() + my_returns['b64-user-data'] = "true" + my_returns['b64-hostname'] = "true" + for k in ('hostname', 'user-data'): + my_returns[k] = base64.b64encode(my_returns[k]) + + dsrc = self._get_ds(mockdata=my_returns) + ret = dsrc.get_data() + self.assertTrue(ret) + self.assertEquals(MOCK_RETURNS['hostname'], + dsrc.metadata['local-hostname']) + self.assertEquals(MOCK_RETURNS['user-data'], dsrc.userdata_raw) + self.assertEquals(MOCK_RETURNS['root_authorized_keys'], + dsrc.metadata['public-keys']) + + def test_b64_keys(self): + my_returns = MOCK_RETURNS.copy() + my_returns['base64_keys'] = 'hostname,ignored' + for k in ('hostname',): + my_returns[k] = base64.b64encode(my_returns[k]) + + dsrc = self._get_ds(mockdata=my_returns) + ret = dsrc.get_data() + self.assertTrue(ret) + self.assertEquals(MOCK_RETURNS['hostname'], + dsrc.metadata['local-hostname']) + self.assertEquals(MOCK_RETURNS['user-data'], dsrc.userdata_raw) + def test_userdata(self): - dsrc = self._get_ds() + dsrc = self._get_ds(mockdata=MOCK_RETURNS) ret = dsrc.get_data() self.assertTrue(ret) - self.assertEquals("%s\n" % mock_returns['user-data'], - dsrc.userdata_raw) + self.assertEquals(MOCK_RETURNS['user-data'], dsrc.userdata_raw) def test_disable_iptables_flag(self): - dsrc = self._get_ds() + dsrc = self._get_ds(mockdata=MOCK_RETURNS) ret = dsrc.get_data() self.assertTrue(ret) - self.assertEquals(mock_returns['disable_iptables_flag'], + self.assertEquals(MOCK_RETURNS['disable_iptables_flag'], dsrc.metadata['iptables_disable']) def test_motd_sys_info(self): - dsrc = self._get_ds() + dsrc = self._get_ds(mockdata=MOCK_RETURNS) ret = dsrc.get_data() self.assertTrue(ret) - self.assertEquals(mock_returns['enable_motd_sys_info'], + self.assertEquals(MOCK_RETURNS['enable_motd_sys_info'], dsrc.metadata['motd_sys_info']) -- cgit v1.2.3