From 53f1938a1c33b4d9e333101d1d614803373a6bc5 Mon Sep 17 00:00:00 2001 From: Harm Weites Date: Fri, 6 Dec 2013 21:25:04 +0000 Subject: new: FreeBSD module to support cloud-init on the FBSD10 platform. In its current form its still missing some modules though. Supported: -SSH-keys -growpart -growfs -adduser -powerstate --- cloudinit/sources/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'cloudinit/sources') diff --git a/cloudinit/sources/__init__.py b/cloudinit/sources/__init__.py index 7dc1fbde..d799a211 100644 --- a/cloudinit/sources/__init__.py +++ b/cloudinit/sources/__init__.py @@ -119,7 +119,7 @@ class DataSource(object): # when the kernel named them 'vda' or 'xvda' # we want to return the correct value for what will actually # exist in this instance - mappings = {"sd": ("vd", "xvd")} + mappings = {"sd": ("vd", "xvd", "vtb")} for (nfrom, tlist) in mappings.iteritems(): if not short_name.startswith(nfrom): continue -- cgit v1.2.3 From 66aa9826b818c3478516104b38039fecbd717b6b Mon Sep 17 00:00:00 2001 From: Paul Querna Date: Thu, 9 Jan 2014 21:14:51 +0000 Subject: Allow a Config Drive source on a partition, if the label matches. --- cloudinit/sources/DataSourceConfigDrive.py | 6 ++++-- tests/unittests/test_datasource/test_configdrive.py | 5 +++-- 2 files changed, 7 insertions(+), 4 deletions(-) (limited to 'cloudinit/sources') diff --git a/cloudinit/sources/DataSourceConfigDrive.py b/cloudinit/sources/DataSourceConfigDrive.py index 4f437244..2a244496 100644 --- a/cloudinit/sources/DataSourceConfigDrive.py +++ b/cloudinit/sources/DataSourceConfigDrive.py @@ -284,8 +284,10 @@ def find_candidate_devs(): # followed by fstype items, but with dupes removed combined = (by_label + [d for d in by_fstype if d not in by_label]) - # We are looking for block device (sda, not sda1), ignore partitions - combined = [d for d in combined if not util.is_partition(d)] + # We are looking for a block device or partition with necessary label or + # an unpartitioned block device. + combined = [d for d in combined + if d in by_label or not util.is_partition(d)] return combined diff --git a/tests/unittests/test_datasource/test_configdrive.py b/tests/unittests/test_datasource/test_configdrive.py index d5935294..3c1e8add 100644 --- a/tests/unittests/test_datasource/test_configdrive.py +++ b/tests/unittests/test_datasource/test_configdrive.py @@ -285,10 +285,11 @@ class TestConfigDriveDataSource(MockerTestCase): self.assertEqual(["/dev/vdb", "/dev/zdd"], ds.find_candidate_devs()) - # verify that partitions are not considered + # verify that partitions are considered, but only if they have a label. devs_with_answers = {"TYPE=vfat": ["/dev/sda1"], "TYPE=iso9660": [], "LABEL=config-2": ["/dev/vdb3"]} - self.assertEqual([], ds.find_candidate_devs()) + self.assertEqual(["/dev/vdb3"], + ds.find_candidate_devs()) finally: util.find_devs_with = orig_find_devs_with -- cgit v1.2.3 From fb55c1079375454d2a2a2f82c6c1812759eeb1f1 Mon Sep 17 00:00:00 2001 From: Ben Howard Date: Fri, 24 Jan 2014 12:29:04 -0700 Subject: Fixes for SmartOS datasource (LP: #1272115): 1. fixed conflation of user-data and cloud-init user-data. Cloud-init user-data is now namespaced as 'cloud-init:user-data'. 2. user-scripts are now fetched from the meta-data service each boot and executed as in the scripts directory 3. datacenter name is now namespaced as sdc:datacenter 4. user-scripts should be shebanged if there is no file magic --- cloudinit/sources/DataSourceSmartOS.py | 45 +++++++- cloudinit/util.py | 72 ++++++++++++ doc/sources/smartos/README.rst | 92 ++++++++++++--- tests/unittests/test_datasource/test_smartos.py | 145 ++++++++++++++++++++++-- 4 files changed, 322 insertions(+), 32 deletions(-) (limited to 'cloudinit/sources') diff --git a/cloudinit/sources/DataSourceSmartOS.py b/cloudinit/sources/DataSourceSmartOS.py index 6593ce6e..6bd4a5c7 100644 --- a/cloudinit/sources/DataSourceSmartOS.py +++ b/cloudinit/sources/DataSourceSmartOS.py @@ -25,7 +25,9 @@ # requests on the console. For example, to get the hostname, you # would send "GET hostname" on /dev/ttyS1. # - +# Certain behavior is defined by the DataDictionary +# http://us-east.manta.joyent.com/jmc/public/mdata/datadict.html +# Comments with "@datadictionary" are snippets of the definition import base64 from cloudinit import log as logging @@ -43,10 +45,11 @@ SMARTOS_ATTRIB_MAP = { 'local-hostname': ('hostname', True), 'public-keys': ('root_authorized_keys', True), 'user-script': ('user-script', False), - 'user-data': ('user-data', False), + 'legacy-user-data': ('user-data', False), + 'user-data': ('cloud-init:user-data', False), 'iptables_disable': ('iptables_disable', True), 'motd_sys_info': ('motd_sys_info', True), - 'availability_zone': ('datacenter_name', True), + 'availability_zone': ('sdc:datacenter_name', True), 'vendordata': ('sdc:operator-script', False), } @@ -71,7 +74,11 @@ BUILTIN_DS_CONFIG = { 'seed_timeout': 60, 'no_base64_decode': ['root_authorized_keys', 'motd_sys_info', - 'iptables_disable'], + 'iptables_disable', + 'user-data', + 'user-script', + 'sdc:datacenter_name', + ], 'base64_keys': [], 'base64_all': False, 'disk_aliases': {'ephemeral0': '/dev/vdb'}, @@ -88,6 +95,11 @@ BUILTIN_CLOUD_CONFIG = { 'device': 'ephemeral0'}], } +# @datadictionary: this is legacy path for placing files from metadata +# per the SmartOS location. It is not preferable, but is done for +# legacy reasons +LEGACY_USER_D = "/var/db" + class DataSourceSmartOS(sources.DataSource): def __init__(self, sys_cfg, distro, paths): @@ -107,6 +119,9 @@ class DataSourceSmartOS(sources.DataSource): self.smartos_no_base64 = self.ds_cfg.get('no_base64_decode') self.b64_keys = self.ds_cfg.get('base64_keys') self.b64_all = self.ds_cfg.get('base64_all') + self.script_base_d = os.path.join(self.paths.get_cpath("scripts")) + self.user_script_d = os.path.join(self.paths.get_cpath("scripts"), + 'per-boot') def __str__(self): root = sources.DataSource.__str__(self) @@ -144,14 +159,32 @@ class DataSourceSmartOS(sources.DataSource): smartos_noun, strip = attribute md[ci_noun] = self.query(smartos_noun, strip=strip) + # @datadictionary: This key has no defined format, but its value + # is written to the file /var/db/mdata-user-data on each boot prior + # to the phase that runs user-script. This file is not to be executed. + # This allows a configuration file of some kind to be injected into + # the machine to be consumed by the user-script when it runs. + u_script = md.get('user-script') + u_script_f = "%s/99_user_script" % self.user_script_d + u_script_l = "%s/user-script" % LEGACY_USER_D + util.write_content(u_script, u_script_f, link=u_script_l, + executable=True) + + # @datadictionary: This key may contain a program that is written + # to a file in the filesystem of the guest on each boot and then + # executed. It may be of any format that would be considered + # executable in the guest instance. + u_data = md.get('legacy-user-data') + u_data_f = "%s/mdata-user-data" % LEGACY_USER_D + util.write_content(u_data, u_data_f) + + # Handle the cloud-init regular meta if not md['local-hostname']: md['local-hostname'] = system_uuid ud = None if md['user-data']: ud = md['user-data'] - elif md['user-script']: - ud = md['user-script'] self.metadata = util.mergemanydict([md, self.metadata]) self.userdata_raw = ud diff --git a/cloudinit/util.py b/cloudinit/util.py index 77f9ab36..5f64cb69 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -1904,3 +1904,75 @@ def expand_dotted_devname(dotted): return toks else: return (dotted, None) + + +def write_executable_content(script, script_f): + """ + This writes executable content and ensures that the shebang + exists. + """ + write_file(script_f, script, mode=0700) + try: + cmd = ["file", "--brief", "--mime-type", script_f] + (f_type, _err) = subp(cmd) + + LOG.debug("script %s mime type is %s" % (script_f, f_type)) + + # if the magic is text/plain, re-write with the shebang + if f_type.strip() == "text/plain": + with open(script_f, 'w') as f: + f.write("#!/bin/bash\n") + f.write(script) + LOG.debug("added shebang to file %s" % script_f) + + except ProcessExecutionError as e: + logexc(LOG, "Failed to identify script type for %s" % script_f, e) + return False + + except IOError as e: + logexc(LOG, "Failed to add shebang to file %s" % script_f, e) + return False + + return True + + +def write_content(content, content_f, link=None, + executable=False): + """ + Write the content to content_f. Under the following rules: + 1. Backup previous content_f + 2. Write the contente + 3. If no content, remove the file + 4. If there is a link, create it + + @param content: what to write + @param content_f: the file name + @param backup_d: the directory to save the backup at + @param link: if defined, location to create a symlink to + @param executable: is the file executable + """ + + if content: + if not executable: + write_file(content_f, content, mode=0400) + else: + w = write_executable_content(content, content_f) + if not w: + LOG.debug("failed to write file to %s" % content_f) + return False + + if not content and os.path.exists(content_f): + os.unlink(content_f) + + if link: + try: + if os.path.islink(link): + os.unlink(link) + if content and os.path.exists(content_f): + ensure_dir(os.path.dirname(link)) + os.symlink(content_f, link) + except IOError as e: + logexc(LOG, "failed establishing content link", e) + return False + + return True diff --git a/doc/sources/smartos/README.rst b/doc/sources/smartos/README.rst index 8b63e520..e63f311f 100644 --- a/doc/sources/smartos/README.rst +++ b/doc/sources/smartos/README.rst @@ -16,11 +16,35 @@ responds with the status and if "SUCCESS" returns until a single ".\n". New versions of the SmartOS tooling will include support for base64 encoded data. -Userdata --------- - -In SmartOS parlance, user-data is a actually meta-data. This userdata can be -provided as key-value pairs. +Meta-data channels +------------------ + +Cloud-init supports three modes of delivering user/meta-data via the flexible +channels of SmartOS. + +* user-data is written to /var/db/user-data + - per the spec, user-data is for consumption by the end-user, not provisioning + tools + - cloud-init entirely ignores this channel other than writting it to disk + - removal of the meta-data key means that /var/db/user-data gets removed + - a backup of previous meta-data is maintained as /var/db/user-data. + - is the epoch time when cloud-init ran + +* user-script is written to /var/lib/cloud/scripts/per-boot/99_user_data + - this is executed each boot + - a link is created to /var/db/user-script + - previous versions of the user-script is written to + /var/lib/cloud/scripts/per-boot.backup/99_user_script.. + - is the epoch time when cloud-init ran. + - when the 'user-script' meta-data key goes missing, the user-script is + removed from the file system, although a backup is maintained. + - if the script is not shebanged (i.e. starts with #!), then + or is not an executable, cloud-init will add a shebang of "#!/bin/bash" + +* cloud-init:user-data is treated like on other Clouds. + - this channel is used for delivering _all_ cloud-init instructions + - scripts delivered over this channel must be well formed (i.e. must have + a shebang) Cloud-init supports reading the traditional meta-data fields supported by the SmartOS tools. These are: @@ -32,19 +56,49 @@ SmartOS tools. These are: Note: At this time iptables_disable and enable_motd_sys_info are read but are not actioned. -user-script ------------ - -SmartOS traditionally supports sending over a user-script for execution at the -rc.local level. Cloud-init supports running user-scripts as if they were -cloud-init user-data. In this sense, anything with a shell interpreter -directive will run. - -user-data and user-script -------------------------- - -In the event that a user defines the meta-data key of "user-data" it will -always supersede any user-script data. This is for consistency. +disabling user-script +--------------------- + +Cloud-init uses the per-boot script functionality to handle the execution +of the user-script. If you want to prevent this use a cloud-config of: + +#cloud-config +cloud_final_modules: + - scripts-per-once + - scripts-per-instance + - scripts-user + - ssh-authkey-fingerprints + - keys-to-console + - phone-home + - final-message + - power-state-change + +Alternatively you can use the json patch method +#cloud-config-jsonp +[ + { "op": "replace", + "path": "/cloud_final_modules", + "value": ["scripts-per-once", + "scripts-per-instance", + "scripts-user", + "ssh-authkey-fingerprints", + "keys-to-console", + "phone-home", + "final-message", + "power-state-change"] + } +] + +The default cloud-config includes "script-per-boot". Cloud-init will still +ingest and write the user-data but will not execute it, when you disable +the per-boot script handling. + +Note: Unless you have an explicit use-case, it is recommended that you not + disable the per-boot script execution, especially if you are using + any of the life-cycle management features of SmartOS. + +The cloud-config needs to be delivered over the cloud-init:user-data channel +in order for cloud-init to ingest it. base64 ------ @@ -54,6 +108,8 @@ are provided by SmartOS: * root_authorized_keys * enable_motd_sys_info * iptables_disable + * user-data + * user-script This list can be changed through system config of variable 'no_base64_decode'. diff --git a/tests/unittests/test_datasource/test_smartos.py b/tests/unittests/test_datasource/test_smartos.py index 956767d8..ae427bb5 100644 --- a/tests/unittests/test_datasource/test_smartos.py +++ b/tests/unittests/test_datasource/test_smartos.py @@ -27,6 +27,10 @@ from cloudinit import helpers from cloudinit.sources import DataSourceSmartOS from mocker import MockerTestCase +import os +import os.path +import re +import stat import uuid MOCK_RETURNS = { @@ -35,7 +39,11 @@ MOCK_RETURNS = { 'disable_iptables_flag': None, 'enable_motd_sys_info': None, 'test-var1': 'some data', - 'user-data': '\n'.join(['#!/bin/sh', '/bin/true', '']), + 'cloud-init:user-data': '\n'.join(['#!/bin/sh', '/bin/true', '']), + 'sdc:datacenter_name': 'somewhere2', + 'sdc:operator-script': '\n'.join(['bin/true', '']), + 'user-data': '\n'.join(['something', '']), + 'user-script': '\n'.join(['/bin/true', '']), } DMI_DATA_RETURN = (str(uuid.uuid4()), 'smartdc') @@ -101,6 +109,7 @@ class TestSmartOSDataSource(MockerTestCase): def setUp(self): # makeDir comes from MockerTestCase self.tmp = self.makeDir() + self.legacy_user_d = self.makeDir() # patch cloud_dir, so our 'seed_dir' is guaranteed empty self.paths = helpers.Paths({'cloud_dir': self.tmp}) @@ -138,6 +147,7 @@ class TestSmartOSDataSource(MockerTestCase): sys_cfg['datasource'] = sys_cfg.get('datasource', {}) sys_cfg['datasource']['SmartOS'] = ds_cfg + self.apply_patches([(mod, 'LEGACY_USER_D', self.legacy_user_d)]) self.apply_patches([(mod, 'get_serial', _get_serial)]) self.apply_patches([(mod, 'dmi_data', _dmi_data)]) dsrc = mod.DataSourceSmartOS(sys_cfg, distro=None, @@ -194,7 +204,7 @@ class TestSmartOSDataSource(MockerTestCase): # metadata provided base64_all of true my_returns = MOCK_RETURNS.copy() my_returns['base64_all'] = "true" - for k in ('hostname', 'user-data'): + for k in ('hostname', 'cloud-init:user-data'): my_returns[k] = base64.b64encode(my_returns[k]) dsrc = self._get_ds(mockdata=my_returns) @@ -202,7 +212,7 @@ class TestSmartOSDataSource(MockerTestCase): self.assertTrue(ret) self.assertEquals(MOCK_RETURNS['hostname'], dsrc.metadata['local-hostname']) - self.assertEquals(MOCK_RETURNS['user-data'], + self.assertEquals(MOCK_RETURNS['cloud-init:user-data'], dsrc.userdata_raw) self.assertEquals(MOCK_RETURNS['root_authorized_keys'], dsrc.metadata['public-keys']) @@ -213,9 +223,9 @@ class TestSmartOSDataSource(MockerTestCase): def test_b64_userdata(self): my_returns = MOCK_RETURNS.copy() - my_returns['b64-user-data'] = "true" + my_returns['b64-cloud-init:user-data'] = "true" my_returns['b64-hostname'] = "true" - for k in ('hostname', 'user-data'): + for k in ('hostname', 'cloud-init:user-data'): my_returns[k] = base64.b64encode(my_returns[k]) dsrc = self._get_ds(mockdata=my_returns) @@ -223,7 +233,8 @@ class TestSmartOSDataSource(MockerTestCase): self.assertTrue(ret) self.assertEquals(MOCK_RETURNS['hostname'], dsrc.metadata['local-hostname']) - self.assertEquals(MOCK_RETURNS['user-data'], dsrc.userdata_raw) + self.assertEquals(MOCK_RETURNS['cloud-init:user-data'], + dsrc.userdata_raw) self.assertEquals(MOCK_RETURNS['root_authorized_keys'], dsrc.metadata['public-keys']) @@ -238,13 +249,131 @@ class TestSmartOSDataSource(MockerTestCase): self.assertTrue(ret) self.assertEquals(MOCK_RETURNS['hostname'], dsrc.metadata['local-hostname']) - self.assertEquals(MOCK_RETURNS['user-data'], dsrc.userdata_raw) + self.assertEquals(MOCK_RETURNS['cloud-init:user-data'], + dsrc.userdata_raw) def test_userdata(self): dsrc = self._get_ds(mockdata=MOCK_RETURNS) ret = dsrc.get_data() self.assertTrue(ret) - self.assertEquals(MOCK_RETURNS['user-data'], dsrc.userdata_raw) + self.assertEquals(MOCK_RETURNS['user-data'], + dsrc.metadata['legacy-user-data']) + self.assertEquals(MOCK_RETURNS['cloud-init:user-data'], + dsrc.userdata_raw) + + def test_sdc_scripts(self): + dsrc = self._get_ds(mockdata=MOCK_RETURNS) + ret = dsrc.get_data() + self.assertTrue(ret) + self.assertEquals(MOCK_RETURNS['user-script'], + dsrc.metadata['user-script']) + + legacy_script_f = "%s/user-script" % self.legacy_user_d + self.assertTrue(os.path.exists(legacy_script_f)) + self.assertTrue(os.path.islink(legacy_script_f)) + user_script_perm = oct(os.stat(legacy_script_f)[stat.ST_MODE])[-3:] + self.assertEquals(user_script_perm, '700') + + def test_scripts_shebanged(self): + dsrc = self._get_ds(mockdata=MOCK_RETURNS) + ret = dsrc.get_data() + self.assertTrue(ret) + self.assertEquals(MOCK_RETURNS['user-script'], + dsrc.metadata['user-script']) + + legacy_script_f = "%s/user-script" % self.legacy_user_d + self.assertTrue(os.path.exists(legacy_script_f)) + self.assertTrue(os.path.islink(legacy_script_f)) + shebang = None + with open(legacy_script_f, 'r') as f: + shebang = f.readlines()[0].strip() + self.assertEquals(shebang, "#!/bin/bash") + user_script_perm = oct(os.stat(legacy_script_f)[stat.ST_MODE])[-3:] + self.assertEquals(user_script_perm, '700') + + def test_scripts_shebang_not_added(self): + """ + Test that the SmartOS requirement that plain text scripts + are executable. This test makes sure that plain texts scripts + with out file magic have it added appropriately by cloud-init. + """ + + my_returns = MOCK_RETURNS.copy() + my_returns['user-script'] = '\n'.join(['#!/usr/bin/perl', + 'print("hi")', '']) + + dsrc = self._get_ds(mockdata=my_returns) + ret = dsrc.get_data() + self.assertTrue(ret) + self.assertEquals(my_returns['user-script'], + dsrc.metadata['user-script']) + + legacy_script_f = "%s/user-script" % self.legacy_user_d + self.assertTrue(os.path.exists(legacy_script_f)) + self.assertTrue(os.path.islink(legacy_script_f)) + shebang = None + with open(legacy_script_f, 'r') as f: + shebang = f.readlines()[0].strip() + self.assertEquals(shebang, "#!/usr/bin/perl") + + def test_scripts_removed(self): + """ + Since SmartOS requires that the user script is fetched + each boot, we want to make sure that the information + is backed-up for user-review later. + + This tests the behavior of when a script is removed. It makes + sure that a) the previous script is backed-up; and 2) that + there is no script remaining. + """ + + script_d = os.path.join(self.tmp, "scripts", "per-boot") + os.makedirs(script_d) + + test_script_f = "%s/99_user_script" % script_d + with open(test_script_f, 'w') as f: + f.write("TEST DATA") + + my_returns = MOCK_RETURNS.copy() + del my_returns['user-script'] + + dsrc = self._get_ds(mockdata=my_returns) + ret = dsrc.get_data() + self.assertTrue(ret) + self.assertFalse(dsrc.metadata['user-script']) + self.assertFalse(os.path.exists(test_script_f)) + + def test_userdata_removed(self): + """ + User-data in the SmartOS world is supposed to be written to a file + each and every boot. This tests to make sure that in the event the + legacy user-data is removed, the existing user-data is backed-up and + there is no /var/db/user-data left. + """ + + user_data_f = "%s/mdata-user-data" % self.legacy_user_d + with open(user_data_f, 'w') as f: + f.write("PREVIOUS") + + my_returns = MOCK_RETURNS.copy() + del my_returns['user-data'] + + dsrc = self._get_ds(mockdata=my_returns) + ret = dsrc.get_data() + self.assertTrue(ret) + self.assertFalse(dsrc.metadata.get('legacy-user-data')) + + found_new = False + for root, _dirs, files in os.walk(self.legacy_user_d): + for name in files: + name_f = os.path.join(root, name) + permissions = oct(os.stat(name_f)[stat.ST_MODE])[-3:] + if re.match(r'.*\/mdata-user-data$', name_f): + found_new = True + print name_f + self.assertEquals(permissions, '400') + + self.assertFalse(found_new) def test_disable_iptables_flag(self): dsrc = self._get_ds(mockdata=MOCK_RETURNS) -- cgit v1.2.3 From 7079fac4646380db1e064a433d7843473bda1542 Mon Sep 17 00:00:00 2001 From: Ben Howard Date: Fri, 24 Jan 2014 12:52:04 -0700 Subject: Fixed flip-flopped comment --- cloudinit/distros/freebsd.py | 2 +- cloudinit/distros/net_util.py | 18 +++++++++--------- cloudinit/sources/DataSourceSmartOS.py | 18 +++++++++--------- 3 files changed, 19 insertions(+), 19 deletions(-) (limited to 'cloudinit/sources') diff --git a/cloudinit/distros/freebsd.py b/cloudinit/distros/freebsd.py index f1650a77..d28860eb 100644 --- a/cloudinit/distros/freebsd.py +++ b/cloudinit/distros/freebsd.py @@ -2,7 +2,7 @@ # # Copyright (C) 2014 Harm Weites # -# Author: Harm Weites +# Author: Harm Weites # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License version 3, as diff --git a/cloudinit/distros/net_util.py b/cloudinit/distros/net_util.py index 5f60666d..b9bcfd8b 100644 --- a/cloudinit/distros/net_util.py +++ b/cloudinit/distros/net_util.py @@ -51,7 +51,7 @@ # # auto lo # iface lo inet loopback -# +# # auto eth0 # iface eth0 inet static # address 10.0.0.1 @@ -64,17 +64,17 @@ # { # "lo": { # "auto": true -# }, +# }, # "eth0": { -# "auto": true, +# "auto": true, # "dns-nameservers": [ -# "98.0.0.1", +# "98.0.0.1", # "98.0.0.2" -# ], -# "broadcast": "10.0.0.255", -# "netmask": "255.255.252.0", -# "bootproto": "static", -# "address": "10.0.0.1", +# ], +# "broadcast": "10.0.0.255", +# "netmask": "255.255.252.0", +# "bootproto": "static", +# "address": "10.0.0.1", # "gateway": "10.0.0.2" # } # } diff --git a/cloudinit/sources/DataSourceSmartOS.py b/cloudinit/sources/DataSourceSmartOS.py index 6bd4a5c7..73dd2ba0 100644 --- a/cloudinit/sources/DataSourceSmartOS.py +++ b/cloudinit/sources/DataSourceSmartOS.py @@ -159,21 +159,21 @@ class DataSourceSmartOS(sources.DataSource): smartos_noun, strip = attribute md[ci_noun] = self.query(smartos_noun, strip=strip) - # @datadictionary: This key has no defined format, but its value - # is written to the file /var/db/mdata-user-data on each boot prior - # to the phase that runs user-script. This file is not to be executed. - # This allows a configuration file of some kind to be injected into - # the machine to be consumed by the user-script when it runs. + # @datadictionary: This key may contain a program that is written + # to a file in the filesystem of the guest on each boot and then + # executed. It may be of any format that would be considered + # executable in the guest instance. u_script = md.get('user-script') u_script_f = "%s/99_user_script" % self.user_script_d u_script_l = "%s/user-script" % LEGACY_USER_D util.write_content(u_script, u_script_f, link=u_script_l, executable=True) - # @datadictionary: This key may contain a program that is written - # to a file in the filesystem of the guest on each boot and then - # executed. It may be of any format that would be considered - # executable in the guest instance. + # @datadictionary: This key has no defined format, but its value + # is written to the file /var/db/mdata-user-data on each boot prior + # to the phase that runs user-script. This file is not to be executed. + # This allows a configuration file of some kind to be injected into + # the machine to be consumed by the user-script when it runs. u_data = md.get('legacy-user-data') u_data_f = "%s/mdata-user-data" % LEGACY_USER_D util.write_content(u_data, u_data_f) -- cgit v1.2.3 From 93c0bcf6a048e278ead6b4392d3507c40441b7bb Mon Sep 17 00:00:00 2001 From: Ben Howard Date: Fri, 24 Jan 2014 15:28:55 -0700 Subject: Make SmartOS script handling self-contained in datasource. --- cloudinit/sources/DataSourceSmartOS.py | 63 +++++++++++++++++++++++++++-- cloudinit/util.py | 72 ---------------------------------- 2 files changed, 60 insertions(+), 75 deletions(-) (limited to 'cloudinit/sources') diff --git a/cloudinit/sources/DataSourceSmartOS.py b/cloudinit/sources/DataSourceSmartOS.py index 73dd2ba0..b0fabe05 100644 --- a/cloudinit/sources/DataSourceSmartOS.py +++ b/cloudinit/sources/DataSourceSmartOS.py @@ -36,6 +36,7 @@ from cloudinit import util import os import os.path import serial +import subprocess LOG = logging.getLogger(__name__) @@ -166,8 +167,8 @@ class DataSourceSmartOS(sources.DataSource): u_script = md.get('user-script') u_script_f = "%s/99_user_script" % self.user_script_d u_script_l = "%s/user-script" % LEGACY_USER_D - util.write_content(u_script, u_script_f, link=u_script_l, - executable=True) + write_boot_content(u_script, u_script_f, link=u_script_l, shebang=True, + mode=0700) # @datadictionary: This key has no defined format, but its value # is written to the file /var/db/mdata-user-data on each boot prior @@ -176,7 +177,7 @@ class DataSourceSmartOS(sources.DataSource): # the machine to be consumed by the user-script when it runs. u_data = md.get('legacy-user-data') u_data_f = "%s/mdata-user-data" % LEGACY_USER_D - util.write_content(u_data, u_data_f) + write_boot_content(u_data, u_data_f) # Handle the cloud-init regular meta if not md['local-hostname']: @@ -312,6 +313,62 @@ def dmi_data(): return (sys_uuid.lower().strip(), sys_type.strip()) +def write_boot_content(content, content_f, link=None, shebang=False, mode=0400): + """ + Write the content to content_f. Under the following rules: + 1. If no content, remove the file + 2. Write the content + 3. If executable and no file magic, add it + 4. If there is a link, create it + + @param content: what to write + @param content_f: the file name + @param backup_d: the directory to save the backup at + @param link: if defined, location to create a symlink to + @param shebang: if no file magic, set shebang + @param mode: file mode + + Becuase of the way that Cloud-init executes scripts (no shell), + a script will fail to execute if does not have a magic bit (shebang) set + for the file. If shebang=True, then the script will be checked for a magic + bit and to the SmartOS default of assuming that bash. + """ + + if not content and os.path.exists(content_f): + os.unlink(content_f) + if link and os.path.islink(link): + os.unlink(link) + if not content: + return + + util.write_file(content_f, content, mode=mode) + + if shebang: + try: + cmd = ["file", "--brief", "--mime-type", content_f] + (f_type, _err) = util.subp(cmd) + LOG.debug("script %s mime type is %s" % (content_f, f_type)) + line_one = content.splitlines()[0] + if f_type.strip() == "text/plain" and "#!" not in line_one: + new_content = "\n".join(["#!/bin/bash", content]) + util.write_file(content_f, new_content, mode=mode) + LOG.debug("added shebang to file %s" % content_f) + + except Exception as e: + util.logexc(LOG, ("Failed to identify script type for %s" % + content_f, e)) + + if link: + try: + if os.path.islink(link): + os.unlink(link) + if content and os.path.exists(content_f): + util.ensure_dir(os.path.dirname(link)) + os.symlink(content_f, link) + except IOError as e: + util.logexc(LOG, "failed establishing content link", e) + + # Used to match classes to dependencies datasources = [ (DataSourceSmartOS, (sources.DEP_FILESYSTEM, sources.DEP_NETWORK)), diff --git a/cloudinit/util.py b/cloudinit/util.py index bf4006cb..d350ba08 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -1907,75 +1907,3 @@ def expand_dotted_devname(dotted): return toks else: return (dotted, None) - - -def write_executable_content(script, script_f): - """ - This writes executable content and ensures that the shebang - exists. - """ - write_file(script_f, script, mode=0700) - try: - cmd = ["file", "--brief", "--mime-type", script_f] - (f_type, _err) = subp(cmd) - - LOG.debug("script %s mime type is %s" % (script_f, f_type)) - - # if the magic is text/plain, re-write with the shebang - if f_type.strip() == "text/plain": - with open(script_f, 'w') as f: - f.write("#!/bin/bash\n") - f.write(script) - LOG.debug("added shebang to file %s" % script_f) - - except ProcessExecutionError as e: - logexc(LOG, "Failed to identify script type for %s" % script_f, e) - return False - - except IOError as e: - logexc(LOG, "Failed to add shebang to file %s" % script_f, e) - return False - - return True - - -def write_content(content, content_f, link=None, - executable=False): - """ - Write the content to content_f. Under the following rules: - 1. Backup previous content_f - 2. Write the contente - 3. If no content, remove the file - 4. If there is a link, create it - - @param content: what to write - @param content_f: the file name - @param backup_d: the directory to save the backup at - @param link: if defined, location to create a symlink to - @param executable: is the file executable - """ - - if content: - if not executable: - write_file(content_f, content, mode=0400) - else: - w = write_executable_content(content, content_f) - if not w: - LOG.debug("failed to write file to %s" % content_f) - return False - - if not content and os.path.exists(content_f): - os.unlink(content_f) - - if link: - try: - if os.path.islink(link): - os.unlink(link) - if content and os.path.exists(content_f): - ensure_dir(os.path.dirname(link)) - os.symlink(content_f, link) - except IOError as e: - logexc(LOG, "failed establishing content link", e) - return False - - return True -- cgit v1.2.3 From 9876ad7d74f90f7c7433fb4dc1fa07e664ff92bc Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Fri, 24 Jan 2014 20:13:38 -0500 Subject: minor changes for pylint, write_boot_content improvement. if write_boot_content is given somethign that starts with #!, then there isn't a reason to invoke 'file' to tell us that it starts with shebang. This way, we only run file in 2 cases: a.) binary content (don't really know if that is supported or not) b.) magic "user meant to run this with /bin/bash but couldn't be bothered to type that" --- cloudinit/sources/DataSourceSmartOS.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) (limited to 'cloudinit/sources') diff --git a/cloudinit/sources/DataSourceSmartOS.py b/cloudinit/sources/DataSourceSmartOS.py index b0fabe05..140c7814 100644 --- a/cloudinit/sources/DataSourceSmartOS.py +++ b/cloudinit/sources/DataSourceSmartOS.py @@ -36,7 +36,6 @@ from cloudinit import util import os import os.path import serial -import subprocess LOG = logging.getLogger(__name__) @@ -313,7 +312,8 @@ def dmi_data(): return (sys_uuid.lower().strip(), sys_type.strip()) -def write_boot_content(content, content_f, link=None, shebang=False, mode=0400): +def write_boot_content(content, content_f, link=None, shebang=False, + mode=0400): """ Write the content to content_f. Under the following rules: 1. If no content, remove the file @@ -343,16 +343,15 @@ def write_boot_content(content, content_f, link=None, shebang=False, mode=0400): util.write_file(content_f, content, mode=mode) - if shebang: + if shebang and not content.startswith("#!"): try: cmd = ["file", "--brief", "--mime-type", content_f] (f_type, _err) = util.subp(cmd) - LOG.debug("script %s mime type is %s" % (content_f, f_type)) - line_one = content.splitlines()[0] - if f_type.strip() == "text/plain" and "#!" not in line_one: + LOG.debug("script %s mime type is %s", content_f, f_type) + if f_type.strip() == "text/plain": new_content = "\n".join(["#!/bin/bash", content]) util.write_file(content_f, new_content, mode=mode) - LOG.debug("added shebang to file %s" % content_f) + LOG.debug("added shebang to file %s", content_f) except Exception as e: util.logexc(LOG, ("Failed to identify script type for %s" % -- cgit v1.2.3 From c1253945761c33bfa89289a63cdb8799fc18d019 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Tue, 28 Jan 2014 14:03:20 -0500 Subject: DataSourceNoCloud: support reading vendor-data Here we add the ability to read vendor-data from a file named vendor-data at the same location as the user-data and meta-data files. At the moment, vendor-data is not read at all from 'seedfrom'. --- cloudinit/sources/DataSourceNoCloud.py | 92 ++++++++++++++++++++++------------ 1 file changed, 60 insertions(+), 32 deletions(-) (limited to 'cloudinit/sources') diff --git a/cloudinit/sources/DataSourceNoCloud.py b/cloudinit/sources/DataSourceNoCloud.py index 4ef92a56..cbaac29f 100644 --- a/cloudinit/sources/DataSourceNoCloud.py +++ b/cloudinit/sources/DataSourceNoCloud.py @@ -50,40 +50,47 @@ class DataSourceNoCloud(sources.DataSource): } found = [] - md = {} - ud = "" + mydata = {'meta-data': {}, 'user-data': "", 'vendor-data': ""} try: # Parse the kernel command line, getting data passed in + md = {} if parse_cmdline_data(self.cmdline_id, md): found.append("cmdline") + mydata.update(md) except: util.logexc(LOG, "Unable to parse command line data") return False # Check to see if the seed dir has data. - seedret = {} - if util.read_optional_seed(seedret, base=self.seed_dir + "/"): - md = util.mergemanydict([md, seedret['meta-data']]) - ud = seedret['user-data'] + pp2d_kwargs = {'required': ['user-data', 'meta-data'], + 'optional': ['vendor-data']} + + try: + seeded = util.pathprefix2dict(self.seed_dir, **pp2d_kwargs) found.append(self.seed_dir) - LOG.debug("Using seeded cache data from %s", self.seed_dir) + LOG.debug("Using seeded data from %s", self.seed_dir) + except ValueError as e: + pass + + if self.seed_dir in found: + mydata = _merge_new_seed(mydata, seeded) # If the datasource config had a 'seedfrom' entry, then that takes # precedence over a 'seedfrom' that was found in a filesystem # but not over external media - if 'seedfrom' in self.ds_cfg and self.ds_cfg['seedfrom']: - found.append("ds_config") - md["seedfrom"] = self.ds_cfg['seedfrom'] + if self.ds_cfg.get('seedfrom'): + found.append("ds_config_seedfrom") + mydata['meta-data']["seedfrom"] = self.ds_cfg['seedfrom'] - # if ds_cfg has 'user-data' and 'meta-data' + # fields appropriately named can also just come from the datasource + # config (ie, 'user-data', 'meta-data', 'vendor-data' there) if 'user-data' in self.ds_cfg and 'meta-data' in self.ds_cfg: - if self.ds_cfg['user-data']: - ud = self.ds_cfg['user-data'] - if self.ds_cfg['meta-data'] is not False: - md = util.mergemanydict([md, self.ds_cfg['meta-data']]) - if 'ds_config' not in found: - found.append("ds_config") + mydata = _merge_new_seed(mydata, self.ds_cfg) + found.append("ds_config") + + def _pp2d_callback(mp, data): + util.pathprefix2dict(mp, **data) label = self.ds_cfg.get('fs_label', "cidata") if label is not None: @@ -102,15 +109,21 @@ class DataSourceNoCloud(sources.DataSource): try: LOG.debug("Attempting to use data from %s", dev) - (newmd, newud) = util.mount_cb(dev, util.read_seeded) - md = util.mergemanydict([newmd, md]) - ud = newud + try: + seeded = util.mount_cb(dev, _pp2d_callback) + except ValueError as e: + if dev in label_list: + LOG.warn("device %s with label=%s not a" + "valid seed.", dev, label) + continue + + mydata = _merge_new_seed(mydata, seeded) # For seed from a device, the default mode is 'net'. # that is more likely to be what is desired. If they want # dsmode of local, then they must specify that. - if 'dsmode' not in md: - md['dsmode'] = "net" + if 'dsmode' not in mydata['meta-data']: + mydata['meta-data'] = "net" LOG.debug("Using data from %s", dev) found.append(dev) @@ -133,8 +146,8 @@ class DataSourceNoCloud(sources.DataSource): # attempt to seed the userdata / metadata from its value # its primarily value is in allowing the user to type less # on the command line, ie: ds=nocloud;s=http://bit.ly/abcdefg - if "seedfrom" in md: - seedfrom = md["seedfrom"] + if "seedfrom" in mydata['meta-data']: + seedfrom = mydata['meta-data']["seedfrom"] seedfound = False for proto in self.supported_seed_starts: if seedfrom.startswith(proto): @@ -144,7 +157,7 @@ class DataSourceNoCloud(sources.DataSource): LOG.debug("Seed from %s not supported by %s", seedfrom, self) return False - if 'network-interfaces' in md: + if 'network-interfaces' in mydata['meta-data']: seeded_interfaces = self.dsmode # This could throw errors, but the user told us to do it @@ -153,25 +166,30 @@ class DataSourceNoCloud(sources.DataSource): LOG.debug("Using seeded cache data from %s", seedfrom) # Values in the command line override those from the seed - md = util.mergemanydict([md, md_seed]) + mydata['meta-data'] = util.mergemanydict([mydata['meta-data'], + md_seed]) + mydata['user-data'] = ud found.append(seedfrom) # Now that we have exhausted any other places merge in the defaults - md = util.mergemanydict([md, defaults]) + mydata['meta-data'] = util.mergemanydict([mydata['meta-data'], + defaults]) # Update the network-interfaces if metadata had 'network-interfaces' # entry and this is the local datasource, or 'seedfrom' was used # and the source of the seed was self.dsmode # ('local' for NoCloud, 'net' for NoCloudNet') - if ('network-interfaces' in md and + if ('network-interfaces' in mydata['meta-data'] and (self.dsmode in ("local", seeded_interfaces))): LOG.debug("Updating network interfaces from %s", self) - self.distro.apply_network(md['network-interfaces']) + self.distro.apply_network( + mydata['meta-data']['network-interfaces']) - if md['dsmode'] == self.dsmode: + if mydata['meta-data']['dsmode'] == self.dsmode: self.seed = ",".join(found) - self.metadata = md - self.userdata_raw = ud + self.metadata = mydata['meta-data'] + self.userdata_raw = mydata['user-data'] + self.vendordata = mydata['vendor-data'] return True LOG.debug("%s: not claiming datasource, dsmode=%s", self, md['dsmode']) @@ -222,6 +240,16 @@ def parse_cmdline_data(ds_id, fill, cmdline=None): return True +def _merge_new_seed(cur, seeded): + ret = cur.copy() + ret['meta-data'] = util.mergemanydict([cur['meta-data'], + util.load_yaml(seeded['meta-data'])]) + ret['user-data'] = seeded['user-data'] + if 'vendor-data' in seeded: + ret['vendor-data'] = seeded['vendor-data'] + return ret + + class DataSourceNoCloudNet(DataSourceNoCloud): def __init__(self, sys_cfg, distro, paths): DataSourceNoCloud.__init__(self, sys_cfg, distro, paths) -- cgit v1.2.3