From aae54b5e8e1aee908b01050aa6c9c821fe94195e Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Fri, 31 Aug 2012 17:01:51 -0700 Subject: Don't look into cloud-archive (after processing) for launch indexes (since they will be handled beforehand) and fix the types being checked on the root of the archive format to be a tuple instead of a list (which oddly causes complaints). --- cloudinit/user_data.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cloudinit/user_data.py b/cloudinit/user_data.py index 5d550e1d..f9f84030 100644 --- a/cloudinit/user_data.py +++ b/cloudinit/user_data.py @@ -54,7 +54,7 @@ ATTACHMENT_FIELD = 'Number-Attachments' # Only the following content types can have there launch index examined # in there payload, evey other content type can still provide a header -EXAMINE_FOR_LAUNCH_INDEX = ["text/cloud-config", "text/cloud-config-archive"] +EXAMINE_FOR_LAUNCH_INDEX = ["text/cloud-config"] class UserDataProcessor(object): @@ -180,7 +180,7 @@ class UserDataProcessor(object): self._process_msg(new_msg, append_msg) def _explode_archive(self, archive, append_msg): - entries = util.load_yaml(archive, default=[], allowed=[list, set]) + entries = util.load_yaml(archive, default=[], allowed=(list, set)) for ent in entries: # ent can be one of: # dict { 'filename' : 'value', 'content' : -- cgit v1.2.3 From ec911fd083db63521aa425203fb30dd6fc7302d5 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Fri, 31 Aug 2012 18:12:30 -0700 Subject: When a parts content type is found to be different than its original content type said it is, make sure we set the new value, also unsure if the old top level message should have the same header (which will flip-flop). --- cloudinit/user_data.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/cloudinit/user_data.py b/cloudinit/user_data.py index f9f84030..803ffc3a 100644 --- a/cloudinit/user_data.py +++ b/cloudinit/user_data.py @@ -84,6 +84,12 @@ class UserDataProcessor(object): if ctype is None: ctype = ctype_orig + if ctype != ctype_orig: + if CONTENT_TYPE in part: + part.replace_header(CONTENT_TYPE, ctype) + else: + part[CONTENT_TYPE] = ctype + if ctype in INCLUDE_TYPES: self._do_include(payload, append_msg) continue @@ -92,6 +98,8 @@ class UserDataProcessor(object): self._explode_archive(payload, append_msg) continue + # Should this be happening, shouldn't + # the part header be modified and not the base? if CONTENT_TYPE in base_msg: base_msg.replace_header(CONTENT_TYPE, ctype) else: -- cgit v1.2.3 From 1d67756be9e768ff9f55e7322c1ab3a6b5cdec34 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Fri, 31 Aug 2012 18:13:18 -0700 Subject: 1. Add a helper for tests to use to load resource/data files from 2. Add a set of tests+data that ensure the launch index filtering works as expected in the various modes including raw yaml and via mime/email message formats. --- tests/data/filter_cloud_multipart.yaml | 30 +++++ tests/data/filter_cloud_multipart_1.email | 11 ++ tests/data/filter_cloud_multipart_2.email | 39 +++++++ tests/data/filter_cloud_multipart_header.email | 11 ++ tests/unittests/helpers.py | 41 +++++++ tests/unittests/test_filters/test_launch_index.py | 133 ++++++++++++++++++++++ 6 files changed, 265 insertions(+) create mode 100644 tests/data/filter_cloud_multipart.yaml create mode 100644 tests/data/filter_cloud_multipart_1.email create mode 100644 tests/data/filter_cloud_multipart_2.email create mode 100644 tests/data/filter_cloud_multipart_header.email create mode 100644 tests/unittests/helpers.py create mode 100644 tests/unittests/test_filters/test_launch_index.py diff --git a/tests/data/filter_cloud_multipart.yaml b/tests/data/filter_cloud_multipart.yaml new file mode 100644 index 00000000..7acc2b9d --- /dev/null +++ b/tests/data/filter_cloud_multipart.yaml @@ -0,0 +1,30 @@ +#cloud-config-archive +--- +- content: "\n blah: true\n launch-index: 3\n" + type: text/cloud-config +- content: "\n blah: true\n launch-index: 4\n" + type: text/cloud-config +- content: The quick brown fox jumps over the lazy dog + filename: b0.txt + launch-index: 0 + type: plain/text +- content: The quick brown fox jumps over the lazy dog + filename: b3.txt + launch-index: 3 + type: plain/text +- content: The quick brown fox jumps over the lazy dog + filename: b2.txt + launch-index: 2 + type: plain/text +- content: '#!/bin/bash \n echo "stuff"' + filename: b2.txt + launch-index: 2 +- content: '#!/bin/bash \n echo "stuff"' + filename: b2.txt + launch-index: 1 +- content: '#!/bin/bash \n echo "stuff"' + filename: b2.txt + # Use a string to see if conversion works + launch-index: "1" +... + diff --git a/tests/data/filter_cloud_multipart_1.email b/tests/data/filter_cloud_multipart_1.email new file mode 100644 index 00000000..6d93b1f1 --- /dev/null +++ b/tests/data/filter_cloud_multipart_1.email @@ -0,0 +1,11 @@ +From nobody Fri Aug 31 17:17:00 2012 +Content-Type: text/plain; charset="us-ascii" +MIME-Version: 1.0 +Content-Transfer-Encoding: 7bit + + +#cloud-config +b: c +launch-index: 2 + + diff --git a/tests/data/filter_cloud_multipart_2.email b/tests/data/filter_cloud_multipart_2.email new file mode 100644 index 00000000..b04068c5 --- /dev/null +++ b/tests/data/filter_cloud_multipart_2.email @@ -0,0 +1,39 @@ +From nobody Fri Aug 31 17:43:04 2012 +Content-Type: multipart/mixed; boundary="===============1668325974==" +MIME-Version: 1.0 + +--===============1668325974== +Content-Type: text/cloud-config; charset="us-ascii" +MIME-Version: 1.0 +Content-Transfer-Encoding: 7bit + + +#cloud-config +b: c +launch-index: 2 + + +--===============1668325974== +Content-Type: text/plain; charset="us-ascii" +MIME-Version: 1.0 +Content-Transfer-Encoding: 7bit + + +#cloud-config-archive +- content: The quick brown fox jumps over the lazy dog + filename: b3.txt + launch-index: 3 + type: plain/text + +--===============1668325974== +Content-Type: text/plain; charset="us-ascii" +MIME-Version: 1.0 +Content-Transfer-Encoding: 7bit + + +#cloud-config +b: c +launch-index: 2 + + +--===============1668325974==-- diff --git a/tests/data/filter_cloud_multipart_header.email b/tests/data/filter_cloud_multipart_header.email new file mode 100644 index 00000000..770f7ef1 --- /dev/null +++ b/tests/data/filter_cloud_multipart_header.email @@ -0,0 +1,11 @@ +From nobody Fri Aug 31 17:17:00 2012 +Content-Type: text/plain; charset="us-ascii" +MIME-Version: 1.0 +Launch-Index: 5 +Content-Transfer-Encoding: 7bit + + +#cloud-config +b: c + + diff --git a/tests/unittests/helpers.py b/tests/unittests/helpers.py new file mode 100644 index 00000000..60891191 --- /dev/null +++ b/tests/unittests/helpers.py @@ -0,0 +1,41 @@ +import os + +from mocker import MockerTestCase + +from cloudinit import helpers as ch + + +class ResourceUsingTestCase(MockerTestCase): + def __init__(self, methodName="runTest"): + MockerTestCase.__init__(self, methodName) + self.resource_path = None + + def resourceLocation(self, subname=None): + if self.resource_path is None: + paths = [ + os.path.join('tests', 'data'), + os.path.join('data'), + os.path.join(os.pardir, 'tests', 'data'), + os.path.join(os.pardir, 'data'), + ] + for p in paths: + if os.path.isdir(p): + self.resource_path = p + break + self.assertTrue((self.resource_path and os.path.isdir(self.resource_path)), + msg="Unable to locate test resource data path!") + if not subname: + return self.resource_path + return os.path.join(self.resource_path, subname) + + def readResource(self, name): + where = self.resourceLocation(name) + with open(where, 'r') as fh: + return fh.read() + + def getCloudPaths(self): + cp = ch.Paths({ + 'cloud_dir': self.makeDir(), + 'templates_dir': self.resourceLocation(), + }) + return cp diff --git a/tests/unittests/test_filters/test_launch_index.py b/tests/unittests/test_filters/test_launch_index.py new file mode 100644 index 00000000..c122609a --- /dev/null +++ b/tests/unittests/test_filters/test_launch_index.py @@ -0,0 +1,133 @@ +import os +import copy +import sys + +import helpers as th + +import itertools + +from cloudinit import user_data as ud +from cloudinit import util +from cloudinit.filters import launch_index + + +def count_messages(root): + am = 0 + for m in root.walk(): + if ud.is_skippable(m): + continue + am += 1 + return am + + + +class TestLaunchFilter(th.ResourceUsingTestCase): + + def assertCounts(self, message, expected_counts): + orig_message = copy.deepcopy(message) + for (index, count) in expected_counts.items(): + filtered_message = launch_index.Filter(util.safe_int(index)).apply(message) + self.assertEquals(count_messages(filtered_message), count) + # Ensure original message still ok/not modified + self.assertTrue(self.equivalentMessage(message, orig_message)) + + def equivalentMessage(self, msg1, msg2): + msg1_count = count_messages(msg1) + msg2_count = count_messages(msg2) + if msg1_count != msg2_count: + return False + # Do some basic payload checking + msg1_msgs = [m for m in msg1.walk()] + msg1_msgs = [m for m in + itertools.ifilterfalse(ud.is_skippable, msg1_msgs)] + msg2_msgs = [m for m in msg2.walk()] + msg2_msgs = [m for m in + itertools.ifilterfalse(ud.is_skippable, msg2_msgs)] + for i in range(0, len(msg2_msgs)): + m1_msg = msg1_msgs[i] + m2_msg = msg2_msgs[i] + if m1_msg.get_charset() != m2_msg.get_charset(): + return False + if m1_msg.is_multipart() != m2_msg.is_multipart(): + return False + if m1_msg.get_payload(decode=True) != m2_msg.get_payload(decode=True): + return False + return True + + def testMultiEmailIndex(self): + test_data = self.readResource('filter_cloud_multipart_2.email') + ud_proc = ud.UserDataProcessor(self.getCloudPaths()) + message = ud_proc.process(test_data) + self.assertTrue(count_messages(message) > 0) + # This file should have the following + # indexes -> amount mapping in it + expected_counts = { + 3: 1, + 2: 2, + None: 3, + -1: 0, + } + self.assertCounts(message, expected_counts) + + def testHeaderEmailIndex(self): + test_data = self.readResource('filter_cloud_multipart_header.email') + ud_proc = ud.UserDataProcessor(self.getCloudPaths()) + message = ud_proc.process(test_data) + self.assertTrue(count_messages(message) > 0) + # This file should have the following + # indexes -> amount mapping in it + expected_counts = { + 5: 1, + -1: 0, + None: 1, + } + self.assertCounts(message, expected_counts) + + def testConfigEmailIndex(self): + test_data = self.readResource('filter_cloud_multipart_1.email') + ud_proc = ud.UserDataProcessor(self.getCloudPaths()) + message = ud_proc.process(test_data) + self.assertTrue(count_messages(message) > 0) + # This file should have the following + # indexes -> amount mapping in it + expected_counts = { + 2: 1, + -1: 0, + None: 1, + } + self.assertCounts(message, expected_counts) + + def testNoneIndex(self): + test_data = self.readResource('filter_cloud_multipart.yaml') + ud_proc = ud.UserDataProcessor(self.getCloudPaths()) + message = ud_proc.process(test_data) + start_count = count_messages(message) + self.assertTrue(start_count > 0) + filtered_message = launch_index.Filter(None).apply(message) + self.assertTrue(self.equivalentMessage(message, filtered_message)) + + def testIndexes(self): + test_data = self.readResource('filter_cloud_multipart.yaml') + ud_proc = ud.UserDataProcessor(self.getCloudPaths()) + message = ud_proc.process(test_data) + start_count = count_messages(message) + self.assertTrue(start_count > 0) + # This file should have the following + # indexes -> amount mapping in it + expected_counts = { + 2:2, + 3:2, + 1:2, + 0:1, + 4:1, + 7:0, + -1:0, + 100:0, + # None should just give all back + None: start_count, + # Non ints should be ignored + 'c': start_count, + # Strings should be converted + '1': 2, + } + self.assertCounts(message, expected_counts) -- cgit v1.2.3 From c2e5c6fd4dc917d2d4fb23569b1615b76c695201 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Fri, 31 Aug 2012 18:22:01 -0700 Subject: Add some examples on how to use the new launch-index support in a cloud-archive format as well as a cloud-config format and explain how this will affect the final userdata available to an instance. --- doc/examples/cloud-config-archive-launch-index.txt | 30 ++++++++++++++++++++++ doc/examples/cloud-config-launch-index.txt | 23 +++++++++++++++++ 2 files changed, 53 insertions(+) create mode 100644 doc/examples/cloud-config-archive-launch-index.txt create mode 100644 doc/examples/cloud-config-launch-index.txt diff --git a/doc/examples/cloud-config-archive-launch-index.txt b/doc/examples/cloud-config-archive-launch-index.txt new file mode 100644 index 00000000..e2ac2869 --- /dev/null +++ b/doc/examples/cloud-config-archive-launch-index.txt @@ -0,0 +1,30 @@ +#cloud-config-archive + +# This is an example of a cloud archive +# format which includes a set of launch indexes +# that will be filtered on (thus only showing +# up in instances with that launch index), this +# is done by adding the 'launch-index' key which +# maps to the integer 'launch-index' that the +# corresponding content should be used with. +# +# It is possible to leave this value out which +# will mean that the content will be applicable +# for all instances + +- type: foo/wark + filename: bar + content: | + This is my payload + hello + launch-index: 1 # I will only be used on launch-index 1 +- this is also payload +- | + multi line payload + here +- + type: text/upstart-job + filename: my-upstart.conf + content: | + whats this, yo? + launch-index: 0 # I will only be used on launch-index 0 diff --git a/doc/examples/cloud-config-launch-index.txt b/doc/examples/cloud-config-launch-index.txt new file mode 100644 index 00000000..e7dfdc0c --- /dev/null +++ b/doc/examples/cloud-config-launch-index.txt @@ -0,0 +1,23 @@ +#cloud-config +# vim: syntax=yaml + +# +# This is the configuration syntax that can be provided to have +# a given set of cloud config data show up on a certain launch +# index (and not other launches) by provided a key here which +# will act as a filter on the instances userdata. When +# this key is left out (or non-integer) then the content +# of this file will always be used for all launch-indexes +# (ie the previous behavior). +launch-index: 5 + +# Upgrade the instance on first boot +# (ie run apt-get upgrade) +# +# Default: false +# +apt_upgrade: true + +# Other yaml keys below... +# ....... +# ....... -- cgit v1.2.3 From a7bd5c448a6eda8b3d841f2dd5c73ed3956fe3c3 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Fri, 31 Aug 2012 18:28:12 -0700 Subject: Fix pylint complaints. --- tests/unittests/helpers.py | 3 ++- tests/unittests/test_filters/test_launch_index.py | 10 ++++++---- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/tests/unittests/helpers.py b/tests/unittests/helpers.py index 60891191..d0f09e70 100644 --- a/tests/unittests/helpers.py +++ b/tests/unittests/helpers.py @@ -22,7 +22,8 @@ class ResourceUsingTestCase(MockerTestCase): if os.path.isdir(p): self.resource_path = p break - self.assertTrue((self.resource_path and os.path.isdir(self.resource_path)), + self.assertTrue((self.resource_path and + os.path.isdir(self.resource_path)), msg="Unable to locate test resource data path!") if not subname: return self.resource_path diff --git a/tests/unittests/test_filters/test_launch_index.py b/tests/unittests/test_filters/test_launch_index.py index c122609a..0c75bf56 100644 --- a/tests/unittests/test_filters/test_launch_index.py +++ b/tests/unittests/test_filters/test_launch_index.py @@ -1,6 +1,4 @@ -import os import copy -import sys import helpers as th @@ -26,7 +24,8 @@ class TestLaunchFilter(th.ResourceUsingTestCase): def assertCounts(self, message, expected_counts): orig_message = copy.deepcopy(message) for (index, count) in expected_counts.items(): - filtered_message = launch_index.Filter(util.safe_int(index)).apply(message) + index = util.safe_int(index) + filtered_message = launch_index.Filter(index).apply(message) self.assertEquals(count_messages(filtered_message), count) # Ensure original message still ok/not modified self.assertTrue(self.equivalentMessage(message, orig_message)) @@ -50,7 +49,9 @@ class TestLaunchFilter(th.ResourceUsingTestCase): return False if m1_msg.is_multipart() != m2_msg.is_multipart(): return False - if m1_msg.get_payload(decode=True) != m2_msg.get_payload(decode=True): + m1_py = m1_msg.get_payload(decode=True) + m2_py = m2_msg.get_payload(decode=True) + if m1_py != m2_py: return False return True @@ -79,6 +80,7 @@ class TestLaunchFilter(th.ResourceUsingTestCase): expected_counts = { 5: 1, -1: 0, + 'c': 1, None: 1, } self.assertCounts(message, expected_counts) -- cgit v1.2.3