From c4f3133dc99b68a41353f763a65dee1b323f8868 Mon Sep 17 00:00:00 2001 From: Soren Hansen Date: Tue, 25 Aug 2009 13:51:13 +0200 Subject: Added basic appliance config handling (specifically package installation and removal) Also added unit tests for the above. --- ec2-run-user-data.py | 72 ++++++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 61 insertions(+), 11 deletions(-) (limited to 'ec2-run-user-data.py') diff --git a/ec2-run-user-data.py b/ec2-run-user-data.py index 51e0d68d..784a1b3f 100755 --- a/ec2-run-user-data.py +++ b/ec2-run-user-data.py @@ -22,20 +22,10 @@ import email import os import subprocess import tempfile +from xml.dom.minidom import parse, parseString import ec2init -content_type_handlers = { 'text/x-shellscript' : handle_shell_script, - 'text/x-ebs-mount-description' : handle_ebs_mount_description } - -def main(): - ec2 = ec2init.EC2Init() - - user_data = ec2.get_user_data() - - msg = email.message_from_string(user_data) - handle_part(msg) - def handle_part(part): if part.is_multipart(): for p in part.get_payload(): @@ -51,6 +41,14 @@ def handle_unknown_payload(payload): # Try to detect magic if payload.startswith('#!'): content_type_handlers['text/x-shellscript'](payload) + return + if payload.startswith(''): + content_type_handlers['text/x-appliance-config'](payload) + + +def handle_appliance_config(payload): + app = ApplianceConfig(payload) + app.handle() def handle_ebs_mount_description(payload): (volume_description, path) = payload.split(':') @@ -79,5 +77,57 @@ def handle_shell_script(payload): os.unlink(path) +content_type_handlers = { 'text/x-shellscript' : handle_shell_script, + 'text/x-ebs-mount-description' : handle_ebs_mount_description, + 'text/x-appliance-config': handle_appliance_config } + +class ApplianceConfig(object): + def __init__(self, data): + self.data = data + + def handle(self): + self.dom = parseString(self.data) + + if self.dom.childNodes[0].tagName == 'appliance': + root = self.dom.childNodes[0] + else: + return + + for node in root.childNodes: + if node.tagName == 'package': + pkg = None + for subnode in node.childNodes: + if subnode.nodeType == root.TEXT_NODE: + pkg = subnode.nodeValue + if not pkg: + # Something's fishy. We should have been passed the name of + # a package. + return + if node.getAttribute('action') == 'remove': + remove_package(pkg) + else: + install_package(pkg) + +def main(): + ec2 = ec2init.EC2Init() + + user_data = ec2.get_user_data() + msg = parse_user_data(user_data) + handle_part(msg) + +def parse_user_data(user_data): + return email.message_from_string(user_data) + +def install_remove_package(pkg, action): + apt_get = subprocess.Popen(['apt-get', action, pkg], stdout=subprocess.PIPE) + logger_process = subprocess.Popen(['logger', '-t', 'user-data'], stdin=apt_get.stdout) + logger_process.communicate() + +def install_package(pkg): + return install_remove_package(pkg, 'install') + +def remove_package(pkg): + return install_remove_package(pkg, 'remove') + if __name__ == '__main__': main() -- cgit v1.2.3 From 76d5d79c0ec119cf5e87e71578125081c40c291d Mon Sep 17 00:00:00 2001 From: Soren Hansen Date: Tue, 25 Aug 2009 14:46:16 +0200 Subject: Added script handling to appliance config handling. Added test cases for this. --- ec2-run-user-data.py | 15 +++++++++++++++ tests.py | 42 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 57 insertions(+) (limited to 'ec2-run-user-data.py') diff --git a/ec2-run-user-data.py b/ec2-run-user-data.py index 784a1b3f..2917b2fe 100755 --- a/ec2-run-user-data.py +++ b/ec2-run-user-data.py @@ -107,6 +107,21 @@ class ApplianceConfig(object): remove_package(pkg) else: install_package(pkg) + elif node.tagName == 'script': + script = '' + for subnode in node.childNodes: + # If someone went through the trouble of wrapping it in CDATA, + # it's probably the script we want to run.. + if subnode.nodeType == root.CDATA_SECTION_NODE: + script = subnode.nodeValue + # ..however, fall back to whatever TEXT_NODE stuff is between + # the ' % script + self.expected_scripts += [script] + self.handle_xml(xml) + self.assertEqual(self.fake_handle_shell_script_counter, 1) + + def testApplianceConfigPackageScriptMultiple(self): + script1 = '''#!/bin/sh +echo hey''' + script2 = '''#!/usr/bin/python +print "hey"''' + xml = '' % (script1, script2) + self.expected_scripts += [script1, script2] + self.handle_xml(xml) + self.assertEqual(self.fake_handle_shell_script_counter, 2) + + def testApplianceConfigPackageScriptCDATA(self): + script = '''#!/bin/sh +echo hey''' + xml = '' % (script, ) + self.expected_scripts += [script] + self.handle_xml(xml) + self.assertEqual(self.fake_handle_shell_script_counter, 1) + class RunUserDataApplianceConfigPackageHandling(unittest.TestCase): def setUp(self): self.fake_install_remove_package_counter = 0 -- cgit v1.2.3 From 59d21bb23db06e5e02cbd91ec531b1506ab97fae Mon Sep 17 00:00:00 2001 From: Soren Hansen Date: Tue, 25 Aug 2009 23:51:16 +0200 Subject: Implement EBS volume mounting This can either be invoked by instrumenting the user-data with a mime part with content-type 'text/x-ebs-mount-description' with a body like so: device=/dev/sde:/var/lib/mysql,/etc/alfresco device=/dev/sdf:/other/things or by using the appliance config XML format like so: /var/lib/mysql /etc/alfresco /other/things In either case, if the volume does not yet have a filesystem, one will be created. For each path that is to live on the volume, a directory is created, and populated with the data currently in the target directory (e.g. /var/lib/mysql is copied to ${ebs_volume_path}/_var_lib_mysql). Once this is done, the directories are bind-mounted to the relevant paths. If the directories in question already exist, they will just be bind-mounted. --- ec2-init-appliance-ebs-volume-mount.sh | 48 ++++++++++++++++++++++++++++++++++ ec2-run-user-data.py | 23 +++++++++++++++- setup.py | 4 ++- tests.py | 37 ++++++++++++++++++++++++-- 4 files changed, 108 insertions(+), 4 deletions(-) create mode 100755 ec2-init-appliance-ebs-volume-mount.sh (limited to 'ec2-run-user-data.py') diff --git a/ec2-init-appliance-ebs-volume-mount.sh b/ec2-init-appliance-ebs-volume-mount.sh new file mode 100755 index 00000000..7106b353 --- /dev/null +++ b/ec2-init-appliance-ebs-volume-mount.sh @@ -0,0 +1,48 @@ +#!/bin/sh + +if [ -n "$EBSMOUNT_DEBUG" ] +then + do="echo" + mktemp_args="-u" +else + do="" + mktemp_args="" +fi + +if [ "$#" -lt 2 ] +then + echo "Usage: $0 [ [...]]" + exit 1 +fi + +ebs_volume_device="$1" +shift + +canonicalise_dir() { + dirname="$1" + echo "${dirname}" | sed -e 's/[^a-zA-Z0-9]/_/g' +} + +# The blkid call will detect whether there's already a filesystem on the EBS volume +if [ -n "$(blkid -p -o udev "${ebs_volume_device}")" ] +then + $do mkfs.ext3 "${ebs_volume_device}" +fi + +tmpdir="$(mktemp -d $mktemp_args --tmpdir=/var/run/ec2-init)" +$do mount ${ebs_volume_device} ${tmpdir} + +for dir in "$@" +do + ebsdir="${tmpdir}/$(canonicalise_dir "${dir}")" + if [ ! -d "${ebsdir}" ] + then + # We bootstrap the storage with the existing data + $do mkdir "${ebsdir}" + $do cp -a ${dir} "${ebsdir}" + $do chown --reference "${dir}" "${ebsdir}" + $do chmod --reference "${dir}" "${ebsdir}" + fi + # Finally, we mount it on top of the old directory. + $do mount --bind "${ebsdir}" "${dir}" +done diff --git a/ec2-run-user-data.py b/ec2-run-user-data.py index 67ecf219..d7e8e632 100755 --- a/ec2-run-user-data.py +++ b/ec2-run-user-data.py @@ -61,7 +61,7 @@ def handle_appliance_config(payload): @handler('text/x-ebs-mount-description') def handle_ebs_mount_description(payload): - (volume_description, path) = payload.split(':') + (volume_description, paths) = payload.split(':') (identifier_type, identifier) = volume_description.split('=') if identifier_type == 'device': @@ -73,6 +73,17 @@ def handle_ebs_mount_description(payload): else: return + mount_ebs_volume(device, paths.split(',')) + +def mount_ebs_volume(device, paths): + if os.path.exists('ec2-init-appliance-ebs-volume-mount.sh'): + helper = './ec2-init-appliance-ebs-volume-mount.sh' + else: + helper = '/usr/share/ec2-init/ec2-init-appliance-ebs-volume-mount.sh' + helper = subprocess.Popen([helper, device] + paths, stdout=subprocess.PIPE) + stdout, stderr = helper.communicate() + return stdout + @handler('text/x-shellscript') def handle_shell_script(payload): (fd, path) = tempfile.mkstemp() @@ -129,6 +140,16 @@ class ApplianceConfig(object): # An empty script? continue content_type_handlers['text/x-shellscript'](script) + elif node.tagName == 'storage': + paths = [] + device = node.getAttribute('device') + for subnode in node.childNodes: + if subnode.tagName == 'path': + for subsubnode in subnode.childNodes: + if subsubnode.nodeType == root.TEXT_NODE: + paths += [subsubnode.nodeValue.strip()] + break + mount_ebs_volume(device, paths) def main(): ec2 = ec2init.EC2Init() diff --git a/setup.py b/setup.py index fa855bd6..dab57103 100755 --- a/setup.py +++ b/setup.py @@ -37,5 +37,7 @@ setup(name='EC2-init', 'ec2-wait-for-meta-data-service.py'], data_files=[('/etc/ec2-init', ['ec2-config.cfg']), ('/etc/ec2-init/templates', glob('templates/*')), - ('/etc/init.d', ['ec2-init'])], + ('/etc/init.d', ['ec2-init']), + ('/usr/share/ec2-init', ['ec2-init-appliance-ebs-volume-mount.sh']), + ], ) diff --git a/tests.py b/tests.py index 33c45ec7..84103745 100644 --- a/tests.py +++ b/tests.py @@ -18,9 +18,42 @@ # along with this program. If not, see . # +import re +import os import unittest -class RunUserDataApplianceConfigScript(unittest.TestCase): +class RunUserDataApplianceTestCase(unittest.TestCase): + def handle_xml(self, xml): + msg = self.ec2_run_user_data.parse_user_data(xml) + self.ec2_run_user_data.handle_part(msg) + +class RunUserDataApplianceConfigEBS(RunUserDataApplianceTestCase): + def setUp(self): + self.ec2_run_user_data = __import__('ec2-run-user-data') + reload(self.ec2_run_user_data) + self.real_mount_ebs_volume = self.ec2_run_user_data.mount_ebs_volume + self.ec2_run_user_data.mount_ebs_volume = self.fake_mount_ebs_volume + + def fake_mount_ebs_volume(self, device, paths): + self.assertEqual(device, '/dev/sdc') + self.assertEqual(paths, ['/etc/alfresco', '/var/lib/mysql']) + + def testApplianceConfigEBS(self): + os.environ['EBSMOUNT_DEBUG'] = 'yes, please' + xml = '/etc/alfresco/var/lib/mysql' + self.handle_xml(xml) + + def testMountEBSVolume(self): + output = self.real_mount_ebs_volume('/dev/sdh', ['/foo', '/bar']) + lines = output.strip().split('\n') + self.assertEqual(len(lines), 11) + match = re.match('mount /dev/sdh (/var/run/ec2-init/tmp.[a-zA-Z0-9]+)', lines[0]) + self.assertNotEqual(match, None) + tmpdir = match.group(1) + for (i, s) in zip(range(10), ['mkdir %s/_foo', 'cp -a /foo %s/_foo', 'chown --reference /foo %s/_foo', 'chmod --reference /foo %s/_foo', 'mount --bind %s/_foo /foo', 'mkdir %s/_bar', 'cp -a /bar %s/_bar', 'chown --reference /bar %s/_bar', 'chmod --reference /bar %s/_bar', 'mount --bind %s/_bar /bar']): + self.assertEqual(s % tmpdir, lines[i+1]) + +class RunUserDataApplianceConfigScript(RunUserDataApplianceTestCase): def setUp(self): self.ec2_run_user_data = __import__('ec2-run-user-data') self.fake_handle_shell_script_counter = 0 @@ -62,7 +95,7 @@ echo hey''' self.handle_xml(xml) self.assertEqual(self.fake_handle_shell_script_counter, 1) -class RunUserDataApplianceConfigPackageHandling(unittest.TestCase): +class RunUserDataApplianceConfigPackageHandling(RunUserDataApplianceTestCase): def setUp(self): self.fake_install_remove_package_counter = 0 -- cgit v1.2.3