Merge with lp:~soren/ec2-init/appliancexml

This gives us the bulk of the magic needed on the instance side
to implement http://wiki.ubuntu.com/VirtualApplianceSpec

4 files changed, 279 insertions, 10 deletions

diff --git a/ec2-init-appliance-ebs-volume-mount.sh b/ec2-init-appliance-ebs-volume-mount.sh
new file mode 100755
index 00000000..7106b353
--- /dev/null
+++ b/ec2-init-appliance-ebs-volume-mount.sh
@@ -0,0 +1,48 @@
+#!/bin/sh
+
+if [ -n "$EBSMOUNT_DEBUG" ]
+then
+    do="echo"
+    mktemp_args="-u"
+else
+    do=""
+    mktemp_args=""
+fi
+
+if [ "$#" -lt 2 ]
+then
+	echo "Usage: $0 <EBS device> <path> [<path> [<path>...]]"
+	exit 1
+fi
+
+ebs_volume_device="$1"
+shift
+
+canonicalise_dir() {
+    dirname="$1"
+    echo "${dirname}" | sed -e 's/[^a-zA-Z0-9]/_/g'
+}
+
+# The blkid call will detect whether there's already a filesystem on the EBS volume
+if [ -n "$(blkid -p -o udev "${ebs_volume_device}")" ]
+then
+    $do mkfs.ext3 "${ebs_volume_device}"
+fi
+
+tmpdir="$(mktemp -d $mktemp_args --tmpdir=/var/run/ec2-init)"
+$do mount ${ebs_volume_device} ${tmpdir}
+
+for dir in "$@"
+do
+    ebsdir="${tmpdir}/$(canonicalise_dir "${dir}")"
+    if [ ! -d "${ebsdir}" ]
+    then
+        # We bootstrap the storage with the existing data
+        $do mkdir "${ebsdir}"
+        $do cp -a ${dir} "${ebsdir}"
+        $do chown --reference "${dir}" "${ebsdir}"
+        $do chmod --reference "${dir}" "${ebsdir}"
+    fi
+    # Finally, we mount it on top of the old directory.
+    $do mount --bind "${ebsdir}" "${dir}"
+done
diff --git a/ec2-run-user-data.py b/ec2-run-user-data.py
index 5af04249..d7e8e632 100755
--- a/ec2-run-user-data.py
+++ b/ec2-run-user-data.py
@@ -22,6 +22,7 @@ import email
 import os
 import subprocess
 import tempfile
+from xml.dom.minidom import parse, parseString
 
 import ec2init
 
@@ -34,14 +35,6 @@ def register_handler(mimetype, func):
     content_type_handlers[mimetype] = func
     return func
 
-def main():
-    ec2 = ec2init.EC2Init()
-
-    user_data = ec2.get_user_data()
-
-    msg = email.message_from_string(user_data)
-    handle_part(msg)
-
 def handle_part(part):
     if part.is_multipart():
         for p in part.get_payload():
@@ -57,10 +50,18 @@ def handle_unknown_payload(payload):
     # Try to detect magic
     if payload.startswith('#!'):
         content_type_handlers['text/x-shellscript'](payload)
+        return
+    if payload.startswith('<appliance>'):
+        content_type_handlers['text/x-appliance-config'](payload)
+
+@handler('text/x-appliance-config')
+def handle_appliance_config(payload):
+    app = ApplianceConfig(payload)
+    app.handle()
 
 @handler('text/x-ebs-mount-description')
 def handle_ebs_mount_description(payload):
-    (volume_description, path) = payload.split(':')
+    (volume_description, paths) = payload.split(':')
     (identifier_type, identifier) = volume_description.split('=')
 
     if identifier_type == 'device':
@@ -72,6 +73,17 @@ def handle_ebs_mount_description(payload):
     else:
         return
 
+    mount_ebs_volume(device, paths.split(','))
+
+def mount_ebs_volume(device, paths):
+    if os.path.exists('ec2-init-appliance-ebs-volume-mount.sh'):
+        helper = './ec2-init-appliance-ebs-volume-mount.sh'
+    else:
+        helper = '/usr/share/ec2-init/ec2-init-appliance-ebs-volume-mount.sh'
+    helper = subprocess.Popen([helper, device] + paths, stdout=subprocess.PIPE)
+    stdout, stderr = helper.communicate()
+    return stdout
+
 @handler('text/x-shellscript')
 def handle_shell_script(payload):
     (fd, path) = tempfile.mkstemp()
@@ -87,5 +99,78 @@ def handle_shell_script(payload):
     
     os.unlink(path)
 
+class ApplianceConfig(object):
+    def __init__(self, data):
+        self.data = data
+
+    def handle(self):
+        self.dom = parseString(self.data)
+
+        if self.dom.childNodes[0].tagName == 'appliance':
+            root = self.dom.childNodes[0]
+        else:
+            return
+
+        for node in root.childNodes:
+            if node.tagName == 'package':
+                pkg = None
+                for subnode in node.childNodes:
+                    if subnode.nodeType == root.TEXT_NODE:
+                        pkg = subnode.nodeValue
+                if not pkg:
+                    # Something's fishy. We should have been passed the name of
+                    # a package.
+                    return
+                if node.getAttribute('action') == 'remove':
+                    remove_package(pkg)
+                else:
+                    install_package(pkg)
+            elif node.tagName == 'script':
+                script = ''
+                for subnode in node.childNodes:
+                    # If someone went through the trouble of wrapping it in CDATA, 
+                    # it's probably the script we want to run..
+                    if subnode.nodeType == root.CDATA_SECTION_NODE:
+                        script = subnode.nodeValue
+                    # ..however, fall back to whatever TEXT_NODE stuff is between
+                    # the <script> tags.
+                    if subnode.nodeType == root.TEXT_NODE and not script:
+                        script = subnode.nodeValue
+                if not script:
+                    # An empty script?
+                    continue
+                content_type_handlers['text/x-shellscript'](script)
+            elif node.tagName == 'storage':
+                paths = []
+                device = node.getAttribute('device')
+                for subnode in node.childNodes:
+                    if subnode.tagName == 'path':
+                        for subsubnode in subnode.childNodes:
+                            if subsubnode.nodeType == root.TEXT_NODE:
+                                paths += [subsubnode.nodeValue.strip()]
+                                break
+                mount_ebs_volume(device, paths)
+
+def main():
+    ec2 = ec2init.EC2Init()
+
+    user_data = ec2.get_user_data()
+    msg = parse_user_data(user_data)
+    handle_part(msg)
+
+def parse_user_data(user_data):
+    return email.message_from_string(user_data)
+
+def install_remove_package(pkg, action):
+    apt_get = subprocess.Popen(['apt-get', action, pkg], stdout=subprocess.PIPE)
+    logger_process = subprocess.Popen(['logger', '-t', 'user-data'], stdin=apt_get.stdout)
+    logger_process.communicate()
+
+def install_package(pkg):
+    return install_remove_package(pkg, 'install')
+
+def remove_package(pkg):
+    return install_remove_package(pkg, 'remove')
+
 if __name__ == '__main__':
     main()
diff --git a/setup.py b/setup.py
index fa855bd6..dab57103 100755
--- a/setup.py
+++ b/setup.py
@@ -37,5 +37,7 @@ setup(name='EC2-init',
                'ec2-wait-for-meta-data-service.py'],
       data_files=[('/etc/ec2-init', ['ec2-config.cfg']),
                   ('/etc/ec2-init/templates', glob('templates/*')),
-                  ('/etc/init.d', ['ec2-init'])],
+                  ('/etc/init.d', ['ec2-init']),
+                  ('/usr/share/ec2-init', ['ec2-init-appliance-ebs-volume-mount.sh']),
+                  ],
       )
diff --git a/tests.py b/tests.py
new file mode 100644
index 00000000..84103745
--- /dev/null
+++ b/tests.py
@@ -0,0 +1,134 @@
+#!/usr/bin/python
+#
+#    Unit tests for EC2-init
+#    Copyright (C) 2008-2009 Canonical Ltd.
+#
+#    Author: Soren Hansen <soren@canonical.com>
+#
+#    This program is free software: you can redistribute it and/or modify
+#    it under the terms of the GNU General Public License version 3, as
+#    published by the Free Software Foundation.
+#
+#    This program is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU General Public License for more details.
+#
+#    You should have received a copy of the GNU General Public License
+#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+import re
+import os
+import unittest
+
+class RunUserDataApplianceTestCase(unittest.TestCase):
+    def handle_xml(self, xml):
+        msg = self.ec2_run_user_data.parse_user_data(xml)
+        self.ec2_run_user_data.handle_part(msg)
+
+class RunUserDataApplianceConfigEBS(RunUserDataApplianceTestCase):
+    def setUp(self):
+        self.ec2_run_user_data = __import__('ec2-run-user-data')
+        reload(self.ec2_run_user_data) 
+        self.real_mount_ebs_volume = self.ec2_run_user_data.mount_ebs_volume
+        self.ec2_run_user_data.mount_ebs_volume  = self.fake_mount_ebs_volume
+    
+    def fake_mount_ebs_volume(self, device, paths):
+        self.assertEqual(device, '/dev/sdc')
+        self.assertEqual(paths, ['/etc/alfresco', '/var/lib/mysql'])
+
+    def testApplianceConfigEBS(self):
+        os.environ['EBSMOUNT_DEBUG'] = 'yes, please'
+        xml = '<appliance><storage device="/dev/sdc"><path>/etc/alfresco</path><path>/var/lib/mysql</path></storage></appliance>'
+        self.handle_xml(xml)
+
+    def testMountEBSVolume(self):
+        output = self.real_mount_ebs_volume('/dev/sdh', ['/foo', '/bar'])
+        lines = output.strip().split('\n')
+        self.assertEqual(len(lines), 11)
+        match = re.match('mount /dev/sdh (/var/run/ec2-init/tmp.[a-zA-Z0-9]+)', lines[0])
+        self.assertNotEqual(match, None)
+        tmpdir = match.group(1)
+        for (i, s) in zip(range(10), ['mkdir %s/_foo', 'cp -a /foo %s/_foo', 'chown --reference /foo %s/_foo', 'chmod --reference /foo %s/_foo', 'mount --bind %s/_foo /foo', 'mkdir %s/_bar', 'cp -a /bar %s/_bar', 'chown --reference /bar %s/_bar', 'chmod --reference /bar %s/_bar', 'mount --bind %s/_bar /bar']):
+            self.assertEqual(s % tmpdir, lines[i+1])
+
+class RunUserDataApplianceConfigScript(RunUserDataApplianceTestCase):
+    def setUp(self):
+        self.ec2_run_user_data = __import__('ec2-run-user-data')
+        self.fake_handle_shell_script_counter = 0
+        self.expected_scripts = []
+        # Override install_remove_package 
+        self.ec2_run_user_data.content_type_handlers['text/x-shellscript'] = self.fake_handle_shell_script
+
+    def fake_handle_shell_script(self, txt):
+        self.fake_handle_shell_script_counter += 1
+        self.assertEqual(self.expected_scripts.pop(0), txt)
+
+    def handle_xml(self, xml):
+        msg = self.ec2_run_user_data.parse_user_data(xml)
+        self.ec2_run_user_data.handle_part(msg)
+
+    def testApplianceConfigPackageScriptSingle(self):
+        script = '''#!/bin/sh
+echo hey'''
+        xml = '<appliance><script>%s</script></appliance>' % script
+        self.expected_scripts += [script]
+        self.handle_xml(xml)
+        self.assertEqual(self.fake_handle_shell_script_counter, 1) 
+
+    def testApplianceConfigPackageScriptMultiple(self):
+        script1 = '''#!/bin/sh
+echo hey'''
+        script2 = '''#!/usr/bin/python
+print "hey"'''
+        xml = '<appliance><script>%s</script><script>%s</script></appliance>' % (script1, script2)
+        self.expected_scripts += [script1, script2]
+        self.handle_xml(xml)
+        self.assertEqual(self.fake_handle_shell_script_counter, 2) 
+
+    def testApplianceConfigPackageScriptCDATA(self):
+        script = '''#!/bin/sh
+echo hey'''
+        xml = '<appliance><script><![CDATA[%s]]></script></appliance>' % (script, )
+        self.expected_scripts += [script]
+        self.handle_xml(xml)
+        self.assertEqual(self.fake_handle_shell_script_counter, 1) 
+
+class RunUserDataApplianceConfigPackageHandling(RunUserDataApplianceTestCase):
+    def setUp(self):
+        self.fake_install_remove_package_counter = 0
+
+        self.ec2_run_user_data = __import__('ec2-run-user-data')
+
+        # Override install_remove_package 
+        self.ec2_run_user_data.install_remove_package = self.fake_install_remove_package
+
+    def fake_install_remove_package(self, package, action):
+        self.fake_install_remove_package_counter += 1
+        mapping = { 'foobarplus': 'install',
+                    'foobarminus': 'remove' }
+        self.assert_(package in mapping)
+        self.assertEqual(action, mapping[package])
+
+    def handle_xml(self, xml):
+        msg = self.ec2_run_user_data.parse_user_data(xml)
+        self.ec2_run_user_data.handle_part(msg)
+
+    def testApplianceConfigPackageInstall(self):
+        xml = '<appliance><package>foobarplus</package></appliance>'
+        self.handle_xml(xml)
+        self.assertEqual(self.fake_install_remove_package_counter, 1) 
+
+    def testApplianceConfigPackageRemove(self):
+        xml = '<appliance><package action="remove">foobarminus</package></appliance>'
+        self.handle_xml(xml)
+        self.assertEqual(self.fake_install_remove_package_counter, 1) 
+
+    def testApplianceConfigPackageInstallAndRemove(self):
+        xml = '<appliance><package>foobarplus</package><package action="remove">foobarminus</package></appliance>'
+        self.handle_xml(xml)
+        self.assertEqual(self.fake_install_remove_package_counter, 2) 
+
+if __name__ == "__main__":
+    unittest.main()