initial re-org, ec2 data source successfully reads from cache if present

6 files changed, 176 insertions, 285 deletions

diff --git a/ec2-init b/ec2-init
deleted file mode 100755
index bee4655b..00000000
--- a/ec2-init
+++ /dev/null
@@ -1,190 +0,0 @@
-#!/bin/sh
-### BEGIN INIT INFO
-# Provides:          ec2-init
-# Required-Start:    $network $local_fs
-# Required-Stop:
-# Should-Start:      $named
-# Should-Stop:
-# Default-Start:     S
-# Default-Stop:      1
-# Short-Description: Initialises system for use on Amazon EC2
-# Description:       Fetches login credentials and handles various quirks
-### END INIT INFO
-
-PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin
-NAME=ec2-init
-
-. /lib/lsb/init-functions
-
-run_once() {
-    per_id=$1
-    action_id=$2
-
-    semaphore="/var/lib/ec2/$action_id.$per_id"
-
-    if ! [ -e "$semaphore" ]
-    then
-        touch "$semaphore"
-        return 0
-    fi
-    return 1
-}
-
-run_once_per_ami() {
-    action_id=$1
-    ami=`ec2-get-info --ami-id | cut -f2 -d\ `
-    run_once $ami $action_id
-}
-
-run_once_ever() {
-    action_id=$1
-    run_once ever $action_id
-}
-
-regenerate_ssh_host_keys() {
-    rm -f /etc/ssh/ssh_host_*_key*
-
-    ssh-keygen -f /etc/ssh/ssh_host_rsa_key -t rsa -N ''
-    ssh-keygen -f /etc/ssh/ssh_host_dsa_key -t dsa -N ''
-
-    # This allows user to get host keys securely through console log
-    echo
-    echo
-    echo "#############################################################"
-    echo "-----BEGIN SSH HOST KEY FINGERPRINTS-----"
-    ssh-keygen -l -f /etc/ssh/ssh_host_rsa_key.pub
-    ssh-keygen -l -f /etc/ssh/ssh_host_dsa_key.pub
-    echo "-----END SSH HOST KEY FINGERPRINTS-----"
-    echo "#############################################################"
-}
-
-# fix LP bug 458850
-# the ephemeral mounts provided in eucalyptus instances differ from
-# those found in ec2 in 2 ways:
-#   1. independent of arch type, the filesystem is on /dev/sda2
-#   2. the filesystem is ext2, not ext3
-fix_euca_fstab() {
-    
-    local edev="/dev/sda2" eedev='\/dev\/sda2' 
-
-    [ -e "${edev}" ] || return 0
-
-    local sops=""; # sed operations
-    local mntinfo="" file_out="" sops="" umdev=${edev}
-
-    # if /dev/sdb is set to mount to /mnt, then we
-    # want to rewrite that to be /dev/sda2
-    mntinfo=$(awk '$2 == "/mnt" { printf("dev=%s fs=%s\n",$1,$3); }' /etc/fstab)
-    case "${mntinfo}" in
-        dev=/dev/sdb\ *)
-            umdev=/dev/sdb;
-            sops="${sops:+${sops};}s,^/dev/sdb,${edev},";;
-    esac
-
-    # if fstab says ext3, but fs on edev is ext2, switch fstab
-    case "${mntinfo}" in
-        *\ fs=ext3)
-            file_out=$(file --special-files "${edev}")
-            case "${file_out}" in
-                *ext2*) sops="${sops:+${sops};}/^${eedev}/s/ext3/ext2/;";;
-            esac
-            ;;
-    esac
-
-    # if there were no sed operations to preform, then nothing to do
-    [ -n "${sops}" ] || return 0
-
-    log_daemon_msg "Fixing fstab for eucalyptus"
-    sed -i "${sops}" /etc/fstab
-    # subsequent boots, /etc/fstab will be updated, and the mount
-    # here isn't needed, but if modifications were made, it is
-    umount "${edev}" >/dev/null 2>&1
-    [ "${edev}" = "${umdev}" ] || umount "${umdev}" >/dev/null 2>&1
-    mount "${edev}"
-    log_end_msg $?
-}
-
-case "$1" in
-    start)
-        if ! ec2-is-compat-env --quiet; then
-           log_daemon_msg "ec2-init disabled"
-           log_end_msg 0
-           exit 0
-        fi
-        if [ ! -d /var/run/ec2 ]; then
-            mkdir /var/run/ec2
-        fi
-
-        log_daemon_msg "Waiting for EC2 meta-data service"
-        if ec2-wait-for-meta-data-service
-        then
-            log_end_msg 0
-        else
-            log_end_msg 1
-            exit 1
-        fi
-
-        # fix euca_fstab for ephemeral mounts one time ever
-        # on rebundle, it should collect the fixed /etc/fstab
-        if run_once_ever euca-fix-fstab-for-ephemeral; then
-            fix_euca_fstab
-        fi
-
-        if run_once_per_ami ssh_host_key_regeneration
-        then
-            # we can't be certain that rsyslog is up (or configured to send
-            # messages to console), but we want to make sure this goes to
-            # console. So write to /dev/console directly through tee.
-            # Change priority of message, so if user.notice (logger's default)
-            # also goes to /dev/console , we could avoid dup messages
-            regenerate_ssh_host_keys 2>&1 | 
-                logger -p user.info -s -t "ec2" 2>&1 |
-                tee /dev/console
-        fi
-
-        if run_once_ever ec2-defaults
-        then
-            log_daemon_msg "Setting EC2 defaults"
-            if ec2-set-defaults 2> /dev/null
-            then
-                log_end_msg 0
-            else
-                log_end_msg 1
-            fi
-        fi
-
-        if run_once_per_ami ssh_authorized_keys
-        then
-            log_daemon_msg "Fetching EC2 SSH keys"
-            if ec2-fetch-credentials 2> /dev/null
-            then
-                log_end_msg 0
-            else
-                log_end_msg 1
-            fi
-        fi
-
-        log_daemon_msg "Setting hostname to EC2 localhostname"
-        if ec2-set-hostname 2> /dev/null
-        then
-            log_end_msg 0
-            invoke-rc.d rsyslog reload
-        else
-            log_end_msg 1
-        fi
-
-        ;;
-    stop)
-        exit 0
-        ;;
-    restart|force-reload)
-        exec $0 start
-        ;;
-    *)
-        N=/etc/init.d/$NAME
-        echo "Usage: $N {start|stop|restart|force-reload|status}" >&2
-        exit 1
-        ;;
-esac
-
-exit 0
diff --git a/ec2init/DataSource.py b/ec2init/DataSource.py
new file mode 100644
index 00000000..da6170fd
--- /dev/null
+++ b/ec2init/DataSource.py
@@ -0,0 +1,8 @@
+
+class DataSource:
+    def __init__(self):
+       pass
+
+    def get_user_data(self):
+        raise Exception("get_user_data Not-implemented")
+
diff --git a/ec2init/DataSourceEc2.py b/ec2init/DataSourceEc2.py
new file mode 100644
index 00000000..c3317272
--- /dev/null
+++ b/ec2init/DataSourceEc2.py
@@ -0,0 +1,122 @@
+import DataSource
+
+import ec2init
+import boto.utils
+import socket
+import urllib2
+import time
+import pickle
+
+class DataSourceEc2(DataSource.DataSource):
+    api_ver  = '2009-04-04'
+    conffile = '/etc/ec2-init/ec2-config.cfg'
+    cachedir = ec2init.cachedir + '/ec2'
+
+    location_locale_map = { 
+        'us' : 'en_US.UTF-8',
+        'eu' : 'en_GB.UTF-8'
+    }
+
+    location_archive_map = { 
+        'us' : 'http://us.ec2.archive.ubuntu.com/ubuntu',
+        'eu' : 'http://eu.ec2.archive.ubuntu.com/ubuntu'
+    }
+
+    def __init__(self):
+        self.meta_data_base_url = 'http://169.254.169.254/%s/meta-data' % self.api_ver
+        self.user_data_base_url = 'http://169.254.169.254/%s/user-data' % self.api_ver
+
+    def get_user_data(self):
+        return("hello")
+
+    def get_data(self):
+        print "checking %s" % self.cachedir + "/user-data.pkl"
+        udf = open(self.cachedir + "/user-data.pkl")
+        self.userdata = pickle.load(udf)
+        udf.close()
+
+        mdf = open(self.cachedir + "/meta-data.pkl")
+        self.metadata = pickle.load(mdf)
+        mdf.close()
+
+        return True
+
+        try:
+            if not self.wait_for_metadata_service():
+                return False
+            self.metadata = boto.utils.get_instance_userdata(api_ver)
+            self.userdata = boto.utils.get_instance_metadata(api_ver)
+        except Exception as e:
+            print e
+            return False
+
+    def wait_or_bail(self):
+        if self.wait_for_metadata_service():
+            return True
+        else:
+            bailout_command = self.get_cfg_option_str('bailout_command')
+            if bailout_command:
+                os.system(bailout_command)
+            return False
+
+    def get_cfg_option_str(self, key, default=None):
+        return self.config.get(key, default)
+
+    def get_ssh_keys(self):
+        conn = urllib2.urlopen('%s/public-keys/' % self.meta_data_base_url)
+        data = conn.read()
+        keyids = [line.split('=')[0] for line in data.split('\n')]
+        return [urllib2.urlopen('%s/public-keys/%d/openssh-key' % (self.meta_data_base_url, int(keyid))).read().rstrip() for keyid in keyids]
+
+#    def get_user_data(self):
+#        return boto.utils.get_instance_userdata()
+#
+#    def get_instance_metadata(self):
+#        self.instance_metadata = getattr(self, 'instance_metadata', boto.utils.get_instance_metadata())
+#        return self.instance_metadata 
+
+    def get_ami_id(self):
+        return self.get_instance_metadata()['ami-id']
+    
+    def get_availability_zone(self):
+        conn = urllib2.urlopen('%s/placement/availability-zone' % self.meta_data_base_url)
+        return conn.read()
+
+    def get_hostname(self):
+        hostname = self.get_instance_metadata()['local-hostname']
+        hostname = hostname.split('.')[0]
+        return hostname
+
+    def get_mirror_from_availability_zone(self, availability_zone):
+        # availability is like 'us-west-1b' or 'eu-west-1a'
+        try:
+            host="%s.ec2.archive.ubuntu.com" % availability_zone[:-1]
+            socket.getaddrinfo(host, None, 0, socket.SOCK_STREAM)
+            return 'http://%s/ubuntu/' % host
+        except:
+            return 'http://archive.ubuntu.com/ubuntu/'
+
+    def wait_for_metadata_service(self, sleeps = 10):
+        sleeptime = 1
+        for x in range(sleeps):
+            s = socket.socket()
+            try:
+                address = '169.254.169.254'
+                port = 80
+                s.connect((address,port))
+                s.close()
+                return True
+            except socket.error, e:
+                print "sleeping %s" % sleeptime
+                time.sleep(sleeptime)
+                #timeout = timeout * 2
+        return False
+
+    def get_location_from_availability_zone(self, availability_zone):
+        if availability_zone.startswith('us-'):
+            return 'us'
+        elif availability_zone.startswith('eu-'):
+            return 'eu'
+        raise Exception('Could not determine location')
+ 
+
diff --git a/ec2init/__init__.py b/ec2init/__init__.py
index 3bd1d23e..b34f15cc 100644
--- a/ec2init/__init__.py
+++ b/ec2init/__init__.py
@@ -1,6 +1,6 @@
 #
 #    Common code for the EC2 initialisation scripts in Ubuntu
-#    Copyright (C) 2008-2009 Canonical Ltd.
+#    Copyright (C) 2008-2009 Canonical Ltd
 #
 #    Author: Soren Hansen <soren@canonical.com>
 #
@@ -18,105 +18,55 @@
 #
 
 import os
-import socket
-import time
-import urllib2
 from   configobj import ConfigObj
 
 import boto.utils
 
-class EC2Init():
-    api_ver  = '2008-02-01'
-    conffile = '/etc/ec2-init/ec2-config.cfg'
+cachedir = '/var/lib/cloud/data/cache'
 
-    location_locale_map = { 
-        'us' : 'en_US.UTF-8',
-        'eu' : 'en_GB.UTF-8'
-    }
+import DataSourceEc2
 
-    location_archive_map = { 
-        'us' : 'http://us.ec2.archive.ubuntu.com/ubuntu',
-        'eu' : 'http://eu.ec2.archive.ubuntu.com/ubuntu'
-    }
+class EC2Init:
+    datasource_list = [ DataSourceEc2.DataSourceEc2 ]
 
-    def __init__(self):
-        self.meta_data_base_url = 'http://169.254.169.254/%s/meta-data' % self.api_ver
-        self.user_data_base_url = 'http://169.254.169.254/%s/user-data' % self.api_ver
-        self.config = ConfigObj(self.conffile)
-
-    def wait_or_bail(self):
-        if self.wait_for_metadata_service():
+    def restore_from_cache(self):
+        try:
+            f=open(cachedir + "/obj.pkl", "rb")
+            data = pickle.load(f)
+            self.datasource = data
             return True
-        else:
-            bailout_command = self.get_cfg_option_str('bailout_command')
-            if bailout_command:
-                os.system(bailout_command)
+        except:
             return False
 
-    def get_cfg_option_bool(self, key, default=None):
-        val = self.config.get(key, default)
-        if val.lower() in ['1', 'on', 'yes']:
+    def write_to_cache(self):
+        try:
+            f=open(cachedir + "/obj.pkl", "wb")
+            data = pickle.dump(self.datasource,f)
+            return True
+        except:
+            return False
+        
+    def get_data_source(self):
+        if self.restore_from_cache():
             return True
-        return False
-
-    def get_cfg_option_str(self, key, default=None):
-        return self.config.get(key, default)
 
-    def get_ssh_keys(self):
-        conn = urllib2.urlopen('%s/public-keys/' % self.meta_data_base_url)
-        data = conn.read()
-        keyids = [line.split('=')[0] for line in data.split('\n')]
-        return [urllib2.urlopen('%s/public-keys/%d/openssh-key' % (self.meta_data_base_url, int(keyid))).read().rstrip() for keyid in keyids]
+        for source in self.datasource_list:
+            try:
+                print "trying + %s" % source
+                s = source()
+                if s.get_data():
+                    self.datasource = s
+                    return
+            except:
+                pass
+        raise Exception("Could not find data source")
 
     def get_user_data(self):
-        return boto.utils.get_instance_userdata()
+        return(self.datasource.get_user_data())
 
-    def get_instance_metadata(self):
-        self.instance_metadata = getattr(self, 'instance_metadata', boto.utils.get_instance_metadata())
-        return self.instance_metadata 
-
-    def get_ami_id(self):
-        return self.get_instance_metadata()['ami-id']
-    
-    def get_availability_zone(self):
-        conn = urllib2.urlopen('%s/placement/availability-zone' % self.meta_data_base_url)
-        return conn.read()
-
-    def get_hostname(self):
-        hostname = self.get_instance_metadata()['local-hostname']
-        hostname = hostname.split('.')[0]
-        return hostname
-
-    def get_mirror_from_availability_zone(self, availability_zone):
-        # availability is like 'us-west-1b' or 'eu-west-1a'
-        try:
-            host="%s.ec2.archive.ubuntu.com" % availability_zone[:-1]
-            socket.getaddrinfo(host, None, 0, socket.SOCK_STREAM)
-            return 'http://%s/ubuntu/' % host
-        except:
-            return 'http://archive.ubuntu.com/ubuntu/'
-
-    def wait_for_metadata_service(self):
-        timeout = 2
-        # This gives us about half an hour before we ultimately bail out
-        for x in range(10):
-            s = socket.socket()
-            try:
-                address = '169.254.169.254'
-                port = 80
-                s.connect((address,port))
-                s.close()
-                return True
-            except socket.error, e:
-                time.sleep(timeout)
-                timeout = timeout * 2
+    def get_cfg_option_bool(self, key, default=None):
+        val = self.config.get(key, default)
+        if val.lower() in ['1', 'on', 'yes']:
+            return True
         return False
 
-    def get_location_from_availability_zone(self, availability_zone):
-        if availability_zone.startswith('us-'):
-            return 'us'
-        elif availability_zone.startswith('eu-'):
-            return 'eu'
-        raise Exception('Could not determine location')
- 
-
diff --git a/install.sh b/install.sh
index aa46d5c1..7ddbdc0a 100755
--- a/install.sh
+++ b/install.sh
@@ -16,14 +16,14 @@ destdir=$(readlink -f ${1})
 cd $(dirname ${0})
 ./setup.py install --root=${destdir} ${DEB_PYTHON_INSTALL_ARGS_ALL}
 
-mkdir -p ${destdir}/usr/lib/pyshared
-for x in ${destdir}/usr/lib/python2.6/dist-packages/*; do
-   [ -d "$x" ] || continue
-   [ ! -d "${destdir}/usr/lib/pyshared/${x##*/}" ] ||
-      rm -Rf "${destdir}/usr/lib/pyshared/${x##*/}"
-   mv $x ${destdir}/usr/lib/pyshared
-done
-rm -Rf ${destdir}/usr/lib/python2.6
+#mkdir -p ${destdir}/usr/share/pyshared
+#for x in ${destdir}/usr/lib/python2.6/dist-packages/*; do
+#   [ -d "$x" ] || continue
+#   [ ! -d "${destdir}/usr/share/pyshared/${x##*/}" ] ||
+#      rm -Rf "${destdir}/usr/share/pyshared/${x##*/}"
+#   mv $x ${destdir}/usr/share/pyshared
+#done
+#rm -Rf ${destdir}/usr/lib/python2.6
 
 for x in "${destdir}/usr/bin/"*.py; do
    [ -f "${x}" ] && mv "${x}" "${x%.py}"
diff --git a/setup.py b/setup.py
index a6f53f8e..aaa7cbeb 100755
--- a/setup.py
+++ b/setup.py
@@ -35,10 +35,11 @@ setup(name='EC2-init',
                'ec2-set-defaults.py',
                'ec2-set-hostname.py',
                'ec2-wait-for-meta-data-service.py',
+               'ec2-init.py',
                'ec2-is-compat-env'],
       data_files=[('/etc/ec2-init', ['ec2-config.cfg']),
                   ('/etc/ec2-init/templates', glob('templates/*')),
-                  ('/etc/init.d', ['ec2-init']),
+                  ('/etc/init', ['ec2init.conf']),
                   ('/usr/share/ec2-init', ['ec2-init-appliance-ebs-volume-mount.sh']),
                   ],
       )