From d50bc13aa357f6c3fa9d55f89f3e6c006102f559 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Tue, 19 Jun 2012 11:24:21 -0700 Subject: Add a mock ec2 metadata server that can be used for testing with --- tools/mock-meta.py | 327 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 327 insertions(+) create mode 100755 tools/mock-meta.py (limited to 'tools') diff --git a/tools/mock-meta.py b/tools/mock-meta.py new file mode 100755 index 00000000..019a24f2 --- /dev/null +++ b/tools/mock-meta.py @@ -0,0 +1,327 @@ +#!/usr/bin/python + +# Provides a somewhat random, somewhat compat, somewhat useful mock version of +# +# http://docs.amazonwebservices.com/AWSEC2/2007-08-29/DeveloperGuide/AESDG-chapter-instancedata.html + +import functools +import httplib +import logging +import sys +import string +import random +import yaml + +from optparse import OptionParser + +from BaseHTTPServer import (HTTPServer, BaseHTTPRequestHandler) + +log = logging.getLogger('meta-server') + +# Constants +EC2_VERSIONS = [ + '1.0', + '2007-01-19', + '2007-03-01', + '2007-08-29', + '2007-10-10', + '2007-12-15', + '2008-02-01', + '2008-09-01', + '2009-04-04', + 'latest', +] + +BLOCK_DEVS = [ + 'ami', + 'root', + 'ephemeral0', +] + +DEV_PREFIX = 'v' +DEV_MAPPINGS = { + 'ephemeral0': '%sda2' % (DEV_PREFIX), + 'root': '/dev/%sda1' % (DEV_PREFIX), + 'ami': '%sda1' % (DEV_PREFIX), + 'swap': '%sda3' % (DEV_PREFIX), +} + +META_CAPABILITIES = [ + 'aki-id', + 'ami-id', + 'ami-launch-index', + 'ami-manifest-path', + 'ari-id', + 'block-device-mapping/', + 'hostname', + 'instance-action', + 'instance-id', + 'instance-type', + 'local-hostname', + 'local-ipv4', + 'placement/', + 'product-codes', + 'public-hostname', + 'public-ipv4', + 'reservation-id', + 'security-groups' +] + +INSTANCE_TYPES = [ + 'm1.small', + 'm1.medium', + 'm1.large', + 'm1.xlarge', +] + +AVAILABILITY_ZONES = [ + "us-east-1a", + "us-east-1b", + "us-east-1c", + 'us-west-1', + "us-east-1d", + 'eu-west-1a', + 'eu-west-1b', +] + +PLACEMENT_CAPABILITIES = { + 'availability-zone': AVAILABILITY_ZONES, +} + + +class WebException(Exception): + def __init__(self, code, msg): + Exception.__init__(self, msg) + self.code = code + + +def yamlify(data): + formatted = yaml.dump(data, + line_break="\n", + indent=4, + explicit_start=True, + explicit_end=True, + default_flow_style=False) + return formatted + + +def format_text(text): + if not len(text): + return "<<" + lines = text.splitlines() + nlines = [] + for line in lines: + nlines.append("<< %s" % line) + return "\n".join(nlines) + + +ID_CHARS = [c for c in (string.ascii_uppercase + string.digits)] +def id_generator(size=6, lower=False): + txt = ''.join(random.choice(ID_CHARS) for x in range(size)) + if lower: + return txt.lower() + else: + return txt + + +class MetaDataHandler(object): + + def __init__(self, opts): + self.opts = opts + self.instances = {} + + def get_data(self, params, who, **kwargs): + if not params: + caps = sorted(META_CAPABILITIES) + return "\n".join(caps) + action = params[0] + action = action.lower() + if action == 'instance-id': + return 'i-%s' % (id_generator(lower=True)) + elif action == 'ami-launch-index': + return "%s" % random.choice([0,1,2,3]) + elif action == 'aki-id': + return 'aki-%s' % (id_generator(lower=True)) + elif action == 'ami-id': + return 'ami-%s' % (id_generator(lower=True)) + elif action == 'ari-id': + return 'ari-%s' % (id_generator(lower=True)) + elif action == 'block-device-mapping': + nparams = params[1:] + if not nparams: + devs = sorted(BLOCK_DEVS) + return "\n".join(devs) + else: + return "%s" % (DEV_MAPPINGS.get(nparams[0].strip(), '')) + elif action in ['hostname', 'local-hostname', 'public-hostname']: + return "%s" % (who) + elif action == 'instance-type': + return random.choice(INSTANCE_TYPES) + elif action == 'ami-manifest-path': + return 'my-amis/spamd-image.manifest.xml' + elif action == 'security-groups': + return 'default' + elif action in ['local-ipv4', 'public-ipv4']: + there_ip = kwargs.get('client_ip', '10.0.0.1') + return "%s" % (there_ip) + elif action == 'reservation-id': + return "r-%s" % (id_generator(lower=True)) + elif action == 'product-codes': + return "%s" % (id_generator(size=8)) + elif action == 'placement': + nparams = params[1:] + if not nparams: + pcaps = sorted(PLACEMENT_CAPABILITIES.keys()) + return "\n".join(pcaps) + else: + pentry = nparams[0].strip().lower() + if pentry == 'availability-zone': + zones = PLACEMENT_CAPABILITIES[pentry] + return "%s" % random.choice(zones) + else: + return "%s" % (PLACEMENT_CAPABILITIES.get(pentry, '')) + else: + return '' + +class UserDataHandler(object): + + def __init__(self, opts): + self.opts = opts + + def _get_user_blob(self, **kwargs): + blob_mp = {} + blob_mp['hostname'] = kwargs.get('who', '') + lines = [] + lines.append("#cloud-config") + lines.append(yamlify(blob_mp)) + blob = "\n".join(lines) + return blob.strip() + + def get_data(self, params, who, **kwargs): + if not params: + return self._get_user_blob(who=who) + return '' + + +# Seem to need to use globals since can't pass +# data into the request handlers instances... +# Puke! +meta_fetcher = None +user_fetcher = None + + +class Ec2Handler(BaseHTTPRequestHandler): + + def _get_versions(self): + versions = [] + for v in EC2_VERSIONS: + if v == 'latest': + continue + else: + versions.append(v) + versions = sorted(versions) + return "\n".join(versions) + + def log_message(self, format, *args): + msg = "%s - %s" % (self.address_string(), format % (args)) + log.info(msg) + + def _find_method(self, path): + # Puke! (globals) + global meta_fetcher + global user_fetcher + func_mapping = { + 'user-data': user_fetcher.get_data, + 'meta-data': meta_fetcher.get_data, + } + segments = [piece for piece in path.split('/') if len(piece)] + if not segments: + return self._get_versions + date = segments[0].strip().lower() + if date not in EC2_VERSIONS: + raise WebException(httplib.BAD_REQUEST, "Unknown date format %r" % date) + if len(segments) < 2: + raise WebException(httplib.BAD_REQUEST, "No action provided") + look_name = segments[1].lower() + if look_name not in func_mapping: + raise WebException(httplib.BAD_REQUEST, "Unknown requested data %r" % look_name) + base_func = func_mapping[look_name] + who = self.address_string() + kwargs = { + 'params': list(segments[2:]), + 'who': self.address_string(), + 'client_ip': self.client_address[0], + } + return functools.partial(base_func, **kwargs) + + def _do_response(self): + who = self.client_address + log.info("Got a call from %s for path %s", who, self.path) + try: + func = self._find_method(self.path) + log.info("Calling into func %s to get your data.", func) + data = func() + if not data: + data = '' + self.send_response(httplib.OK) + self.send_header("Content-Type", "binary/octet-stream") + self.send_header("Content-Length", len(data)) + log.info("Sending data (len=%s):\n%s", len(data), format_text(data)) + self.end_headers() + self.wfile.write(data) + except RuntimeError as e: + log.exception("Error somewhere in the server.") + self.send_error(httplib.INTERNAL_SERVER_ERROR, message=str(e)) + except WebException as e: + code = e.code + log.exception(str(e)) + self.send_error(code, message=str(e)) + + def do_GET(self): + self._do_response() + + def do_POST(self): + self._do_response() + + +def setup_logging(log_level, format='%(levelname)s: @%(name)s : %(message)s'): + root_logger = logging.getLogger() + console_logger = logging.StreamHandler(sys.stdout) + console_logger.setFormatter(logging.Formatter(format)) + root_logger.addHandler(console_logger) + root_logger.setLevel(log_level) + + +def extract_opts(): + parser = OptionParser() + parser.add_option("-p", "--port", dest="port", action="store", type=int, default=80, + help="port from which to serve traffic (default: %default)", metavar="PORT") + (options, args) = parser.parse_args() + out = dict() + out['extra'] = args + out['port'] = options.port + return out + + +def setup_fetchers(opts): + global meta_fetcher + global user_fetcher + meta_fetcher = MetaDataHandler(opts) + user_fetcher = UserDataHandler(opts) + + +def run_server(): + # Using global here since it doesn't seem like we + # can pass opts into a request handler constructor... + opts = extract_opts() + setup_logging(logging.DEBUG) + setup_fetchers(opts) + log.info("CLI opts: %s", opts) + server = HTTPServer(('0.0.0.0', opts['port']), Ec2Handler) + sa = server.socket.getsockname() + log.info("Serving server on %s using port %s ...", sa[0], sa[1]) + server.serve_forever() + + +if __name__ == '__main__': + run_server() -- cgit v1.2.3 From 25403168cea7b0ece56bc98a0b3ed05043182c91 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Wed, 20 Jun 2012 16:24:24 -0400 Subject: update tools/bddeb and debian.trunk packaging * debian.trunk/changelog: increase debian version to '1' to avoid lintian error * debian.trunk/control: bump standards version * debian.trunk/rules: remove cloud-init-run-module symlink (been deprecated for some time) * tools/bddeb: read version from ChangeLog rather than setup.py --- debian.trunk/changelog | 2 +- debian.trunk/control | 2 +- debian.trunk/rules | 1 - setup.py | 3 +-- tools/bddeb | 6 +++--- 5 files changed, 6 insertions(+), 8 deletions(-) (limited to 'tools') diff --git a/debian.trunk/changelog b/debian.trunk/changelog index 53e3678c..a36d00d6 100644 --- a/debian.trunk/changelog +++ b/debian.trunk/changelog @@ -1,4 +1,4 @@ -cloud-init (VERSION~REVNO-0) UNRELEASED; urgency=low +cloud-init (VERSION~REVNO-1) UNRELEASED; urgency=low * build diff --git a/debian.trunk/control b/debian.trunk/control index f2eec1e4..f0dcef6a 100644 --- a/debian.trunk/control +++ b/debian.trunk/control @@ -10,7 +10,7 @@ Build-Depends: cdbs, pylint, python-mocker, XS-Python-Version: all -Standards-Version: 3.9.1 +Standards-Version: 3.9.3 Package: cloud-init Architecture: all diff --git a/debian.trunk/rules b/debian.trunk/rules index 19384687..0f79136c 100755 --- a/debian.trunk/rules +++ b/debian.trunk/rules @@ -13,7 +13,6 @@ cloud-init-fixups: for x in $(DEB_DESTDIR)/usr/bin/*.py; do mv "$$x" "$${x%.py}"; done install -d $(DEB_DESTDIR)/etc/rsyslog.d cp tools/21-cloudinit.conf $(DEB_DESTDIR)/etc/rsyslog.d/21-cloudinit.conf - ln -sf cloud-init-per $(DEB_DESTDIR)/usr/bin/cloud-init-run-module # You only need to run this immediately after checking out the package from # revision control. diff --git a/setup.py b/setup.py index 96f889d8..9ee58b9a 100755 --- a/setup.py +++ b/setup.py @@ -70,8 +70,7 @@ setup(name='cloud-init', author_email='scott.moser@canonical.com', url='http://launchpad.net/cloud-init/', packages=find_packages(), - scripts=['bin/cloud-init.py', - 'bin/cloud-init-cfg.py', + scripts=['bin/cloud-init', 'tools/cloud-init-per', ], data_files=[('/etc/cloud', glob('config/*.cfg')), diff --git a/tools/bddeb b/tools/bddeb index 598f71bb..b162b06f 100755 --- a/tools/bddeb +++ b/tools/bddeb @@ -8,8 +8,8 @@ set -e trap "rm -Rf '${TEMP_D}'" exit files=$(bzr ls --versioned) revno=$(bzr revno) -version=$(awk \ - -F= '$1 ~ /version$/ { gsub("[^0-9.]","",$2); print $2; }' setup.py) +version=$(awk -F: \ + '$1 ~ /[0-9][.][0-9]+[.][0-9]+/ { print $1 ; exit(0); }' ChangeLog ) mkdir "${TEMP_D}/cloud-init" otar="$TEMP_D/cloud-init_$version~bzr${revno}.orig.tar.gz" tar -czf - ${files} > "$otar" @@ -25,7 +25,7 @@ debuild "$@" #for x in ../*.deb; do # echo wrote ${x##*/} #done -debname="cloud-init_${version}~bzr${revno}-0_all.deb" +debname="cloud-init_${version}~bzr${revno}-1_all.deb" mv "../$debname" "$start" link="$start/cloud-init_all.deb" echo "wrote $debname" -- cgit v1.2.3 From f7e638f6f58188cd4be1921cb045608f3c00d9c4 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Wed, 20 Jun 2012 23:59:24 -0700 Subject: Return a empty json map as default instead of an empty string for unknown fields --- tools/mock-meta.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'tools') diff --git a/tools/mock-meta.py b/tools/mock-meta.py index 019a24f2..d4677af6 100755 --- a/tools/mock-meta.py +++ b/tools/mock-meta.py @@ -181,7 +181,7 @@ class MetaDataHandler(object): else: return "%s" % (PLACEMENT_CAPABILITIES.get(pentry, '')) else: - return '' + return '{}' class UserDataHandler(object): -- cgit v1.2.3 From 67232af3cba2c7bc99c2ca67b83470b38d6db103 Mon Sep 17 00:00:00 2001 From: harlowja Date: Sat, 23 Jun 2012 14:59:16 -0700 Subject: 1. Separate the pep8 check from the pylint check a. This allows them to be run as different tools (if desired) 2. Adjust the makefile to have a 'make pep8' section which can run this new script --- Makefile | 5 ++++- tools/run-pep8 | 28 ++++++++++++++++++++++++++++ 2 files changed, 32 insertions(+), 1 deletion(-) create mode 100755 tools/run-pep8 (limited to 'tools') diff --git a/Makefile b/Makefile index ab23bf1f..c20dfbd3 100644 --- a/Makefile +++ b/Makefile @@ -4,6 +4,9 @@ PY_FILES+="bin/cloud-init" all: test +pep8: + $(CWD)/tools/run-pep8 $(PY_FILES) + pylint: $(CWD)/tools/run-pylint $(PY_FILES) @@ -20,5 +23,5 @@ clean: rm -rf /var/log/cloud-init.log \ /var/lib/cloud/ -.PHONY: test pylint pyflakes 2to3 clean +.PHONY: test pylint pyflakes 2to3 clean pep8 diff --git a/tools/run-pep8 b/tools/run-pep8 new file mode 100755 index 00000000..e7707985 --- /dev/null +++ b/tools/run-pep8 @@ -0,0 +1,28 @@ +#!/bin/bash + +ci_files='cloud*.py cloudinit/*.py cloudinit/config/*.py' +test_files=$(find tests -name "*.py") +def_files="$ci_files $test_files" + +if [ $# -eq 0 ]; then + files=( ) + for f in $def_files; do + [ -f "$f" ] || { echo "failed, $f not a file" 1>&2; exit 1; } + files[${#files[@]}]=${f} + done +else + files=( "$@" ); +fi + +cmd=( + pep8 + + --ignore=E501 # Line too long (these are caught by pylint) + + "${files[@]}" +) + +echo -e "\nRunning pep8:" +echo "${cmd[@]}" +"${cmd[@]}" + -- cgit v1.2.3 From a14e3090400d24cc34a5dc3fc1a12186175a164c Mon Sep 17 00:00:00 2001 From: harlowja Date: Sat, 23 Jun 2012 15:00:06 -0700 Subject: Remove the pep8 tool from being ran in a script that has a name that seems to just say it will run pylint. Put the pep8 tool in a 'run-pep8' script. --- tools/run-pylint | 13 +------------ 1 file changed, 1 insertion(+), 12 deletions(-) (limited to 'tools') diff --git a/tools/run-pylint b/tools/run-pylint index 46748ffb..dd6369aa 100755 --- a/tools/run-pylint +++ b/tools/run-pylint @@ -1,6 +1,6 @@ #!/bin/bash -ci_files='cloud*.py cloudinit/*.py cloudinit/CloudConfig/*.py' +ci_files='cloud*.py cloudinit/*.py cloudinit/config/*.py' test_files=$(find tests -name "*.py") def_files="$ci_files $test_files" @@ -38,14 +38,3 @@ echo -e "\nRunning pylint:" echo "${cmd[@]}" "${cmd[@]}" -cmd=( - pep8 - - --ignore=E501 # Line too long (these are caught by pylint above) - - "${files[@]}" -) - -echo -e "\nRunning pep8:" -echo "${cmd[@]}" -"${cmd[@]}" -- cgit v1.2.3 From ffe4cde6d3fd95131b4434ca2d88af469dacfd51 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Mon, 25 Jun 2012 12:28:04 -0700 Subject: Add warning when hitting a unknown api. --- tools/mock-meta.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) (limited to 'tools') diff --git a/tools/mock-meta.py b/tools/mock-meta.py index d4677af6..0f13acd6 100755 --- a/tools/mock-meta.py +++ b/tools/mock-meta.py @@ -6,10 +6,11 @@ import functools import httplib +import json import logging -import sys -import string import random +import string +import sys import yaml from optparse import OptionParser @@ -88,6 +89,8 @@ PLACEMENT_CAPABILITIES = { 'availability-zone': AVAILABILITY_ZONES, } +NOT_IMPL_RESPONSE = json.dumps({}) + class WebException(Exception): def __init__(self, code, msg): @@ -181,7 +184,11 @@ class MetaDataHandler(object): else: return "%s" % (PLACEMENT_CAPABILITIES.get(pentry, '')) else: - return '{}' + log.warn(("Did not implement action %s, " + "returning empty response: %r"), + action, NOT_IMPL_RESPONSE) + return NOT_IMPL_RESPONSE + class UserDataHandler(object): -- cgit v1.2.3 From 6d74afcd0e11de5cb56623d9121472e9212e8481 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Mon, 25 Jun 2012 12:28:59 -0700 Subject: 1. Moving around the packages dir. 2. Adjusting the bddep shell script 3. Starting to add a brpm --- debian.trunk/changelog | 5 ---- debian.trunk/compat | 1 - debian.trunk/control | 31 --------------------- debian.trunk/copyright | 51 --------------------------------- debian.trunk/dirs | 5 ---- debian.trunk/pycompat | 1 - debian.trunk/rules | 28 ------------------- packages/bddeb | 62 +++++++++++++++++++++++++++++++++++++++++ packages/brpm | 3 ++ packages/debian.trunk/changelog | 5 ++++ packages/debian.trunk/compat | 1 + packages/debian.trunk/control | 31 +++++++++++++++++++++ packages/debian.trunk/copyright | 51 +++++++++++++++++++++++++++++++++ packages/debian.trunk/dirs | 5 ++++ packages/debian.trunk/pycompat | 1 + packages/debian.trunk/rules | 28 +++++++++++++++++++ packages/make-dist-tarball | 25 +++++++++++++++++ tools/bddeb | 33 ---------------------- tools/make-dist-tarball | 25 ----------------- 19 files changed, 212 insertions(+), 180 deletions(-) delete mode 100644 debian.trunk/changelog delete mode 100644 debian.trunk/compat delete mode 100644 debian.trunk/control delete mode 100644 debian.trunk/copyright delete mode 100644 debian.trunk/dirs delete mode 100644 debian.trunk/pycompat delete mode 100755 debian.trunk/rules create mode 100755 packages/bddeb create mode 100755 packages/brpm create mode 100644 packages/debian.trunk/changelog create mode 100644 packages/debian.trunk/compat create mode 100644 packages/debian.trunk/control create mode 100644 packages/debian.trunk/copyright create mode 100644 packages/debian.trunk/dirs create mode 100644 packages/debian.trunk/pycompat create mode 100755 packages/debian.trunk/rules create mode 100755 packages/make-dist-tarball delete mode 100755 tools/bddeb delete mode 100755 tools/make-dist-tarball (limited to 'tools') diff --git a/debian.trunk/changelog b/debian.trunk/changelog deleted file mode 100644 index a36d00d6..00000000 --- a/debian.trunk/changelog +++ /dev/null @@ -1,5 +0,0 @@ -cloud-init (VERSION~REVNO-1) UNRELEASED; urgency=low - - * build - - -- Scott Moser Fri, 16 Dec 2011 11:50:25 -0500 diff --git a/debian.trunk/compat b/debian.trunk/compat deleted file mode 100644 index 7ed6ff82..00000000 --- a/debian.trunk/compat +++ /dev/null @@ -1 +0,0 @@ -5 diff --git a/debian.trunk/control b/debian.trunk/control deleted file mode 100644 index f0dcef6a..00000000 --- a/debian.trunk/control +++ /dev/null @@ -1,31 +0,0 @@ -Source: cloud-init -Section: admin -Priority: extra -Maintainer: Scott Moser -Build-Depends: cdbs, - debhelper (>= 5.0.38), - python (>= 2.6.6-3~), - python-nose, - pyflakes, - pylint, - python-mocker, -XS-Python-Version: all -Standards-Version: 3.9.3 - -Package: cloud-init -Architecture: all -Depends: cloud-utils, - procps, - python, - python-boto (>=2.0), - python-cheetah, - python-configobj, - python-oauth, - python-software-properties, - python-yaml, - ${misc:Depends}, - ${python:Depends} -XB-Python-Version: ${python:Versions} -Description: Init scripts for cloud instances - Cloud instances need special scripts to run during initialisation - to retrieve and install ssh keys and to let the user run various scripts. diff --git a/debian.trunk/copyright b/debian.trunk/copyright deleted file mode 100644 index dc993525..00000000 --- a/debian.trunk/copyright +++ /dev/null @@ -1,51 +0,0 @@ -Format-Specification: http://svn.debian.org/wsvn/dep/web/deps/dep5.mdwn?op=file&rev=135 -Name: cloud-init -Maintainer: Scott Moser -Source: https://launchpad.net/cloud-init - -This package was debianized by Soren Hansen on -Thu, 04 Sep 2008 12:49:15 +0200 as ec2-init. It was later renamed to -cloud-utils by Scott Moser - -Upstream Author: Scott Moser - Soren Hansen - Chuck Short - -Copyright: 2010, Canonical Ltd. -License: GPL-3 - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License version 3, as - published by the Free Software Foundation. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program. If not, see . - - The complete text of the GPL version 3 can be seen in - /usr/share/common-licenses/GPL-3. - -Files: cloudinit/boto_utils.py -Copyright: 2006,2007, Mitch Garnaat http://garnaat.org/ -License: MIT - Permission is hereby granted, free of charge, to any person obtaining a - copy of this software and associated documentation files (the - "Software"), to deal in the Software without restriction, including - without limitation the rights to use, copy, modify, merge, publish, dis- - tribute, sublicense, and/or sell copies of the Software, and to permit - persons to whom the Software is furnished to do so, subject to the fol- - lowing conditions: - - The above copyright notice and this permission notice shall be included - in all copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS - OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- - ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT - SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, - WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - IN THE SOFTWARE. diff --git a/debian.trunk/dirs b/debian.trunk/dirs deleted file mode 100644 index f3de468d..00000000 --- a/debian.trunk/dirs +++ /dev/null @@ -1,5 +0,0 @@ -var/lib/cloud -usr/bin -etc/init -usr/share/doc/cloud -etc/cloud diff --git a/debian.trunk/pycompat b/debian.trunk/pycompat deleted file mode 100644 index 0cfbf088..00000000 --- a/debian.trunk/pycompat +++ /dev/null @@ -1 +0,0 @@ -2 diff --git a/debian.trunk/rules b/debian.trunk/rules deleted file mode 100755 index 0f79136c..00000000 --- a/debian.trunk/rules +++ /dev/null @@ -1,28 +0,0 @@ -#!/usr/bin/make -f - -DEB_PYTHON2_MODULE_PACKAGES = cloud-init - -binary-install/cloud-init::cloud-init-fixups - -include /usr/share/cdbs/1/rules/debhelper.mk -include /usr/share/cdbs/1/class/python-distutils.mk - -DEB_DH_INSTALL_SOURCEDIR := debian/tmp - -cloud-init-fixups: - for x in $(DEB_DESTDIR)/usr/bin/*.py; do mv "$$x" "$${x%.py}"; done - install -d $(DEB_DESTDIR)/etc/rsyslog.d - cp tools/21-cloudinit.conf $(DEB_DESTDIR)/etc/rsyslog.d/21-cloudinit.conf - -# You only need to run this immediately after checking out the package from -# revision control. -# http://bugs.debian.org/cgi-bin/bugreport.cgi?bug=572204 -quilt-setup: - @[ ! -d .pc ] || { echo ".pc exists. remove it and re-run to start fresh"; exit 1; } - set -e; for patch in $$(quilt series | tac); do \ - patch -p1 -R --no-backup-if-mismatch <"debian/patches/$$patch"; \ - done - quilt push -a - -.PHONY: quilt-setup - diff --git a/packages/bddeb b/packages/bddeb new file mode 100755 index 00000000..74932985 --- /dev/null +++ b/packages/bddeb @@ -0,0 +1,62 @@ +#!/bin/sh + +# Ensure we can find the setup.py file which +# should be at the root of the cloud-init tree +PKG_DIR=`pwd` +ROOT="$PKG_DIR/../" +if [ ! -e "$ROOT/setup.py" ]; then + echo "No setup.py found at $ROOT" + exit 1 +fi +echo "Using root directory $ROOT for building your debian package" + +# Ensure tempdir is cleaned and ready to go +TEMP_D=$(mktemp -d "${TMPDIR:-/tmp}/${0##*/}.XXXXXXX") +echo "With $TEMP_D as a temporary workspace" +rm -Rf "${TEMP_D}"; mkdir "${TEMP_D}" +set -e + +# Ensure tempdir removed if breaks... +trap "rm -Rf '${TEMP_D}'" EXIT SIGINT SIGTERM + +# Go to the root and start making a package! +cd $ROOT +files=$(bzr ls --versioned) +revno=$(bzr revno) +version=$(awk -F: \ + '$1 ~ /[0-9][.][0-9]+[.][0-9]+/ { print $1 ; exit(0); }' ChangeLog ) + +# Ensure that the version found in 'Changelog' is the same in the python +# cloud-init version directory +py_version=$(python -c 'from cloudinit import version; import sys; \ +sys.stdout.write(version.version_string())') + +# Canonicalize the changelog version +ch_version=$(python -c "from distutils import version; import sys; \ +sys.stdout.write(str(version.StrictVersion('$version')));") + +if [ "$py_version" != "$ch_version" ]; then + echo "Cloud-init python version $py_version" \ + " != changelog version $ch_version" + echo "Please ensure they are the same!" + exit 1 +fi + +mkdir "${TEMP_D}/cloud-init" +otar="$TEMP_D/cloud-init_$version~bzr${revno}.orig.tar.gz" +tar -czf - ${files} > "$otar" +tar -C "${TEMP_D}/cloud-init" -xzf - < "$otar" + +if [ ! -d "${TEMP_D}/cloud-init/debian" ]; then + rsync -a $PKG_DIR/debian.trunk/ "${TEMP_D}/cloud-init/debian" +fi + +sed -i -e "s,VERSION,$version," -e "s,REVNO,bzr$revno," \ + "$TEMP_D/cloud-init/debian/changelog" + +cd "${TEMP_D}/cloud-init" +debuild "$@" +debname="cloud-init_${version}~bzr${revno}-1_all.deb" +mv "../$debname" "$PKG_DIR" +echo "Wrote $debname to $PKG_DIR" + diff --git a/packages/brpm b/packages/brpm new file mode 100755 index 00000000..829303ea --- /dev/null +++ b/packages/brpm @@ -0,0 +1,3 @@ +#!/bin/sh + + diff --git a/packages/debian.trunk/changelog b/packages/debian.trunk/changelog new file mode 100644 index 00000000..a36d00d6 --- /dev/null +++ b/packages/debian.trunk/changelog @@ -0,0 +1,5 @@ +cloud-init (VERSION~REVNO-1) UNRELEASED; urgency=low + + * build + + -- Scott Moser Fri, 16 Dec 2011 11:50:25 -0500 diff --git a/packages/debian.trunk/compat b/packages/debian.trunk/compat new file mode 100644 index 00000000..7ed6ff82 --- /dev/null +++ b/packages/debian.trunk/compat @@ -0,0 +1 @@ +5 diff --git a/packages/debian.trunk/control b/packages/debian.trunk/control new file mode 100644 index 00000000..f0dcef6a --- /dev/null +++ b/packages/debian.trunk/control @@ -0,0 +1,31 @@ +Source: cloud-init +Section: admin +Priority: extra +Maintainer: Scott Moser +Build-Depends: cdbs, + debhelper (>= 5.0.38), + python (>= 2.6.6-3~), + python-nose, + pyflakes, + pylint, + python-mocker, +XS-Python-Version: all +Standards-Version: 3.9.3 + +Package: cloud-init +Architecture: all +Depends: cloud-utils, + procps, + python, + python-boto (>=2.0), + python-cheetah, + python-configobj, + python-oauth, + python-software-properties, + python-yaml, + ${misc:Depends}, + ${python:Depends} +XB-Python-Version: ${python:Versions} +Description: Init scripts for cloud instances + Cloud instances need special scripts to run during initialisation + to retrieve and install ssh keys and to let the user run various scripts. diff --git a/packages/debian.trunk/copyright b/packages/debian.trunk/copyright new file mode 100644 index 00000000..dc993525 --- /dev/null +++ b/packages/debian.trunk/copyright @@ -0,0 +1,51 @@ +Format-Specification: http://svn.debian.org/wsvn/dep/web/deps/dep5.mdwn?op=file&rev=135 +Name: cloud-init +Maintainer: Scott Moser +Source: https://launchpad.net/cloud-init + +This package was debianized by Soren Hansen on +Thu, 04 Sep 2008 12:49:15 +0200 as ec2-init. It was later renamed to +cloud-utils by Scott Moser + +Upstream Author: Scott Moser + Soren Hansen + Chuck Short + +Copyright: 2010, Canonical Ltd. +License: GPL-3 + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License version 3, as + published by the Free Software Foundation. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + + The complete text of the GPL version 3 can be seen in + /usr/share/common-licenses/GPL-3. + +Files: cloudinit/boto_utils.py +Copyright: 2006,2007, Mitch Garnaat http://garnaat.org/ +License: MIT + Permission is hereby granted, free of charge, to any person obtaining a + copy of this software and associated documentation files (the + "Software"), to deal in the Software without restriction, including + without limitation the rights to use, copy, modify, merge, publish, dis- + tribute, sublicense, and/or sell copies of the Software, and to permit + persons to whom the Software is furnished to do so, subject to the fol- + lowing conditions: + + The above copyright notice and this permission notice shall be included + in all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS + OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- + ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT + SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + IN THE SOFTWARE. diff --git a/packages/debian.trunk/dirs b/packages/debian.trunk/dirs new file mode 100644 index 00000000..f3de468d --- /dev/null +++ b/packages/debian.trunk/dirs @@ -0,0 +1,5 @@ +var/lib/cloud +usr/bin +etc/init +usr/share/doc/cloud +etc/cloud diff --git a/packages/debian.trunk/pycompat b/packages/debian.trunk/pycompat new file mode 100644 index 00000000..0cfbf088 --- /dev/null +++ b/packages/debian.trunk/pycompat @@ -0,0 +1 @@ +2 diff --git a/packages/debian.trunk/rules b/packages/debian.trunk/rules new file mode 100755 index 00000000..0f79136c --- /dev/null +++ b/packages/debian.trunk/rules @@ -0,0 +1,28 @@ +#!/usr/bin/make -f + +DEB_PYTHON2_MODULE_PACKAGES = cloud-init + +binary-install/cloud-init::cloud-init-fixups + +include /usr/share/cdbs/1/rules/debhelper.mk +include /usr/share/cdbs/1/class/python-distutils.mk + +DEB_DH_INSTALL_SOURCEDIR := debian/tmp + +cloud-init-fixups: + for x in $(DEB_DESTDIR)/usr/bin/*.py; do mv "$$x" "$${x%.py}"; done + install -d $(DEB_DESTDIR)/etc/rsyslog.d + cp tools/21-cloudinit.conf $(DEB_DESTDIR)/etc/rsyslog.d/21-cloudinit.conf + +# You only need to run this immediately after checking out the package from +# revision control. +# http://bugs.debian.org/cgi-bin/bugreport.cgi?bug=572204 +quilt-setup: + @[ ! -d .pc ] || { echo ".pc exists. remove it and re-run to start fresh"; exit 1; } + set -e; for patch in $$(quilt series | tac); do \ + patch -p1 -R --no-backup-if-mismatch <"debian/patches/$$patch"; \ + done + quilt push -a + +.PHONY: quilt-setup + diff --git a/packages/make-dist-tarball b/packages/make-dist-tarball new file mode 100755 index 00000000..622283bd --- /dev/null +++ b/packages/make-dist-tarball @@ -0,0 +1,25 @@ +#!/bin/sh + +Usage() { + cat <&2 ; exit 1; } + +tmpd=$(mktemp -d ); +trap "rm -Rf '${tmpd}'" 0 + +out=${topdir}/cloud-init-${tag}.tar.gz + +cd ${tmpd} && + bzr branch -r "tag:${tag}" "${topdir}" ./cloud-init-${tag} && + tar czf "${out}" cloud-init-${tag}/ --exclude cloud-init-${tag}/.bzr && + echo "Wrote ${out}" diff --git a/tools/bddeb b/tools/bddeb deleted file mode 100755 index b162b06f..00000000 --- a/tools/bddeb +++ /dev/null @@ -1,33 +0,0 @@ -#!/bin/sh - -TEMP_D=$(mktemp -d "${TMPDIR:-/tmp}/${0##*/}.XXXXXXX") -#TEMP_D=/tmp/my.d -start=${PWD} -rm -Rf "${TEMP_D}"; mkdir "${TEMP_D}" -set -e -trap "rm -Rf '${TEMP_D}'" exit -files=$(bzr ls --versioned) -revno=$(bzr revno) -version=$(awk -F: \ - '$1 ~ /[0-9][.][0-9]+[.][0-9]+/ { print $1 ; exit(0); }' ChangeLog ) -mkdir "${TEMP_D}/cloud-init" -otar="$TEMP_D/cloud-init_$version~bzr${revno}.orig.tar.gz" -tar -czf - ${files} > "$otar" -tar -C "${TEMP_D}/cloud-init" -xzf - <"$otar" - -if [ ! -d "${TEMP_D}/cloud-init/debian" ]; then - rsync -a debian.trunk/ "${TEMP_D}/cloud-init/debian" -fi -sed -i -e "s,VERSION,$version," -e "s,REVNO,bzr$revno," \ - "$TEMP_D/cloud-init/debian/changelog" -cd "${TEMP_D}/cloud-init" -debuild "$@" -#for x in ../*.deb; do -# echo wrote ${x##*/} -#done -debname="cloud-init_${version}~bzr${revno}-1_all.deb" -mv "../$debname" "$start" -link="$start/cloud-init_all.deb" -echo "wrote $debname" -[ ! -e "$link" -o -L "$link" ] - { ln -sf "$debname" "$link" && echo "linked ${link##*/}"; } diff --git a/tools/make-dist-tarball b/tools/make-dist-tarball deleted file mode 100755 index d6d53aa7..00000000 --- a/tools/make-dist-tarball +++ /dev/null @@ -1,25 +0,0 @@ -#!/bin/sh - -Usage() { - cat <&2 ; exit 1; } - -tmpd=$(mktemp -d ); -trap "rm -Rf '${tmpd}'" 0 - -out=${topdir}/cloud-init-${tag}.tar.gz - -cd ${tmpd} && - bzr branch -r "tag:${tag}" "${topdir}" ./cloud-init-${tag} && - tar czf "${out}" cloud-init-${tag}/ --exclude cloud-init-${tag}/.bzr && - echo "wrote ${out}" -- cgit v1.2.3 From d45b0aacf88cf640dde538b3ee92958099129f74 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Mon, 25 Jun 2012 13:07:17 -0700 Subject: This tool knows how to extract the current version from the 'changelog' file. --- tools/read-version | 51 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) create mode 100755 tools/read-version (limited to 'tools') diff --git a/tools/read-version b/tools/read-version new file mode 100755 index 00000000..4458c712 --- /dev/null +++ b/tools/read-version @@ -0,0 +1,51 @@ +#!/usr/bin/python +# vi: ts=4 expandtab + +import os +import sys +import re + +def parse_versions(fn): + with open(fn, 'r') as fh: + lines = fh.read().splitlines() + versions = [] + for line in lines: + line = line.strip() + if line.startswith("-") or not line: + continue + if not re.match(r"[\d]", line): + continue + line = line.strip(":") + if (re.match(r"^[\d+]\.[\d+]\.[\d+]$", line) or + re.match(r"^[\d+]\.[\d+]$", line)): + versions.append(line) + return versions + +def find_changelog(args): + p_files = [] + if args: + p_files.append(args[0]) + p_files.append(os.path.join(os.pardir, "ChangeLog")) + p_files.append(os.path.join(os.getcwd(), 'ChangeLog')) + found = None + for fn in p_files: + if os.path.isfile(fn): + found = fn + break + return found + + +if __name__ == '__main__': + run_args = sys.argv[1:] + fn = find_changelog(run_args) + if not fn: + sys.stderr.write("'ChangeLog' file not found!\n") + sys.exit(1) + else: + versions = parse_versions(fn) + if not versions: + sys.stderr.write("No versions found in %s!\n" % (fn)) + sys.exit(1) + else: + sys.stdout.write(versions[0].strip()) + sys.exit(0) -- cgit v1.2.3 From 3b05473d57f5aaadf4cd0b57b7138d9ff2141808 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Mon, 25 Jun 2012 16:59:15 -0700 Subject: Add a simple tool that will parse the requires file --- tools/read-dependencies | 45 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) create mode 100755 tools/read-dependencies (limited to 'tools') diff --git a/tools/read-dependencies b/tools/read-dependencies new file mode 100755 index 00000000..72e1e095 --- /dev/null +++ b/tools/read-dependencies @@ -0,0 +1,45 @@ +#!/usr/bin/python +# vi: ts=4 expandtab + +import os +import sys +import re + + +def parse_requires(fn): + requires = [] + with open(fn, 'r') as fh: + lines = fh.read().splitlines() + for line in lines: + line = line.strip() + if not line or line[0] == '#': + continue + else: + requires.append(line) + return requires + + +def find_requires(args): + p_files = [] + if args: + p_files.append(args[0]) + p_files.append(os.path.join(os.pardir, "Requires")) + p_files.append(os.path.join(os.getcwd(), 'Requires')) + found = None + for fn in p_files: + if os.path.isfile(fn): + found = fn + break + return found + + +if __name__ == '__main__': + run_args = sys.argv[1:] + fn = find_requires(run_args) + if not fn: + sys.stderr.write("'Requires' file not found!\n") + sys.exit(1) + else: + deps = parse_requires(fn) + for entry in deps: + print entry -- cgit v1.2.3 From 060844af4ec1f9d2d2d7c8763988b4131043dc91 Mon Sep 17 00:00:00 2001 From: harlowja Date: Tue, 26 Jun 2012 07:55:21 -0700 Subject: Add check that the changelog version is the same as the code version --- tools/read-version | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) (limited to 'tools') diff --git a/tools/read-version b/tools/read-version index 4458c712..e6167a2c 100755 --- a/tools/read-version +++ b/tools/read-version @@ -5,6 +5,15 @@ import os import sys import re +from distutils import version as ver + +possible_topdir = os.path.normpath(os.path.join(os.path.abspath( + sys.argv[0]), os.pardir, os.pardir)) +if os.path.exists(os.path.join(possible_topdir, "cloudinit", "__init__.py")): + sys.path.insert(0, possible_topdir) + +from cloudinit import version as cver + def parse_versions(fn): with open(fn, 'r') as fh: lines = fh.read().splitlines() @@ -47,5 +56,15 @@ if __name__ == '__main__': sys.stderr.write("No versions found in %s!\n" % (fn)) sys.exit(1) else: - sys.stdout.write(versions[0].strip()) + # Check that the code version is the same + # as the version we found! + ch_ver = versions[0].strip() + code_ver = cver.version() + ch_ver_obj = ver.StrictVersion(ch_ver) + if ch_ver_obj != code_ver: + sys.stderr.write(("Code version %s does not match" + " changelog version %s\n") % + (code_ver, ch_ver_obj)) + sys.exit(1) + sys.stdout.write(ch_ver) sys.exit(0) -- cgit v1.2.3 From bae7dfccda6662a40091b92c8c85ec6c37c5268d Mon Sep 17 00:00:00 2001 From: harlowja Date: Sun, 1 Jul 2012 12:43:33 -0700 Subject: Add a nova/openstack based extension to pep8 via hacking.py --- tools/hacking.py | 175 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ tools/run-pep8 | 13 ++++- 2 files changed, 185 insertions(+), 3 deletions(-) create mode 100755 tools/hacking.py (limited to 'tools') diff --git a/tools/hacking.py b/tools/hacking.py new file mode 100755 index 00000000..d0c27d25 --- /dev/null +++ b/tools/hacking.py @@ -0,0 +1,175 @@ +#!/usr/bin/env python +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright (c) 2012, Cloudscaling +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""cloudinit HACKING file compliance testing (based off of nova hacking.py) + +built on top of pep8.py +""" + +import inspect +import logging +import os +import re +import sys +import tokenize +import warnings + +import pep8 + +# Don't need this for testing +logging.disable('LOG') + +# N1xx comments +# N2xx except +# N3xx imports +# N4xx docstrings +# N[5-9]XX (future use) + +DOCSTRING_TRIPLE = ['"""', "'''"] +VERBOSE_MISSING_IMPORT = False +_missingImport = set([]) + + +def import_normalize(line): + # convert "from x import y" to "import x.y" + # handle "from x import y as z" to "import x.y as z" + split_line = line.split() + if (line.startswith("from ") and "," not in line and + split_line[2] == "import" and split_line[3] != "*" and + split_line[1] != "__future__" and + (len(split_line) == 4 or + (len(split_line) == 6 and split_line[4] == "as"))): + return "import %s.%s" % (split_line[1], split_line[3]) + else: + return line + + +def cloud_import_alphabetical(physical_line, line_number, lines): + """Check for imports in alphabetical order. + + HACKING guide recommendation for imports: + imports in human alphabetical order + N306 + """ + # handle import x + # use .lower since capitalization shouldn't dictate order + split_line = import_normalize(physical_line.strip()).lower().split() + split_previous = import_normalize(lines[line_number - 2] + ).strip().lower().split() + # with or without "as y" + length = [2, 4] + if (len(split_line) in length and len(split_previous) in length and + split_line[0] == "import" and split_previous[0] == "import"): + if split_line[1] < split_previous[1]: + return (0, "N306: imports not in alphabetical order (%s, %s)" + % (split_previous[1], split_line[1])) + + +def cloud_docstring_start_space(physical_line): + """Check for docstring not start with space. + + HACKING guide recommendation for docstring: + Docstring should not start with space + N401 + """ + pos = max([physical_line.find(i) for i in DOCSTRING_TRIPLE]) # start + if (pos != -1 and len(physical_line) > pos + 1): + if (physical_line[pos + 3] == ' '): + return (pos, "N401: one line docstring should not start with" + " a space") + + +def cloud_todo_format(physical_line): + """Check for 'TODO()'. + + HACKING guide recommendation for TODO: + Include your name with TODOs as in "#TODO(termie)" + N101 + """ + pos = physical_line.find('TODO') + pos1 = physical_line.find('TODO(') + pos2 = physical_line.find('#') # make sure it's a comment + if (pos != pos1 and pos2 >= 0 and pos2 < pos): + return pos, "N101: Use TODO(NAME)" + + +def cloud_docstring_one_line(physical_line): + """Check one line docstring end. + + HACKING guide recommendation for one line docstring: + A one line docstring looks like this and ends in a period. + N402 + """ + pos = max([physical_line.find(i) for i in DOCSTRING_TRIPLE]) # start + end = max([physical_line[-4:-1] == i for i in DOCSTRING_TRIPLE]) # end + if (pos != -1 and end and len(physical_line) > pos + 4): + if (physical_line[-5] != '.'): + return pos, "N402: one line docstring needs a period" + + +def cloud_docstring_multiline_end(physical_line): + """Check multi line docstring end. + + HACKING guide recommendation for docstring: + Docstring should end on a new line + N403 + """ + pos = max([physical_line.find(i) for i in DOCSTRING_TRIPLE]) # start + if (pos != -1 and len(physical_line) == pos): + print physical_line + if (physical_line[pos + 3] == ' '): + return (pos, "N403: multi line docstring end on new line") + + + +current_file = "" + + +def readlines(filename): + """Record the current file being tested.""" + pep8.current_file = filename + return open(filename).readlines() + + +def add_cloud(): + """Monkey patch pep8 for cloud-init guidelines. + + Look for functions that start with cloud_ + and add them to pep8 module. + + Assumes you know how to write pep8.py checks + """ + for name, function in globals().items(): + if not inspect.isfunction(function): + continue + if name.startswith("cloud_"): + exec("pep8.%s = %s" % (name, name)) + +if __name__ == "__main__": + # NOVA based 'hacking.py' error codes start with an N + pep8.ERRORCODE_REGEX = re.compile(r'[EWN]\d{3}') + add_cloud() + pep8.current_file = current_file + pep8.readlines = readlines + try: + pep8._main() + finally: + if len(_missingImport) > 0: + print >> sys.stderr, ("%i imports missing in this test environment" + % len(_missingImport)) + diff --git a/tools/run-pep8 b/tools/run-pep8 index e7707985..ea46c117 100755 --- a/tools/run-pep8 +++ b/tools/run-pep8 @@ -12,17 +12,24 @@ if [ $# -eq 0 ]; then done else files=( "$@" ); -fi +fi + +if [ -f 'hacking.py' ] +then + base=`pwd` +else + base=`pwd`/tools/ +fi cmd=( - pep8 + ${base}/hacking.py --ignore=E501 # Line too long (these are caught by pylint) "${files[@]}" ) -echo -e "\nRunning pep8:" +echo -e "\nRunning 'cloudinit' pep8:" echo "${cmd[@]}" "${cmd[@]}" -- cgit v1.2.3 From f99dd7c2e43f9117c66e06fc31d10f559c912e86 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Mon, 2 Jul 2012 19:02:16 -0700 Subject: Update the mock metadata server to return a file (or the default generated content) for the userdata blob. --- tools/mock-meta.py | 27 ++++++++++++++++++++------- 1 file changed, 20 insertions(+), 7 deletions(-) (limited to 'tools') diff --git a/tools/mock-meta.py b/tools/mock-meta.py index 0f13acd6..8a73b43c 100755 --- a/tools/mock-meta.py +++ b/tools/mock-meta.py @@ -196,18 +196,26 @@ class UserDataHandler(object): self.opts = opts def _get_user_blob(self, **kwargs): - blob_mp = {} - blob_mp['hostname'] = kwargs.get('who', '') - lines = [] - lines.append("#cloud-config") - lines.append(yamlify(blob_mp)) - blob = "\n".join(lines) + blob = None + if self.opts['user_data_file']: + with open(opts['user_data_file'], 'rb') as fh: + blob = fh.read() + blob = blob.strip() + if not blob: + blob_mp = { + 'hostname': kwargs.get('who', 'localhost'), + } + lines = [ + "#cloud-config", + yamlify(blob_mp), + ] + blob = "\n".join(lines) return blob.strip() def get_data(self, params, who, **kwargs): if not params: return self._get_user_blob(who=who) - return '' + return NOT_IMPL_RESPONSE # Seem to need to use globals since can't pass @@ -303,10 +311,15 @@ def extract_opts(): parser = OptionParser() parser.add_option("-p", "--port", dest="port", action="store", type=int, default=80, help="port from which to serve traffic (default: %default)", metavar="PORT") + parser.add_option("-f", '--user-data-file', dest='user_data_file', action='store', + help="user data blob to serve back to incoming requests", metavar='FILE') (options, args) = parser.parse_args() out = dict() out['extra'] = args out['port'] = options.port + out['user_data_file'] = None + if options.user_data_file: + out['user_data_file'] = options.user_data_file return out -- cgit v1.2.3 From 7e1a250d70c39607bc1f3e3134d6331cd372f511 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Mon, 2 Jul 2012 19:11:46 -0700 Subject: Add a check on the filename provided ensuring it actually exists. --- tools/mock-meta.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) (limited to 'tools') diff --git a/tools/mock-meta.py b/tools/mock-meta.py index 8a73b43c..5f421c98 100755 --- a/tools/mock-meta.py +++ b/tools/mock-meta.py @@ -8,6 +8,7 @@ import functools import httplib import json import logging +import os import random import string import sys @@ -312,13 +313,15 @@ def extract_opts(): parser.add_option("-p", "--port", dest="port", action="store", type=int, default=80, help="port from which to serve traffic (default: %default)", metavar="PORT") parser.add_option("-f", '--user-data-file', dest='user_data_file', action='store', - help="user data blob to serve back to incoming requests", metavar='FILE') + help="user data filename to serve back to incoming requests", metavar='FILE') (options, args) = parser.parse_args() out = dict() out['extra'] = args out['port'] = options.port out['user_data_file'] = None if options.user_data_file: + if not os.path.isfile(options.user_data_file): + parser.error("Option -f specified a non-existent file") out['user_data_file'] = options.user_data_file return out -- cgit v1.2.3 From 21117bb5c26abcb42a7dcc5f318190e734c849bd Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Tue, 3 Jul 2012 12:46:55 -0700 Subject: 1. Update the mock ec2 data with some of the pubkey code from smosers ec2 metadata server. 2. Allow the setting of the ip addr (not just to 0.0.0.0) 3. Add comment as to how to use this for the 169 'magic' addr --- tools/mock-meta.py | 78 ++++++++++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 70 insertions(+), 8 deletions(-) (limited to 'tools') diff --git a/tools/mock-meta.py b/tools/mock-meta.py index 5f421c98..7c38ec48 100755 --- a/tools/mock-meta.py +++ b/tools/mock-meta.py @@ -4,6 +4,19 @@ # # http://docs.amazonwebservices.com/AWSEC2/2007-08-29/DeveloperGuide/AESDG-chapter-instancedata.html +""" +To use this to mimic the EC2 metadata service entirely, run it like: + # Where 'eth0' is *some* interface. + sudo ifconfig eth0:0 169.254.169.254 netmask 255.255.255.255 + + sudo ./mock-meta -a 169.254.169.254 -p 80 + +Then: + wget -q http://169.254.169.254/latest/meta-data/instance-id -O -; echo + curl --silent http://169.254.169.254/latest/meta-data/instance-id ; echo + ec2metadata --instance-id +""" + import functools import httplib import json @@ -20,7 +33,6 @@ from BaseHTTPServer import (HTTPServer, BaseHTTPRequestHandler) log = logging.getLogger('meta-server') -# Constants EC2_VERSIONS = [ '1.0', '2007-01-19', @@ -69,6 +81,14 @@ META_CAPABILITIES = [ 'security-groups' ] +PUB_KEYS = { + 'brickies': [ + 'ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEA3I7VUf2l5gSn5uavROsc5HRDpZdQueUq5ozemNSj8T7enqKHOEaFoU2VoPgGEWC9RyzSQVeyD6s7APMcE82EtmW4skVEgEGSbDc1pvxzxtchBj78hJP6Cf5TCMFSXw+Fz5rF1dR23QDbN1mkHs7adr8GW4kSWqU7Q7NDwfIrJJtO7Hi42GyXtvEONHbiRPOe8stqUly7MvUoN+5kfjBM8Qqpfl2+FNhTYWpMfYdPUnE7u536WqzFmsaqJctz3gBxH9Ex7dFtrxR4qiqEr9Qtlu3xGn7Bw07/+i1D+ey3ONkZLN+LQ714cgj8fRS4Hj29SCmXp5Kt5/82cD/VN3NtHw== brickies', + '', + ], +} + + INSTANCE_TYPES = [ 'm1.small', 'm1.medium', @@ -136,6 +156,8 @@ class MetaDataHandler(object): def get_data(self, params, who, **kwargs): if not params: + # Show the root level capabilities when + # no params are passed... caps = sorted(META_CAPABILITIES) return "\n".join(caps) action = params[0] @@ -172,6 +194,43 @@ class MetaDataHandler(object): return "r-%s" % (id_generator(lower=True)) elif action == 'product-codes': return "%s" % (id_generator(size=8)) + elif action == 'public-keys': + nparams = params[1:] + # public-keys is messed up. a list of /latest/meta-data/public-keys/ + # shows something like: '0=brickies' + # but a GET to /latest/meta-data/public-keys/0=brickies will fail + # you have to know to get '/latest/meta-data/public-keys/0', then + # from there you get a 'openssh-key', which you can get. + # this hunk of code just re-works the object for that. + key_ids = sorted(list(PUB_KEYS.keys())) + if nparams: + mybe_key = nparams[0] + try: + key_id = int(mybe_key) + key_name = key_ids[key_id] + except: + raise WebException(httplib.BAD_REQUEST, "Unknown key id %r" % mybe_key) + # Extract the possible sub-params + key_info = { + "openssh-key": "\n".join(PUB_KEYS[key_name]), + } + result = dict(key_info) + for k in nparams[1:]: + try: + result = result.get(k) + except (AttributeError, TypeError): + result = None + break + if isinstance(result, (dict)): + result = json.dumps(result) + if result is None: + result = '' + return str(result) + else: + contents = [] + for (i, key_id) in enumerate(key_ids): + contents.append("%s=%s" % (i, key_id)) + return "\n".join(contents) elif action == 'placement': nparams = params[1:] if not nparams: @@ -198,10 +257,8 @@ class UserDataHandler(object): def _get_user_blob(self, **kwargs): blob = None - if self.opts['user_data_file']: - with open(opts['user_data_file'], 'rb') as fh: - blob = fh.read() - blob = blob.strip() + if self.opts['user_data_file'] is not None: + blob = self.opts['user_data_file'] if not blob: blob_mp = { 'hostname': kwargs.get('who', 'localhost'), @@ -312,6 +369,8 @@ def extract_opts(): parser = OptionParser() parser.add_option("-p", "--port", dest="port", action="store", type=int, default=80, help="port from which to serve traffic (default: %default)", metavar="PORT") + parser.add_option("-a", "--addr", dest="address", action="store", type=str, default='0.0.0.0', + help="address from which to serve traffic (default: %default)", metavar="ADDRESS") parser.add_option("-f", '--user-data-file', dest='user_data_file', action='store', help="user data filename to serve back to incoming requests", metavar='FILE') (options, args) = parser.parse_args() @@ -319,10 +378,12 @@ def extract_opts(): out['extra'] = args out['port'] = options.port out['user_data_file'] = None + out['address'] = options.address if options.user_data_file: if not os.path.isfile(options.user_data_file): parser.error("Option -f specified a non-existent file") - out['user_data_file'] = options.user_data_file + with open(options.user_data_file, 'rb') as fh: + out['user_data_file'] = fh.read() return out @@ -340,9 +401,10 @@ def run_server(): setup_logging(logging.DEBUG) setup_fetchers(opts) log.info("CLI opts: %s", opts) - server = HTTPServer(('0.0.0.0', opts['port']), Ec2Handler) + server_address = (opts['address'], opts['port']) + server = HTTPServer(server_address, Ec2Handler) sa = server.socket.getsockname() - log.info("Serving server on %s using port %s ...", sa[0], sa[1]) + log.info("Serving ec2 metadata on %s using port %s ...", sa[0], sa[1]) server.serve_forever() -- cgit v1.2.3 From 101d19df1c71b559bf34cfec87894f84ea041810 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Tue, 3 Jul 2012 19:59:04 -0700 Subject: 1. Cleanup of some mock functionality 2. Adding in returning the 'public-keys' to the metadata 'list' response 3. Adding in sending back the running users keys (useful for testing) along with 'brickies' 4. Add in a traverse function that can walk down a dictionary (if possible) --- tools/mock-meta.py | 103 ++++++++++++++++++++++++++++++++++------------------- 1 file changed, 66 insertions(+), 37 deletions(-) (limited to 'tools') diff --git a/tools/mock-meta.py b/tools/mock-meta.py index 7c38ec48..5bbe62cc 100755 --- a/tools/mock-meta.py +++ b/tools/mock-meta.py @@ -43,16 +43,15 @@ EC2_VERSIONS = [ '2008-02-01', '2008-09-01', '2009-04-04', - 'latest', ] BLOCK_DEVS = [ 'ami', - 'root', 'ephemeral0', + 'root', ] -DEV_PREFIX = 'v' +DEV_PREFIX = 'v' # This seems to vary alot depending on images... DEV_MAPPINGS = { 'ephemeral0': '%sda2' % (DEV_PREFIX), 'root': '/dev/%sda1' % (DEV_PREFIX), @@ -77,22 +76,28 @@ META_CAPABILITIES = [ 'product-codes', 'public-hostname', 'public-ipv4', + 'public-keys/', 'reservation-id', 'security-groups' ] PUB_KEYS = { 'brickies': [ - 'ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEA3I7VUf2l5gSn5uavROsc5HRDpZdQueUq5ozemNSj8T7enqKHOEaFoU2VoPgGEWC9RyzSQVeyD6s7APMcE82EtmW4skVEgEGSbDc1pvxzxtchBj78hJP6Cf5TCMFSXw+Fz5rF1dR23QDbN1mkHs7adr8GW4kSWqU7Q7NDwfIrJJtO7Hi42GyXtvEONHbiRPOe8stqUly7MvUoN+5kfjBM8Qqpfl2+FNhTYWpMfYdPUnE7u536WqzFmsaqJctz3gBxH9Ex7dFtrxR4qiqEr9Qtlu3xGn7Bw07/+i1D+ey3ONkZLN+LQ714cgj8fRS4Hj29SCmXp5Kt5/82cD/VN3NtHw== brickies', + ('ssh-rsa ' + 'AAAAB3NzaC1yc2EAAAABIwAAAQEA3I7VUf2l5gSn5uavROsc5HRDpZdQueUq5ozemNSj8T' + '7enqKHOEaFoU2VoPgGEWC9RyzSQVeyD6s7APMcE82EtmW4skVEgEGSbDc1pvxzxtchBj78' + 'hJP6Cf5TCMFSXw+Fz5rF1dR23QDbN1mkHs7adr8GW4kSWqU7Q7NDwfIrJJtO7Hi42GyXtv' + 'EONHbiRPOe8stqUly7MvUoN+5kfjBM8Qqpfl2+FNhTYWpMfYdPUnE7u536WqzFmsaqJctz' + '3gBxH9Ex7dFtrxR4qiqEr9Qtlu3xGn7Bw07/+i1D+ey3ONkZLN+LQ714cgj8fRS4Hj29SC' + 'mXp5Kt5/82cD/VN3NtHw== brickies'), '', ], } - INSTANCE_TYPES = [ - 'm1.small', - 'm1.medium', 'm1.large', + 'm1.medium', + 'm1.small', 'm1.xlarge', ] @@ -100,10 +105,10 @@ AVAILABILITY_ZONES = [ "us-east-1a", "us-east-1b", "us-east-1c", - 'us-west-1', "us-east-1d", 'eu-west-1a', 'eu-west-1b', + 'us-west-1', ] PLACEMENT_CAPABILITIES = { @@ -139,6 +144,17 @@ def format_text(text): return "\n".join(nlines) +def traverse(keys, mp): + result = dict(mp) + for k in keys: + try: + result = result.get(k) + except (AttributeError, TypeError): + result = None + break + return result + + ID_CHARS = [c for c in (string.ascii_uppercase + string.digits)] def id_generator(size=6, lower=False): txt = ''.join(random.choice(ID_CHARS) for x in range(size)) @@ -148,6 +164,23 @@ def id_generator(size=6, lower=False): return txt +def get_ssh_keys(): + keys = {} + keys.update(PUB_KEYS) + + # Nice helper to add in the 'running' users key (if they have one) + key_pth = os.path.expanduser('~/.ssh/id_rsa.pub') + if not os.path.isfile(key_pth): + key_pth = os.path.expanduser('~/.ssh/id_dsa.pub') + + if os.path.isfile(key_pth): + with open(key_pth, 'rb') as fh: + contents = fh.read() + keys[os.getlogin()] = [contents, ''] + + return keys + + class MetaDataHandler(object): def __init__(self, opts): @@ -165,7 +198,7 @@ class MetaDataHandler(object): if action == 'instance-id': return 'i-%s' % (id_generator(lower=True)) elif action == 'ami-launch-index': - return "%s" % random.choice([0,1,2,3]) + return "%s" % random.choice([0, 1, 2, 3]) elif action == 'aki-id': return 'aki-%s' % (id_generator(lower=True)) elif action == 'ami-id': @@ -175,11 +208,15 @@ class MetaDataHandler(object): elif action == 'block-device-mapping': nparams = params[1:] if not nparams: - devs = sorted(BLOCK_DEVS) - return "\n".join(devs) + return "\n".join(BLOCK_DEVS) else: - return "%s" % (DEV_MAPPINGS.get(nparams[0].strip(), '')) + subvalue = traverse(nparams, DEV_MAPPINGS) + if not subvalue: + return "\n".join(sorted(list(DEV_MAPPINGS.keys()))) + else: + return str(subvalue) elif action in ['hostname', 'local-hostname', 'public-hostname']: + # Just echo back there own hostname that they called in on.. return "%s" % (who) elif action == 'instance-type': return random.choice(INSTANCE_TYPES) @@ -188,21 +225,23 @@ class MetaDataHandler(object): elif action == 'security-groups': return 'default' elif action in ['local-ipv4', 'public-ipv4']: - there_ip = kwargs.get('client_ip', '10.0.0.1') - return "%s" % (there_ip) + # Just echo back there own ip that they called in on... + return "%s" % (kwargs.get('client_ip', '10.0.0.1')) elif action == 'reservation-id': return "r-%s" % (id_generator(lower=True)) elif action == 'product-codes': return "%s" % (id_generator(size=8)) elif action == 'public-keys': nparams = params[1:] - # public-keys is messed up. a list of /latest/meta-data/public-keys/ + # This is a weird kludge, why amazon why!!! + # public-keys is messed up, a list of /latest/meta-data/public-keys/ # shows something like: '0=brickies' # but a GET to /latest/meta-data/public-keys/0=brickies will fail # you have to know to get '/latest/meta-data/public-keys/0', then # from there you get a 'openssh-key', which you can get. # this hunk of code just re-works the object for that. - key_ids = sorted(list(PUB_KEYS.keys())) + avail_keys = get_ssh_keys() + key_ids = sorted(list(avail_keys.keys())) if nparams: mybe_key = nparams[0] try: @@ -211,21 +250,15 @@ class MetaDataHandler(object): except: raise WebException(httplib.BAD_REQUEST, "Unknown key id %r" % mybe_key) # Extract the possible sub-params - key_info = { - "openssh-key": "\n".join(PUB_KEYS[key_name]), - } - result = dict(key_info) - for k in nparams[1:]: - try: - result = result.get(k) - except (AttributeError, TypeError): - result = None - break + result = traverse(nparams[1:], { + "openssh-key": "\n".join(avail_keys[key_name]), + }) if isinstance(result, (dict)): - result = json.dumps(result) - if result is None: + # TODO: This might not be right?? + result = "\n".join(sorted(result.keys())) + if not result: result = '' - return str(result) + return result else: contents = [] for (i, key_id) in enumerate(key_ids): @@ -286,12 +319,7 @@ user_fetcher = None class Ec2Handler(BaseHTTPRequestHandler): def _get_versions(self): - versions = [] - for v in EC2_VERSIONS: - if v == 'latest': - continue - else: - versions.append(v) + versions = ['latest'] + EC2_VERSIONS versions = sorted(versions) return "\n".join(versions) @@ -308,11 +336,12 @@ class Ec2Handler(BaseHTTPRequestHandler): 'meta-data': meta_fetcher.get_data, } segments = [piece for piece in path.split('/') if len(piece)] + log.info("Received segments %s", segments) if not segments: return self._get_versions date = segments[0].strip().lower() - if date not in EC2_VERSIONS: - raise WebException(httplib.BAD_REQUEST, "Unknown date format %r" % date) + if date not in self._get_versions(): + raise WebException(httplib.BAD_REQUEST, "Unknown version format %r" % date) if len(segments) < 2: raise WebException(httplib.BAD_REQUEST, "No action provided") look_name = segments[1].lower() -- cgit v1.2.3 From 32ea55364e7688110646818dd651ed476b3c57f1 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Tue, 3 Jul 2012 20:14:57 -0700 Subject: When the hostname is also the ip (thus no hostname) just use 'localhost' --- tools/mock-meta.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) (limited to 'tools') diff --git a/tools/mock-meta.py b/tools/mock-meta.py index 5bbe62cc..247cff14 100755 --- a/tools/mock-meta.py +++ b/tools/mock-meta.py @@ -349,10 +349,14 @@ class Ec2Handler(BaseHTTPRequestHandler): raise WebException(httplib.BAD_REQUEST, "Unknown requested data %r" % look_name) base_func = func_mapping[look_name] who = self.address_string() + ip_from = self.client_address[0] + if who == ip_from: + # Nothing resolved, so just use 'localhost' + who = 'localhost' kwargs = { 'params': list(segments[2:]), - 'who': self.address_string(), - 'client_ip': self.client_address[0], + 'who': who, + 'client_ip': ip_from, } return functools.partial(base_func, **kwargs) -- cgit v1.2.3 From 3cd17b470b70cf4c15f47bfe4f84844b078cd914 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Tue, 3 Jul 2012 20:29:36 -0700 Subject: Remove the useless function printout. --- tools/mock-meta.py | 1 - 1 file changed, 1 deletion(-) (limited to 'tools') diff --git a/tools/mock-meta.py b/tools/mock-meta.py index 247cff14..4548e4ae 100755 --- a/tools/mock-meta.py +++ b/tools/mock-meta.py @@ -365,7 +365,6 @@ class Ec2Handler(BaseHTTPRequestHandler): log.info("Got a call from %s for path %s", who, self.path) try: func = self._find_method(self.path) - log.info("Calling into func %s to get your data.", func) data = func() if not data: data = '' -- cgit v1.2.3