Update build tools to work with git

- Update HACKING.rst to include git instructions
- update MANIFEST.in and .gitignore to ignore git-related things
- replaced tarball generation scripts with git-based script
- have the spec files correctly identify themselves as cheetah templates
- make brpm work with git

5 files changed, 138 insertions, 205 deletions

diff --git a/packages/bddeb b/packages/bddeb
index 3c77ce1d..46c07c88 100755
--- a/packages/bddeb
+++ b/packages/bddeb
@@ -180,13 +180,10 @@ def main():
 
     with util.tempdir() as tdir:
 
-        cmd = [util.abs_join(find_root(), 'tools', 'read-version')]
+        # output like 0.7.6-1022-g36e92d3
+        cmd = ['git', 'describe', '--long']
         (sysout, _stderr) = util.subp(cmd)
-        version = sysout.strip()
-
-        cmd = ['bzr', 'revno']
-        (sysout, _stderr) = util.subp(cmd)
-        revno = sysout.strip()
+        version, extra = sysout.strip().split("-", 1)
 
         # This is really only a temporary archive
         # since we will extract it then add in the debian
@@ -212,12 +209,12 @@ def main():
             append_requires=['cloud-utils | cloud-guest-utils']
         else:
             append_requires=[]
-        write_debian_folder(xdir, version, revno, pkgmap,
+        write_debian_folder(xdir, version, extra, pkgmap,
                             pyver=pyver, append_requires=append_requires)
 
         # The naming here seems to follow some debian standard
         # so it will whine if it is changed...
-        tar_fn = "cloud-init_%s~bzr%s.orig.tar.gz" % (version, revno)
+        tar_fn = "cloud-init_%s+%s~bddeb.orig.tar.gz" % (version, extra)
         print("Archiving the adjusted source into %r" %
               (util.abs_join(tdir, tar_fn)))
         cmd = ['tar', '-czvf',
diff --git a/packages/brpm b/packages/brpm
index 45e47610..5d16eb71 100755
--- a/packages/brpm
+++ b/packages/brpm
@@ -1,27 +1,24 @@
-#!/usr/bin/python
+#!/usr/bin/env python
 
 import argparse
-import contextlib
 import glob
 import os
 import shutil
-import subprocess
 import sys
 import tempfile
-import re
-
-from datetime import datetime
+import time
 
 
 def find_root():
     # expected path is in <top_dir>/packages/
     top_dir = os.environ.get("CLOUD_INIT_TOP_D", None)
     if top_dir is None:
-       top_dir = os.path.dirname(os.path.dirname(os.path.abspath(sys.argv[0])))
+        top_dir = os.path.dirname(
+            os.path.dirname(os.path.abspath(sys.argv[0])))
     if os.path.isfile(os.path.join(top_dir, 'setup.py')):
         return os.path.abspath(top_dir)
     raise OSError(("Unable to determine where your cloud-init topdir is."
-                       " set CLOUD_INIT_TOP_D?"))
+                   " set CLOUD_INIT_TOP_D?"))
 
 
 # Use the util functions from cloudinit
@@ -30,34 +27,15 @@ sys.path.insert(0, find_root())
 from cloudinit import templater
 from cloudinit import util
 
-# Mapping of expected packages to there full name...
-# this is a translation of the 'requires'
-# file pypi package name to a redhat/fedora package name.
-PKG_MP = {
+# Map python requirements to package names.  If a match isn't found
+# here, we assume 'python-<pypi_name>'.
+PACKAGE_MAP = {
     'redhat': {
-        'argparse': 'python-argparse',
-        'cheetah': 'python-cheetah',
-        'jinja2': 'python-jinja2',
-        'configobj': 'python-configobj',
-        'jsonpatch': 'python-jsonpatch',
-        'oauthlib': 'python-oauthlib',
-        'prettytable': 'python-prettytable',
         'pyserial': 'pyserial',
         'pyyaml': 'PyYAML',
-        'requests': 'python-requests',
-        'six': 'python-six',
     },
     'suse': {
-        'argparse': 'python-argparse',
-        'cheetah': 'python-cheetah',
-        'configobj': 'python-configobj',
-        'jsonpatch': 'python-jsonpatch',
-        'oauthlib': 'python-oauthlib',
-        'prettytable': 'python-prettytable',
-        'pyserial': 'python-pyserial',
         'pyyaml': 'python-yaml',
-        'requests': 'python-requests',
-        'six': 'python-six',
     }
 }
 
@@ -65,113 +43,79 @@ PKG_MP = {
 RPM_BUILD_SUBDIRS = ['BUILD', 'RPMS', 'SOURCES', 'SPECS', 'SRPMS']
 
 
-def get_log_header(version):
-    # Try to find the version in the tags output
-    cmd = ['bzr', 'tags']
-    (stdout, _stderr) = util.subp(cmd)
-    a_rev = None
-    for t in stdout.splitlines():
-        ver, rev = t.split(None)
-        if ver == version:
-            a_rev = rev
-            break
-    if not a_rev:
-        return None
-    
-    # Extract who made that tag as the header
-    cmd = ['bzr', 'log', '-r%s' % (a_rev), '--timezone=utc']
+def read_dependencies():
+    '''Returns the Python depedencies from requirements.txt.  This explicitly
+    removes 'argparse' from the list of requirements for python >= 2.7,
+    because with 2.7 argparse became part of the standard library.'''
+    cmd = [util.abs_join(find_root(), 'tools', 'read-dependencies')]
     (stdout, _stderr) = util.subp(cmd)
-    kvs = {
-        'comment': version,
-    }
+    return [p.lower().strip() for p in stdout.splitlines()
+            if p != 'argparse' or (p == 'argparse'
+                                   and sys.version_info[0:2] < (2, 7))]
 
-    for line in stdout.splitlines():
-        if line.startswith('committer:'):
-            kvs['who'] = line[len('committer:'):].strip()
-        if line.startswith('timestamp:'):
-            ts = line[len('timestamp:'):]
-            ts = ts.strip()
-            # http://bugs.python.org/issue6641
-            ts = ts.replace("+0000", '').strip()
-            ds = datetime.strptime(ts, '%a %Y-%m-%d %H:%M:%S')
-            kvs['ds'] = ds
 
-    return format_change_line(**kvs)
+def translate_dependencies(deps, distro):
+    '''Maps python requirements into package names.  We assume
+    python-<pypi_name> for packages not listed explicitly in
+    PACKAGE_MAP.'''
+    return [PACKAGE_MAP[distro][req]
+            if req in PACKAGE_MAP[distro] else 'python-%s' % req
+            for req in deps]
 
 
-def format_change_line(ds, who, comment=None):
-    # Rpmbuild seems to be pretty strict about the date format
-    d = ds.strftime("%a %b %d %Y")
-    d += " - %s" % (who)
-    if comment:
-        d += " - %s" % (comment)
-    return "* %s" % (d)
-
-
-def generate_spec_contents(args, tmpl_fn, top_dir, arc_fn):
-
+def read_version():
+    '''Read version information.  We parse the version itself from
+    the changelog, and then ask git for the commit id and distance
+    from the last tag.'''
     # Figure out the version and revno
     cmd = [util.abs_join(find_root(), 'tools', 'read-version')]
     (stdout, _stderr) = util.subp(cmd)
     version = stdout.strip()
-    
-    cmd = ['bzr', 'revno']
+
+    cmd = ['git', 'describe', '--tags']
     (stdout, _stderr) = util.subp(cmd)
-    revno = stdout.strip()
+    git_version = stdout.strip()
+
+    try:
+        _version, distance, revno = git_version.split('-')
+    except ValueError:
+        distance = None
+        revno = None
+
+    return (version, distance, revno)
+
+
+def generate_spec_contents(args, tmpl_fn, top_dir, arc_fn):
+
+    # This will get us something like ('0.7.6', None, None) for a
+    # tagged commit, and something like ('0.7.6', '1026', 'gd1d5796')
+    # for an untagged commited.
+    version, distance, revno = read_version()
 
     # Tmpl params
     subs = {}
     subs['version'] = version
     subs['revno'] = revno
-    subs['release'] = "bzr%s" % (revno)
+    subs['distance'] = distance
+
+    if distance is not None:
+        now = time.strftime('%Y%m%d', time.localtime())
+        release = '.%sgit%s' % (now, revno)
+    else:
+        release = ''
+
     if args.sub_release is not None:
-        subs['subrelease'] = "." + str(args.sub_release)
+        subs['subrelease'] = release + "." + str(args.sub_release)
     else:
-        subs['subrelease'] = ''
-    subs['archive_name'] = arc_fn
+        subs['subrelease'] = release
 
-    cmd = [util.abs_join(find_root(), 'tools', 'read-dependencies')]
-    (stdout, _stderr) = util.subp(cmd)
-    pkgs = [p.lower().strip() for p in stdout.splitlines()]
+    subs['archive_name'] = arc_fn
+    subs['source_name'] = os.path.basename(arc_fn).replace('.tar.gz', '')
 
     # Map to known packages
-    requires = []
-    for p in pkgs:
-        if p == 'argparse' and sys.version_info[0:2] >= (2, 7):
-            # Not needed on anything but 2.6 or older.
-            continue
-        tgt_pkg = PKG_MP[args.distro].get(p)
-        if not tgt_pkg:
-            raise RuntimeError(("Do not know how to translate pypi dependency"
-                                " %r to a known package") % (p))
-        else:
-            requires.append(tgt_pkg)
-    subs['requires'] = requires
-
-    # Format a nice changelog (as best as we can)
-    changelog = util.load_file(util.abs_join(find_root(), 'ChangeLog'))
-    changelog_lines = []
-    missing_versions = 0
-    for line in changelog.splitlines():
-        if not line.strip():
-            continue
-        if re.match(r"^\s*[\d][.][\d][.][\d]:\s*", line):
-            line = line.strip(":")
-            header = get_log_header(line)
-            if not header:
-                missing_versions += 1
-                if missing_versions == 1:
-                    # Must be using a new 'dev'/'trunk' release
-                    changelog_lines.append(format_change_line(datetime.now(),
-                                                              '??'))
-                else:
-                    sys.stderr.write(("Changelog version line %s does not "
-                                      "have a corresponding tag!\n") % (line))
-            else:
-                changelog_lines.append(header)
-        else:
-            changelog_lines.append(line)
-    subs['changelog'] = "\n".join(changelog_lines)
+    python_deps = read_dependencies()
+    package_deps = translate_dependencies(python_deps, args.distro)
+    subs['requires'] = package_deps
 
     if args.boot == 'sysvinit':
         subs['sysvinit'] = True
@@ -183,21 +127,23 @@ def generate_spec_contents(args, tmpl_fn, top_dir, arc_fn):
     else:
         subs['systemd'] = False
 
-    subs['defines'] = ["_topdir %s" % (top_dir)]
     subs['init_sys'] = args.boot
     subs['patches'] = [os.path.basename(p) for p in args.patches]
     return templater.render_from_file(tmpl_fn, params=subs)
 
 
 def main():
-    
+
     parser = argparse.ArgumentParser()
     parser.add_argument("-d", "--distro", dest="distro",
                         help="select distro (default: %(default)s)",
                         metavar="DISTRO", default='redhat',
                         choices=('redhat', 'suse'))
+    parser.add_argument('--srpm',
+                        help='Produce a source rpm',
+                        action='store_true')
     parser.add_argument("-b", "--boot", dest="boot",
-                        help="select boot type (default: %(default)s)", 
+                        help="select boot type (default: %(default)s)",
                         metavar="TYPE", default='sysvinit',
                         choices=('sysvinit', 'systemd'))
     parser.add_argument("-v", "--verbose", dest="verbose",
@@ -221,57 +167,63 @@ def main():
     if args.verbose:
         capture = False
 
-    # Clean out the root dir and make sure the dirs we want are in place
-    root_dir = os.path.expanduser("~/rpmbuild")
-    if os.path.isdir(root_dir):
-        shutil.rmtree(root_dir)
-
-    arc_dir = util.abs_join(root_dir, 'SOURCES')
-    build_dirs = [root_dir, arc_dir]
-    for dname in RPM_BUILD_SUBDIRS:
-        build_dirs.append(util.abs_join(root_dir, dname))
-    build_dirs.sort()
-    util.ensure_dirs(build_dirs)
-
-    # Archive the code
-    cmd = [util.abs_join(find_root(), 'tools', 'make-tarball')]
-    (stdout, _stderr) = util.subp(cmd)
-    archive_fn = stdout.strip()
-    real_archive_fn = os.path.join(arc_dir, os.path.basename(archive_fn))
-    shutil.move(archive_fn, real_archive_fn)
-    print("Archived the code in %r" % (real_archive_fn))
-
-    # Form the spec file to be used
-    tmpl_fn = util.abs_join(find_root(), 'packages',
-                            args.distro, 'cloud-init.spec.in')
-    contents = generate_spec_contents(args, tmpl_fn, root_dir,
-                                      os.path.basename(archive_fn))
-    spec_fn = util.abs_join(root_dir, 'cloud-init.spec')
-    util.write_file(spec_fn, contents)
-    print("Created spec file at %r" % (spec_fn))
-    print(contents)
-    for p in args.patches:
-        util.copy(p, util.abs_join(arc_dir, os.path.basename(p)))
-
-    # Now build it!
-    print("Running 'rpmbuild' in %r" % (root_dir))
-    cmd = ['rpmbuild', '-ba', spec_fn]
-    util.subp(cmd, capture=capture)
-
-    # Copy the items built to our local dir
-    globs = [] 
-    globs.extend(glob.glob("%s/*.rpm" %
-                           (util.abs_join(root_dir, 'RPMS', 'noarch'))))
-    globs.extend(glob.glob("%s/*.rpm" %
-                           (util.abs_join(root_dir, 'RPMS', 'x86_64'))))
-    globs.extend(glob.glob("%s/*.rpm" %
-                           (util.abs_join(root_dir, 'RPMS'))))
-    globs.extend(glob.glob("%s/*.rpm" %
-                           (util.abs_join(root_dir, 'SRPMS'))))
-    for rpm_fn in globs:
-        tgt_fn = util.abs_join(os.getcwd(), os.path.basename(rpm_fn))
-        shutil.move(rpm_fn, tgt_fn)
-        print("Wrote out %s package %r" % (args.distro, tgt_fn))
+    workdir = None
+    try:
+        workdir = tempfile.mkdtemp(prefix='rpmbuild')
+        os.environ['HOME'] = workdir
+        topdir = os.path.join(workdir, 'rpmbuild')
+        build_dirs = [os.path.join(topdir, dir)
+                      for dir in RPM_BUILD_SUBDIRS]
+        util.ensure_dirs(build_dirs)
+
+        # Archive the code
+        cmd = [util.abs_join(find_root(), 'tools', 'make-tarball')]
+        (stdout, _stderr) = util.subp(cmd)
+        archive_fn = stdout.strip()
+        print "Archived source as %s" % archive_fn
+        real_archive_fn = os.path.join(topdir, 'SOURCES',
+                                       os.path.basename(archive_fn))
+        shutil.move(archive_fn, real_archive_fn)
+        print("Archived the code in %r" % (real_archive_fn))
+
+        # Form the spec file to be used
+        tmpl_fn = util.abs_join(find_root(), 'packages',
+                                args.distro, 'cloud-init.spec.in')
+        contents = generate_spec_contents(args, tmpl_fn, topdir,
+                                          os.path.basename(archive_fn))
+        spec_fn = util.abs_join(topdir, 'SPECS', 'cloud-init.spec')
+        util.write_file(spec_fn, contents)
+        print("Created spec file at %r" % (spec_fn))
+        for p in args.patches:
+            util.copy(p, util.abs_join(topdir, 'SOURCES', os.path.basename(p)))
+
+        # Now build it!
+        print("Running 'rpmbuild' in %r" % (topdir))
+
+        if args.srpm:
+            cmd = ['rpmbuild', '-bs', '--nodeps', spec_fn]
+        else:
+            cmd = ['rpmbuild', '-ba', spec_fn]
+
+        util.subp(cmd, capture=capture)
+
+        # Copy the items built to our local dir
+        globs = []
+        globs.extend(glob.glob("%s/*.rpm" %
+                               (util.abs_join(topdir, 'RPMS', 'noarch'))))
+        globs.extend(glob.glob("%s/*.rpm" %
+                               (util.abs_join(topdir, 'RPMS', 'x86_64'))))
+        globs.extend(glob.glob("%s/*.rpm" %
+                               (util.abs_join(topdir, 'RPMS'))))
+        globs.extend(glob.glob("%s/*.rpm" %
+                               (util.abs_join(topdir, 'SRPMS'))))
+        for rpm_fn in globs:
+            tgt_fn = util.abs_join(os.getcwd(), os.path.basename(rpm_fn))
+            shutil.move(rpm_fn, tgt_fn)
+            print("Wrote out %s package %r" % (args.distro, tgt_fn))
+    finally:
+        if workdir is not None:
+            shutil.rmtree(workdir)
 
     return 0
 
diff --git a/packages/debian/changelog.in b/packages/debian/changelog.in
index c9affe47..544d23cf 100644
--- a/packages/debian/changelog.in
+++ b/packages/debian/changelog.in
@@ -1,5 +1,5 @@
 ## template:basic
-cloud-init (${version}~bzr${revision}-1) UNRELEASED; urgency=low
+cloud-init (${version}+${revision}~bddeb-1) UNRELEASED; urgency=low
 
   * build
 
diff --git a/packages/redhat/cloud-init.spec.in b/packages/redhat/cloud-init.spec.in
index 254d209b..c30d33ca 100644
--- a/packages/redhat/cloud-init.spec.in
+++ b/packages/redhat/cloud-init.spec.in
@@ -1,17 +1,13 @@
-## This is a cheetah template
+## template: cheetah
 %{!?python_sitelib: %global python_sitelib %(%{__python} -c "from distutils.sysconfig import get_python_lib; print get_python_lib()")}
 
 # See: http://www.zarb.org/~jasonc/macros.php
 # Or: http://fedoraproject.org/wiki/Packaging:ScriptletSnippets
 # Or: http://www.rpm.org/max-rpm/ch-rpm-inside.html
 
-#for $d in $defines
-%define ${d}
-#end for
-
 Name:           cloud-init
 Version:        ${version}
-Release:        ${release}${subrelease}%{?dist}
+Release:        1${subrelease}%{?dist}
 Summary:        Cloud instance init scripts
 
 Group:          System Environment/Base
@@ -22,9 +18,9 @@ Source0:        ${archive_name}
 BuildArch:      noarch
 BuildRoot:      %{_tmppath}
 
-BuildRequires:        python-devel
-BuildRequires:        python-setuptools
-BuildRequires:        python-cheetah
+BuildRequires:  python-devel
+BuildRequires:  python-setuptools
+BuildRequires:  python-cheetah
 
 # System util packages needed
 Requires:       shadow-utils
@@ -68,7 +64,7 @@ need special scripts to run during initialization to retrieve and install
 ssh keys and to let the user run various scripts.
 
 %prep
-%setup -q -n %{name}-%{version}~${release}
+%setup -q -n ${source_name}
 
 # Custom patches activation
 #set $size = 0
@@ -198,7 +194,3 @@ fi
 
 # Python code is here...
 %{python_sitelib}/*
-
-%changelog
-
-${changelog}
diff --git a/packages/suse/cloud-init.spec.in b/packages/suse/cloud-init.spec.in
index 53e6ad13..f994a0cf 100644
--- a/packages/suse/cloud-init.spec.in
+++ b/packages/suse/cloud-init.spec.in
@@ -1,16 +1,12 @@
-## This is a cheetah template
+## template: cheetah
 
 # See: http://www.zarb.org/~jasonc/macros.php
 # Or: http://fedoraproject.org/wiki/Packaging:ScriptletSnippets
 # Or: http://www.rpm.org/max-rpm/ch-rpm-inside.html
 
-#for $d in $defines
-%define ${d}
-#end for
-
 Name:           cloud-init
 Version:        ${version}
-Release:        ${release}${subrelease}%{?dist}
+Release:        1${subrelease}%{?dist}
 Summary:        Cloud instance init scripts
 
 Group:          System/Management
@@ -63,7 +59,7 @@ need special scripts to run during initialization to retrieve and install
 ssh keys and to let the user run various scripts.
 
 %prep
-%setup -q -n %{name}-%{version}~${release}
+%setup -q -n ${source_name}
 
 # Custom patches activation
 #set $size = 0
@@ -157,7 +153,3 @@ mkdir -p %{buildroot}/var/lib/cloud
 %{python_sitelib}/*
 
 /var/lib/cloud
-
-%changelog
-
-${changelog}