adjust tools and version information.

upstream snapshots are versioned in the format 'X.Y.Z+<distance>.g<commit>'
where X.Y.Z are major, minor, and micro.  Distance is number of commits
since last annotated tag, and commit is the git commit.

bddeb and brpm will now create and use the "upstream version" like above.

Things changed here:
 - tools/make-tarball
   update cloudinit/version.py to contain the full version
   support --output
   support '--long' to always create the long format version string.

 - bddeb:
   - use quilt debian source format
   - use read-version and long version in changelog.

 - brpm:
   - change to use read-version and upstream long version in the spec.
   - flake8 changes

 - tools/read-version
   - read version from git or from cloudinit/version.
   - provide --json output with more nicely formed data.

8 files changed, 205 insertions, 150 deletions

diff --git a/cloudinit/version.py b/cloudinit/version.py
index 3d1d1d23..01785eb8 100644
--- a/cloudinit/version.py
+++ b/cloudinit/version.py
@@ -16,12 +16,17 @@
 #    You should have received a copy of the GNU General Public License
 #    along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-from distutils import version as vr
+__VERSION__ = "0.7.6"
+__EXPORT_VERSION__ = "@@EXPORT_VERSION@@"
 
 
-def version():
-    return vr.StrictVersion("0.7.7")
+def version_string():
+    if not __EXPORT_VERSION__.startswith("@@"):
+        return __EXPORT_VERSION__
+    return __VERSION__
 
 
-def version_string():
-    return str(version())
+def full_version_string():
+    if __EXPORT_VERSION__.startswith("@@"):
+        raise ValueError("No full version available")
+    return __EXPORT_VERSION__
diff --git a/packages/bddeb b/packages/bddeb
index 46c07c88..94496c74 100755
--- a/packages/bddeb
+++ b/packages/bddeb
@@ -1,6 +1,7 @@
 #!/usr/bin/env python3
 
-import glob
+import argparse
+import json
 import os
 import shutil
 import sys
@@ -15,15 +16,13 @@ def find_root():
     if os.path.isfile(os.path.join(top_dir, 'setup.py')):
         return os.path.abspath(top_dir)
     raise OSError(("Unable to determine where your cloud-init topdir is."
-                       " set CLOUD_INIT_TOP_D?"))
-
-# Use the util functions from cloudinit
-sys.path.insert(0, find_root())
+                   " set CLOUD_INIT_TOP_D?"))
 
-from cloudinit import templater
-from cloudinit import util
-
-import argparse
+if "avoid-pep8-E402-import-not-top-of-file":
+    # Use the util functions from cloudinit
+    sys.path.insert(0, find_root())
+    from cloudinit import templater
+    from cloudinit import util
 
 # Package names that will showup in requires to what we can actually
 # use in our debian 'control' file, this is a translation of the 'requires'
@@ -58,27 +57,37 @@ NONSTD_NAMED_PACKAGES = {
 DEBUILD_ARGS = ["-S", "-d"]
 
 
-def write_debian_folder(root, version, revno, pkgmap,
-                        pyver="3", append_requires=[]):
+def run_helper(helper, args=None, strip=True):
+    if args is None:
+        args = []
+    cmd = [util.abs_join(find_root(), 'tools', helper)] + args
+    (stdout, _stderr) = util.subp(cmd)
+    if strip:
+        stdout = stdout.strip()
+    return stdout
+
+
+def write_debian_folder(root, version_data, pkgmap, pyver="3",
+                        append_requires=[]):
     deb_dir = util.abs_join(root, 'debian')
-    os.makedirs(deb_dir)
+
+    # Just copy debian/ dir and then update files
+    pdeb_d = util.abs_join(find_root(), 'packages', 'debian')
+    util.subp(['cp', '-a', pdeb_d, deb_dir])
 
     # Fill in the change log template
     templater.render_to_file(util.abs_join(find_root(),
                              'packages', 'debian', 'changelog.in'),
                              util.abs_join(deb_dir, 'changelog'),
-                             params={
-                                 'version': version,
-                                 'revision': revno,
-                             })
+                             params=version_data)
 
     # Write out the control file template
-    cmd = [util.abs_join(find_root(), 'tools', 'read-dependencies')]
-    (stdout, _stderr) = util.subp(cmd)
-    pypi_pkgs = [p.lower().strip() for p in stdout.splitlines()]
+    reqs = run_helper('read-dependencies').splitlines()
+    test_reqs = run_helper(
+        'read-dependencies', ['test-requirements.txt']).splitlines()
 
-    (stdout, _stderr) = util.subp(cmd + ['test-requirements.txt'])
-    pypi_test_pkgs = [p.lower().strip() for p in stdout.splitlines()]
+    pypi_pkgs = [p.lower().strip() for p in reqs]
+    pypi_test_pkgs = [p.lower().strip() for p in test_reqs]
 
     # Map to known packages
     requires = append_requires
@@ -109,11 +118,9 @@ def write_debian_folder(root, version, revno, pkgmap,
                              util.abs_join(deb_dir, 'rules'),
                              params={'python': python, 'pyver': pyver})
 
-    # Just copy any other files directly (including .in)
-    pdeb_d = util.abs_join(find_root(), 'packages', 'debian')
-    for f in [os.path.join(pdeb_d, f) for f in os.listdir(pdeb_d)]:
-        if os.path.isfile(f):
-            shutil.copy(f, util.abs_join(deb_dir, os.path.basename(f)))
+
+def read_version():
+    return json.loads(run_helper('read-version', ['--json']))
 
 
 def main():
@@ -140,11 +147,10 @@ def main():
                         default=os.environ.get("INIT_SYSTEM",
                                                "upstart,systemd"))
 
-
     for ent in DEBUILD_ARGS:
         parser.add_argument(ent, dest="debuild_args", action='append_const',
-            const=ent, help=("pass through '%s' to debuild" % ent),
-            default=[])
+                            const=ent, default=[],
+                            help=("pass through '%s' to debuild" % ent))
 
     parser.add_argument("--sign", default=False, action='store_true',
                         help="sign result. do not pass -us -uc to debuild")
@@ -181,54 +187,30 @@ def main():
     with util.tempdir() as tdir:
 
         # output like 0.7.6-1022-g36e92d3
-        cmd = ['git', 'describe', '--long']
-        (sysout, _stderr) = util.subp(cmd)
-        version, extra = sysout.strip().split("-", 1)
+        ver_data = read_version()
 
         # This is really only a temporary archive
         # since we will extract it then add in the debian
         # folder, then re-archive it for debian happiness
         print("Creating a temporary tarball using the 'make-tarball' helper")
-        cmd = [util.abs_join(find_root(), 'tools', 'make-tarball')]
-        (sysout, _stderr) = util.subp(cmd)
-        arch_fn = sysout.strip()
-        tmp_arch_fn = util.abs_join(tdir, os.path.basename(arch_fn))
-        shutil.move(arch_fn, tmp_arch_fn)
-
-        print("Extracting temporary tarball %r" % (tmp_arch_fn))
-        cmd = ['tar', '-xvzf', tmp_arch_fn, '-C', tdir]
+        tarball = "cloud-init_%s.orig.tar.gz" % ver_data['version_long']
+        tarball_fp = util.abs_join(tdir, tarball)
+        run_helper('make-tarball', ['--long', '--output=' + tarball_fp])
+
+        print("Extracting temporary tarball %r" % (tarball))
+        cmd = ['tar', '-xvzf', tarball_fp, '-C', tdir]
         util.subp(cmd, capture=capture)
-        extracted_name = tmp_arch_fn[:-len('.tar.gz')]
-        os.remove(tmp_arch_fn)
 
-        xdir = util.abs_join(tdir, 'cloud-init')
-        shutil.move(extracted_name, xdir)
+        xdir = util.abs_join(tdir, "cloud-init-%s" % ver_data['version_long'])
 
         print("Creating a debian/ folder in %r" % (xdir))
         if args.cloud_utils:
-            append_requires=['cloud-utils | cloud-guest-utils']
+            append_requires = ['cloud-utils | cloud-guest-utils']
         else:
-            append_requires=[]
-        write_debian_folder(xdir, version, extra, pkgmap,
+            append_requires = []
+        write_debian_folder(xdir, ver_data, pkgmap,
                             pyver=pyver, append_requires=append_requires)
 
-        # The naming here seems to follow some debian standard
-        # so it will whine if it is changed...
-        tar_fn = "cloud-init_%s+%s~bddeb.orig.tar.gz" % (version, extra)
-        print("Archiving the adjusted source into %r" %
-              (util.abs_join(tdir, tar_fn)))
-        cmd = ['tar', '-czvf',
-               util.abs_join(tdir, tar_fn),
-               '-C', xdir]
-        cmd.extend(os.listdir(xdir))
-        util.subp(cmd, capture=capture)
-
-        # Copy it locally for reference
-        shutil.copy(util.abs_join(tdir, tar_fn),
-                    util.abs_join(os.getcwd(), tar_fn))
-        print("Copied that archive to %r for local usage (if desired)." %
-              (util.abs_join(os.getcwd(), tar_fn)))
-
         print("Running 'debuild %s' in %r" % (' '.join(args.debuild_args),
                                               xdir))
         with util.chdir(xdir):
diff --git a/packages/brpm b/packages/brpm
index 5d16eb71..14d75f2b 100755
--- a/packages/brpm
+++ b/packages/brpm
@@ -2,11 +2,11 @@
 
 import argparse
 import glob
+import json
 import os
 import shutil
 import sys
 import tempfile
-import time
 
 
 def find_root():
@@ -21,11 +21,11 @@ def find_root():
                    " set CLOUD_INIT_TOP_D?"))
 
 
-# Use the util functions from cloudinit
-sys.path.insert(0, find_root())
-
-from cloudinit import templater
-from cloudinit import util
+if "avoid-pep8-E402-import-not-top-of-file":
+    # Use the util functions from cloudinit
+    sys.path.insert(0, find_root())
+    from cloudinit import templater
+    from cloudinit import util
 
 # Map python requirements to package names.  If a match isn't found
 # here, we assume 'python-<pypi_name>'.
@@ -43,15 +43,24 @@ PACKAGE_MAP = {
 RPM_BUILD_SUBDIRS = ['BUILD', 'RPMS', 'SOURCES', 'SPECS', 'SRPMS']
 
 
+def run_helper(helper, args=None, strip=True):
+    if args is None:
+        args = []
+    cmd = [util.abs_join(find_root(), 'tools', helper)] + args
+    (stdout, _stderr) = util.subp(cmd)
+    if strip:
+        stdout = stdout.strip()
+    return stdout
+
+
 def read_dependencies():
     '''Returns the Python depedencies from requirements.txt.  This explicitly
     removes 'argparse' from the list of requirements for python >= 2.7,
     because with 2.7 argparse became part of the standard library.'''
-    cmd = [util.abs_join(find_root(), 'tools', 'read-dependencies')]
-    (stdout, _stderr) = util.subp(cmd)
+    stdout = run_helper('read-dependencies')
     return [p.lower().strip() for p in stdout.splitlines()
-            if p != 'argparse' or (p == 'argparse'
-                                   and sys.version_info[0:2] < (2, 7))]
+            if p != 'argparse' or (p == 'argparse' and
+                                   sys.version_info[0:2] < (2, 7))]
 
 
 def translate_dependencies(deps, distro):
@@ -64,53 +73,22 @@ def translate_dependencies(deps, distro):
 
 
 def read_version():
-    '''Read version information.  We parse the version itself from
-    the changelog, and then ask git for the commit id and distance
-    from the last tag.'''
-    # Figure out the version and revno
-    cmd = [util.abs_join(find_root(), 'tools', 'read-version')]
-    (stdout, _stderr) = util.subp(cmd)
-    version = stdout.strip()
+    return json.loads(run_helper('read-version', ['--json']))
 
-    cmd = ['git', 'describe', '--tags']
-    (stdout, _stderr) = util.subp(cmd)
-    git_version = stdout.strip()
 
-    try:
-        _version, distance, revno = git_version.split('-')
-    except ValueError:
-        distance = None
-        revno = None
-
-    return (version, distance, revno)
-
-
-def generate_spec_contents(args, tmpl_fn, top_dir, arc_fn):
-
-    # This will get us something like ('0.7.6', None, None) for a
-    # tagged commit, and something like ('0.7.6', '1026', 'gd1d5796')
-    # for an untagged commited.
-    version, distance, revno = read_version()
+def generate_spec_contents(args, version_data, tmpl_fn, top_dir, arc_fn):
 
     # Tmpl params
     subs = {}
-    subs['version'] = version
-    subs['revno'] = revno
-    subs['distance'] = distance
-
-    if distance is not None:
-        now = time.strftime('%Y%m%d', time.localtime())
-        release = '.%sgit%s' % (now, revno)
-    else:
-        release = ''
 
     if args.sub_release is not None:
-        subs['subrelease'] = release + "." + str(args.sub_release)
+        subs['subrelease'] = str(args.sub_release)
     else:
-        subs['subrelease'] = release
+        subs['subrelease'] = ""
 
     subs['archive_name'] = arc_fn
     subs['source_name'] = os.path.basename(arc_fn).replace('.tar.gz', '')
+    subs.update(version_data)
 
     # Map to known packages
     python_deps = read_dependencies()
@@ -176,20 +154,19 @@ def main():
                       for dir in RPM_BUILD_SUBDIRS]
         util.ensure_dirs(build_dirs)
 
+        version_data = read_version()
+
         # Archive the code
-        cmd = [util.abs_join(find_root(), 'tools', 'make-tarball')]
-        (stdout, _stderr) = util.subp(cmd)
-        archive_fn = stdout.strip()
-        print "Archived source as %s" % archive_fn
-        real_archive_fn = os.path.join(topdir, 'SOURCES',
-                                       os.path.basename(archive_fn))
-        shutil.move(archive_fn, real_archive_fn)
+        archive_fn = "cloud-init-%s.tar.gz" % version_data['version_long']
+        real_archive_fn = os.path.join(topdir, 'SOURCES', archive_fn)
+        archive_fn = run_helper(
+            'make-tarball', ['--long', '--output=' + real_archive_fn])
         print("Archived the code in %r" % (real_archive_fn))
 
         # Form the spec file to be used
         tmpl_fn = util.abs_join(find_root(), 'packages',
                                 args.distro, 'cloud-init.spec.in')
-        contents = generate_spec_contents(args, tmpl_fn, topdir,
+        contents = generate_spec_contents(args, version_data, tmpl_fn, topdir,
                                           os.path.basename(archive_fn))
         spec_fn = util.abs_join(topdir, 'SPECS', 'cloud-init.spec')
         util.write_file(spec_fn, contents)
diff --git a/packages/debian/changelog.in b/packages/debian/changelog.in
index 544d23cf..f8e98258 100644
--- a/packages/debian/changelog.in
+++ b/packages/debian/changelog.in
@@ -1,5 +1,5 @@
 ## template:basic
-cloud-init (${version}+${revision}~bddeb-1) UNRELEASED; urgency=low
+cloud-init (${version_long}-1~bddeb) UNRELEASED; urgency=low
 
   * build
 
diff --git a/packages/debian/source/format b/packages/debian/source/format
new file mode 100644
index 00000000..163aaf8d
--- /dev/null
+++ b/packages/debian/source/format
@@ -0,0 +1 @@
+3.0 (quilt)
diff --git a/packages/redhat/cloud-init.spec.in b/packages/redhat/cloud-init.spec.in
index c30d33ca..2f02f017 100644
--- a/packages/redhat/cloud-init.spec.in
+++ b/packages/redhat/cloud-init.spec.in
@@ -6,7 +6,7 @@
 # Or: http://www.rpm.org/max-rpm/ch-rpm-inside.html
 
 Name:           cloud-init
-Version:        ${version}
+Version:        ${version_long}
 Release:        1${subrelease}%{?dist}
 Summary:        Cloud instance init scripts
 
diff --git a/tools/make-tarball b/tools/make-tarball
index 4828a622..bd7399c1 100755
--- a/tools/make-tarball
+++ b/tools/make-tarball
@@ -1,16 +1,53 @@
 #!/bin/sh
 set -e
 
+TEMP_D=""
+cleanup() {
+    [ -z "$TEMP_D" ] || rm -Rf "${TEMP_D}"
+}
+trap cleanup EXIT
+
+Usage() {
+    cat <<EOF
+Usage: ${0##*/} [revision]
+    create a tarball of revision (default HEAD)
+
+    options:
+      -o | --output FILE   write to file
+EOF
+}
+
+short_opts="ho:v"
+long_opts="help,output:,long,verbose"
+getopt_out=$(getopt --name "${0##*/}" \
+    --options "${short_opts}" --long "${long_opts}" -- "$@") &&
+    eval set -- "${getopt_out}" || { Usage 1>&2; exit 1; }
+
+long_opt=""
+while [ $# -ne 0 ]; do
+    cur=$1; next=$2
+    case "$cur" in
+        -o|--output) output=$next; shift;;
+           --long) long_opt="--long";;
+        --) shift; break;;
+    esac
+    shift;
+done
+
 rev=${1:-HEAD}
-revname=$(git describe $rev)
+git_describe=$(git describe ${long_opt} $rev)
 
-# revname could be 0.7.5 or 0.7.5-NNN-gHASH
+# git_describe could be 0.7.5 or 0.7.5-NNN-gHASH
 # turn that into 0.7.5 or 0.7.5+NNN.gHASH
-case "$revname" in
-   *-*) revname=$(echo "$revname" | sed -e 's/-/+/' -e 's/-/./')
+case "$git_describe" in
+   *-*) version=$(echo "$git_describe" | sed -e 's/-/+/' -e 's/-/./');;
+   *) version=${git_describe};;
 esac
 
-archive_base="cloud-init-$revname"
+archive_base="cloud-init-$version"
+if [ -z "$output" ]; then
+    output="$archive_base.tar.gz"
+fi
 
 # when building an archiving from HEAD, ensure that there aren't any
 # uncomitted changes in the working directory (because these would not
@@ -25,9 +62,11 @@ if [ "$rev" = HEAD ] && ! git diff-index --quiet HEAD --; then
     fi
 fi
 
-git archive \
-    --format=tar.gz \
-    --prefix="$archive_base/" "$rev" \
-    "--output=$archive_base.tar.gz"
+git archive --format=tar --prefix="$archive_base/" "$rev" |
+    ( cd "$TEMP_D" && tar xpf - )
+
+sed -i "s,@@EXPORT_VERSION@@,$version," "$archive_base/cloudinit/version.py"
+
+( cd "$TEMP_D" && tar cpzf - "$archive_base/" ) > "$output"
 
-echo "${archive_base}.tar.gz"
+echo "$output"
diff --git a/tools/read-version b/tools/read-version
index d02651e9..e585ab2e 100755
--- a/tools/read-version
+++ b/tools/read-version
@@ -1,26 +1,77 @@
 #!/usr/bin/env python
 
 import os
-import re
+import json
 import sys
 
-if 'CLOUD_INIT_TOP_D' in os.environ:
-    topd = os.path.realpath(os.environ.get('CLOUD_INIT_TOP_D'))
-else:
-    topd = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
+if "avoid-pep8-E402-import-not-top-of-file":
+    _tdir = os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))
+    sys.path.insert(0, _tdir)
+    from cloudinit import version as ci_version
+    from cloudinit import util
+
+
+use_long = '--long' in sys.argv or os.environ.get('CI_RV_LONG')
+use_tags = '--tags' in sys.argv or os.environ.get('CI_RV_TAGS')
+output_json = '--json' in sys.argv
+
+src_version = ci_version.version_string()
+version_long = None
+
+if os.path.isdir(os.path.join(_tdir, ".git")):
+    def fix_git_version(ver):
+        ver = ver.strip()
+        if "-" in ver:
+            # change X.Y.Z-1023-gHASH to X.Y.Z+1023.ghash
+            return "{0}+{1}.{2}".format(*ver.split("-"))
+        return ver
+
+    flags = []
+    if use_tags:
+        flags = ['--tags']
+    cmd = ['git', 'describe'] + flags
 
-for fname in ("setup.py", "ChangeLog"):
-    if not os.path.isfile(os.path.join(topd, fname)):
-        sys.stderr.write("Unable to locate '%s' file that should "
-                         "exist in cloud-init root directory." % fname)
+    version = fix_git_version(util.subp(cmd)[0])
+
+    if not version.startswith(src_version):
+        sys.stderr.write("git describe version (%s) differs from "
+                         "cloudinit.version (%s)\n" % (version, src_version))
         sys.exit(1)
 
-vermatch = re.compile(r"^[0-9]+[.][0-9]+[.][0-9]+:$")
+    version_long = fix_git_version(util.subp(cmd + ["--long"])[0])
+else:
+    version = src_version
+    try:
+        version_long = ci_version.full_version_string()
+    except ValueError:
+        pass
+
+# version is X.Y.Z[+xxx.gHASH]
+# version_long is None or X.Y.Z+xxx.gHASH
+release = version.partition("+")[0]
+extra = None
+commit = None
+distance = None
 
-with open(os.path.join(topd, "ChangeLog"), "r") as fp:
-    for line in fp:
-        if vermatch.match(line):
-            sys.stdout.write(line.strip()[:-1] + "\n")
-            break
+if version_long:
+    info = version_long.partition("+")[2]
+    extra = "+" + info
+    distance, commit = info.split(".")
+    # remove the 'g' from gHASH
+    commit = commit[1:]
+
+data = {
+    'release': release,
+    'version': version,
+    'version_long': version_long,
+    'extra': extra,
+    'commit': commit,
+    'distance': distance,
+}
+
+if output_json:
+    sys.stdout.write(json.dumps(data, indent=1) + "\n")
+else:
+    sys.stdout.write(release + "\n")
 
 sys.exit(0)