adjust tools and version information.

upstream snapshots are versioned in the format 'X.Y.Z+<distance>.g<commit>'
where X.Y.Z are major, minor, and micro.  Distance is number of commits
since last annotated tag, and commit is the git commit.

bddeb and brpm will now create and use the "upstream version" like above.

Things changed here:
 - tools/make-tarball
   update cloudinit/version.py to contain the full version
   support --output
   support '--long' to always create the long format version string.

 - bddeb:
   - use quilt debian source format
   - use read-version and long version in changelog.

 - brpm:
   - change to use read-version and upstream long version in the spec.
   - flake8 changes

 - tools/read-version
   - read version from git or from cloudinit/version.
   - provide --json output with more nicely formed data.

5 files changed, 80 insertions, 120 deletions

diff --git a/packages/bddeb b/packages/bddeb
index 46c07c88..94496c74 100755
--- a/packages/bddeb
+++ b/packages/bddeb
@@ -1,6 +1,7 @@
 #!/usr/bin/env python3
 
-import glob
+import argparse
+import json
 import os
 import shutil
 import sys
@@ -15,15 +16,13 @@ def find_root():
     if os.path.isfile(os.path.join(top_dir, 'setup.py')):
         return os.path.abspath(top_dir)
     raise OSError(("Unable to determine where your cloud-init topdir is."
-                       " set CLOUD_INIT_TOP_D?"))
-
-# Use the util functions from cloudinit
-sys.path.insert(0, find_root())
+                   " set CLOUD_INIT_TOP_D?"))
 
-from cloudinit import templater
-from cloudinit import util
-
-import argparse
+if "avoid-pep8-E402-import-not-top-of-file":
+    # Use the util functions from cloudinit
+    sys.path.insert(0, find_root())
+    from cloudinit import templater
+    from cloudinit import util
 
 # Package names that will showup in requires to what we can actually
 # use in our debian 'control' file, this is a translation of the 'requires'
@@ -58,27 +57,37 @@ NONSTD_NAMED_PACKAGES = {
 DEBUILD_ARGS = ["-S", "-d"]
 
 
-def write_debian_folder(root, version, revno, pkgmap,
-                        pyver="3", append_requires=[]):
+def run_helper(helper, args=None, strip=True):
+    if args is None:
+        args = []
+    cmd = [util.abs_join(find_root(), 'tools', helper)] + args
+    (stdout, _stderr) = util.subp(cmd)
+    if strip:
+        stdout = stdout.strip()
+    return stdout
+
+
+def write_debian_folder(root, version_data, pkgmap, pyver="3",
+                        append_requires=[]):
     deb_dir = util.abs_join(root, 'debian')
-    os.makedirs(deb_dir)
+
+    # Just copy debian/ dir and then update files
+    pdeb_d = util.abs_join(find_root(), 'packages', 'debian')
+    util.subp(['cp', '-a', pdeb_d, deb_dir])
 
     # Fill in the change log template
     templater.render_to_file(util.abs_join(find_root(),
                              'packages', 'debian', 'changelog.in'),
                              util.abs_join(deb_dir, 'changelog'),
-                             params={
-                                 'version': version,
-                                 'revision': revno,
-                             })
+                             params=version_data)
 
     # Write out the control file template
-    cmd = [util.abs_join(find_root(), 'tools', 'read-dependencies')]
-    (stdout, _stderr) = util.subp(cmd)
-    pypi_pkgs = [p.lower().strip() for p in stdout.splitlines()]
+    reqs = run_helper('read-dependencies').splitlines()
+    test_reqs = run_helper(
+        'read-dependencies', ['test-requirements.txt']).splitlines()
 
-    (stdout, _stderr) = util.subp(cmd + ['test-requirements.txt'])
-    pypi_test_pkgs = [p.lower().strip() for p in stdout.splitlines()]
+    pypi_pkgs = [p.lower().strip() for p in reqs]
+    pypi_test_pkgs = [p.lower().strip() for p in test_reqs]
 
     # Map to known packages
     requires = append_requires
@@ -109,11 +118,9 @@ def write_debian_folder(root, version, revno, pkgmap,
                              util.abs_join(deb_dir, 'rules'),
                              params={'python': python, 'pyver': pyver})
 
-    # Just copy any other files directly (including .in)
-    pdeb_d = util.abs_join(find_root(), 'packages', 'debian')
-    for f in [os.path.join(pdeb_d, f) for f in os.listdir(pdeb_d)]:
-        if os.path.isfile(f):
-            shutil.copy(f, util.abs_join(deb_dir, os.path.basename(f)))
+
+def read_version():
+    return json.loads(run_helper('read-version', ['--json']))
 
 
 def main():
@@ -140,11 +147,10 @@ def main():
                         default=os.environ.get("INIT_SYSTEM",
                                                "upstart,systemd"))
 
-
     for ent in DEBUILD_ARGS:
         parser.add_argument(ent, dest="debuild_args", action='append_const',
-            const=ent, help=("pass through '%s' to debuild" % ent),
-            default=[])
+                            const=ent, default=[],
+                            help=("pass through '%s' to debuild" % ent))
 
     parser.add_argument("--sign", default=False, action='store_true',
                         help="sign result. do not pass -us -uc to debuild")
@@ -181,54 +187,30 @@ def main():
     with util.tempdir() as tdir:
 
         # output like 0.7.6-1022-g36e92d3
-        cmd = ['git', 'describe', '--long']
-        (sysout, _stderr) = util.subp(cmd)
-        version, extra = sysout.strip().split("-", 1)
+        ver_data = read_version()
 
         # This is really only a temporary archive
         # since we will extract it then add in the debian
         # folder, then re-archive it for debian happiness
         print("Creating a temporary tarball using the 'make-tarball' helper")
-        cmd = [util.abs_join(find_root(), 'tools', 'make-tarball')]
-        (sysout, _stderr) = util.subp(cmd)
-        arch_fn = sysout.strip()
-        tmp_arch_fn = util.abs_join(tdir, os.path.basename(arch_fn))
-        shutil.move(arch_fn, tmp_arch_fn)
-
-        print("Extracting temporary tarball %r" % (tmp_arch_fn))
-        cmd = ['tar', '-xvzf', tmp_arch_fn, '-C', tdir]
+        tarball = "cloud-init_%s.orig.tar.gz" % ver_data['version_long']
+        tarball_fp = util.abs_join(tdir, tarball)
+        run_helper('make-tarball', ['--long', '--output=' + tarball_fp])
+
+        print("Extracting temporary tarball %r" % (tarball))
+        cmd = ['tar', '-xvzf', tarball_fp, '-C', tdir]
         util.subp(cmd, capture=capture)
-        extracted_name = tmp_arch_fn[:-len('.tar.gz')]
-        os.remove(tmp_arch_fn)
 
-        xdir = util.abs_join(tdir, 'cloud-init')
-        shutil.move(extracted_name, xdir)
+        xdir = util.abs_join(tdir, "cloud-init-%s" % ver_data['version_long'])
 
         print("Creating a debian/ folder in %r" % (xdir))
         if args.cloud_utils:
-            append_requires=['cloud-utils | cloud-guest-utils']
+            append_requires = ['cloud-utils | cloud-guest-utils']
         else:
-            append_requires=[]
-        write_debian_folder(xdir, version, extra, pkgmap,
+            append_requires = []
+        write_debian_folder(xdir, ver_data, pkgmap,
                             pyver=pyver, append_requires=append_requires)
 
-        # The naming here seems to follow some debian standard
-        # so it will whine if it is changed...
-        tar_fn = "cloud-init_%s+%s~bddeb.orig.tar.gz" % (version, extra)
-        print("Archiving the adjusted source into %r" %
-              (util.abs_join(tdir, tar_fn)))
-        cmd = ['tar', '-czvf',
-               util.abs_join(tdir, tar_fn),
-               '-C', xdir]
-        cmd.extend(os.listdir(xdir))
-        util.subp(cmd, capture=capture)
-
-        # Copy it locally for reference
-        shutil.copy(util.abs_join(tdir, tar_fn),
-                    util.abs_join(os.getcwd(), tar_fn))
-        print("Copied that archive to %r for local usage (if desired)." %
-              (util.abs_join(os.getcwd(), tar_fn)))
-
         print("Running 'debuild %s' in %r" % (' '.join(args.debuild_args),
                                               xdir))
         with util.chdir(xdir):
diff --git a/packages/brpm b/packages/brpm
index 5d16eb71..14d75f2b 100755
--- a/packages/brpm
+++ b/packages/brpm
@@ -2,11 +2,11 @@
 
 import argparse
 import glob
+import json
 import os
 import shutil
 import sys
 import tempfile
-import time
 
 
 def find_root():
@@ -21,11 +21,11 @@ def find_root():
                    " set CLOUD_INIT_TOP_D?"))
 
 
-# Use the util functions from cloudinit
-sys.path.insert(0, find_root())
-
-from cloudinit import templater
-from cloudinit import util
+if "avoid-pep8-E402-import-not-top-of-file":
+    # Use the util functions from cloudinit
+    sys.path.insert(0, find_root())
+    from cloudinit import templater
+    from cloudinit import util
 
 # Map python requirements to package names.  If a match isn't found
 # here, we assume 'python-<pypi_name>'.
@@ -43,15 +43,24 @@ PACKAGE_MAP = {
 RPM_BUILD_SUBDIRS = ['BUILD', 'RPMS', 'SOURCES', 'SPECS', 'SRPMS']
 
 
+def run_helper(helper, args=None, strip=True):
+    if args is None:
+        args = []
+    cmd = [util.abs_join(find_root(), 'tools', helper)] + args
+    (stdout, _stderr) = util.subp(cmd)
+    if strip:
+        stdout = stdout.strip()
+    return stdout
+
+
 def read_dependencies():
     '''Returns the Python depedencies from requirements.txt.  This explicitly
     removes 'argparse' from the list of requirements for python >= 2.7,
     because with 2.7 argparse became part of the standard library.'''
-    cmd = [util.abs_join(find_root(), 'tools', 'read-dependencies')]
-    (stdout, _stderr) = util.subp(cmd)
+    stdout = run_helper('read-dependencies')
     return [p.lower().strip() for p in stdout.splitlines()
-            if p != 'argparse' or (p == 'argparse'
-                                   and sys.version_info[0:2] < (2, 7))]
+            if p != 'argparse' or (p == 'argparse' and
+                                   sys.version_info[0:2] < (2, 7))]
 
 
 def translate_dependencies(deps, distro):
@@ -64,53 +73,22 @@ def translate_dependencies(deps, distro):
 
 
 def read_version():
-    '''Read version information.  We parse the version itself from
-    the changelog, and then ask git for the commit id and distance
-    from the last tag.'''
-    # Figure out the version and revno
-    cmd = [util.abs_join(find_root(), 'tools', 'read-version')]
-    (stdout, _stderr) = util.subp(cmd)
-    version = stdout.strip()
+    return json.loads(run_helper('read-version', ['--json']))
 
-    cmd = ['git', 'describe', '--tags']
-    (stdout, _stderr) = util.subp(cmd)
-    git_version = stdout.strip()
 
-    try:
-        _version, distance, revno = git_version.split('-')
-    except ValueError:
-        distance = None
-        revno = None
-
-    return (version, distance, revno)
-
-
-def generate_spec_contents(args, tmpl_fn, top_dir, arc_fn):
-
-    # This will get us something like ('0.7.6', None, None) for a
-    # tagged commit, and something like ('0.7.6', '1026', 'gd1d5796')
-    # for an untagged commited.
-    version, distance, revno = read_version()
+def generate_spec_contents(args, version_data, tmpl_fn, top_dir, arc_fn):
 
     # Tmpl params
     subs = {}
-    subs['version'] = version
-    subs['revno'] = revno
-    subs['distance'] = distance
-
-    if distance is not None:
-        now = time.strftime('%Y%m%d', time.localtime())
-        release = '.%sgit%s' % (now, revno)
-    else:
-        release = ''
 
     if args.sub_release is not None:
-        subs['subrelease'] = release + "." + str(args.sub_release)
+        subs['subrelease'] = str(args.sub_release)
     else:
-        subs['subrelease'] = release
+        subs['subrelease'] = ""
 
     subs['archive_name'] = arc_fn
     subs['source_name'] = os.path.basename(arc_fn).replace('.tar.gz', '')
+    subs.update(version_data)
 
     # Map to known packages
     python_deps = read_dependencies()
@@ -176,20 +154,19 @@ def main():
                       for dir in RPM_BUILD_SUBDIRS]
         util.ensure_dirs(build_dirs)
 
+        version_data = read_version()
+
         # Archive the code
-        cmd = [util.abs_join(find_root(), 'tools', 'make-tarball')]
-        (stdout, _stderr) = util.subp(cmd)
-        archive_fn = stdout.strip()
-        print "Archived source as %s" % archive_fn
-        real_archive_fn = os.path.join(topdir, 'SOURCES',
-                                       os.path.basename(archive_fn))
-        shutil.move(archive_fn, real_archive_fn)
+        archive_fn = "cloud-init-%s.tar.gz" % version_data['version_long']
+        real_archive_fn = os.path.join(topdir, 'SOURCES', archive_fn)
+        archive_fn = run_helper(
+            'make-tarball', ['--long', '--output=' + real_archive_fn])
         print("Archived the code in %r" % (real_archive_fn))
 
         # Form the spec file to be used
         tmpl_fn = util.abs_join(find_root(), 'packages',
                                 args.distro, 'cloud-init.spec.in')
-        contents = generate_spec_contents(args, tmpl_fn, topdir,
+        contents = generate_spec_contents(args, version_data, tmpl_fn, topdir,
                                           os.path.basename(archive_fn))
         spec_fn = util.abs_join(topdir, 'SPECS', 'cloud-init.spec')
         util.write_file(spec_fn, contents)
diff --git a/packages/debian/changelog.in b/packages/debian/changelog.in
index 544d23cf..f8e98258 100644
--- a/packages/debian/changelog.in
+++ b/packages/debian/changelog.in
@@ -1,5 +1,5 @@
 ## template:basic
-cloud-init (${version}+${revision}~bddeb-1) UNRELEASED; urgency=low
+cloud-init (${version_long}-1~bddeb) UNRELEASED; urgency=low
 
   * build
 
diff --git a/packages/debian/source/format b/packages/debian/source/format
new file mode 100644
index 00000000..163aaf8d
--- /dev/null
+++ b/packages/debian/source/format
@@ -0,0 +1 @@
+3.0 (quilt)
diff --git a/packages/redhat/cloud-init.spec.in b/packages/redhat/cloud-init.spec.in
index c30d33ca..2f02f017 100644
--- a/packages/redhat/cloud-init.spec.in
+++ b/packages/redhat/cloud-init.spec.in
@@ -6,7 +6,7 @@
 # Or: http://www.rpm.org/max-rpm/ch-rpm-inside.html
 
 Name:           cloud-init
-Version:        ${version}
+Version:        ${version_long}
 Release:        1${subrelease}%{?dist}
 Summary:        Cloud instance init scripts