From 72d6adcb2e4cb5911f7809b89835965d4bf04476 Mon Sep 17 00:00:00 2001 From: Lars Kellogg-Stedman Date: Fri, 22 Jul 2016 15:09:24 -0400 Subject: Update build tools to work with git - Update HACKING.rst to include git instructions - update MANIFEST.in and .gitignore to ignore git-related things - replaced tarball generation scripts with git-based script - have the spec files correctly identify themselves as cheetah templates - make brpm work with git --- packages/brpm | 294 ++++++++++++++++++++++++---------------------------------- 1 file changed, 123 insertions(+), 171 deletions(-) (limited to 'packages/brpm') diff --git a/packages/brpm b/packages/brpm index 45e47610..5d16eb71 100755 --- a/packages/brpm +++ b/packages/brpm @@ -1,27 +1,24 @@ -#!/usr/bin/python +#!/usr/bin/env python import argparse -import contextlib import glob import os import shutil -import subprocess import sys import tempfile -import re - -from datetime import datetime +import time def find_root(): # expected path is in /packages/ top_dir = os.environ.get("CLOUD_INIT_TOP_D", None) if top_dir is None: - top_dir = os.path.dirname(os.path.dirname(os.path.abspath(sys.argv[0]))) + top_dir = os.path.dirname( + os.path.dirname(os.path.abspath(sys.argv[0]))) if os.path.isfile(os.path.join(top_dir, 'setup.py')): return os.path.abspath(top_dir) raise OSError(("Unable to determine where your cloud-init topdir is." - " set CLOUD_INIT_TOP_D?")) + " set CLOUD_INIT_TOP_D?")) # Use the util functions from cloudinit @@ -30,34 +27,15 @@ sys.path.insert(0, find_root()) from cloudinit import templater from cloudinit import util -# Mapping of expected packages to there full name... -# this is a translation of the 'requires' -# file pypi package name to a redhat/fedora package name. -PKG_MP = { +# Map python requirements to package names. If a match isn't found +# here, we assume 'python-'. +PACKAGE_MAP = { 'redhat': { - 'argparse': 'python-argparse', - 'cheetah': 'python-cheetah', - 'jinja2': 'python-jinja2', - 'configobj': 'python-configobj', - 'jsonpatch': 'python-jsonpatch', - 'oauthlib': 'python-oauthlib', - 'prettytable': 'python-prettytable', 'pyserial': 'pyserial', 'pyyaml': 'PyYAML', - 'requests': 'python-requests', - 'six': 'python-six', }, 'suse': { - 'argparse': 'python-argparse', - 'cheetah': 'python-cheetah', - 'configobj': 'python-configobj', - 'jsonpatch': 'python-jsonpatch', - 'oauthlib': 'python-oauthlib', - 'prettytable': 'python-prettytable', - 'pyserial': 'python-pyserial', 'pyyaml': 'python-yaml', - 'requests': 'python-requests', - 'six': 'python-six', } } @@ -65,113 +43,79 @@ PKG_MP = { RPM_BUILD_SUBDIRS = ['BUILD', 'RPMS', 'SOURCES', 'SPECS', 'SRPMS'] -def get_log_header(version): - # Try to find the version in the tags output - cmd = ['bzr', 'tags'] - (stdout, _stderr) = util.subp(cmd) - a_rev = None - for t in stdout.splitlines(): - ver, rev = t.split(None) - if ver == version: - a_rev = rev - break - if not a_rev: - return None - - # Extract who made that tag as the header - cmd = ['bzr', 'log', '-r%s' % (a_rev), '--timezone=utc'] +def read_dependencies(): + '''Returns the Python depedencies from requirements.txt. This explicitly + removes 'argparse' from the list of requirements for python >= 2.7, + because with 2.7 argparse became part of the standard library.''' + cmd = [util.abs_join(find_root(), 'tools', 'read-dependencies')] (stdout, _stderr) = util.subp(cmd) - kvs = { - 'comment': version, - } + return [p.lower().strip() for p in stdout.splitlines() + if p != 'argparse' or (p == 'argparse' + and sys.version_info[0:2] < (2, 7))] - for line in stdout.splitlines(): - if line.startswith('committer:'): - kvs['who'] = line[len('committer:'):].strip() - if line.startswith('timestamp:'): - ts = line[len('timestamp:'):] - ts = ts.strip() - # http://bugs.python.org/issue6641 - ts = ts.replace("+0000", '').strip() - ds = datetime.strptime(ts, '%a %Y-%m-%d %H:%M:%S') - kvs['ds'] = ds - return format_change_line(**kvs) +def translate_dependencies(deps, distro): + '''Maps python requirements into package names. We assume + python- for packages not listed explicitly in + PACKAGE_MAP.''' + return [PACKAGE_MAP[distro][req] + if req in PACKAGE_MAP[distro] else 'python-%s' % req + for req in deps] -def format_change_line(ds, who, comment=None): - # Rpmbuild seems to be pretty strict about the date format - d = ds.strftime("%a %b %d %Y") - d += " - %s" % (who) - if comment: - d += " - %s" % (comment) - return "* %s" % (d) - - -def generate_spec_contents(args, tmpl_fn, top_dir, arc_fn): - +def read_version(): + '''Read version information. We parse the version itself from + the changelog, and then ask git for the commit id and distance + from the last tag.''' # Figure out the version and revno cmd = [util.abs_join(find_root(), 'tools', 'read-version')] (stdout, _stderr) = util.subp(cmd) version = stdout.strip() - - cmd = ['bzr', 'revno'] + + cmd = ['git', 'describe', '--tags'] (stdout, _stderr) = util.subp(cmd) - revno = stdout.strip() + git_version = stdout.strip() + + try: + _version, distance, revno = git_version.split('-') + except ValueError: + distance = None + revno = None + + return (version, distance, revno) + + +def generate_spec_contents(args, tmpl_fn, top_dir, arc_fn): + + # This will get us something like ('0.7.6', None, None) for a + # tagged commit, and something like ('0.7.6', '1026', 'gd1d5796') + # for an untagged commited. + version, distance, revno = read_version() # Tmpl params subs = {} subs['version'] = version subs['revno'] = revno - subs['release'] = "bzr%s" % (revno) + subs['distance'] = distance + + if distance is not None: + now = time.strftime('%Y%m%d', time.localtime()) + release = '.%sgit%s' % (now, revno) + else: + release = '' + if args.sub_release is not None: - subs['subrelease'] = "." + str(args.sub_release) + subs['subrelease'] = release + "." + str(args.sub_release) else: - subs['subrelease'] = '' - subs['archive_name'] = arc_fn + subs['subrelease'] = release - cmd = [util.abs_join(find_root(), 'tools', 'read-dependencies')] - (stdout, _stderr) = util.subp(cmd) - pkgs = [p.lower().strip() for p in stdout.splitlines()] + subs['archive_name'] = arc_fn + subs['source_name'] = os.path.basename(arc_fn).replace('.tar.gz', '') # Map to known packages - requires = [] - for p in pkgs: - if p == 'argparse' and sys.version_info[0:2] >= (2, 7): - # Not needed on anything but 2.6 or older. - continue - tgt_pkg = PKG_MP[args.distro].get(p) - if not tgt_pkg: - raise RuntimeError(("Do not know how to translate pypi dependency" - " %r to a known package") % (p)) - else: - requires.append(tgt_pkg) - subs['requires'] = requires - - # Format a nice changelog (as best as we can) - changelog = util.load_file(util.abs_join(find_root(), 'ChangeLog')) - changelog_lines = [] - missing_versions = 0 - for line in changelog.splitlines(): - if not line.strip(): - continue - if re.match(r"^\s*[\d][.][\d][.][\d]:\s*", line): - line = line.strip(":") - header = get_log_header(line) - if not header: - missing_versions += 1 - if missing_versions == 1: - # Must be using a new 'dev'/'trunk' release - changelog_lines.append(format_change_line(datetime.now(), - '??')) - else: - sys.stderr.write(("Changelog version line %s does not " - "have a corresponding tag!\n") % (line)) - else: - changelog_lines.append(header) - else: - changelog_lines.append(line) - subs['changelog'] = "\n".join(changelog_lines) + python_deps = read_dependencies() + package_deps = translate_dependencies(python_deps, args.distro) + subs['requires'] = package_deps if args.boot == 'sysvinit': subs['sysvinit'] = True @@ -183,21 +127,23 @@ def generate_spec_contents(args, tmpl_fn, top_dir, arc_fn): else: subs['systemd'] = False - subs['defines'] = ["_topdir %s" % (top_dir)] subs['init_sys'] = args.boot subs['patches'] = [os.path.basename(p) for p in args.patches] return templater.render_from_file(tmpl_fn, params=subs) def main(): - + parser = argparse.ArgumentParser() parser.add_argument("-d", "--distro", dest="distro", help="select distro (default: %(default)s)", metavar="DISTRO", default='redhat', choices=('redhat', 'suse')) + parser.add_argument('--srpm', + help='Produce a source rpm', + action='store_true') parser.add_argument("-b", "--boot", dest="boot", - help="select boot type (default: %(default)s)", + help="select boot type (default: %(default)s)", metavar="TYPE", default='sysvinit', choices=('sysvinit', 'systemd')) parser.add_argument("-v", "--verbose", dest="verbose", @@ -221,57 +167,63 @@ def main(): if args.verbose: capture = False - # Clean out the root dir and make sure the dirs we want are in place - root_dir = os.path.expanduser("~/rpmbuild") - if os.path.isdir(root_dir): - shutil.rmtree(root_dir) - - arc_dir = util.abs_join(root_dir, 'SOURCES') - build_dirs = [root_dir, arc_dir] - for dname in RPM_BUILD_SUBDIRS: - build_dirs.append(util.abs_join(root_dir, dname)) - build_dirs.sort() - util.ensure_dirs(build_dirs) - - # Archive the code - cmd = [util.abs_join(find_root(), 'tools', 'make-tarball')] - (stdout, _stderr) = util.subp(cmd) - archive_fn = stdout.strip() - real_archive_fn = os.path.join(arc_dir, os.path.basename(archive_fn)) - shutil.move(archive_fn, real_archive_fn) - print("Archived the code in %r" % (real_archive_fn)) - - # Form the spec file to be used - tmpl_fn = util.abs_join(find_root(), 'packages', - args.distro, 'cloud-init.spec.in') - contents = generate_spec_contents(args, tmpl_fn, root_dir, - os.path.basename(archive_fn)) - spec_fn = util.abs_join(root_dir, 'cloud-init.spec') - util.write_file(spec_fn, contents) - print("Created spec file at %r" % (spec_fn)) - print(contents) - for p in args.patches: - util.copy(p, util.abs_join(arc_dir, os.path.basename(p))) - - # Now build it! - print("Running 'rpmbuild' in %r" % (root_dir)) - cmd = ['rpmbuild', '-ba', spec_fn] - util.subp(cmd, capture=capture) - - # Copy the items built to our local dir - globs = [] - globs.extend(glob.glob("%s/*.rpm" % - (util.abs_join(root_dir, 'RPMS', 'noarch')))) - globs.extend(glob.glob("%s/*.rpm" % - (util.abs_join(root_dir, 'RPMS', 'x86_64')))) - globs.extend(glob.glob("%s/*.rpm" % - (util.abs_join(root_dir, 'RPMS')))) - globs.extend(glob.glob("%s/*.rpm" % - (util.abs_join(root_dir, 'SRPMS')))) - for rpm_fn in globs: - tgt_fn = util.abs_join(os.getcwd(), os.path.basename(rpm_fn)) - shutil.move(rpm_fn, tgt_fn) - print("Wrote out %s package %r" % (args.distro, tgt_fn)) + workdir = None + try: + workdir = tempfile.mkdtemp(prefix='rpmbuild') + os.environ['HOME'] = workdir + topdir = os.path.join(workdir, 'rpmbuild') + build_dirs = [os.path.join(topdir, dir) + for dir in RPM_BUILD_SUBDIRS] + util.ensure_dirs(build_dirs) + + # Archive the code + cmd = [util.abs_join(find_root(), 'tools', 'make-tarball')] + (stdout, _stderr) = util.subp(cmd) + archive_fn = stdout.strip() + print "Archived source as %s" % archive_fn + real_archive_fn = os.path.join(topdir, 'SOURCES', + os.path.basename(archive_fn)) + shutil.move(archive_fn, real_archive_fn) + print("Archived the code in %r" % (real_archive_fn)) + + # Form the spec file to be used + tmpl_fn = util.abs_join(find_root(), 'packages', + args.distro, 'cloud-init.spec.in') + contents = generate_spec_contents(args, tmpl_fn, topdir, + os.path.basename(archive_fn)) + spec_fn = util.abs_join(topdir, 'SPECS', 'cloud-init.spec') + util.write_file(spec_fn, contents) + print("Created spec file at %r" % (spec_fn)) + for p in args.patches: + util.copy(p, util.abs_join(topdir, 'SOURCES', os.path.basename(p))) + + # Now build it! + print("Running 'rpmbuild' in %r" % (topdir)) + + if args.srpm: + cmd = ['rpmbuild', '-bs', '--nodeps', spec_fn] + else: + cmd = ['rpmbuild', '-ba', spec_fn] + + util.subp(cmd, capture=capture) + + # Copy the items built to our local dir + globs = [] + globs.extend(glob.glob("%s/*.rpm" % + (util.abs_join(topdir, 'RPMS', 'noarch')))) + globs.extend(glob.glob("%s/*.rpm" % + (util.abs_join(topdir, 'RPMS', 'x86_64')))) + globs.extend(glob.glob("%s/*.rpm" % + (util.abs_join(topdir, 'RPMS')))) + globs.extend(glob.glob("%s/*.rpm" % + (util.abs_join(topdir, 'SRPMS')))) + for rpm_fn in globs: + tgt_fn = util.abs_join(os.getcwd(), os.path.basename(rpm_fn)) + shutil.move(rpm_fn, tgt_fn) + print("Wrote out %s package %r" % (args.distro, tgt_fn)) + finally: + if workdir is not None: + shutil.rmtree(workdir) return 0 -- cgit v1.2.3