tools/run-centos: cleanups and move to using read-dependencies

These changes are all in an effort to get tools/run-centos using
read-dependencies rather than the 'setup-centos' script with a separate
set of dependencies listed.

 - tools/read-dependencies: support taking multiple --requirements
   options.  This allows run-centos to get both test and build
   dependencies.  Ultimately, I think it might be nicer for
   read-dependencies to take a list of "goals" (build, test, run or
   test-tox) rather than having the caller need to know to provide
   multiple --requirements.

 - packages/pkg-deps.json: drop the version on the sudo package.
   centos 6 has newer (1.8.6p3) version than listed, so its not a problem.

 - test_handler_disk_setup.py: a test case here was using assertLogs
   which is not present in the version of unittest2 that is available in
   centos 6 epel. We just adjust it to use with_logs = True.

 - tools/run-cents:
   - improve usage with example
   - add 'inside_as_cd' to provide the dir you want to cd first to.
   - avoid the intermediate tarball on disk in the container.
   - add 'prep' subcommand and use it to install pre-dependencies.
   - use read-dependencies.

1 files changed, 23 insertions, 9 deletions

diff --git a/tools/read-dependencies b/tools/read-dependencies
index 4ba2c1bc..8a585343 100755
--- a/tools/read-dependencies
+++ b/tools/read-dependencies
@@ -18,6 +18,7 @@ import re
 import subprocess
 import sys
 
+DEFAULT_REQUIREMENTS = 'requirements.txt'
 
 # Map the appropriate package dir needed for each distro choice
 DISTRO_PKG_TYPE_MAP = {
@@ -51,8 +52,9 @@ def get_parser():
     """Return an argument parser for this command."""
     parser = ArgumentParser(description=__doc__)
     parser.add_argument(
-        '-r', '--requirements-file', type=str, dest='req_file',
-        default='requirements.txt', help='The pip-style requirements file')
+        '-r', '--requirements-file', type=str, dest='req_files',
+        action='append', default=None,
+        help='pip-style requirements file [default=%s]' % DEFAULT_REQUIREMENTS)
     parser.add_argument(
         '-d', '--distro', type=str, choices=DISTRO_PKG_TYPE_MAP.keys(),
         help='The name of the distro to generate package deps for.')
@@ -144,12 +146,24 @@ def main(distro):
         topd = os.path.realpath(os.environ.get('CLOUD_INIT_TOP_D'))
     else:
         topd = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
-    req_path = os.path.join(topd, args.req_file)
-    if not os.path.isfile(req_path):
-        sys.stderr.write("Unable to locate '%s' file that should "
-                         "exist in cloud-init root directory." % req_path)
-        return 1
-    pip_pkg_names = parse_pip_requirements(req_path)
+
+    if args.req_files is None:
+        args.req_files = [os.path.join(topd, DEFAULT_REQUIREMENTS)]
+        if not os.path.isfile(args.req_files[0]):
+            sys.stderr.write("Unable to locate '%s' file that should "
+                             "exist in cloud-init root directory." %
+                             args.req_files[0])
+            sys.exit(1)
+
+    bad_files = [r for r in args.req_files if not os.path.isfile(r)]
+    if bad_files:
+        sys.stderr.write(
+            "Unable to find requirements files: %s\n" % ','.join(bad_files))
+        sys.exit(1)
+
+    pip_pkg_names = set()
+    for req_path in args.req_files:
+        pip_pkg_names.update(set(parse_pip_requirements(req_path)))
     deps_from_json = get_package_deps_from_json(topd, args.distro)
     renames = deps_from_json.get('renames', {})
     translated_pip_names = translate_pip_to_system_pkg(
@@ -174,7 +188,7 @@ def pkg_install(pkg_list, distro, dry_run=False):
     """Install a list of packages using the DISTRO_INSTALL_PKG_CMD."""
     print('Installing deps: {0}{1}'.format(
           '(dryrun)' if dry_run else '', ' '.join(pkg_list)))
-    pkg_list.extend(EXTRA_SYSTEM_BASE_PKGS)
+    pkg_list = list(pkg_list) + EXTRA_SYSTEM_BASE_PKGS
     install_cmd = []
     if dry_run:
        install_cmd.append('echo')