pkg build ci: Add make ci-deps-<distro> target to install pkgs

This change adds a couple of makefile targets for ci environments to
install all necessary dependencies for package builds and test runs.

It adds a number of arguments to ./tools/read-dependencies to facilitate
reading pip dependencies, translating pip deps to system package names and
optionally installing needed system-package dependencies on the local
system. This relocates all package dependency and translation logic into
./tools/read-dependencies instead of duplication found in packages/brpm
and packages/bddeb.

In this branch, we also define buildrequires as including all runtime
requires when rendering cloud-init.spec.in and debian/control files
because our package build infrastructure will also be running all unit
test during the package build process so we need runtime deps at build
time.

Additionally, this branch converts
packages/(redhat|suse)/cloud-init.spec.in from cheetah templates to jinja
to allow building python3 envs.

8 files changed, 354 insertions, 175 deletions

diff --git a/Makefile b/Makefile
index a3bfaf79..c752530c 100644
--- a/Makefile
+++ b/Makefile
@@ -53,6 +53,14 @@ unittest: clean_pyc
 unittest3: clean_pyc
 	nosetests3 $(noseopts) tests/unittests
 
+ci-deps-ubuntu:
+	@$(PYVER) $(CWD)/tools/read-dependencies --distro ubuntu --install --python-version 3
+	@$(PYVER) $(CWD)/tools/read-dependencies --distro ubuntu --requirements-file test-requirements.txt --install --python-version 3
+
+ci-deps-centos:
+	@$(PYVER) $(CWD)/tools/read-dependencies --distro centos --install
+	@$(PYVER) $(CWD)/tools/read-dependencies --distro centos --requirements-file test-requirements.txt --install
+
 pip-requirements:
 	@echo "Installing cloud-init dependencies..."
 	$(PIP_INSTALL) -r "$@.txt" -q
diff --git a/packages/bddeb b/packages/bddeb
index f415209f..e45af6ee 100755
--- a/packages/bddeb
+++ b/packages/bddeb
@@ -24,19 +24,6 @@ if "avoid-pep8-E402-import-not-top-of-file":
     from cloudinit import templater
     from cloudinit import util
 
-# Package names that will showup in requires which have unique package names.
-# Format is '<pypi-name>': {'<python_major_version>': <pkg_name_or_none>, ...}.
-NONSTD_NAMED_PACKAGES = {
-    'argparse': {'2': 'python-argparse', '3': None},
-    'contextlib2': {'2': 'python-contextlib2', '3': None},
-    'cheetah': {'2': 'python-cheetah', '3': None},
-    'pyserial': {'2': 'python-serial', '3': 'python3-serial'},
-    'pyyaml': {'2': 'python-yaml', '3': 'python3-yaml'},
-    'six': {'2': 'python-six', '3': 'python3-six'},
-    'pep8': {'2': 'pep8', '3': 'python3-pep8'},
-    'pyflakes': {'2': 'pyflakes', '3': 'pyflakes'},
-}
-
 DEBUILD_ARGS = ["-S", "-d"]
 
 
@@ -59,7 +46,6 @@ def write_debian_folder(root, templ_data, is_python2, cloud_util_deps):
     else:
         pyver = "3"
         python = "python3"
-    pkgfmt = "{}-{}"
 
     deb_dir = util.abs_join(root, 'debian')
 
@@ -74,30 +60,23 @@ def write_debian_folder(root, templ_data, is_python2, cloud_util_deps):
                              params=templ_data)
 
     # Write out the control file template
-    reqs = run_helper('read-dependencies').splitlines()
+    reqs_output = run_helper(
+        'read-dependencies',
+        args=['--distro', 'debian', '--python-version', pyver])
+    reqs = reqs_output.splitlines()
     test_reqs = run_helper(
-        'read-dependencies', ['test-requirements.txt']).splitlines()
-
-    pypi_pkgs = [p.lower().strip() for p in reqs]
-    pypi_test_pkgs = [p.lower().strip() for p in test_reqs]
+        'read-dependencies',
+        ['--requirements-file', 'test-requirements.txt',
+         '--system-pkg-names', '--python-version', pyver]).splitlines()
 
-    # Map to known packages
     requires = ['cloud-utils | cloud-guest-utils'] if cloud_util_deps else []
-    test_requires = []
-    lists = ((pypi_pkgs, requires), (pypi_test_pkgs, test_requires))
-    for pypilist, target in lists:
-        for p in pypilist:
-            if p in NONSTD_NAMED_PACKAGES:
-                if NONSTD_NAMED_PACKAGES[p][pyver]:
-                    target.append(NONSTD_NAMED_PACKAGES[p][pyver])
-            else:  # Then standard package prefix
-                target.append(pkgfmt.format(python, p))
-
+    # We consolidate all deps as Build-Depends as our package build runs all
+    # tests so we need all runtime dependencies anyway.
+    requires.extend(reqs + test_reqs + [python])
     templater.render_to_file(util.abs_join(find_root(),
                                            'packages', 'debian', 'control.in'),
                              util.abs_join(deb_dir, 'control'),
-                             params={'requires': ','.join(requires),
-                                     'test_requires': ','.join(test_requires),
+                             params={'build_depends': ','.join(requires),
                                      'python': python})
 
     templater.render_to_file(util.abs_join(find_root(),
diff --git a/packages/brpm b/packages/brpm
index 89696ab8..3439cf35 100755
--- a/packages/brpm
+++ b/packages/brpm
@@ -27,17 +27,6 @@ if "avoid-pep8-E402-import-not-top-of-file":
     from cloudinit import templater
     from cloudinit import util
 
-# Map python requirements to package names.  If a match isn't found
-# here, we assume 'python-<pypi_name>'.
-PACKAGE_MAP = {
-    'redhat': {
-        'pyserial': 'pyserial',
-        'pyyaml': 'PyYAML',
-    },
-    'suse': {
-        'pyyaml': 'python-yaml',
-    }
-}
 
 # Subdirectories of the ~/rpmbuild dir
 RPM_BUILD_SUBDIRS = ['BUILD', 'RPMS', 'SOURCES', 'SPECS', 'SRPMS']
@@ -53,23 +42,18 @@ def run_helper(helper, args=None, strip=True):
     return stdout
 
 
-def read_dependencies():
-    '''Returns the Python depedencies from requirements.txt.  This explicitly
-    removes 'argparse' from the list of requirements for python >= 2.7,
-    because with 2.7 argparse became part of the standard library.'''
-    stdout = run_helper('read-dependencies')
-    return [p.lower().strip() for p in stdout.splitlines()
-            if p != 'argparse' or (p == 'argparse' and
-                                   sys.version_info[0:2] < (2, 7))]
+def read_dependencies(requirements_file='requirements.txt'):
+    """Returns the Python package depedencies from requirements.txt files.
 
-
-def translate_dependencies(deps, distro):
-    '''Maps python requirements into package names.  We assume
-    python-<pypi_name> for packages not listed explicitly in
-    PACKAGE_MAP.'''
-    return [PACKAGE_MAP[distro][req]
-            if req in PACKAGE_MAP[distro] else 'python-%s' % req
-            for req in deps]
+    @returns a tuple of (requirements, test_requirements)
+    """
+    pkg_deps = run_helper(
+        'read-dependencies', args=['--distro', 'redhat']).splitlines()
+    test_deps = run_helper(
+        'read-dependencies', args=[
+            '--requirements-file', 'test-requirements.txt',
+            '--system-pkg-names']).splitlines()
+    return (pkg_deps, test_deps)
 
 
 def read_version():
@@ -99,10 +83,9 @@ def generate_spec_contents(args, version_data, tmpl_fn, top_dir, arc_fn):
         rpm_upstream_version = version_data['version']
     subs['rpm_upstream_version'] = rpm_upstream_version
 
-    # Map to known packages
-    python_deps = read_dependencies()
-    package_deps = translate_dependencies(python_deps, args.distro)
-    subs['requires'] = package_deps
+    deps, test_deps = read_dependencies()
+    subs['buildrequires'] = deps + test_deps
+    subs['requires'] = deps
 
     if args.boot == 'sysvinit':
         subs['sysvinit'] = True
diff --git a/packages/debian/control.in b/packages/debian/control.in
index 6c39d531..265b261f 100644
--- a/packages/debian/control.in
+++ b/packages/debian/control.in
@@ -3,20 +3,13 @@ Source: cloud-init
 Section: admin
 Priority: optional
 Maintainer: Scott Moser <smoser@ubuntu.com>
-Build-Depends: debhelper (>= 9),
-               dh-python,
-               dh-systemd,
-               ${python},
-               ${test_requires},
-               ${requires}
+Build-Depends: ${build_depends}
 XS-Python-Version: all
 Standards-Version: 3.9.6
 
 Package: cloud-init
 Architecture: all
-Depends: procps,
-         ${python},
-         ${misc:Depends},
+Depends: ${misc:Depends},
          ${${python}:Depends}
 Recommends: eatmydata, sudo, software-properties-common, gdisk
 XB-Python-Version: ${python:Versions}
diff --git a/packages/pkg-deps.json b/packages/pkg-deps.json
new file mode 100644
index 00000000..8b8f3c37
--- /dev/null
+++ b/packages/pkg-deps.json
@@ -0,0 +1,88 @@
+{
+   "debian" : {
+      "build-requires" : [
+         "debhelper",
+         "dh-python",
+         "dh-systemd"
+      ],
+      "renames" : {
+         "pyyaml" : {
+            "2" : "python-yaml",
+            "3" : "python3-yaml"
+         },
+         "contextlib2" : {
+            "2" : "python-contextlib2"
+         },
+         "pyserial" : {
+            "2" : "python-serial",
+            "3" : "python3-serial"
+         }
+      },
+      "requires" : [
+         "procps"
+      ]
+   },
+   "redhat" : {
+      "build-requires" : [
+         "python-devel",
+         "python-setuptools"
+      ],
+      "renames" : {
+         "jinja2" : {
+            "3" : "python34-jinja2"
+         },
+         "jsonschema" : {
+            "3" : "python34-jsonschema"
+         },
+         "prettytable" : {
+            "3" : "python34-prettytable"
+         },
+         "pyflakes" : {
+            "2" : "pyflakes",
+            "3" : "python34-pyflakes"
+         },
+         "pyyaml" : {
+            "2" : "PyYAML",
+            "3" : "python34-PyYAML"
+         },
+         "pyserial" : {
+            "2" : "pyserial"
+         },
+         "requests" : {
+            "3" : "python34-requests"
+         },
+         "six" : {
+            "3" : "python34-six"
+         }
+      },
+      "requires" : [
+         "e2fsprogs",
+         "iproute",
+         "net-tools",
+         "procps",
+         "rsyslog",
+         "shadow-utils",
+         "sudo >= 1.7.2p2-3"
+      ]
+   },
+   "suse" : {
+      "renames" : {
+         "pyyaml" : {
+            "2" : "python-yaml"
+         }
+      },
+      "build-requires" : [
+         "fdupes",
+         "filesystem",
+         "python-devel",
+         "python-setuptools"
+      ],
+      "requires" : [
+         "iproute2",
+         "e2fsprogs",
+         "net-tools",
+         "procps",
+         "sudo"
+      ]
+   }
+}
diff --git a/packages/redhat/cloud-init.spec.in b/packages/redhat/cloud-init.spec.in
index 3e92c98f..9f75c4b8 100644
--- a/packages/redhat/cloud-init.spec.in
+++ b/packages/redhat/cloud-init.spec.in
@@ -1,4 +1,4 @@
-## template: cheetah
+## template: jinja
 %{!?python_sitelib: %global python_sitelib %(%{__python} -c "from distutils.sysconfig import get_python_lib; print get_python_lib()")}
 
 %define use_systemd (0%{?fedora} && 0%{?fedora} >= 18) || (0%{?rhel} && 0%{?rhel} >= 7)
@@ -14,20 +14,18 @@
 # Or: http://www.rpm.org/max-rpm/ch-rpm-inside.html
 
 Name:           cloud-init
-Version:        ${rpm_upstream_version}
-Release:        1${subrelease}%{?dist}
+Version:        {{rpm_upstream_version}}
+Release:        1{{subrelease}}%{?dist}
 Summary:        Cloud instance init scripts
 
 Group:          System Environment/Base
 License:        Dual-licesed GPLv3 or Apache 2.0
 URL:            http://launchpad.net/cloud-init
 
-Source0:        ${archive_name}
+Source0:        {{archive_name}}
 BuildArch:      noarch
 BuildRoot:      %{_tmppath}
 
-BuildRequires:  python-devel
-BuildRequires:  python-setuptools
 %if "%{?el6}" == "1"
 BuildRequires:  python-argparse
 %endif
@@ -46,40 +44,30 @@ Requires(preun):    chkconfig
 # These are runtime dependencies, but declared as BuildRequires so that
 # - tests can be run here.
 # - parts of cloud-init such (setup.py) use these dependencies.
-#for $r in $requires
-BuildRequires:  ${r}
-#end for
+{% for r in requires %}
+BuildRequires:  {{r}}
+{% endfor %}
 
 # System util packages needed
 %ifarch %{?ix86} x86_64 ia64
 Requires:       dmidecode
 %endif
-Requires:       shadow-utils
-Requires:       rsyslog
-Requires:       iproute
-Requires:       e2fsprogs
-Requires:       net-tools
-Requires:       procps
-Requires:       shadow-utils
-Requires:       sudo >= 1.7.2p2-3
-
-Requires:  python-setuptools
+
 # python2.6 needs argparse
 %if "%{?el6}" == "1"
 Requires:  python-argparse
 %endif
 
-# Install pypi 'dynamic' requirements
-#for $r in $requires
-Requires:       ${r}
-#end for
+
+# Install 'dynamic' runtime reqs from *requirements.txt and pkg-deps.json
+{% for r in requires %}
+Requires:       {{r}}
+{% endfor %}
 
 # Custom patches
-#set $size = 0
-#for $p in $patches
-Patch${size}: $p
-#set $size += 1
-#end for
+{% for p in patches %}
+Patch{{loop.index0}}: {{p}}
+{% endfor %}
 
 %if "%{init_system}" == "systemd"
 Requires(post):       systemd
@@ -98,14 +86,12 @@ need special scripts to run during initialization to retrieve and install
 ssh keys and to let the user run various scripts.
 
 %prep
-%setup -q -n ${source_name}
+%setup -q -n {{source_name}}
 
 # Custom patches activation
-#set $size = 0
-#for $p in $patches
-%patch${size} -p1
-#set $size += 1
-#end for
+{% for p in patches %}
+%patch{{loop.index0}} -p1
+{% endfor %}
 
 %build
 %{__python} setup.py build
@@ -113,34 +99,34 @@ ssh keys and to let the user run various scripts.
 %install
 
 %{__python} setup.py install -O1 \
-            --skip-build --root \$RPM_BUILD_ROOT \
+            --skip-build --root $RPM_BUILD_ROOT \
             --init-system=%{init_system}
 
 # Note that /etc/rsyslog.d didn't exist by default until F15.
 # el6 request: https://bugzilla.redhat.com/show_bug.cgi?id=740420
-mkdir -p \$RPM_BUILD_ROOT/%{_sysconfdir}/rsyslog.d
+mkdir -p $RPM_BUILD_ROOT/%{_sysconfdir}/rsyslog.d
 cp -p tools/21-cloudinit.conf \
-      \$RPM_BUILD_ROOT/%{_sysconfdir}/rsyslog.d/21-cloudinit.conf
+      $RPM_BUILD_ROOT/%{_sysconfdir}/rsyslog.d/21-cloudinit.conf
 
 # Remove the tests
-rm -rf \$RPM_BUILD_ROOT%{python_sitelib}/tests
+rm -rf $RPM_BUILD_ROOT%{python_sitelib}/tests
 
 # Required dirs...
-mkdir -p \$RPM_BUILD_ROOT/%{_sharedstatedir}/cloud
-mkdir -p \$RPM_BUILD_ROOT/%{_libexecdir}/%{name}
+mkdir -p $RPM_BUILD_ROOT/%{_sharedstatedir}/cloud
+mkdir -p $RPM_BUILD_ROOT/%{_libexecdir}/%{name}
 
 %if "%{init_system}" == "systemd"
-mkdir -p         \$RPM_BUILD_ROOT/%{_unitdir}
-cp -p systemd/*  \$RPM_BUILD_ROOT/%{_unitdir}
+mkdir -p         $RPM_BUILD_ROOT/%{_unitdir}
+cp -p systemd/*  $RPM_BUILD_ROOT/%{_unitdir}
 %endif
 
 %clean
-rm -rf \$RPM_BUILD_ROOT
+rm -rf $RPM_BUILD_ROOT
 
 %post
 
 %if "%{init_system}" == "systemd"
-if [ \$1 -eq 1 ]
+if [ $1 -eq 1 ]
 then
     /bin/systemctl enable cloud-config.service     >/dev/null 2>&1 || :
     /bin/systemctl enable cloud-final.service      >/dev/null 2>&1 || :
@@ -157,7 +143,7 @@ fi
 %preun
 
 %if "%{init_system}" == "systemd"
-if [ \$1 -eq 0 ]
+if [ $1 -eq 0 ]
 then
     /bin/systemctl --no-reload disable cloud-config.service >/dev/null 2>&1 || :
     /bin/systemctl --no-reload disable cloud-final.service  >/dev/null 2>&1 || :
@@ -165,7 +151,7 @@ then
     /bin/systemctl --no-reload disable cloud-init-local.service >/dev/null 2>&1 || :
 fi
 %else
-if [ \$1 -eq 0 ]
+if [ $1 -eq 0 ]
 then
     /sbin/service cloud-init stop >/dev/null 2>&1 || :
     /sbin/chkconfig --del cloud-init || :
diff --git a/packages/suse/cloud-init.spec.in b/packages/suse/cloud-init.spec.in
index 6ce0be8c..86e18b1b 100644
--- a/packages/suse/cloud-init.spec.in
+++ b/packages/suse/cloud-init.spec.in
@@ -1,19 +1,19 @@
-## template: cheetah
+## template: jinja
 
 # See: http://www.zarb.org/~jasonc/macros.php
 # Or: http://fedoraproject.org/wiki/Packaging:ScriptletSnippets
 # Or: http://www.rpm.org/max-rpm/ch-rpm-inside.html
 
 Name:           cloud-init
-Version:        ${version}
-Release:        1${subrelease}%{?dist}
+Version:        {{version}}
+Release:        1{{subrelease}}%{?dist}
 Summary:        Cloud instance init scripts
 
 Group:          System/Management
 License:        Dual licensed GPLv3 or Apache 2.0
 URL:            http://launchpad.net/cloud-init
 
-Source0:        ${archive_name}
+Source0:        {{archive_name}}
 BuildRoot:      %{_tmppath}/%{name}-%{version}-build
 
 %if 0%{?suse_version} && 0%{?suse_version} <= 1110
@@ -22,11 +22,9 @@ BuildRoot:      %{_tmppath}/%{name}-%{version}-build
 BuildArch:      noarch
 %endif
 
-BuildRequires:        fdupes
-BuildRequires:        filesystem
-BuildRequires:        python-devel
-BuildRequires:        python-setuptools
-BuildRequires:        python-cheetah
+{% for r in buildrequires %}
+BuildRequires:        {{r}}
+{% endfor %}
 
 %if 0%{?suse_version} && 0%{?suse_version} <= 1210
   %define initsys sysvinit
@@ -34,24 +32,15 @@ BuildRequires:        python-cheetah
   %define initsys systemd
 %endif
 
-# System util packages needed
-Requires:       iproute2
-Requires:       e2fsprogs
-Requires:       net-tools
-Requires:       procps
-Requires:       sudo
-
 # Install pypi 'dynamic' requirements
-#for $r in $requires
-Requires:       ${r}
-#end for
+{% for r in requires %}
+Requires:       {{r}}
+{% endfor %}
 
 # Custom patches
-#set $size = 0
-#for $p in $patches
-Patch${size}: $p
-#set $size += 1
-#end for
+{% for p in patches %}
+Patch{{loop.index0}: {{p}}
+{% endfor %}
 
 %description
 Cloud-init is a set of init scripts for cloud instances.  Cloud instances
@@ -59,14 +48,13 @@ need special scripts to run during initialization to retrieve and install
 ssh keys and to let the user run various scripts.
 
 %prep
-%setup -q -n ${source_name}
+%setup -q -n {{source_name}}
 
 # Custom patches activation
-#set $size = 0
-#for $p in $patches
-%patch${size} -p1
-#set $size += 1
-#end for
+{% for p in patches %}
+%patch{{loop.index0}} -p1
+end for
+{% endfor %}
 
 %build
 %{__python} setup.py build
@@ -95,7 +83,7 @@ rm -r %{buildroot}/%{python_sitelib}/tests
    mkdir -p %{buildroot}/%{_sbindir}
    pushd %{buildroot}/%{_initddir}
    for file in * ; do
-      ln -s %{_initddir}/\${file} %{buildroot}/%{_sbindir}/rc\${file}
+      ln -s %{_initddir}/${file} %{buildroot}/%{_sbindir}/rc${file}
    done
    popd
 %endif
@@ -104,7 +92,7 @@ rm -r %{buildroot}/%{python_sitelib}/tests
 mkdir -p %{buildroot}/%{_defaultdocdir}
 mv %{buildroot}/usr/share/doc/cloud-init %{buildroot}/%{_defaultdocdir}
 for doc in TODO LICENSE ChangeLog requirements.txt; do
-   cp \${doc} %{buildroot}/%{_defaultdocdir}/cloud-init
+   cp ${doc} %{buildroot}/%{_defaultdocdir}/cloud-init
 done
 
 # Remove duplicate files
diff --git a/tools/read-dependencies b/tools/read-dependencies
index f4349055..4ba2c1bc 100755
--- a/tools/read-dependencies
+++ b/tools/read-dependencies
@@ -1,43 +1,197 @@
 #!/usr/bin/env python
+"""List pip dependencies or system package dependencies for cloud-init."""
 
 # You might be tempted to rewrite this as a shell script, but you
 # would be surprised to discover that things like 'egrep' or 'sed' may
 # differ between Linux and *BSD.
 
+try:
+    from argparse import ArgumentParser
+except ImportError:
+    raise RuntimeError(
+        'Could not import python-argparse. Please install python-argparse '
+        'package to continue')
+
+import json
 import os
 import re
-import sys
 import subprocess
+import sys
 
-if 'CLOUD_INIT_TOP_D' in os.environ:
-    topd = os.path.realpath(os.environ.get('CLOUD_INIT_TOP_D'))
-else:
-    topd = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
 
-for fname in ("setup.py", "requirements.txt"):
-    if not os.path.isfile(os.path.join(topd, fname)):
-        sys.stderr.write("Unable to locate '%s' file that should "
-                         "exist in cloud-init root directory." % fname)
-        sys.exit(1)
+# Map the appropriate package dir needed for each distro choice
+DISTRO_PKG_TYPE_MAP = {
+    'centos': 'redhat',
+    'redhat': 'redhat',
+    'debian': 'debian',
+    'ubuntu': 'debian',
+    'opensuse': 'suse',
+    'suse': 'suse'
+}
+
+DISTRO_INSTALL_PKG_CMD = {
+    'centos': ['yum', 'install', '--assumeyes'],
+    'redhat': ['yum', 'install', '--assumeyes'],
+    'debian': ['apt', 'install', '-y'],
+    'ubuntu': ['apt', 'install', '-y'],
+    'opensuse': ['zypper', 'install'],
+    'suse': ['zypper', 'install']
+}
+
+
+# List of base system packages required to start using make
+EXTRA_SYSTEM_BASE_PKGS = ['make', 'sudo', 'tar']
+
+
+# JSON definition of distro-specific package dependencies
+DISTRO_PKG_DEPS_PATH = "packages/pkg-deps.json"
+
+
+def get_parser():
+    """Return an argument parser for this command."""
+    parser = ArgumentParser(description=__doc__)
+    parser.add_argument(
+        '-r', '--requirements-file', type=str, dest='req_file',
+        default='requirements.txt', help='The pip-style requirements file')
+    parser.add_argument(
+        '-d', '--distro', type=str, choices=DISTRO_PKG_TYPE_MAP.keys(),
+        help='The name of the distro to generate package deps for.')
+    parser.add_argument(
+        '--dry-run', action='store_true', default=False, dest='dry_run',
+        help='Dry run the install, making no package changes.')
+    parser.add_argument(
+        '-s', '--system-pkg-names', action='store_true', default=False,
+        dest='system_pkg_names',
+        help='The name of the distro to generate package deps for.')
+    parser.add_argument(
+        '-i', '--install', action='store_true', default=False,
+        dest='install',
+        help='When specified, install the required system packages.')
+    parser.add_argument(
+        '-v', '--python-version', type=str, dest='python_version', default="2",
+        choices=["2", "3"],
+        help='The version of python we want to generate system package '
+             'dependencies for.')
+    return parser
+
 
-if len(sys.argv) > 1:
-    reqfile = sys.argv[1]
-else:
-    reqfile = "requirements.txt"
+def get_package_deps_from_json(topdir, distro):
+    """Get a dict of build and runtime package requirements for a distro.
+
+    @param topdir: The root directory in which to search for the
+        DISTRO_PKG_DEPS_PATH json blob of package requirements information.
+    @param distro: The specific distribution shortname to pull dependencies
+        for.
+    @return: Dict containing "requires", "build-requires" and "rename" lists
+        for a given distribution.
+    """
+    with open(os.path.join(topdir, DISTRO_PKG_DEPS_PATH), 'r') as stream:
+        deps = json.loads(stream.read())
+    if distro is None:
+        return {}
+    return deps[DISTRO_PKG_TYPE_MAP[distro]]
+
+
+def parse_pip_requirements(requirements_path):
+    """Return the pip requirement names from pip-style requirements_path."""
+    dep_names = []
+    with open(requirements_path, "r") as fp:
+        for line in fp:
+            line = line.strip()
+            if not line or line.startswith("#"):
+                continue
+
+            # remove pip-style markers
+            dep = line.split(';')[0]
+
+            # remove version requirements
+            if re.search('[>=.<]+', dep):
+                dep_names.append(re.split(r'[>=.<]+', dep)[0].strip())
+            else:
+                dep_names.append(dep)
+    return dep_names
+
+
+def translate_pip_to_system_pkg(pip_requires, renames, python_ver="2"):
+    """Translate pip package names to distro-specific package names.
+
+    @param pip_requires: List of versionless pip package names to translate.
+    @param renames: Dict containg special case renames from pip name to system
+        package name for the distro.
+    """
+    if python_ver == "2":
+        prefix = "python-"
+    else:
+        prefix = "python3-"
+    standard_pkg_name = "{0}{1}"
+    translated_names = []
+    for pip_name in pip_requires:
+        pip_name = pip_name.lower()
+        # Find a rename if present for the distro package and python version
+        rename = renames.get(pip_name, {}).get(python_ver, None)
+        if rename:
+            translated_names.append(rename)
+        else:
+            translated_names.append(
+                standard_pkg_name.format(prefix, pip_name))
+    return translated_names
+
+
+def main(distro):
+    parser = get_parser()
+    args = parser.parse_args()
+    if 'CLOUD_INIT_TOP_D' in os.environ:
+        topd = os.path.realpath(os.environ.get('CLOUD_INIT_TOP_D'))
+    else:
+        topd = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
+    req_path = os.path.join(topd, args.req_file)
+    if not os.path.isfile(req_path):
+        sys.stderr.write("Unable to locate '%s' file that should "
+                         "exist in cloud-init root directory." % req_path)
+        return 1
+    pip_pkg_names = parse_pip_requirements(req_path)
+    deps_from_json = get_package_deps_from_json(topd, args.distro)
+    renames = deps_from_json.get('renames', {})
+    translated_pip_names = translate_pip_to_system_pkg(
+        pip_pkg_names, renames, args.python_version)
+    all_deps = []
+    if args.distro:
+        all_deps.extend(
+            translated_pip_names + deps_from_json['requires'] +
+            deps_from_json['build-requires'])
+    else:
+        if args.system_pkg_names:
+            all_deps = translated_pip_names
+        else:
+            all_deps = pip_pkg_names
+    if args.install:
+        pkg_install(all_deps, args.distro, args.dry_run)
+    else:
+        print('\n'.join(all_deps))
 
-with open(os.path.join(topd, reqfile), "r") as fp:
-    for line in fp:
-        line = line.strip()
-        if not line or line.startswith("#"):
-            continue
 
-        # remove pip-style markers
-        dep = line.split(';')[0]
+def pkg_install(pkg_list, distro, dry_run=False):
+    """Install a list of packages using the DISTRO_INSTALL_PKG_CMD."""
+    print('Installing deps: {0}{1}'.format(
+          '(dryrun)' if dry_run else '', ' '.join(pkg_list)))
+    pkg_list.extend(EXTRA_SYSTEM_BASE_PKGS)
+    install_cmd = []
+    if dry_run:
+       install_cmd.append('echo')
+    if os.geteuid() != 0:
+        install_cmd.append('sudo')
+    install_cmd.extend(DISTRO_INSTALL_PKG_CMD[distro])
+    if distro in ['centos', 'redhat']:
+        # CentOS and Redhat need epel-release to access oauthlib and jsonschema
+        subprocess.check_call(install_cmd + ['epel-release'])
+    if distro in ['suse', 'opensuse', 'redhat', 'centos']:
+        pkg_list.append('rpm-build')
+    subprocess.check_call(install_cmd + pkg_list)
 
-        # remove version requirements
-        dep = re.split("[>=.<]*", dep)[0].strip()
-        print(dep)
 
-sys.exit(0)
+if __name__ == "__main__":
+    parser = get_parser()
+    args = parser.parse_args()
+    sys.exit(main(args.distro))
 
 # vi: ts=4 expandtab