#!/usr/bin/env python """List pip dependencies or system package dependencies for cloud-init.""" # You might be tempted to rewrite this as a shell script, but you # would be surprised to discover that things like 'egrep' or 'sed' may # differ between Linux and *BSD. try: from argparse import ArgumentParser except ImportError: raise RuntimeError( 'Could not import python-argparse. Please install python-argparse ' 'package to continue') import json import os import re import subprocess import sys # Map the appropriate package dir needed for each distro choice DISTRO_PKG_TYPE_MAP = { 'centos': 'redhat', 'redhat': 'redhat', 'debian': 'debian', 'ubuntu': 'debian', 'opensuse': 'suse', 'suse': 'suse' } DISTRO_INSTALL_PKG_CMD = { 'centos': ['yum', 'install', '--assumeyes'], 'redhat': ['yum', 'install', '--assumeyes'], 'debian': ['apt', 'install', '-y'], 'ubuntu': ['apt', 'install', '-y'], 'opensuse': ['zypper', 'install'], 'suse': ['zypper', 'install'] } # List of base system packages required to start using make EXTRA_SYSTEM_BASE_PKGS = ['make', 'sudo', 'tar'] # JSON definition of distro-specific package dependencies DISTRO_PKG_DEPS_PATH = "packages/pkg-deps.json" def get_parser(): """Return an argument parser for this command.""" parser = ArgumentParser(description=__doc__) parser.add_argument( '-r', '--requirements-file', type=str, dest='req_file', default='requirements.txt', help='The pip-style requirements file') parser.add_argument( '-d', '--distro', type=str, choices=DISTRO_PKG_TYPE_MAP.keys(), help='The name of the distro to generate package deps for.') parser.add_argument( '--dry-run', action='store_true', default=False, dest='dry_run', help='Dry run the install, making no package changes.') parser.add_argument( '-s', '--system-pkg-names', action='store_true', default=False, dest='system_pkg_names', help='The name of the distro to generate package deps for.') parser.add_argument( '-i', '--install', action='store_true', default=False, dest='install', help='When specified, install the required system packages.') parser.add_argument( '-v', '--python-version', type=str, dest='python_version', default="2", choices=["2", "3"], help='The version of python we want to generate system package ' 'dependencies for.') return parser def get_package_deps_from_json(topdir, distro): """Get a dict of build and runtime package requirements for a distro. @param topdir: The root directory in which to search for the DISTRO_PKG_DEPS_PATH json blob of package requirements information. @param distro: The specific distribution shortname to pull dependencies for. @return: Dict containing "requires", "build-requires" and "rename" lists for a given distribution. """ with open(os.path.join(topdir, DISTRO_PKG_DEPS_PATH), 'r') as stream: deps = json.loads(stream.read()) if distro is None: return {} return deps[DISTRO_PKG_TYPE_MAP[distro]] def parse_pip_requirements(requirements_path): """Return the pip requirement names from pip-style requirements_path.""" dep_names = [] with open(requirements_path, "r") as fp: for line in fp: line = line.strip() if not line or line.startswith("#"): continue # remove pip-style markers dep = line.split(';')[0] # remove version requirements if re.search('[>=.<]+', dep): dep_names.append(re.split(r'[>=.<]+', dep)[0].strip()) else: dep_names.append(dep) return dep_names def translate_pip_to_system_pkg(pip_requires, renames, python_ver="2"): """Translate pip package names to distro-specific package names. @param pip_requires: List of versionless pip package names to translate. @param renames: Dict containg special case renames from pip name to system package name for the distro. """ if python_ver == "2": prefix = "python-" else: prefix = "python3-" standard_pkg_name = "{0}{1}" translated_names = [] for pip_name in pip_requires: pip_name = pip_name.lower() # Find a rename if present for the distro package and python version rename = renames.get(pip_name, {}).get(python_ver, None) if rename: translated_names.append(rename) else: translated_names.append( standard_pkg_name.format(prefix, pip_name)) return translated_names def main(distro): parser = get_parser() args = parser.parse_args() if 'CLOUD_INIT_TOP_D' in os.environ: topd = os.path.realpath(os.environ.get('CLOUD_INIT_TOP_D')) else: topd = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) req_path = os.path.join(topd, args.req_file) if not os.path.isfile(req_path): sys.stderr.write("Unable to locate '%s' file that should " "exist in cloud-init root directory." % req_path) return 1 pip_pkg_names = parse_pip_requirements(req_path) deps_from_json = get_package_deps_from_json(topd, args.distro) renames = deps_from_json.get('renames', {}) translated_pip_names = translate_pip_to_system_pkg( pip_pkg_names, renames, args.python_version) all_deps = [] if args.distro: all_deps.extend( translated_pip_names + deps_from_json['requires'] + deps_from_json['build-requires']) else: if args.system_pkg_names: all_deps = translated_pip_names else: all_deps = pip_pkg_names if args.install: pkg_install(all_deps, args.distro, args.dry_run) else: print('\n'.join(all_deps)) def pkg_install(pkg_list, distro, dry_run=False): """Install a list of packages using the DISTRO_INSTALL_PKG_CMD.""" print('Installing deps: {0}{1}'.format( '(dryrun)' if dry_run else '', ' '.join(pkg_list))) pkg_list.extend(EXTRA_SYSTEM_BASE_PKGS) install_cmd = [] if dry_run: install_cmd.append('echo') if os.geteuid() != 0: install_cmd.append('sudo') install_cmd.extend(DISTRO_INSTALL_PKG_CMD[distro]) if distro in ['centos', 'redhat']: # CentOS and Redhat need epel-release to access oauthlib and jsonschema subprocess.check_call(install_cmd + ['epel-release']) if distro in ['suse', 'opensuse', 'redhat', 'centos']: pkg_list.append('rpm-build') subprocess.check_call(install_cmd + pkg_list) if __name__ == "__main__": parser = get_parser() args = parser.parse_args() sys.exit(main(args.distro)) # vi: ts=4 expandtab