#!/usr/bin/env python """List pip dependencies or system package dependencies for cloud-init.""" # You might be tempted to rewrite this as a shell script, but you # would be surprised to discover that things like 'egrep' or 'sed' may # differ between Linux and *BSD. try: from argparse import ArgumentParser except ImportError: raise RuntimeError( 'Could not import python-argparse. Please install python-argparse ' 'package to continue') import json import os import re import subprocess import sys DEFAULT_REQUIREMENTS = 'requirements.txt' # Map the appropriate package dir needed for each distro choice DISTRO_PKG_TYPE_MAP = { 'centos': 'redhat', 'redhat': 'redhat', 'debian': 'debian', 'ubuntu': 'debian', 'opensuse': 'suse', 'suse': 'suse' } DISTRO_INSTALL_PKG_CMD = { 'centos': ['yum', 'install', '--assumeyes'], 'redhat': ['yum', 'install', '--assumeyes'], 'debian': ['apt', 'install', '-y'], 'ubuntu': ['apt', 'install', '-y'], 'opensuse': ['zypper', 'install'], 'suse': ['zypper', 'install'] } # List of base system packages required to enable ci automation CI_SYSTEM_BASE_PKGS = { 'common': ['make', 'sudo', 'tar'], 'redhat': ['python-tox'], 'centos': ['python-tox'], 'ubuntu': ['devscripts', 'python3-dev', 'libssl-dev', 'tox', 'sbuild'], 'debian': ['devscripts', 'python3-dev', 'libssl-dev', 'tox', 'sbuild']} # JSON definition of distro-specific package dependencies DISTRO_PKG_DEPS_PATH = "packages/pkg-deps.json" def get_parser(): """Return an argument parser for this command.""" parser = ArgumentParser(description=__doc__) parser.add_argument( '-r', '--requirements-file', type=str, dest='req_files', action='append', default=None, help='pip-style requirements file [default=%s]' % DEFAULT_REQUIREMENTS) parser.add_argument( '-d', '--distro', type=str, choices=DISTRO_PKG_TYPE_MAP.keys(), help='The name of the distro to generate package deps for.') parser.add_argument( '--dry-run', action='store_true', default=False, dest='dry_run', help='Dry run the install, making no package changes.') parser.add_argument( '-s', '--system-pkg-names', action='store_true', default=False, dest='system_pkg_names', help='The name of the distro to generate package deps for.') parser.add_argument( '-i', '--install', action='store_true', default=False, dest='install', help='When specified, install the required system packages.') parser.add_argument( '-t', '--test-distro', action='store_true', default=False, dest='test_distro', help='Additionally install continuous integration system packages ' 'required for build and test automation.') parser.add_argument( '-v', '--python-version', type=str, dest='python_version', default=None, choices=["2", "3"], help='Override the version of python we want to generate system ' 'package dependencies for. Defaults to the version of python ' 'this script is called with') return parser def get_package_deps_from_json(topdir, distro): """Get a dict of build and runtime package requirements for a distro. @param topdir: The root directory in which to search for the DISTRO_PKG_DEPS_PATH json blob of package requirements information. @param distro: The specific distribution shortname to pull dependencies for. @return: Dict containing "requires", "build-requires" and "rename" lists for a given distribution. """ with open(os.path.join(topdir, DISTRO_PKG_DEPS_PATH), 'r') as stream: deps = json.loads(stream.read()) if distro is None: return {} return deps[DISTRO_PKG_TYPE_MAP[distro]] def parse_pip_requirements(requirements_path): """Return the pip requirement names from pip-style requirements_path.""" dep_names = [] with open(requirements_path, "r") as fp: for line in fp: line = line.strip() if not line or line.startswith("#"): continue # remove pip-style markers dep = line.split(';')[0] # remove version requirements if re.search('[>=.<]+', dep): dep_names.append(re.split(r'[>=.<]+', dep)[0].strip()) else: dep_names.append(dep) return dep_names def translate_pip_to_system_pkg(pip_requires, renames, python_ver): """Translate pip package names to distro-specific package names. @param pip_requires: List of versionless pip package names to translate. @param renames: Dict containg special case renames from pip name to system package name for the distro. @param python_ver: Optional python version string "2" or "3". When None, use the python version that is calling this script via sys.version_info. """ if python_ver is None: python_ver = str(sys.version_info[0]) if python_ver == "2": prefix = "python-" else: prefix = "python3-" standard_pkg_name = "{0}{1}" translated_names = [] for pip_name in pip_requires: pip_name = pip_name.lower() # Find a rename if present for the distro package and python version rename = renames.get(pip_name, {}).get(python_ver, None) if rename: translated_names.append(rename) else: translated_names.append( standard_pkg_name.format(prefix, pip_name)) return translated_names def main(distro): parser = get_parser() args = parser.parse_args() if 'CLOUD_INIT_TOP_D' in os.environ: topd = os.path.realpath(os.environ.get('CLOUD_INIT_TOP_D')) else: topd = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) if args.test_distro: # Give us all the system deps we need for continuous integration if args.req_files: sys.stderr.write( "Parameter --test-distro overrides --requirements-file. Use " "one or the other.\n") sys.exit(1) args.req_files = [os.path.join(topd, DEFAULT_REQUIREMENTS), os.path.join(topd, 'test-' + DEFAULT_REQUIREMENTS)] args.install = True if args.req_files is None: args.req_files = [os.path.join(topd, DEFAULT_REQUIREMENTS)] if not os.path.isfile(args.req_files[0]): sys.stderr.write("Unable to locate '%s' file that should " "exist in cloud-init root directory." % args.req_files[0]) sys.exit(1) bad_files = [r for r in args.req_files if not os.path.isfile(r)] if bad_files: sys.stderr.write( "Unable to find requirements files: %s\n" % ','.join(bad_files)) sys.exit(1) pip_pkg_names = set() for req_path in args.req_files: pip_pkg_names.update(set(parse_pip_requirements(req_path))) deps_from_json = get_package_deps_from_json(topd, args.distro) renames = deps_from_json.get('renames', {}) translated_pip_names = translate_pip_to_system_pkg( pip_pkg_names, renames, args.python_version) all_deps = [] if args.distro: all_deps.extend( translated_pip_names + deps_from_json['requires'] + deps_from_json['build-requires']) else: if args.system_pkg_names: all_deps = translated_pip_names else: all_deps = pip_pkg_names if args.install: pkg_install(all_deps, args.distro, args.test_distro, args.dry_run) else: print('\n'.join(all_deps)) def pkg_install(pkg_list, distro, test_distro=False, dry_run=False): """Install a list of packages using the DISTRO_INSTALL_PKG_CMD.""" if test_distro: pkg_list = list(pkg_list) + CI_SYSTEM_BASE_PKGS['common'] distro_base_pkgs = CI_SYSTEM_BASE_PKGS.get(distro, []) pkg_list += distro_base_pkgs print('Installing deps: {0}{1}'.format( '(dryrun)' if dry_run else '', ' '.join(pkg_list))) install_cmd = [] if dry_run: install_cmd.append('echo') if os.geteuid() != 0: install_cmd.append('sudo') install_cmd.extend(DISTRO_INSTALL_PKG_CMD[distro]) if distro in ['centos', 'redhat']: # CentOS and Redhat need epel-release to access oauthlib and jsonschema subprocess.check_call(install_cmd + ['epel-release']) if distro in ['suse', 'opensuse', 'redhat', 'centos']: pkg_list.append('rpm-build') subprocess.check_call(install_cmd + pkg_list) if __name__ == "__main__": parser = get_parser() args = parser.parse_args() sys.exit(main(args.distro)) # vi: ts=4 expandtab