From 7dcc15d02d997e88c075976a55bf07e789cd18fd Mon Sep 17 00:00:00 2001 From: Chad Smith Date: Wed, 7 Jun 2017 18:13:09 -0600 Subject: makefile: fix python 2/3 detection in the Makefile Fix detection of python in a non-python3 environment. The old path always used python3. The 2 fixes here are: a.) escape the '$' before the subshell. b.) use shell builtin 'command -v' rather than 'which' in case 'which' is not available. --- Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'Makefile') diff --git a/Makefile b/Makefile index 09cd1479..821be4b5 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,7 @@ CWD=$(shell pwd) PYVER ?= $(shell for p in python3 python2; do \ - out=$(which $$p 2>&1) && echo $$p && exit; done; \ - exit 1) + out=$$(command -v $$p 2>&1) && echo $$p && exit; done; exit 1) + noseopts ?= -v YAML_FILES=$(shell find cloudinit bin tests tools -name "*.yaml" -type f ) -- cgit v1.2.3 From e7c95208451422cfd3da7edfbd67dd271e7aa337 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Wed, 7 Jun 2017 13:03:13 -0400 Subject: Makefile: add deb-src and srpm targets. use PYVER more places. This just adds targets for deb-src and srpm, and uses PYVER anywhere where we run a python program. --- Makefile | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) (limited to 'Makefile') diff --git a/Makefile b/Makefile index 821be4b5..66d1dcad 100644 --- a/Makefile +++ b/Makefile @@ -79,15 +79,25 @@ yaml: @$(PYVER) $(CWD)/tools/validate-yaml.py $(YAML_FILES) rpm: - ./packages/brpm --distro $(distro) + $(PYVER) ./packages/brpm --distro=$(distro) + +srpm: + $(PYVER) ./packages/brpm --srpm --distro=$(distro) deb: @which debuild || \ { echo "Missing devscripts dependency. Install with:"; \ echo sudo apt-get install devscripts; exit 1; } - ./packages/bddeb + $(PYVER) ./packages/bddeb + +deb-src: + @which debuild || \ + { echo "Missing devscripts dependency. Install with:"; \ + echo sudo apt-get install devscripts; exit 1; } + $(PYVER) ./packages/bddeb -S -d + -.PHONY: test pyflakes pyflakes3 clean pep8 rpm deb yaml check_version -.PHONY: pip-test-requirements pip-requirements clean_pyc unittest unittest3 -.PHONY: style-check +.PHONY: test pyflakes pyflakes3 clean pep8 rpm srpm deb deb-src yaml +.PHONY: check_version pip-test-requirements pip-requirements clean_pyc +.PHONY: unittest unittest3 style-check -- cgit v1.2.3 From 41d46bfb85929c79dabcec3cf21c8d71401fd2b8 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Wed, 28 Sep 2016 13:20:55 -0700 Subject: cloud.cfg: move to a template. setup.py changes along the way. Here we move the config/cloud.cfg to be rendered as a template. That allows us to maintain deltas between distros in one place. Currently we use 'variant' variable to make decisions. A tools/render-cloudcfg is provided to render the file. There were changes to setup.py, MANIFEST.in to allow us to put all files into a virtual env installation and to render the cloud-config file in 'install' or 'bdist' targets. We have also included some config changes that were found in the redhat distro spec. * include some config changes from the redhat distro spec. The rendered cloud.cfg has some differences. Ubuntu: white space and comment changes only. Freebsd: - whitespace changes and comment changes - datasource_list definition moved to be closer to 'datasource'. - enable modules: migrator, write_files - move package-update-upgrade-install to final. The initial work was done by Josh Harlow. --- MANIFEST.in | 11 ++- Makefile | 3 + cloudinit/util.py | 29 ++++++- config/cloud.cfg | 117 ---------------------------- config/cloud.cfg-freebsd | 88 --------------------- config/cloud.cfg.tmpl | 194 +++++++++++++++++++++++++++++++++++++++++++++++ setup.py | 168 ++++++++++++++++++++++++---------------- tools/render-cloudcfg | 43 +++++++++++ 8 files changed, 379 insertions(+), 274 deletions(-) delete mode 100644 config/cloud.cfg delete mode 100644 config/cloud.cfg-freebsd create mode 100644 config/cloud.cfg.tmpl create mode 100755 tools/render-cloudcfg (limited to 'Makefile') diff --git a/MANIFEST.in b/MANIFEST.in index 94264640..1a4d7711 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,6 +1,15 @@ -include *.py MANIFEST.in ChangeLog +include *.py MANIFEST.in LICENSE* ChangeLog global-include *.txt *.rst *.ini *.in *.conf *.cfg *.sh +graft config +graft doc +graft packages +graft systemd +graft sysvinit +graft templates +graft tests graft tools +graft udev +graft upstart prune build prune dist prune .tox diff --git a/Makefile b/Makefile index 66d1dcad..a3bfaf79 100644 --- a/Makefile +++ b/Makefile @@ -69,6 +69,9 @@ check_version: "not equal to code version '$(CODE_VERSION)'"; exit 2; \ else true; fi +config/cloud.cfg: + $(PYVER) ./tools/render-cloudcfg config/cloud.cfg.tmpl config/cloud.cfg + clean_pyc: @find . -type f -name "*.pyc" -delete diff --git a/cloudinit/util.py b/cloudinit/util.py index 135e4608..b8c3e4ee 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -592,13 +592,40 @@ def get_cfg_option_int(yobj, key, default=0): def system_info(): - return { + info = { 'platform': platform.platform(), 'release': platform.release(), 'python': platform.python_version(), 'uname': platform.uname(), 'dist': platform.linux_distribution(), # pylint: disable=W1505 } + plat = info['platform'].lower() + # Try to get more info about what it actually is, in a format + # that we can easily use across linux and variants... + if plat.startswith('darwin'): + info['variant'] = 'darwin' + elif plat.endswith("bsd"): + info['variant'] = 'bsd' + elif plat.startswith('win'): + info['variant'] = 'windows' + elif 'linux' in plat: + # Try to get a single string out of these... + linux_dist, _version, _id = info['dist'] + linux_dist = linux_dist.lower() + if linux_dist in ('ubuntu', 'linuxmint', 'mint'): + info['variant'] = 'ubuntu' + else: + for prefix, variant in [('redhat', 'rhel'), + ('centos', 'centos'), + ('fedora', 'fedora'), + ('debian', 'debian')]: + if linux_dist.startswith(prefix): + info['variant'] = variant + if 'variant' not in info: + info['variant'] = 'linux' + if 'variant' not in info: + info['variant'] = 'unknown' + return info def get_cfg_option_list(yobj, key, default=None): diff --git a/config/cloud.cfg b/config/cloud.cfg deleted file mode 100644 index 1b93e7f9..00000000 --- a/config/cloud.cfg +++ /dev/null @@ -1,117 +0,0 @@ -# The top level settings are used as module -# and system configuration. - -# A set of users which may be applied and/or used by various modules -# when a 'default' entry is found it will reference the 'default_user' -# from the distro configuration specified below -users: - - default - -# If this is set, 'root' will not be able to ssh in and they -# will get a message to login instead as the above $user (ubuntu) -disable_root: true - -# This will cause the set+update hostname module to not operate (if true) -preserve_hostname: false - -# Example datasource config -# datasource: -# Ec2: -# metadata_urls: [ 'blah.com' ] -# timeout: 5 # (defaults to 50 seconds) -# max_wait: 10 # (defaults to 120 seconds) - -# The modules that run in the 'init' stage -cloud_init_modules: - - migrator - - ubuntu-init-switch - - seed_random - - bootcmd - - write-files - - growpart - - resizefs - - disk_setup - - mounts - - set_hostname - - update_hostname - - update_etc_hosts - - ca-certs - - rsyslog - - users-groups - - ssh - -# The modules that run in the 'config' stage -cloud_config_modules: -# Emit the cloud config ready event -# this can be used by upstart jobs for 'start on cloud-config'. - - emit_upstart - - snap_config - - ssh-import-id - - locale - - set-passwords - - grub-dpkg - - apt-pipelining - - apt-configure - - ntp - - timezone - - disable-ec2-metadata - - runcmd - - byobu - -# The modules that run in the 'final' stage -cloud_final_modules: - - snappy - - package-update-upgrade-install - - fan - - landscape - - lxd - - puppet - - chef - - salt-minion - - mcollective - - rightscale_userdata - - scripts-vendor - - scripts-per-once - - scripts-per-boot - - scripts-per-instance - - scripts-user - - ssh-authkey-fingerprints - - keys-to-console - - phone-home - - final-message - - power-state-change - -# System and/or distro specific settings -# (not accessible to handlers/transforms) -system_info: - # This will affect which distro class gets used - distro: ubuntu - # Default user name + that default users groups (if added/used) - default_user: - name: ubuntu - lock_passwd: True - gecos: Ubuntu - groups: [adm, audio, cdrom, dialout, dip, floppy, lxd, netdev, plugdev, sudo, video] - sudo: ["ALL=(ALL) NOPASSWD:ALL"] - shell: /bin/bash - # Other config here will be given to the distro class and/or path classes - paths: - cloud_dir: /var/lib/cloud/ - templates_dir: /etc/cloud/templates/ - upstart_dir: /etc/init/ - package_mirrors: - - arches: [i386, amd64] - failsafe: - primary: http://archive.ubuntu.com/ubuntu - security: http://security.ubuntu.com/ubuntu - search: - primary: - - http://%(ec2_region)s.ec2.archive.ubuntu.com/ubuntu/ - - http://%(availability_zone)s.clouds.archive.ubuntu.com/ubuntu/ - - http://%(region)s.clouds.archive.ubuntu.com/ubuntu/ - security: [] - - arches: [armhf, armel, default] - failsafe: - primary: http://ports.ubuntu.com/ubuntu-ports - security: http://ports.ubuntu.com/ubuntu-ports - ssh_svcname: ssh diff --git a/config/cloud.cfg-freebsd b/config/cloud.cfg-freebsd deleted file mode 100644 index d666c397..00000000 --- a/config/cloud.cfg-freebsd +++ /dev/null @@ -1,88 +0,0 @@ -# The top level settings are used as module -# and system configuration. - -syslog_fix_perms: root:wheel - -# This should not be required, but leave it in place until the real cause of -# not beeing able to find -any- datasources is resolved. -datasource_list: ['ConfigDrive', 'Azure', 'OpenStack', 'Ec2'] - -# A set of users which may be applied and/or used by various modules -# when a 'default' entry is found it will reference the 'default_user' -# from the distro configuration specified below -users: - - default - -# If this is set, 'root' will not be able to ssh in and they -# will get a message to login instead as the above $user (ubuntu) -disable_root: false - -# This will cause the set+update hostname module to not operate (if true) -preserve_hostname: false - -# Example datasource config -# datasource: -# Ec2: -# metadata_urls: [ 'blah.com' ] -# timeout: 5 # (defaults to 50 seconds) -# max_wait: 10 # (defaults to 120 seconds) - -# The modules that run in the 'init' stage -cloud_init_modules: -# - migrator - - seed_random - - bootcmd -# - write-files - - growpart - - resizefs - - set_hostname - - update_hostname -# - update_etc_hosts -# - ca-certs -# - rsyslog - - users-groups - - ssh - -# The modules that run in the 'config' stage -cloud_config_modules: -# - disk_setup -# - mounts - - ssh-import-id - - locale - - set-passwords - - package-update-upgrade-install -# - landscape - - timezone -# - puppet -# - chef -# - salt-minion -# - mcollective - - disable-ec2-metadata - - runcmd -# - byobu - -# The modules that run in the 'final' stage -cloud_final_modules: - - rightscale_userdata - - scripts-vendor - - scripts-per-once - - scripts-per-boot - - scripts-per-instance - - scripts-user - - ssh-authkey-fingerprints - - keys-to-console - - phone-home - - final-message - - power-state-change - -# System and/or distro specific settings -# (not accessible to handlers/transforms) -system_info: - distro: freebsd - default_user: - name: freebsd - lock_passwd: True - gecos: FreeBSD - groups: [wheel] - sudo: ["ALL=(ALL) NOPASSWD:ALL"] - shell: /bin/tcsh diff --git a/config/cloud.cfg.tmpl b/config/cloud.cfg.tmpl new file mode 100644 index 00000000..5af2a88f --- /dev/null +++ b/config/cloud.cfg.tmpl @@ -0,0 +1,194 @@ +## template:jinja +# The top level settings are used as module +# and system configuration. + +{% if variant in ["bsd"] %} +syslog_fix_perms: root:wheel +{% endif %} +# A set of users which may be applied and/or used by various modules +# when a 'default' entry is found it will reference the 'default_user' +# from the distro configuration specified below +users: + - default + +# If this is set, 'root' will not be able to ssh in and they +# will get a message to login instead as the default $user +{% if variant in ["bsd"] %} +disable_root: false +{% else %} +disable_root: true +{% endif %} + +{% if variant in ["centos", "fedora", "rhel"] %} +mount_default_fields: [~, ~, 'auto', 'defaults,nofail', '0', '2'] +resize_rootfs_tmp: /dev +ssh_deletekeys: 0 +ssh_genkeytypes: ~ +ssh_pwauth: 0 + +{% endif %} +# This will cause the set+update hostname module to not operate (if true) +preserve_hostname: false + +{% if variant in ["bsd"] %} +# This should not be required, but leave it in place until the real cause of +# not beeing able to find -any- datasources is resolved. +datasource_list: ['ConfigDrive', 'Azure', 'OpenStack', 'Ec2'] +{% endif %} +# Example datasource config +# datasource: +# Ec2: +# metadata_urls: [ 'blah.com' ] +# timeout: 5 # (defaults to 50 seconds) +# max_wait: 10 # (defaults to 120 seconds) + +# The modules that run in the 'init' stage +cloud_init_modules: + - migrator +{% if variant in ["ubuntu", "unknown", "debian"] %} + - ubuntu-init-switch +{% endif %} + - seed_random + - bootcmd + - write-files + - growpart + - resizefs +{% if variant not in ["bsd"] %} + - disk_setup + - mounts +{% endif %} + - set_hostname + - update_hostname +{% if variant not in ["bsd"] %} + - update_etc_hosts + - ca-certs + - rsyslog +{% endif %} + - users-groups + - ssh + +# The modules that run in the 'config' stage +cloud_config_modules: +{% if variant in ["ubuntu", "unknown", "debian"] %} +# Emit the cloud config ready event +# this can be used by upstart jobs for 'start on cloud-config'. + - emit_upstart + - snap_config +{% endif %} + - ssh-import-id + - locale + - set-passwords +{% if variant in ["rhel", "fedora"] %} + - spacewalk + - yum-add-repo +{% endif %} +{% if variant in ["ubuntu", "unknown", "debian"] %} + - grub-dpkg + - apt-pipelining + - apt-configure +{% endif %} +{% if variant not in ["bsd"] %} + - ntp +{% endif %} + - timezone + - disable-ec2-metadata + - runcmd +{% if variant in ["ubuntu", "unknown", "debian"] %} + - byobu +{% endif %} + +# The modules that run in the 'final' stage +cloud_final_modules: +{% if variant in ["ubuntu", "unknown", "debian"] %} + - snappy +{% endif %} + - package-update-upgrade-install +{% if variant in ["ubuntu", "unknown", "debian"] %} + - fan + - landscape + - lxd +{% endif %} +{% if variant not in ["bsd"] %} + - puppet + - chef + - salt-minion + - mcollective +{% endif %} + - rightscale_userdata + - scripts-vendor + - scripts-per-once + - scripts-per-boot + - scripts-per-instance + - scripts-user + - ssh-authkey-fingerprints + - keys-to-console + - phone-home + - final-message + - power-state-change + +# System and/or distro specific settings +# (not accessible to handlers/transforms) +system_info: + # This will affect which distro class gets used +{% if variant in ["centos", "debian", "fedora", "rhel", "ubuntu"] %} + distro: {{ variant }} +{% elif variant in ["bsd"] %} + distro: freebsd +{% else %} + # Unknown/fallback distro. + distro: ubuntu +{% endif %} +{% if variant in ["ubuntu", "unknown", "debian"] %} + # Default user name + that default users groups (if added/used) + default_user: + name: ubuntu + lock_passwd: True + gecos: Ubuntu + groups: [adm, audio, cdrom, dialout, dip, floppy, lxd, netdev, plugdev, sudo, video] + sudo: ["ALL=(ALL) NOPASSWD:ALL"] + shell: /bin/bash + # Other config here will be given to the distro class and/or path classes + paths: + cloud_dir: /var/lib/cloud/ + templates_dir: /etc/cloud/templates/ + upstart_dir: /etc/init/ + package_mirrors: + - arches: [i386, amd64] + failsafe: + primary: http://archive.ubuntu.com/ubuntu + security: http://security.ubuntu.com/ubuntu + search: + primary: + - http://%(ec2_region)s.ec2.archive.ubuntu.com/ubuntu/ + - http://%(availability_zone)s.clouds.archive.ubuntu.com/ubuntu/ + - http://%(region)s.clouds.archive.ubuntu.com/ubuntu/ + security: [] + - arches: [armhf, armel, default] + failsafe: + primary: http://ports.ubuntu.com/ubuntu-ports + security: http://ports.ubuntu.com/ubuntu-ports + ssh_svcname: ssh +{% elif variant in ["centos", "rhel", "fedora"] %} + # Default user name + that default users groups (if added/used) + default_user: + name: {{ variant }} + lock_passwd: True + gecos: {{ variant }} Cloud User + groups: [wheel, adm, systemd-journal] + sudo: ["ALL=(ALL) NOPASSWD:ALL"] + shell: /bin/bash + # Other config here will be given to the distro class and/or path classes + paths: + cloud_dir: /var/lib/cloud/ + templates_dir: /etc/cloud/templates/ + ssh_svcname: sshd +{% elif variant in ["bsd"] %} + # Default user name + that default users groups (if added/used) + default_user: + name: freebsd + lock_passwd: True + gecos: FreeBSD + groups: [wheel] + sudo: ["ALL=(ALL) NOPASSWD:ALL"] + shell: /bin/tcsh +{% endif %} diff --git a/setup.py b/setup.py index 4616599b..d5223285 100755 --- a/setup.py +++ b/setup.py @@ -10,8 +10,11 @@ from glob import glob +import atexit import os +import shutil import sys +import tempfile import setuptools from setuptools.command.install import install @@ -53,47 +56,15 @@ def pkg_config_read(library, var): cmd = ['pkg-config', '--variable=%s' % var, library] try: (path, err) = tiny_p(cmd) + path = path.strip() except Exception: - return fallbacks[library][var] - return str(path).strip() + path = fallbacks[library][var] + if path.startswith("/"): + path = path[1:] + return path -INITSYS_FILES = { - 'sysvinit': [f for f in glob('sysvinit/redhat/*') if is_f(f)], - 'sysvinit_freebsd': [f for f in glob('sysvinit/freebsd/*') if is_f(f)], - 'sysvinit_deb': [f for f in glob('sysvinit/debian/*') if is_f(f)], - 'sysvinit_openrc': [f for f in glob('sysvinit/gentoo/*') if is_f(f)], - 'systemd': [f for f in (glob('systemd/*.service') + - glob('systemd/*.target')) if is_f(f)], - 'systemd.generators': [f for f in glob('systemd/*-generator') if is_f(f)], - 'upstart': [f for f in glob('upstart/*') if is_f(f)], -} -INITSYS_ROOTS = { - 'sysvinit': '/etc/rc.d/init.d', - 'sysvinit_freebsd': '/usr/local/etc/rc.d', - 'sysvinit_deb': '/etc/init.d', - 'sysvinit_openrc': '/etc/init.d', - 'systemd': pkg_config_read('systemd', 'systemdsystemunitdir'), - 'systemd.generators': pkg_config_read('systemd', - 'systemdsystemgeneratordir'), - 'upstart': '/etc/init/', -} -INITSYS_TYPES = sorted([f.partition(".")[0] for f in INITSYS_ROOTS.keys()]) - -# Install everything in the right location and take care of Linux (default) and -# FreeBSD systems. -USR = "/usr" -ETC = "/etc" -USR_LIB_EXEC = "/usr/lib" -LIB = "/lib" -if os.uname()[0] == 'FreeBSD': - USR = "/usr/local" - USR_LIB_EXEC = "/usr/local/lib" -elif os.path.isfile('/etc/redhat-release'): - USR_LIB_EXEC = "/usr/libexec" - -# Avoid having datafiles installed in a virtualenv... def in_virtualenv(): try: if sys.real_prefix == sys.prefix: @@ -116,6 +87,66 @@ def read_requires(): return str(deps).splitlines() +def render_cloud_cfg(): + """render cloud.cfg into a tmpdir under same dir as setup.py + + This is rendered to a temporary directory under the top level + directory with the name 'cloud.cfg'. The reason for not just rendering + to config/cloud.cfg is for a.) don't want to write over contents + in that file if user had something there. b.) debuild will complain + that files are different outside of the debian directory.""" + + # older versions of tox use bdist (xenial), and then install from there. + # newer versions just use install. + if not (sys.argv[1] == 'install' or sys.argv[1].startswith('bdist*')): + return 'config/cloud.cfg.tmpl' + topdir = os.path.dirname(sys.argv[0]) + tmpd = tempfile.mkdtemp(dir=topdir) + atexit.register(shutil.rmtree, tmpd) + fpath = os.path.join(tmpd, 'cloud.cfg') + tiny_p([sys.executable, './tools/render-cloudcfg', + 'config/cloud.cfg.tmpl', fpath]) + # relpath is relative to setup.py + relpath = os.path.join(os.path.basename(tmpd), 'cloud.cfg') + return relpath + + +INITSYS_FILES = { + 'sysvinit': [f for f in glob('sysvinit/redhat/*') if is_f(f)], + 'sysvinit_freebsd': [f for f in glob('sysvinit/freebsd/*') if is_f(f)], + 'sysvinit_deb': [f for f in glob('sysvinit/debian/*') if is_f(f)], + 'sysvinit_openrc': [f for f in glob('sysvinit/gentoo/*') if is_f(f)], + 'systemd': [f for f in (glob('systemd/*.service') + + glob('systemd/*.target')) if is_f(f)], + 'systemd.generators': [f for f in glob('systemd/*-generator') if is_f(f)], + 'upstart': [f for f in glob('upstart/*') if is_f(f)], +} +INITSYS_ROOTS = { + 'sysvinit': 'etc/rc.d/init.d', + 'sysvinit_freebsd': 'usr/local/etc/rc.d', + 'sysvinit_deb': 'etc/init.d', + 'sysvinit_openrc': 'etc/init.d', + 'systemd': pkg_config_read('systemd', 'systemdsystemunitdir'), + 'systemd.generators': pkg_config_read('systemd', + 'systemdsystemgeneratordir'), + 'upstart': 'etc/init/', +} +INITSYS_TYPES = sorted([f.partition(".")[0] for f in INITSYS_ROOTS.keys()]) + + +# Install everything in the right location and take care of Linux (default) and +# FreeBSD systems. +USR = "usr" +ETC = "etc" +USR_LIB_EXEC = "usr/lib" +LIB = "lib" +if os.uname()[0] == 'FreeBSD': + USR = "usr/local" + USR_LIB_EXEC = "usr/local/lib" +elif os.path.isfile('/etc/redhat-release'): + USR_LIB_EXEC = "usr/libexec" + + # TODO: Is there a better way to do this?? class InitsysInstallData(install): init_system = None @@ -155,36 +186,39 @@ class InitsysInstallData(install): self.distribution.reinitialize_command('install_data', True) -if in_virtualenv(): - data_files = [] - cmdclass = {} -else: - data_files = [ - (ETC + '/cloud', glob('config/*.cfg')), - (ETC + '/cloud/cloud.cfg.d', glob('config/cloud.cfg.d/*')), - (ETC + '/cloud/templates', glob('templates/*')), - (USR_LIB_EXEC + '/cloud-init', ['tools/ds-identify', - 'tools/uncloud-init', - 'tools/write-ssh-key-fingerprints']), - (USR + '/share/doc/cloud-init', [f for f in glob('doc/*') if is_f(f)]), - (USR + '/share/doc/cloud-init/examples', - [f for f in glob('doc/examples/*') if is_f(f)]), - (USR + '/share/doc/cloud-init/examples/seed', - [f for f in glob('doc/examples/seed/*') if is_f(f)]), - ] - if os.uname()[0] != 'FreeBSD': - data_files.extend([ - (ETC + '/NetworkManager/dispatcher.d/', - ['tools/hook-network-manager']), - (ETC + '/dhcp/dhclient-exit-hooks.d/', ['tools/hook-dhclient']), - (LIB + '/udev/rules.d', [f for f in glob('udev/*.rules')]) - ]) - # Use a subclass for install that handles - # adding on the right init system configuration files - cmdclass = { - 'install': InitsysInstallData, - } - +if not in_virtualenv(): + USR = "/" + USR + ETC = "/" + ETC + USR_LIB_EXEC = "/" + USR_LIB_EXEC + LIB = "/" + LIB + for k in INITSYS_ROOTS.keys(): + INITSYS_ROOTS[k] = "/" + INITSYS_ROOTS[k] + +data_files = [ + (ETC + '/cloud', [render_cloud_cfg()]), + (ETC + '/cloud/cloud.cfg.d', glob('config/cloud.cfg.d/*')), + (ETC + '/cloud/templates', glob('templates/*')), + (USR_LIB_EXEC + '/cloud-init', ['tools/ds-identify', + 'tools/uncloud-init', + 'tools/write-ssh-key-fingerprints']), + (USR + '/share/doc/cloud-init', [f for f in glob('doc/*') if is_f(f)]), + (USR + '/share/doc/cloud-init/examples', + [f for f in glob('doc/examples/*') if is_f(f)]), + (USR + '/share/doc/cloud-init/examples/seed', + [f for f in glob('doc/examples/seed/*') if is_f(f)]), +] +if os.uname()[0] != 'FreeBSD': + data_files.extend([ + (ETC + '/NetworkManager/dispatcher.d/', + ['tools/hook-network-manager']), + (ETC + '/dhcp/dhclient-exit-hooks.d/', ['tools/hook-dhclient']), + (LIB + '/udev/rules.d', [f for f in glob('udev/*.rules')]) + ]) +# Use a subclass for install that handles +# adding on the right init system configuration files +cmdclass = { + 'install': InitsysInstallData, +} requirements = read_requires() if sys.version_info < (3,): diff --git a/tools/render-cloudcfg b/tools/render-cloudcfg new file mode 100755 index 00000000..e624541a --- /dev/null +++ b/tools/render-cloudcfg @@ -0,0 +1,43 @@ +#!/usr/bin/env python3 + +import argparse +import os +import sys + +if "avoid-pep8-E402-import-not-top-of-file": + _tdir = os.path.abspath(os.path.join(os.path.dirname(__file__), "..")) + sys.path.insert(0, _tdir) + from cloudinit import templater + from cloudinit import util + from cloudinit.atomic_helper import write_file + + +def main(): + parser = argparse.ArgumentParser() + variants = ["bsd", "centos", "fedora", "rhel", "ubuntu", "unknown"] + platform = util.system_info() + parser.add_argument( + "--variant", default=platform['variant'], action="store", + help="define the variant.", choices=variants) + parser.add_argument( + "template", nargs="?", action="store", + default='./config/cloud.cfg.tmpl', + help="Path to the cloud.cfg template") + parser.add_argument( + "output", nargs="?", action="store", default="-", + help="Output file. Use '-' to write to stdout") + + args = parser.parse_args() + + with open(args.template, 'r') as fh: + contents = fh.read() + tpl_params = {'variant': args.variant} + contents = (templater.render_string(contents, tpl_params)).rstrip() + "\n" + util.load_yaml(contents) + if args.output == "-": + sys.stdout.write(contents) + else: + write_file(args.output, contents, omode="w") + +if __name__ == '__main__': + main() -- cgit v1.2.3 From 744e648eaf6325758282ef23bffcc4194faa6bac Mon Sep 17 00:00:00 2001 From: Chad Smith Date: Wed, 7 Jun 2017 17:26:52 -0600 Subject: pkg build ci: Add make ci-deps- target to install pkgs This change adds a couple of makefile targets for ci environments to install all necessary dependencies for package builds and test runs. It adds a number of arguments to ./tools/read-dependencies to facilitate reading pip dependencies, translating pip deps to system package names and optionally installing needed system-package dependencies on the local system. This relocates all package dependency and translation logic into ./tools/read-dependencies instead of duplication found in packages/brpm and packages/bddeb. In this branch, we also define buildrequires as including all runtime requires when rendering cloud-init.spec.in and debian/control files because our package build infrastructure will also be running all unit test during the package build process so we need runtime deps at build time. Additionally, this branch converts packages/(redhat|suse)/cloud-init.spec.in from cheetah templates to jinja to allow building python3 envs. --- Makefile | 8 ++ packages/bddeb | 43 ++------ packages/brpm | 45 +++----- packages/debian/control.in | 11 +- packages/pkg-deps.json | 88 ++++++++++++++++ packages/redhat/cloud-init.spec.in | 78 ++++++-------- packages/suse/cloud-init.spec.in | 52 ++++------ tools/read-dependencies | 204 ++++++++++++++++++++++++++++++++----- 8 files changed, 354 insertions(+), 175 deletions(-) create mode 100644 packages/pkg-deps.json (limited to 'Makefile') diff --git a/Makefile b/Makefile index a3bfaf79..c752530c 100644 --- a/Makefile +++ b/Makefile @@ -53,6 +53,14 @@ unittest: clean_pyc unittest3: clean_pyc nosetests3 $(noseopts) tests/unittests +ci-deps-ubuntu: + @$(PYVER) $(CWD)/tools/read-dependencies --distro ubuntu --install --python-version 3 + @$(PYVER) $(CWD)/tools/read-dependencies --distro ubuntu --requirements-file test-requirements.txt --install --python-version 3 + +ci-deps-centos: + @$(PYVER) $(CWD)/tools/read-dependencies --distro centos --install + @$(PYVER) $(CWD)/tools/read-dependencies --distro centos --requirements-file test-requirements.txt --install + pip-requirements: @echo "Installing cloud-init dependencies..." $(PIP_INSTALL) -r "$@.txt" -q diff --git a/packages/bddeb b/packages/bddeb index f415209f..e45af6ee 100755 --- a/packages/bddeb +++ b/packages/bddeb @@ -24,19 +24,6 @@ if "avoid-pep8-E402-import-not-top-of-file": from cloudinit import templater from cloudinit import util -# Package names that will showup in requires which have unique package names. -# Format is '': {'': , ...}. -NONSTD_NAMED_PACKAGES = { - 'argparse': {'2': 'python-argparse', '3': None}, - 'contextlib2': {'2': 'python-contextlib2', '3': None}, - 'cheetah': {'2': 'python-cheetah', '3': None}, - 'pyserial': {'2': 'python-serial', '3': 'python3-serial'}, - 'pyyaml': {'2': 'python-yaml', '3': 'python3-yaml'}, - 'six': {'2': 'python-six', '3': 'python3-six'}, - 'pep8': {'2': 'pep8', '3': 'python3-pep8'}, - 'pyflakes': {'2': 'pyflakes', '3': 'pyflakes'}, -} - DEBUILD_ARGS = ["-S", "-d"] @@ -59,7 +46,6 @@ def write_debian_folder(root, templ_data, is_python2, cloud_util_deps): else: pyver = "3" python = "python3" - pkgfmt = "{}-{}" deb_dir = util.abs_join(root, 'debian') @@ -74,30 +60,23 @@ def write_debian_folder(root, templ_data, is_python2, cloud_util_deps): params=templ_data) # Write out the control file template - reqs = run_helper('read-dependencies').splitlines() + reqs_output = run_helper( + 'read-dependencies', + args=['--distro', 'debian', '--python-version', pyver]) + reqs = reqs_output.splitlines() test_reqs = run_helper( - 'read-dependencies', ['test-requirements.txt']).splitlines() - - pypi_pkgs = [p.lower().strip() for p in reqs] - pypi_test_pkgs = [p.lower().strip() for p in test_reqs] + 'read-dependencies', + ['--requirements-file', 'test-requirements.txt', + '--system-pkg-names', '--python-version', pyver]).splitlines() - # Map to known packages requires = ['cloud-utils | cloud-guest-utils'] if cloud_util_deps else [] - test_requires = [] - lists = ((pypi_pkgs, requires), (pypi_test_pkgs, test_requires)) - for pypilist, target in lists: - for p in pypilist: - if p in NONSTD_NAMED_PACKAGES: - if NONSTD_NAMED_PACKAGES[p][pyver]: - target.append(NONSTD_NAMED_PACKAGES[p][pyver]) - else: # Then standard package prefix - target.append(pkgfmt.format(python, p)) - + # We consolidate all deps as Build-Depends as our package build runs all + # tests so we need all runtime dependencies anyway. + requires.extend(reqs + test_reqs + [python]) templater.render_to_file(util.abs_join(find_root(), 'packages', 'debian', 'control.in'), util.abs_join(deb_dir, 'control'), - params={'requires': ','.join(requires), - 'test_requires': ','.join(test_requires), + params={'build_depends': ','.join(requires), 'python': python}) templater.render_to_file(util.abs_join(find_root(), diff --git a/packages/brpm b/packages/brpm index 89696ab8..3439cf35 100755 --- a/packages/brpm +++ b/packages/brpm @@ -27,17 +27,6 @@ if "avoid-pep8-E402-import-not-top-of-file": from cloudinit import templater from cloudinit import util -# Map python requirements to package names. If a match isn't found -# here, we assume 'python-'. -PACKAGE_MAP = { - 'redhat': { - 'pyserial': 'pyserial', - 'pyyaml': 'PyYAML', - }, - 'suse': { - 'pyyaml': 'python-yaml', - } -} # Subdirectories of the ~/rpmbuild dir RPM_BUILD_SUBDIRS = ['BUILD', 'RPMS', 'SOURCES', 'SPECS', 'SRPMS'] @@ -53,23 +42,18 @@ def run_helper(helper, args=None, strip=True): return stdout -def read_dependencies(): - '''Returns the Python depedencies from requirements.txt. This explicitly - removes 'argparse' from the list of requirements for python >= 2.7, - because with 2.7 argparse became part of the standard library.''' - stdout = run_helper('read-dependencies') - return [p.lower().strip() for p in stdout.splitlines() - if p != 'argparse' or (p == 'argparse' and - sys.version_info[0:2] < (2, 7))] +def read_dependencies(requirements_file='requirements.txt'): + """Returns the Python package depedencies from requirements.txt files. - -def translate_dependencies(deps, distro): - '''Maps python requirements into package names. We assume - python- for packages not listed explicitly in - PACKAGE_MAP.''' - return [PACKAGE_MAP[distro][req] - if req in PACKAGE_MAP[distro] else 'python-%s' % req - for req in deps] + @returns a tuple of (requirements, test_requirements) + """ + pkg_deps = run_helper( + 'read-dependencies', args=['--distro', 'redhat']).splitlines() + test_deps = run_helper( + 'read-dependencies', args=[ + '--requirements-file', 'test-requirements.txt', + '--system-pkg-names']).splitlines() + return (pkg_deps, test_deps) def read_version(): @@ -99,10 +83,9 @@ def generate_spec_contents(args, version_data, tmpl_fn, top_dir, arc_fn): rpm_upstream_version = version_data['version'] subs['rpm_upstream_version'] = rpm_upstream_version - # Map to known packages - python_deps = read_dependencies() - package_deps = translate_dependencies(python_deps, args.distro) - subs['requires'] = package_deps + deps, test_deps = read_dependencies() + subs['buildrequires'] = deps + test_deps + subs['requires'] = deps if args.boot == 'sysvinit': subs['sysvinit'] = True diff --git a/packages/debian/control.in b/packages/debian/control.in index 6c39d531..265b261f 100644 --- a/packages/debian/control.in +++ b/packages/debian/control.in @@ -3,20 +3,13 @@ Source: cloud-init Section: admin Priority: optional Maintainer: Scott Moser -Build-Depends: debhelper (>= 9), - dh-python, - dh-systemd, - ${python}, - ${test_requires}, - ${requires} +Build-Depends: ${build_depends} XS-Python-Version: all Standards-Version: 3.9.6 Package: cloud-init Architecture: all -Depends: procps, - ${python}, - ${misc:Depends}, +Depends: ${misc:Depends}, ${${python}:Depends} Recommends: eatmydata, sudo, software-properties-common, gdisk XB-Python-Version: ${python:Versions} diff --git a/packages/pkg-deps.json b/packages/pkg-deps.json new file mode 100644 index 00000000..8b8f3c37 --- /dev/null +++ b/packages/pkg-deps.json @@ -0,0 +1,88 @@ +{ + "debian" : { + "build-requires" : [ + "debhelper", + "dh-python", + "dh-systemd" + ], + "renames" : { + "pyyaml" : { + "2" : "python-yaml", + "3" : "python3-yaml" + }, + "contextlib2" : { + "2" : "python-contextlib2" + }, + "pyserial" : { + "2" : "python-serial", + "3" : "python3-serial" + } + }, + "requires" : [ + "procps" + ] + }, + "redhat" : { + "build-requires" : [ + "python-devel", + "python-setuptools" + ], + "renames" : { + "jinja2" : { + "3" : "python34-jinja2" + }, + "jsonschema" : { + "3" : "python34-jsonschema" + }, + "prettytable" : { + "3" : "python34-prettytable" + }, + "pyflakes" : { + "2" : "pyflakes", + "3" : "python34-pyflakes" + }, + "pyyaml" : { + "2" : "PyYAML", + "3" : "python34-PyYAML" + }, + "pyserial" : { + "2" : "pyserial" + }, + "requests" : { + "3" : "python34-requests" + }, + "six" : { + "3" : "python34-six" + } + }, + "requires" : [ + "e2fsprogs", + "iproute", + "net-tools", + "procps", + "rsyslog", + "shadow-utils", + "sudo >= 1.7.2p2-3" + ] + }, + "suse" : { + "renames" : { + "pyyaml" : { + "2" : "python-yaml" + } + }, + "build-requires" : [ + "fdupes", + "filesystem", + "python-devel", + "python-setuptools" + ], + "requires" : [ + "iproute2", + "e2fsprogs", + "net-tools", + "procps", + "sudo" + ] + } +} diff --git a/packages/redhat/cloud-init.spec.in b/packages/redhat/cloud-init.spec.in index 3e92c98f..9f75c4b8 100644 --- a/packages/redhat/cloud-init.spec.in +++ b/packages/redhat/cloud-init.spec.in @@ -1,4 +1,4 @@ -## template: cheetah +## template: jinja %{!?python_sitelib: %global python_sitelib %(%{__python} -c "from distutils.sysconfig import get_python_lib; print get_python_lib()")} %define use_systemd (0%{?fedora} && 0%{?fedora} >= 18) || (0%{?rhel} && 0%{?rhel} >= 7) @@ -14,20 +14,18 @@ # Or: http://www.rpm.org/max-rpm/ch-rpm-inside.html Name: cloud-init -Version: ${rpm_upstream_version} -Release: 1${subrelease}%{?dist} +Version: {{rpm_upstream_version}} +Release: 1{{subrelease}}%{?dist} Summary: Cloud instance init scripts Group: System Environment/Base License: Dual-licesed GPLv3 or Apache 2.0 URL: http://launchpad.net/cloud-init -Source0: ${archive_name} +Source0: {{archive_name}} BuildArch: noarch BuildRoot: %{_tmppath} -BuildRequires: python-devel -BuildRequires: python-setuptools %if "%{?el6}" == "1" BuildRequires: python-argparse %endif @@ -46,40 +44,30 @@ Requires(preun): chkconfig # These are runtime dependencies, but declared as BuildRequires so that # - tests can be run here. # - parts of cloud-init such (setup.py) use these dependencies. -#for $r in $requires -BuildRequires: ${r} -#end for +{% for r in requires %} +BuildRequires: {{r}} +{% endfor %} # System util packages needed %ifarch %{?ix86} x86_64 ia64 Requires: dmidecode %endif -Requires: shadow-utils -Requires: rsyslog -Requires: iproute -Requires: e2fsprogs -Requires: net-tools -Requires: procps -Requires: shadow-utils -Requires: sudo >= 1.7.2p2-3 - -Requires: python-setuptools + # python2.6 needs argparse %if "%{?el6}" == "1" Requires: python-argparse %endif -# Install pypi 'dynamic' requirements -#for $r in $requires -Requires: ${r} -#end for + +# Install 'dynamic' runtime reqs from *requirements.txt and pkg-deps.json +{% for r in requires %} +Requires: {{r}} +{% endfor %} # Custom patches -#set $size = 0 -#for $p in $patches -Patch${size}: $p -#set $size += 1 -#end for +{% for p in patches %} +Patch{{loop.index0}}: {{p}} +{% endfor %} %if "%{init_system}" == "systemd" Requires(post): systemd @@ -98,14 +86,12 @@ need special scripts to run during initialization to retrieve and install ssh keys and to let the user run various scripts. %prep -%setup -q -n ${source_name} +%setup -q -n {{source_name}} # Custom patches activation -#set $size = 0 -#for $p in $patches -%patch${size} -p1 -#set $size += 1 -#end for +{% for p in patches %} +%patch{{loop.index0}} -p1 +{% endfor %} %build %{__python} setup.py build @@ -113,34 +99,34 @@ ssh keys and to let the user run various scripts. %install %{__python} setup.py install -O1 \ - --skip-build --root \$RPM_BUILD_ROOT \ + --skip-build --root $RPM_BUILD_ROOT \ --init-system=%{init_system} # Note that /etc/rsyslog.d didn't exist by default until F15. # el6 request: https://bugzilla.redhat.com/show_bug.cgi?id=740420 -mkdir -p \$RPM_BUILD_ROOT/%{_sysconfdir}/rsyslog.d +mkdir -p $RPM_BUILD_ROOT/%{_sysconfdir}/rsyslog.d cp -p tools/21-cloudinit.conf \ - \$RPM_BUILD_ROOT/%{_sysconfdir}/rsyslog.d/21-cloudinit.conf + $RPM_BUILD_ROOT/%{_sysconfdir}/rsyslog.d/21-cloudinit.conf # Remove the tests -rm -rf \$RPM_BUILD_ROOT%{python_sitelib}/tests +rm -rf $RPM_BUILD_ROOT%{python_sitelib}/tests # Required dirs... -mkdir -p \$RPM_BUILD_ROOT/%{_sharedstatedir}/cloud -mkdir -p \$RPM_BUILD_ROOT/%{_libexecdir}/%{name} +mkdir -p $RPM_BUILD_ROOT/%{_sharedstatedir}/cloud +mkdir -p $RPM_BUILD_ROOT/%{_libexecdir}/%{name} %if "%{init_system}" == "systemd" -mkdir -p \$RPM_BUILD_ROOT/%{_unitdir} -cp -p systemd/* \$RPM_BUILD_ROOT/%{_unitdir} +mkdir -p $RPM_BUILD_ROOT/%{_unitdir} +cp -p systemd/* $RPM_BUILD_ROOT/%{_unitdir} %endif %clean -rm -rf \$RPM_BUILD_ROOT +rm -rf $RPM_BUILD_ROOT %post %if "%{init_system}" == "systemd" -if [ \$1 -eq 1 ] +if [ $1 -eq 1 ] then /bin/systemctl enable cloud-config.service >/dev/null 2>&1 || : /bin/systemctl enable cloud-final.service >/dev/null 2>&1 || : @@ -157,7 +143,7 @@ fi %preun %if "%{init_system}" == "systemd" -if [ \$1 -eq 0 ] +if [ $1 -eq 0 ] then /bin/systemctl --no-reload disable cloud-config.service >/dev/null 2>&1 || : /bin/systemctl --no-reload disable cloud-final.service >/dev/null 2>&1 || : @@ -165,7 +151,7 @@ then /bin/systemctl --no-reload disable cloud-init-local.service >/dev/null 2>&1 || : fi %else -if [ \$1 -eq 0 ] +if [ $1 -eq 0 ] then /sbin/service cloud-init stop >/dev/null 2>&1 || : /sbin/chkconfig --del cloud-init || : diff --git a/packages/suse/cloud-init.spec.in b/packages/suse/cloud-init.spec.in index 6ce0be8c..86e18b1b 100644 --- a/packages/suse/cloud-init.spec.in +++ b/packages/suse/cloud-init.spec.in @@ -1,19 +1,19 @@ -## template: cheetah +## template: jinja # See: http://www.zarb.org/~jasonc/macros.php # Or: http://fedoraproject.org/wiki/Packaging:ScriptletSnippets # Or: http://www.rpm.org/max-rpm/ch-rpm-inside.html Name: cloud-init -Version: ${version} -Release: 1${subrelease}%{?dist} +Version: {{version}} +Release: 1{{subrelease}}%{?dist} Summary: Cloud instance init scripts Group: System/Management License: Dual licensed GPLv3 or Apache 2.0 URL: http://launchpad.net/cloud-init -Source0: ${archive_name} +Source0: {{archive_name}} BuildRoot: %{_tmppath}/%{name}-%{version}-build %if 0%{?suse_version} && 0%{?suse_version} <= 1110 @@ -22,11 +22,9 @@ BuildRoot: %{_tmppath}/%{name}-%{version}-build BuildArch: noarch %endif -BuildRequires: fdupes -BuildRequires: filesystem -BuildRequires: python-devel -BuildRequires: python-setuptools -BuildRequires: python-cheetah +{% for r in buildrequires %} +BuildRequires: {{r}} +{% endfor %} %if 0%{?suse_version} && 0%{?suse_version} <= 1210 %define initsys sysvinit @@ -34,24 +32,15 @@ BuildRequires: python-cheetah %define initsys systemd %endif -# System util packages needed -Requires: iproute2 -Requires: e2fsprogs -Requires: net-tools -Requires: procps -Requires: sudo - # Install pypi 'dynamic' requirements -#for $r in $requires -Requires: ${r} -#end for +{% for r in requires %} +Requires: {{r}} +{% endfor %} # Custom patches -#set $size = 0 -#for $p in $patches -Patch${size}: $p -#set $size += 1 -#end for +{% for p in patches %} +Patch{{loop.index0}: {{p}} +{% endfor %} %description Cloud-init is a set of init scripts for cloud instances. Cloud instances @@ -59,14 +48,13 @@ need special scripts to run during initialization to retrieve and install ssh keys and to let the user run various scripts. %prep -%setup -q -n ${source_name} +%setup -q -n {{source_name}} # Custom patches activation -#set $size = 0 -#for $p in $patches -%patch${size} -p1 -#set $size += 1 -#end for +{% for p in patches %} +%patch{{loop.index0}} -p1 +end for +{% endfor %} %build %{__python} setup.py build @@ -95,7 +83,7 @@ rm -r %{buildroot}/%{python_sitelib}/tests mkdir -p %{buildroot}/%{_sbindir} pushd %{buildroot}/%{_initddir} for file in * ; do - ln -s %{_initddir}/\${file} %{buildroot}/%{_sbindir}/rc\${file} + ln -s %{_initddir}/${file} %{buildroot}/%{_sbindir}/rc${file} done popd %endif @@ -104,7 +92,7 @@ rm -r %{buildroot}/%{python_sitelib}/tests mkdir -p %{buildroot}/%{_defaultdocdir} mv %{buildroot}/usr/share/doc/cloud-init %{buildroot}/%{_defaultdocdir} for doc in TODO LICENSE ChangeLog requirements.txt; do - cp \${doc} %{buildroot}/%{_defaultdocdir}/cloud-init + cp ${doc} %{buildroot}/%{_defaultdocdir}/cloud-init done # Remove duplicate files diff --git a/tools/read-dependencies b/tools/read-dependencies index f4349055..4ba2c1bc 100755 --- a/tools/read-dependencies +++ b/tools/read-dependencies @@ -1,43 +1,197 @@ #!/usr/bin/env python +"""List pip dependencies or system package dependencies for cloud-init.""" # You might be tempted to rewrite this as a shell script, but you # would be surprised to discover that things like 'egrep' or 'sed' may # differ between Linux and *BSD. +try: + from argparse import ArgumentParser +except ImportError: + raise RuntimeError( + 'Could not import python-argparse. Please install python-argparse ' + 'package to continue') + +import json import os import re -import sys import subprocess +import sys -if 'CLOUD_INIT_TOP_D' in os.environ: - topd = os.path.realpath(os.environ.get('CLOUD_INIT_TOP_D')) -else: - topd = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) -for fname in ("setup.py", "requirements.txt"): - if not os.path.isfile(os.path.join(topd, fname)): - sys.stderr.write("Unable to locate '%s' file that should " - "exist in cloud-init root directory." % fname) - sys.exit(1) +# Map the appropriate package dir needed for each distro choice +DISTRO_PKG_TYPE_MAP = { + 'centos': 'redhat', + 'redhat': 'redhat', + 'debian': 'debian', + 'ubuntu': 'debian', + 'opensuse': 'suse', + 'suse': 'suse' +} + +DISTRO_INSTALL_PKG_CMD = { + 'centos': ['yum', 'install', '--assumeyes'], + 'redhat': ['yum', 'install', '--assumeyes'], + 'debian': ['apt', 'install', '-y'], + 'ubuntu': ['apt', 'install', '-y'], + 'opensuse': ['zypper', 'install'], + 'suse': ['zypper', 'install'] +} + + +# List of base system packages required to start using make +EXTRA_SYSTEM_BASE_PKGS = ['make', 'sudo', 'tar'] + + +# JSON definition of distro-specific package dependencies +DISTRO_PKG_DEPS_PATH = "packages/pkg-deps.json" + + +def get_parser(): + """Return an argument parser for this command.""" + parser = ArgumentParser(description=__doc__) + parser.add_argument( + '-r', '--requirements-file', type=str, dest='req_file', + default='requirements.txt', help='The pip-style requirements file') + parser.add_argument( + '-d', '--distro', type=str, choices=DISTRO_PKG_TYPE_MAP.keys(), + help='The name of the distro to generate package deps for.') + parser.add_argument( + '--dry-run', action='store_true', default=False, dest='dry_run', + help='Dry run the install, making no package changes.') + parser.add_argument( + '-s', '--system-pkg-names', action='store_true', default=False, + dest='system_pkg_names', + help='The name of the distro to generate package deps for.') + parser.add_argument( + '-i', '--install', action='store_true', default=False, + dest='install', + help='When specified, install the required system packages.') + parser.add_argument( + '-v', '--python-version', type=str, dest='python_version', default="2", + choices=["2", "3"], + help='The version of python we want to generate system package ' + 'dependencies for.') + return parser + -if len(sys.argv) > 1: - reqfile = sys.argv[1] -else: - reqfile = "requirements.txt" +def get_package_deps_from_json(topdir, distro): + """Get a dict of build and runtime package requirements for a distro. + + @param topdir: The root directory in which to search for the + DISTRO_PKG_DEPS_PATH json blob of package requirements information. + @param distro: The specific distribution shortname to pull dependencies + for. + @return: Dict containing "requires", "build-requires" and "rename" lists + for a given distribution. + """ + with open(os.path.join(topdir, DISTRO_PKG_DEPS_PATH), 'r') as stream: + deps = json.loads(stream.read()) + if distro is None: + return {} + return deps[DISTRO_PKG_TYPE_MAP[distro]] + + +def parse_pip_requirements(requirements_path): + """Return the pip requirement names from pip-style requirements_path.""" + dep_names = [] + with open(requirements_path, "r") as fp: + for line in fp: + line = line.strip() + if not line or line.startswith("#"): + continue + + # remove pip-style markers + dep = line.split(';')[0] + + # remove version requirements + if re.search('[>=.<]+', dep): + dep_names.append(re.split(r'[>=.<]+', dep)[0].strip()) + else: + dep_names.append(dep) + return dep_names + + +def translate_pip_to_system_pkg(pip_requires, renames, python_ver="2"): + """Translate pip package names to distro-specific package names. + + @param pip_requires: List of versionless pip package names to translate. + @param renames: Dict containg special case renames from pip name to system + package name for the distro. + """ + if python_ver == "2": + prefix = "python-" + else: + prefix = "python3-" + standard_pkg_name = "{0}{1}" + translated_names = [] + for pip_name in pip_requires: + pip_name = pip_name.lower() + # Find a rename if present for the distro package and python version + rename = renames.get(pip_name, {}).get(python_ver, None) + if rename: + translated_names.append(rename) + else: + translated_names.append( + standard_pkg_name.format(prefix, pip_name)) + return translated_names + + +def main(distro): + parser = get_parser() + args = parser.parse_args() + if 'CLOUD_INIT_TOP_D' in os.environ: + topd = os.path.realpath(os.environ.get('CLOUD_INIT_TOP_D')) + else: + topd = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) + req_path = os.path.join(topd, args.req_file) + if not os.path.isfile(req_path): + sys.stderr.write("Unable to locate '%s' file that should " + "exist in cloud-init root directory." % req_path) + return 1 + pip_pkg_names = parse_pip_requirements(req_path) + deps_from_json = get_package_deps_from_json(topd, args.distro) + renames = deps_from_json.get('renames', {}) + translated_pip_names = translate_pip_to_system_pkg( + pip_pkg_names, renames, args.python_version) + all_deps = [] + if args.distro: + all_deps.extend( + translated_pip_names + deps_from_json['requires'] + + deps_from_json['build-requires']) + else: + if args.system_pkg_names: + all_deps = translated_pip_names + else: + all_deps = pip_pkg_names + if args.install: + pkg_install(all_deps, args.distro, args.dry_run) + else: + print('\n'.join(all_deps)) -with open(os.path.join(topd, reqfile), "r") as fp: - for line in fp: - line = line.strip() - if not line or line.startswith("#"): - continue - # remove pip-style markers - dep = line.split(';')[0] +def pkg_install(pkg_list, distro, dry_run=False): + """Install a list of packages using the DISTRO_INSTALL_PKG_CMD.""" + print('Installing deps: {0}{1}'.format( + '(dryrun)' if dry_run else '', ' '.join(pkg_list))) + pkg_list.extend(EXTRA_SYSTEM_BASE_PKGS) + install_cmd = [] + if dry_run: + install_cmd.append('echo') + if os.geteuid() != 0: + install_cmd.append('sudo') + install_cmd.extend(DISTRO_INSTALL_PKG_CMD[distro]) + if distro in ['centos', 'redhat']: + # CentOS and Redhat need epel-release to access oauthlib and jsonschema + subprocess.check_call(install_cmd + ['epel-release']) + if distro in ['suse', 'opensuse', 'redhat', 'centos']: + pkg_list.append('rpm-build') + subprocess.check_call(install_cmd + pkg_list) - # remove version requirements - dep = re.split("[>=.<]*", dep)[0].strip() - print(dep) -sys.exit(0) +if __name__ == "__main__": + parser = get_parser() + args = parser.parse_args() + sys.exit(main(args.distro)) # vi: ts=4 expandtab -- cgit v1.2.3 From b23d9d7c5c112612dbaaf8c8371c9e735500b2eb Mon Sep 17 00:00:00 2001 From: Chad Smith Date: Wed, 14 Jun 2017 17:11:43 -0600 Subject: ci deps: Add --test-distro to read-dependencies to install all deps read-dependencies now takes --test-distro param to indicate we want to install all system package depenencies to allow for testing and building for our continous integration environment. It allows us to install all needed deps on a fresh system with: python3 ./tools/read-dependencies --distro ubuntu --test-distro [--dry-run]. Additionally read-dependencies now looks at what version of python is running the script (py2 vs p3) and opts to install python 2 or 3 system deps respectively. This behavior can still be overridden with python3 ./tools/read-dependencies ... --python-version 2. There are also some distro-specific packaging and test dependencies, like devscripts, tox and libssl-dev on debian or ubuntu. Those pkg dependencies have now been broken out from common pkg deps to avoid trying to install them on centos/redhat/suse. --- Makefile | 6 ++---- packages/bddeb | 4 +++- tools/read-dependencies | 46 +++++++++++++++++++++++++++++++++++++--------- tools/run-centos | 5 +---- 4 files changed, 43 insertions(+), 18 deletions(-) (limited to 'Makefile') diff --git a/Makefile b/Makefile index c752530c..e9f54982 100644 --- a/Makefile +++ b/Makefile @@ -54,12 +54,10 @@ unittest3: clean_pyc nosetests3 $(noseopts) tests/unittests ci-deps-ubuntu: - @$(PYVER) $(CWD)/tools/read-dependencies --distro ubuntu --install --python-version 3 - @$(PYVER) $(CWD)/tools/read-dependencies --distro ubuntu --requirements-file test-requirements.txt --install --python-version 3 + @$(PYVER) $(CWD)/tools/read-dependencies --distro-ubuntu --test-distro ci-deps-centos: - @$(PYVER) $(CWD)/tools/read-dependencies --distro centos --install - @$(PYVER) $(CWD)/tools/read-dependencies --distro centos --requirements-file test-requirements.txt --install + @$(PYVER) $(CWD)/tools/read-dependencies --distro centos --test-distro pip-requirements: @echo "Installing cloud-init dependencies..." diff --git a/packages/bddeb b/packages/bddeb index e45af6ee..609a94fb 100755 --- a/packages/bddeb +++ b/packages/bddeb @@ -72,7 +72,9 @@ def write_debian_folder(root, templ_data, is_python2, cloud_util_deps): requires = ['cloud-utils | cloud-guest-utils'] if cloud_util_deps else [] # We consolidate all deps as Build-Depends as our package build runs all # tests so we need all runtime dependencies anyway. - requires.extend(reqs + test_reqs + [python]) + # NOTE: python package was moved to the front after debuild -S would fail with + # 'Please add apropriate interpreter' errors (as in debian bug 861132) + requires.extend([python] + reqs + test_reqs) templater.render_to_file(util.abs_join(find_root(), 'packages', 'debian', 'control.in'), util.abs_join(deb_dir, 'control'), diff --git a/tools/read-dependencies b/tools/read-dependencies index 8a585343..2a648680 100755 --- a/tools/read-dependencies +++ b/tools/read-dependencies @@ -40,8 +40,13 @@ DISTRO_INSTALL_PKG_CMD = { } -# List of base system packages required to start using make -EXTRA_SYSTEM_BASE_PKGS = ['make', 'sudo', 'tar'] +# List of base system packages required to enable ci automation +CI_SYSTEM_BASE_PKGS = { + 'common': ['make', 'sudo', 'tar'], + 'redhat': ['python-tox'], + 'centos': ['python-tox'], + 'ubuntu': ['devscripts', 'python3-dev', 'libssl-dev', 'tox', 'sbuild'], + 'debian': ['devscripts', 'python3-dev', 'libssl-dev', 'tox', 'sbuild']} # JSON definition of distro-specific package dependencies @@ -70,10 +75,16 @@ def get_parser(): dest='install', help='When specified, install the required system packages.') parser.add_argument( - '-v', '--python-version', type=str, dest='python_version', default="2", + '-t', '--test-distro', action='store_true', default=False, + dest='test_distro', + help='Additionally install continuous integration system packages ' + 'required for build and test automation.') + parser.add_argument( + '-v', '--python-version', type=str, dest='python_version', default=None, choices=["2", "3"], - help='The version of python we want to generate system package ' - 'dependencies for.') + help='Override the version of python we want to generate system ' + 'package dependencies for. Defaults to the version of python ' + 'this script is called with') return parser @@ -114,13 +125,17 @@ def parse_pip_requirements(requirements_path): return dep_names -def translate_pip_to_system_pkg(pip_requires, renames, python_ver="2"): +def translate_pip_to_system_pkg(pip_requires, renames, python_ver): """Translate pip package names to distro-specific package names. @param pip_requires: List of versionless pip package names to translate. @param renames: Dict containg special case renames from pip name to system package name for the distro. + @param python_ver: Optional python version string "2" or "3". When None, + use the python version that is calling this script via sys.version_info. """ + if python_ver is None: + python_ver = str(sys.version_info[0]) if python_ver == "2": prefix = "python-" else: @@ -147,6 +162,16 @@ def main(distro): else: topd = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) + if args.test_distro: + # Give us all the system deps we need for continuous integration + if args.req_files: + sys.stderr.write( + "Parameter --test-distro overrides --requirements-file. Use " + "one or the other.\n") + sys.exit(1) + args.req_files = [os.path.join(topd, DEFAULT_REQUIREMENTS), + os.path.join(topd, 'test-' + DEFAULT_REQUIREMENTS)] + args.install = True if args.req_files is None: args.req_files = [os.path.join(topd, DEFAULT_REQUIREMENTS)] if not os.path.isfile(args.req_files[0]): @@ -179,16 +204,19 @@ def main(distro): else: all_deps = pip_pkg_names if args.install: - pkg_install(all_deps, args.distro, args.dry_run) + pkg_install(all_deps, args.distro, args.test_distro, args.dry_run) else: print('\n'.join(all_deps)) -def pkg_install(pkg_list, distro, dry_run=False): +def pkg_install(pkg_list, distro, test_distro=False, dry_run=False): """Install a list of packages using the DISTRO_INSTALL_PKG_CMD.""" + if test_distro: + pkg_list = list(pkg_list) + CI_SYSTEM_BASE_PKGS['common'] + distro_base_pkgs = CI_SYSTEM_BASE_PKGS.get(distro, []) + pkg_list += distro_base_pkgs print('Installing deps: {0}{1}'.format( '(dryrun)' if dry_run else '', ' '.join(pkg_list))) - pkg_list = list(pkg_list) + EXTRA_SYSTEM_BASE_PKGS install_cmd = [] if dry_run: install_cmd.append('echo') diff --git a/tools/run-centos b/tools/run-centos index 99ba6be0..b10e3bc4 100755 --- a/tools/run-centos +++ b/tools/run-centos @@ -221,10 +221,7 @@ main() { } inside_as_cd "$name" root "$cdir" \ - ./tools/read-dependencies \ - --requirements-file=requirements.txt \ - --requirements-file=test-requirements.txt \ - --distro=centos --install || { + ./tools/read-dependencies --distro=centos --test-distro || { errorrc "FAIL: failed to install dependencies with read-dependencies" return } -- cgit v1.2.3 From 664a22021ce278dc4120a5737b21bf6fe5448166 Mon Sep 17 00:00:00 2001 From: Chad Smith Date: Fri, 28 Jul 2017 13:48:17 -0600 Subject: makefile: fix ci-deps-ubuntu target Remove errant hyphen in read-dependencies --distro ubuntu parameter for ci-deps-ubuntu. All system packages can now be installed for ubuntu with make ci-deps-ubuntu". [author: powersj, review:blackboxsw] --- Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'Makefile') diff --git a/Makefile b/Makefile index e9f54982..f280911f 100644 --- a/Makefile +++ b/Makefile @@ -46,7 +46,7 @@ pyflakes: pyflakes3: @$(CWD)/tools/run-pyflakes3 - + unittest: clean_pyc nosetests $(noseopts) tests/unittests @@ -54,7 +54,7 @@ unittest3: clean_pyc nosetests3 $(noseopts) tests/unittests ci-deps-ubuntu: - @$(PYVER) $(CWD)/tools/read-dependencies --distro-ubuntu --test-distro + @$(PYVER) $(CWD)/tools/read-dependencies --distro ubuntu --test-distro ci-deps-centos: @$(PYVER) $(CWD)/tools/read-dependencies --distro centos --test-distro -- cgit v1.2.3