summaryrefslogtreecommitdiff
path: root/packages
diff options
context:
space:
mode:
authorChad Smith <chad.smith@canonical.com>2017-06-07 17:26:52 -0600
committerScott Moser <smoser@brickies.net>2017-06-13 22:13:34 -0400
commit744e648eaf6325758282ef23bffcc4194faa6bac (patch)
tree0b497453074cfa817ff1c96afd1d1f749dd1e651 /packages
parent11121fe4d5af0554140d88685029fa248fa0c7c9 (diff)
downloadvyos-cloud-init-744e648eaf6325758282ef23bffcc4194faa6bac.tar.gz
vyos-cloud-init-744e648eaf6325758282ef23bffcc4194faa6bac.zip
pkg build ci: Add make ci-deps-<distro> target to install pkgs
This change adds a couple of makefile targets for ci environments to install all necessary dependencies for package builds and test runs. It adds a number of arguments to ./tools/read-dependencies to facilitate reading pip dependencies, translating pip deps to system package names and optionally installing needed system-package dependencies on the local system. This relocates all package dependency and translation logic into ./tools/read-dependencies instead of duplication found in packages/brpm and packages/bddeb. In this branch, we also define buildrequires as including all runtime requires when rendering cloud-init.spec.in and debian/control files because our package build infrastructure will also be running all unit test during the package build process so we need runtime deps at build time. Additionally, this branch converts packages/(redhat|suse)/cloud-init.spec.in from cheetah templates to jinja to allow building python3 envs.
Diffstat (limited to 'packages')
-rwxr-xr-xpackages/bddeb43
-rwxr-xr-xpackages/brpm45
-rw-r--r--packages/debian/control.in11
-rw-r--r--packages/pkg-deps.json88
-rw-r--r--packages/redhat/cloud-init.spec.in78
-rw-r--r--packages/suse/cloud-init.spec.in52
6 files changed, 167 insertions, 150 deletions
diff --git a/packages/bddeb b/packages/bddeb
index f415209f..e45af6ee 100755
--- a/packages/bddeb
+++ b/packages/bddeb
@@ -24,19 +24,6 @@ if "avoid-pep8-E402-import-not-top-of-file":
from cloudinit import templater
from cloudinit import util
-# Package names that will showup in requires which have unique package names.
-# Format is '<pypi-name>': {'<python_major_version>': <pkg_name_or_none>, ...}.
-NONSTD_NAMED_PACKAGES = {
- 'argparse': {'2': 'python-argparse', '3': None},
- 'contextlib2': {'2': 'python-contextlib2', '3': None},
- 'cheetah': {'2': 'python-cheetah', '3': None},
- 'pyserial': {'2': 'python-serial', '3': 'python3-serial'},
- 'pyyaml': {'2': 'python-yaml', '3': 'python3-yaml'},
- 'six': {'2': 'python-six', '3': 'python3-six'},
- 'pep8': {'2': 'pep8', '3': 'python3-pep8'},
- 'pyflakes': {'2': 'pyflakes', '3': 'pyflakes'},
-}
-
DEBUILD_ARGS = ["-S", "-d"]
@@ -59,7 +46,6 @@ def write_debian_folder(root, templ_data, is_python2, cloud_util_deps):
else:
pyver = "3"
python = "python3"
- pkgfmt = "{}-{}"
deb_dir = util.abs_join(root, 'debian')
@@ -74,30 +60,23 @@ def write_debian_folder(root, templ_data, is_python2, cloud_util_deps):
params=templ_data)
# Write out the control file template
- reqs = run_helper('read-dependencies').splitlines()
+ reqs_output = run_helper(
+ 'read-dependencies',
+ args=['--distro', 'debian', '--python-version', pyver])
+ reqs = reqs_output.splitlines()
test_reqs = run_helper(
- 'read-dependencies', ['test-requirements.txt']).splitlines()
-
- pypi_pkgs = [p.lower().strip() for p in reqs]
- pypi_test_pkgs = [p.lower().strip() for p in test_reqs]
+ 'read-dependencies',
+ ['--requirements-file', 'test-requirements.txt',
+ '--system-pkg-names', '--python-version', pyver]).splitlines()
- # Map to known packages
requires = ['cloud-utils | cloud-guest-utils'] if cloud_util_deps else []
- test_requires = []
- lists = ((pypi_pkgs, requires), (pypi_test_pkgs, test_requires))
- for pypilist, target in lists:
- for p in pypilist:
- if p in NONSTD_NAMED_PACKAGES:
- if NONSTD_NAMED_PACKAGES[p][pyver]:
- target.append(NONSTD_NAMED_PACKAGES[p][pyver])
- else: # Then standard package prefix
- target.append(pkgfmt.format(python, p))
-
+ # We consolidate all deps as Build-Depends as our package build runs all
+ # tests so we need all runtime dependencies anyway.
+ requires.extend(reqs + test_reqs + [python])
templater.render_to_file(util.abs_join(find_root(),
'packages', 'debian', 'control.in'),
util.abs_join(deb_dir, 'control'),
- params={'requires': ','.join(requires),
- 'test_requires': ','.join(test_requires),
+ params={'build_depends': ','.join(requires),
'python': python})
templater.render_to_file(util.abs_join(find_root(),
diff --git a/packages/brpm b/packages/brpm
index 89696ab8..3439cf35 100755
--- a/packages/brpm
+++ b/packages/brpm
@@ -27,17 +27,6 @@ if "avoid-pep8-E402-import-not-top-of-file":
from cloudinit import templater
from cloudinit import util
-# Map python requirements to package names. If a match isn't found
-# here, we assume 'python-<pypi_name>'.
-PACKAGE_MAP = {
- 'redhat': {
- 'pyserial': 'pyserial',
- 'pyyaml': 'PyYAML',
- },
- 'suse': {
- 'pyyaml': 'python-yaml',
- }
-}
# Subdirectories of the ~/rpmbuild dir
RPM_BUILD_SUBDIRS = ['BUILD', 'RPMS', 'SOURCES', 'SPECS', 'SRPMS']
@@ -53,23 +42,18 @@ def run_helper(helper, args=None, strip=True):
return stdout
-def read_dependencies():
- '''Returns the Python depedencies from requirements.txt. This explicitly
- removes 'argparse' from the list of requirements for python >= 2.7,
- because with 2.7 argparse became part of the standard library.'''
- stdout = run_helper('read-dependencies')
- return [p.lower().strip() for p in stdout.splitlines()
- if p != 'argparse' or (p == 'argparse' and
- sys.version_info[0:2] < (2, 7))]
+def read_dependencies(requirements_file='requirements.txt'):
+ """Returns the Python package depedencies from requirements.txt files.
-
-def translate_dependencies(deps, distro):
- '''Maps python requirements into package names. We assume
- python-<pypi_name> for packages not listed explicitly in
- PACKAGE_MAP.'''
- return [PACKAGE_MAP[distro][req]
- if req in PACKAGE_MAP[distro] else 'python-%s' % req
- for req in deps]
+ @returns a tuple of (requirements, test_requirements)
+ """
+ pkg_deps = run_helper(
+ 'read-dependencies', args=['--distro', 'redhat']).splitlines()
+ test_deps = run_helper(
+ 'read-dependencies', args=[
+ '--requirements-file', 'test-requirements.txt',
+ '--system-pkg-names']).splitlines()
+ return (pkg_deps, test_deps)
def read_version():
@@ -99,10 +83,9 @@ def generate_spec_contents(args, version_data, tmpl_fn, top_dir, arc_fn):
rpm_upstream_version = version_data['version']
subs['rpm_upstream_version'] = rpm_upstream_version
- # Map to known packages
- python_deps = read_dependencies()
- package_deps = translate_dependencies(python_deps, args.distro)
- subs['requires'] = package_deps
+ deps, test_deps = read_dependencies()
+ subs['buildrequires'] = deps + test_deps
+ subs['requires'] = deps
if args.boot == 'sysvinit':
subs['sysvinit'] = True
diff --git a/packages/debian/control.in b/packages/debian/control.in
index 6c39d531..265b261f 100644
--- a/packages/debian/control.in
+++ b/packages/debian/control.in
@@ -3,20 +3,13 @@ Source: cloud-init
Section: admin
Priority: optional
Maintainer: Scott Moser <smoser@ubuntu.com>
-Build-Depends: debhelper (>= 9),
- dh-python,
- dh-systemd,
- ${python},
- ${test_requires},
- ${requires}
+Build-Depends: ${build_depends}
XS-Python-Version: all
Standards-Version: 3.9.6
Package: cloud-init
Architecture: all
-Depends: procps,
- ${python},
- ${misc:Depends},
+Depends: ${misc:Depends},
${${python}:Depends}
Recommends: eatmydata, sudo, software-properties-common, gdisk
XB-Python-Version: ${python:Versions}
diff --git a/packages/pkg-deps.json b/packages/pkg-deps.json
new file mode 100644
index 00000000..8b8f3c37
--- /dev/null
+++ b/packages/pkg-deps.json
@@ -0,0 +1,88 @@
+{
+ "debian" : {
+ "build-requires" : [
+ "debhelper",
+ "dh-python",
+ "dh-systemd"
+ ],
+ "renames" : {
+ "pyyaml" : {
+ "2" : "python-yaml",
+ "3" : "python3-yaml"
+ },
+ "contextlib2" : {
+ "2" : "python-contextlib2"
+ },
+ "pyserial" : {
+ "2" : "python-serial",
+ "3" : "python3-serial"
+ }
+ },
+ "requires" : [
+ "procps"
+ ]
+ },
+ "redhat" : {
+ "build-requires" : [
+ "python-devel",
+ "python-setuptools"
+ ],
+ "renames" : {
+ "jinja2" : {
+ "3" : "python34-jinja2"
+ },
+ "jsonschema" : {
+ "3" : "python34-jsonschema"
+ },
+ "prettytable" : {
+ "3" : "python34-prettytable"
+ },
+ "pyflakes" : {
+ "2" : "pyflakes",
+ "3" : "python34-pyflakes"
+ },
+ "pyyaml" : {
+ "2" : "PyYAML",
+ "3" : "python34-PyYAML"
+ },
+ "pyserial" : {
+ "2" : "pyserial"
+ },
+ "requests" : {
+ "3" : "python34-requests"
+ },
+ "six" : {
+ "3" : "python34-six"
+ }
+ },
+ "requires" : [
+ "e2fsprogs",
+ "iproute",
+ "net-tools",
+ "procps",
+ "rsyslog",
+ "shadow-utils",
+ "sudo >= 1.7.2p2-3"
+ ]
+ },
+ "suse" : {
+ "renames" : {
+ "pyyaml" : {
+ "2" : "python-yaml"
+ }
+ },
+ "build-requires" : [
+ "fdupes",
+ "filesystem",
+ "python-devel",
+ "python-setuptools"
+ ],
+ "requires" : [
+ "iproute2",
+ "e2fsprogs",
+ "net-tools",
+ "procps",
+ "sudo"
+ ]
+ }
+}
diff --git a/packages/redhat/cloud-init.spec.in b/packages/redhat/cloud-init.spec.in
index 3e92c98f..9f75c4b8 100644
--- a/packages/redhat/cloud-init.spec.in
+++ b/packages/redhat/cloud-init.spec.in
@@ -1,4 +1,4 @@
-## template: cheetah
+## template: jinja
%{!?python_sitelib: %global python_sitelib %(%{__python} -c "from distutils.sysconfig import get_python_lib; print get_python_lib()")}
%define use_systemd (0%{?fedora} && 0%{?fedora} >= 18) || (0%{?rhel} && 0%{?rhel} >= 7)
@@ -14,20 +14,18 @@
# Or: http://www.rpm.org/max-rpm/ch-rpm-inside.html
Name: cloud-init
-Version: ${rpm_upstream_version}
-Release: 1${subrelease}%{?dist}
+Version: {{rpm_upstream_version}}
+Release: 1{{subrelease}}%{?dist}
Summary: Cloud instance init scripts
Group: System Environment/Base
License: Dual-licesed GPLv3 or Apache 2.0
URL: http://launchpad.net/cloud-init
-Source0: ${archive_name}
+Source0: {{archive_name}}
BuildArch: noarch
BuildRoot: %{_tmppath}
-BuildRequires: python-devel
-BuildRequires: python-setuptools
%if "%{?el6}" == "1"
BuildRequires: python-argparse
%endif
@@ -46,40 +44,30 @@ Requires(preun): chkconfig
# These are runtime dependencies, but declared as BuildRequires so that
# - tests can be run here.
# - parts of cloud-init such (setup.py) use these dependencies.
-#for $r in $requires
-BuildRequires: ${r}
-#end for
+{% for r in requires %}
+BuildRequires: {{r}}
+{% endfor %}
# System util packages needed
%ifarch %{?ix86} x86_64 ia64
Requires: dmidecode
%endif
-Requires: shadow-utils
-Requires: rsyslog
-Requires: iproute
-Requires: e2fsprogs
-Requires: net-tools
-Requires: procps
-Requires: shadow-utils
-Requires: sudo >= 1.7.2p2-3
-
-Requires: python-setuptools
+
# python2.6 needs argparse
%if "%{?el6}" == "1"
Requires: python-argparse
%endif
-# Install pypi 'dynamic' requirements
-#for $r in $requires
-Requires: ${r}
-#end for
+
+# Install 'dynamic' runtime reqs from *requirements.txt and pkg-deps.json
+{% for r in requires %}
+Requires: {{r}}
+{% endfor %}
# Custom patches
-#set $size = 0
-#for $p in $patches
-Patch${size}: $p
-#set $size += 1
-#end for
+{% for p in patches %}
+Patch{{loop.index0}}: {{p}}
+{% endfor %}
%if "%{init_system}" == "systemd"
Requires(post): systemd
@@ -98,14 +86,12 @@ need special scripts to run during initialization to retrieve and install
ssh keys and to let the user run various scripts.
%prep
-%setup -q -n ${source_name}
+%setup -q -n {{source_name}}
# Custom patches activation
-#set $size = 0
-#for $p in $patches
-%patch${size} -p1
-#set $size += 1
-#end for
+{% for p in patches %}
+%patch{{loop.index0}} -p1
+{% endfor %}
%build
%{__python} setup.py build
@@ -113,34 +99,34 @@ ssh keys and to let the user run various scripts.
%install
%{__python} setup.py install -O1 \
- --skip-build --root \$RPM_BUILD_ROOT \
+ --skip-build --root $RPM_BUILD_ROOT \
--init-system=%{init_system}
# Note that /etc/rsyslog.d didn't exist by default until F15.
# el6 request: https://bugzilla.redhat.com/show_bug.cgi?id=740420
-mkdir -p \$RPM_BUILD_ROOT/%{_sysconfdir}/rsyslog.d
+mkdir -p $RPM_BUILD_ROOT/%{_sysconfdir}/rsyslog.d
cp -p tools/21-cloudinit.conf \
- \$RPM_BUILD_ROOT/%{_sysconfdir}/rsyslog.d/21-cloudinit.conf
+ $RPM_BUILD_ROOT/%{_sysconfdir}/rsyslog.d/21-cloudinit.conf
# Remove the tests
-rm -rf \$RPM_BUILD_ROOT%{python_sitelib}/tests
+rm -rf $RPM_BUILD_ROOT%{python_sitelib}/tests
# Required dirs...
-mkdir -p \$RPM_BUILD_ROOT/%{_sharedstatedir}/cloud
-mkdir -p \$RPM_BUILD_ROOT/%{_libexecdir}/%{name}
+mkdir -p $RPM_BUILD_ROOT/%{_sharedstatedir}/cloud
+mkdir -p $RPM_BUILD_ROOT/%{_libexecdir}/%{name}
%if "%{init_system}" == "systemd"
-mkdir -p \$RPM_BUILD_ROOT/%{_unitdir}
-cp -p systemd/* \$RPM_BUILD_ROOT/%{_unitdir}
+mkdir -p $RPM_BUILD_ROOT/%{_unitdir}
+cp -p systemd/* $RPM_BUILD_ROOT/%{_unitdir}
%endif
%clean
-rm -rf \$RPM_BUILD_ROOT
+rm -rf $RPM_BUILD_ROOT
%post
%if "%{init_system}" == "systemd"
-if [ \$1 -eq 1 ]
+if [ $1 -eq 1 ]
then
/bin/systemctl enable cloud-config.service >/dev/null 2>&1 || :
/bin/systemctl enable cloud-final.service >/dev/null 2>&1 || :
@@ -157,7 +143,7 @@ fi
%preun
%if "%{init_system}" == "systemd"
-if [ \$1 -eq 0 ]
+if [ $1 -eq 0 ]
then
/bin/systemctl --no-reload disable cloud-config.service >/dev/null 2>&1 || :
/bin/systemctl --no-reload disable cloud-final.service >/dev/null 2>&1 || :
@@ -165,7 +151,7 @@ then
/bin/systemctl --no-reload disable cloud-init-local.service >/dev/null 2>&1 || :
fi
%else
-if [ \$1 -eq 0 ]
+if [ $1 -eq 0 ]
then
/sbin/service cloud-init stop >/dev/null 2>&1 || :
/sbin/chkconfig --del cloud-init || :
diff --git a/packages/suse/cloud-init.spec.in b/packages/suse/cloud-init.spec.in
index 6ce0be8c..86e18b1b 100644
--- a/packages/suse/cloud-init.spec.in
+++ b/packages/suse/cloud-init.spec.in
@@ -1,19 +1,19 @@
-## template: cheetah
+## template: jinja
# See: http://www.zarb.org/~jasonc/macros.php
# Or: http://fedoraproject.org/wiki/Packaging:ScriptletSnippets
# Or: http://www.rpm.org/max-rpm/ch-rpm-inside.html
Name: cloud-init
-Version: ${version}
-Release: 1${subrelease}%{?dist}
+Version: {{version}}
+Release: 1{{subrelease}}%{?dist}
Summary: Cloud instance init scripts
Group: System/Management
License: Dual licensed GPLv3 or Apache 2.0
URL: http://launchpad.net/cloud-init
-Source0: ${archive_name}
+Source0: {{archive_name}}
BuildRoot: %{_tmppath}/%{name}-%{version}-build
%if 0%{?suse_version} && 0%{?suse_version} <= 1110
@@ -22,11 +22,9 @@ BuildRoot: %{_tmppath}/%{name}-%{version}-build
BuildArch: noarch
%endif
-BuildRequires: fdupes
-BuildRequires: filesystem
-BuildRequires: python-devel
-BuildRequires: python-setuptools
-BuildRequires: python-cheetah
+{% for r in buildrequires %}
+BuildRequires: {{r}}
+{% endfor %}
%if 0%{?suse_version} && 0%{?suse_version} <= 1210
%define initsys sysvinit
@@ -34,24 +32,15 @@ BuildRequires: python-cheetah
%define initsys systemd
%endif
-# System util packages needed
-Requires: iproute2
-Requires: e2fsprogs
-Requires: net-tools
-Requires: procps
-Requires: sudo
-
# Install pypi 'dynamic' requirements
-#for $r in $requires
-Requires: ${r}
-#end for
+{% for r in requires %}
+Requires: {{r}}
+{% endfor %}
# Custom patches
-#set $size = 0
-#for $p in $patches
-Patch${size}: $p
-#set $size += 1
-#end for
+{% for p in patches %}
+Patch{{loop.index0}: {{p}}
+{% endfor %}
%description
Cloud-init is a set of init scripts for cloud instances. Cloud instances
@@ -59,14 +48,13 @@ need special scripts to run during initialization to retrieve and install
ssh keys and to let the user run various scripts.
%prep
-%setup -q -n ${source_name}
+%setup -q -n {{source_name}}
# Custom patches activation
-#set $size = 0
-#for $p in $patches
-%patch${size} -p1
-#set $size += 1
-#end for
+{% for p in patches %}
+%patch{{loop.index0}} -p1
+end for
+{% endfor %}
%build
%{__python} setup.py build
@@ -95,7 +83,7 @@ rm -r %{buildroot}/%{python_sitelib}/tests
mkdir -p %{buildroot}/%{_sbindir}
pushd %{buildroot}/%{_initddir}
for file in * ; do
- ln -s %{_initddir}/\${file} %{buildroot}/%{_sbindir}/rc\${file}
+ ln -s %{_initddir}/${file} %{buildroot}/%{_sbindir}/rc${file}
done
popd
%endif
@@ -104,7 +92,7 @@ rm -r %{buildroot}/%{python_sitelib}/tests
mkdir -p %{buildroot}/%{_defaultdocdir}
mv %{buildroot}/usr/share/doc/cloud-init %{buildroot}/%{_defaultdocdir}
for doc in TODO LICENSE ChangeLog requirements.txt; do
- cp \${doc} %{buildroot}/%{_defaultdocdir}/cloud-init
+ cp ${doc} %{buildroot}/%{_defaultdocdir}/cloud-init
done
# Remove duplicate files