1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
|
language: python
dist: bionic
# We use two different caching strategies. The default is to cache pip
# packages (as most of our jobs use pip packages), which is configured here.
# For the integration tests, we instead want to cache the lxd images and
# package build schroot.
#
# We cache the lxd images because this saves a few seconds in the general
# case, but provides substantial speed-ups when cloud-images.ubuntu.com, the
# source of the images, is under heavy load. The directory in which the lxd
# images are stored (/var/snap/lxd/common/lxd/images/) is not
# readable/writeable by the default user (which is a requirement for caching),
# so we instead cache the `lxd_images/` directory. We move lxd images out of
# there before we run tests and back in once tests are complete. We _move_ the
# images out and only copy the most recent lxd image back into the cache, to
# avoid our cache growing without bound. (We only need the most recent lxd
# image because the integration tests only use a single image.)
#
# We cache the package build schroot because it saves 2-3 minutes per build.
# Without caching, we have to perform a debootstrap for every build. We update
# the schroot before storing it back in the cache, to ensure that we aren't
# just using an increasingly-old schroot as time passes. The cached schroot is
# stored as a tarball, to preserve permissions/ownership.
cache: pip
install:
# Required so `git describe` will definitely find a tag; see
# https://github.com/travis-ci/travis-ci/issues/7422
- git fetch --unshallow
- pip install tox
script:
- tox
matrix:
fast_finish: true
allow_failures:
- name: "Integration Tests (WIP)"
include:
- python: 3.6
env:
TOXENV=py3
PYTEST_ADDOPTS=-v # List all tests run by pytest
- if: NOT branch =~ /^ubuntu\//
cache:
- directories:
- lxd_images
- chroots
before_cache:
- |
# Find the most recent image file
latest_file="$(sudo ls -Art /var/snap/lxd/common/lxd/images/ | tail -n 1)"
# This might be <hash>.rootfs or <hash>, normalise
latest_file="$(basename $latest_file .rootfs)"
# Find all files with that prefix and copy them to our cache dir
sudo find /var/snap/lxd/common/lxd/images/ -name $latest_file* -print -exec cp {} "$TRAVIS_BUILD_DIR/lxd_images/" \;
install:
- git fetch --unshallow
- sudo apt-get install -y --install-recommends sbuild ubuntu-dev-tools fakeroot tox debhelper
- pip install .
- pip install tox
# bionic has lxd from deb installed, remove it first to ensure
# pylxd talks only to the lxd from snap
- sudo apt remove --purge lxd lxd-client
- sudo rm -Rf /var/lib/lxd
- sudo snap install lxd
- sudo lxd init --auto
- sudo mkdir --mode=1777 -p /var/snap/lxd/common/consoles
# Move any cached lxd images into lxd's image dir
- sudo find "$TRAVIS_BUILD_DIR/lxd_images/" -type f -print -exec mv {} /var/snap/lxd/common/lxd/images/ \;
- sudo usermod -a -G lxd $USER
- sudo sbuild-adduser $USER
- cp /usr/share/doc/sbuild/examples/example.sbuildrc /home/$USER/.sbuildrc
script:
# Ubuntu LTS: Build
- ./packages/bddeb -S -d --release xenial
- |
needs_caching=false
if [ -e "$TRAVIS_BUILD_DIR/chroots/xenial-amd64.tar" ]; then
# If we have a cached chroot, move it into place
sudo mkdir -p /var/lib/schroot/chroots/xenial-amd64
sudo tar --sparse --xattrs --preserve-permissions --numeric-owner -xf "$TRAVIS_BUILD_DIR/chroots/xenial-amd64.tar" -C /var/lib/schroot/chroots/xenial-amd64
# Write its configuration
cat > sbuild-xenial-amd64 << EOM
[xenial-amd64]
description=xenial-amd64
groups=sbuild,root,admin
root-groups=sbuild,root,admin
# Uncomment these lines to allow members of these groups to access
# the -source chroots directly (useful for automated updates, etc).
#source-root-users=sbuild,root,admin
#source-root-groups=sbuild,root,admin
type=directory
profile=sbuild
union-type=overlay
directory=/var/lib/schroot/chroots/xenial-amd64
EOM
sudo mv sbuild-xenial-amd64 /etc/schroot/chroot.d/
sudo chown root /etc/schroot/chroot.d/sbuild-xenial-amd64
# And ensure it's up-to-date.
before_pkgs="$(sudo schroot -c source:xenial-amd64 -d / dpkg -l | sha256sum)"
sudo schroot -c source:xenial-amd64 -d / -- sh -c "apt-get update && apt-get -qqy upgrade"
after_pkgs=$(sudo schroot -c source:xenial-amd64 -d / dpkg -l | sha256sum)
if [ "$before_pkgs" != "$after_pkgs" ]; then
needs_caching=true
fi
else
# Otherwise, create the chroot
sudo -E su $USER -c 'mk-sbuild xenial'
needs_caching=true
fi
# If there are changes to the schroot (or it's entirely new),
# tar up the schroot (to preserve ownership/permissions) and
# move it into the cached dir; no need to compress it because
# Travis will do that anyway
if [ "$needs_caching" = "true" ]; then
sudo tar --sparse --xattrs --xattrs-include=* -cf "$TRAVIS_BUILD_DIR/chroots/xenial-amd64.tar" -C /var/lib/schroot/chroots/xenial-amd64 .
fi
# Use sudo to get a new shell where we're in the sbuild group
- sudo -E su $USER -c 'sbuild --nolog --no-run-lintian --verbose --dist=xenial cloud-init_*.dsc'
# Ubuntu LTS: Integration
- sg lxd -c 'tox -e citest -- run --verbose --preserve-data --data-dir results --os-name xenial --test modules/apt_configure_sources_list.yaml --test modules/ntp_servers --test modules/set_password_list --test modules/user_groups --deb cloud-init_*_all.deb'
- name: "Integration Tests (WIP)"
if: NOT branch =~ /^ubuntu\//
cache:
- directories:
- lxd_images
- chroots
before_cache:
- |
# Find the most recent image file
latest_file="$(sudo ls -Art /var/snap/lxd/common/lxd/images/ | tail -n 1)"
# This might be <hash>.rootfs or <hash>, normalise
latest_file="$(basename $latest_file .rootfs)"
# Find all files with that prefix and copy them to our cache dir
sudo find /var/snap/lxd/common/lxd/images/ -name $latest_file* -print -exec cp {} "$TRAVIS_BUILD_DIR/lxd_images/" \;
install:
- git fetch --unshallow
- sudo apt-get install -y --install-recommends sbuild ubuntu-dev-tools fakeroot tox debhelper
- pip install .
- pip install tox
# bionic has lxd from deb installed, remove it first to ensure
# pylxd talks only to the lxd from snap
- sudo apt remove --purge lxd lxd-client
- sudo rm -Rf /var/lib/lxd
- sudo snap install lxd
- sudo lxd init --auto
- sudo mkdir --mode=1777 -p /var/snap/lxd/common/consoles
# Move any cached lxd images into lxd's image dir
- sudo find "$TRAVIS_BUILD_DIR/lxd_images/" -type f -print -exec mv {} /var/snap/lxd/common/lxd/images/ \;
- sudo usermod -a -G lxd $USER
- sudo sbuild-adduser $USER
- cp /usr/share/doc/sbuild/examples/example.sbuildrc /home/$USER/.sbuildrc
script:
# Ubuntu LTS: Build
- ./packages/bddeb -S -d --release xenial
- |
needs_caching=false
if [ -e "$TRAVIS_BUILD_DIR/chroots/xenial-amd64.tar" ]; then
# If we have a cached chroot, move it into place
sudo mkdir -p /var/lib/schroot/chroots/xenial-amd64
sudo tar --sparse --xattrs --preserve-permissions --numeric-owner -xf "$TRAVIS_BUILD_DIR/chroots/xenial-amd64.tar" -C /var/lib/schroot/chroots/xenial-amd64
# Write its configuration
cat > sbuild-xenial-amd64 << EOM
[xenial-amd64]
description=xenial-amd64
groups=sbuild,root,admin
root-groups=sbuild,root,admin
# Uncomment these lines to allow members of these groups to access
# the -source chroots directly (useful for automated updates, etc).
#source-root-users=sbuild,root,admin
#source-root-groups=sbuild,root,admin
type=directory
profile=sbuild
union-type=overlay
directory=/var/lib/schroot/chroots/xenial-amd64
EOM
sudo mv sbuild-xenial-amd64 /etc/schroot/chroot.d/
sudo chown root /etc/schroot/chroot.d/sbuild-xenial-amd64
# And ensure it's up-to-date.
before_pkgs="$(sudo schroot -c source:xenial-amd64 -d / dpkg -l | sha256sum)"
sudo schroot -c source:xenial-amd64 -d / -- sh -c "apt-get update && apt-get -qqy upgrade"
after_pkgs=$(sudo schroot -c source:xenial-amd64 -d / dpkg -l | sha256sum)
if [ "$before_pkgs" != "$after_pkgs" ]; then
needs_caching=true
fi
else
# Otherwise, create the chroot
sudo -E su $USER -c 'mk-sbuild xenial'
needs_caching=true
fi
# If there are changes to the schroot (or it's entirely new),
# tar up the schroot (to preserve ownership/permissions) and
# move it into the cached dir; no need to compress it because
# Travis will do that anyway
if [ "$needs_caching" = "true" ]; then
sudo tar --sparse --xattrs --xattrs-include=* -cf "$TRAVIS_BUILD_DIR/chroots/xenial-amd64.tar" -C /var/lib/schroot/chroots/xenial-amd64 .
fi
# Use sudo to get a new shell where we're in the sbuild group
- sudo -E su $USER -c 'sbuild --nolog --no-run-lintian --verbose --dist=xenial cloud-init_*.dsc'
- sg lxd -c 'CLOUD_INIT_CLOUD_INIT_SOURCE="$(ls *.deb)" tox -e integration-tests-ci' &
- |
SECONDS=0
while [ -e /proc/$! ]; do
if [ "$SECONDS" -gt "570" ]; then
echo -n '.'
SECONDS=0
fi
sleep 10
done
- python: 3.5
env:
TOXENV=xenial
PYTEST_ADDOPTS=-v # List all tests run by pytest
dist: xenial
- python: 3.6
env: TOXENV=flake8
- python: 3.6
env: TOXENV=pylint
- python: 3.6
env: TOXENV=doc
|
