diff options
Diffstat (limited to '.travis.yml')
| -rw-r--r-- | .travis.yml | 105 |
1 files changed, 90 insertions, 15 deletions
diff --git a/.travis.yml b/.travis.yml index d2651c0b..4c5bf4c4 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,6 +1,29 @@ language: python dist: bionic +# We use two different caching strategies. The default is to cache pip +# packages (as most of our jobs use pip packages), which is configured here. +# For the integration tests, we instead want to cache the lxd images and +# package build schroot. +# +# We cache the lxd images because this saves a few seconds in the general +# case, but provides substantial speed-ups when cloud-images.ubuntu.com, the +# source of the images, is under heavy load. The directory in which the lxd +# images are stored (/var/snap/lxd/common/lxd/images/) is not +# readable/writeable by the default user (which is a requirement for caching), +# so we instead cache the `lxd_images/` directory. We move lxd images out of +# there before we run tests and back in once tests are complete. We _move_ the +# images out and only copy the most recent lxd image back into the cache, to +# avoid our cache growing without bound. (We only need the most recent lxd +# image because the integration tests only use a single image.) +# +# We cache the package build schroot because it saves 2-3 minutes per build. +# Without caching, we have to perform a debootstrap for every build. We update +# the schroot before storing it back in the cache, to ensure that we aren't +# just using an increasingly-old schroot as time passes. The cached schroot is +# stored as a tarball, to preserve permissions/ownership. +cache: pip + install: # Required so `git describe` will definitely find a tag; see # https://github.com/travis-ci/travis-ci/issues/7422 @@ -16,13 +39,23 @@ matrix: - python: 3.6 env: TOXENV=py3 - NOSE_VERBOSE=2 # List all tests run by nose - - install: + PYTEST_ADDOPTS=-v # List all tests run by pytest + - if: NOT branch =~ /^ubuntu\// + cache: + - directories: + - lxd_images + - chroots + before_cache: + - | + # Find the most recent image file + latest_file="$(sudo ls -Art /var/snap/lxd/common/lxd/images/ | tail -n 1)" + # This might be <hash>.rootfs or <hash>, normalise + latest_file="$(basename $latest_file .rootfs)" + # Find all files with that prefix and copy them to our cache dir + sudo find /var/snap/lxd/common/lxd/images/ -name $latest_file* -print -exec cp {} "$TRAVIS_BUILD_DIR/lxd_images/" \; + install: - git fetch --unshallow - - sudo apt-get build-dep -y cloud-init - - sudo apt-get install -y --install-recommends sbuild ubuntu-dev-tools fakeroot tox - # These are build deps but not pulled in by the build-dep call above - - sudo apt-get install -y --install-recommends dh-systemd python3-coverage python3-contextlib2 + - sudo apt-get install -y --install-recommends sbuild ubuntu-dev-tools fakeroot tox debhelper - pip install . - pip install tox # bionic has lxd from deb installed, remove it first to ensure @@ -32,26 +65,68 @@ matrix: - sudo snap install lxd - sudo lxd init --auto - sudo mkdir --mode=1777 -p /var/snap/lxd/common/consoles + # Move any cached lxd images into lxd's image dir + - sudo find "$TRAVIS_BUILD_DIR/lxd_images/" -type f -print -exec mv {} /var/snap/lxd/common/lxd/images/ \; - sudo usermod -a -G lxd $USER - sudo sbuild-adduser $USER - cp /usr/share/doc/sbuild/examples/example.sbuildrc /home/$USER/.sbuildrc script: # Ubuntu LTS: Build - - ./packages/bddeb -S - # Use this to get a new shell where we're in the sbuild group - - sudo -E su $USER -c 'mk-sbuild xenial' - - sudo -E su $USER -c 'sbuild --nolog --verbose --dist=xenial cloud-init_*.dsc' + - ./packages/bddeb -S -d --release xenial + - | + needs_caching=false + if [ -e "$TRAVIS_BUILD_DIR/chroots/xenial-amd64.tar" ]; then + # If we have a cached chroot, move it into place + sudo mkdir -p /var/lib/schroot/chroots/xenial-amd64 + sudo tar --sparse --xattrs --preserve-permissions --numeric-owner -xf "$TRAVIS_BUILD_DIR/chroots/xenial-amd64.tar" -C /var/lib/schroot/chroots/xenial-amd64 + # Write its configuration + cat > sbuild-xenial-amd64 << EOM + [xenial-amd64] + description=xenial-amd64 + groups=sbuild,root,admin + root-groups=sbuild,root,admin + # Uncomment these lines to allow members of these groups to access + # the -source chroots directly (useful for automated updates, etc). + #source-root-users=sbuild,root,admin + #source-root-groups=sbuild,root,admin + type=directory + profile=sbuild + union-type=overlay + directory=/var/lib/schroot/chroots/xenial-amd64 + EOM + sudo mv sbuild-xenial-amd64 /etc/schroot/chroot.d/ + sudo chown root /etc/schroot/chroot.d/sbuild-xenial-amd64 + # And ensure it's up-to-date. + before_pkgs="$(sudo schroot -c source:xenial-amd64 -d / dpkg -l | sha256sum)" + sudo schroot -c source:xenial-amd64 -d / -- sh -c "apt-get update && apt-get -qqy upgrade" + after_pkgs=$(sudo schroot -c source:xenial-amd64 -d / dpkg -l | sha256sum) + if [ "$before_pkgs" != "$after_pkgs" ]; then + needs_caching=true + fi + else + # Otherwise, create the chroot + sudo -E su $USER -c 'mk-sbuild xenial' + needs_caching=true + fi + # If there are changes to the schroot (or it's entirely new), + # tar up the schroot (to preserve ownership/permissions) and + # move it into the cached dir; no need to compress it because + # Travis will do that anyway + if [ "$needs_caching" = "true" ]; then + sudo tar --sparse --xattrs --xattrs-include=* -cf "$TRAVIS_BUILD_DIR/chroots/xenial-amd64.tar" -C /var/lib/schroot/chroots/xenial-amd64 . + fi + # Use sudo to get a new shell where we're in the sbuild group + - sudo -E su $USER -c 'sbuild --nolog --no-run-lintian --verbose --dist=xenial cloud-init_*.dsc' # Ubuntu LTS: Integration - sg lxd -c 'tox -e citest -- run --verbose --preserve-data --data-dir results --os-name xenial --test modules/apt_configure_sources_list.yaml --test modules/ntp_servers --test modules/set_password_list --test modules/user_groups --deb cloud-init_*_all.deb' - python: 3.5 env: TOXENV=xenial - NOSE_VERBOSE=2 # List all tests run by nose - # Travis doesn't support Python 3.4 on bionic, so use xenial + PYTEST_ADDOPTS=-v # List all tests run by pytest dist: xenial - python: 3.6 - env: TOXENV=pycodestyle - - python: 3.6 - env: TOXENV=pyflakes + env: TOXENV=flake8 - python: 3.6 env: TOXENV=pylint + - python: 3.6 + env: TOXENV=doc |