mirror of
1
0
Fork 0

Unify Vagrant and Travis-CI tests

This patch makes the tests (including the test driver) run entirely
inside Vagrant, which avoids calling the very slow `vagrant` driver many
times for running the tests. On my machine, `./test` runs in 22 seconds,
down from hundreds of seconds prior to this patch.

This also has the nice side effect of matching how the Travis CI tests
were run, so there's no need for a separate `test_travis` anymore.
This commit is contained in:
Anish Athalye 2020-01-03 15:20:00 -05:00
parent 1e1885c45a
commit 6d24613b0b
9 changed files with 49 additions and 169 deletions

View File

@ -13,4 +13,4 @@ python:
sudo: false
script:
- ./test/test_travis
- ./test/test

View File

@ -36,14 +36,24 @@ git submodule update --init --recursive
Running the Tests
-----------------
Before running the tests, the virtual machine must be running. It can be
started by running `vagrant up`.
Before running the tests, you must SSH into the VM. Start it with `vagrant up`
and SSH in with `vagrant ssh`. All following commands must be run inside the
VM.
The test suite can be run by running `./test`. Selected tests can be run by
passing paths to the tests as arguments to `./test`.
First, you must install a version of Python to test against, using `pyenv
install -s {version}`. You can choose any version you like, e.g. `3.8.1`. It
isn't particularly important to test against all supported versions of Python
in the VM, because they will be tested by CI. Once you've installed a specific
version of Python, activate it with `pyenv global {version}`.
Tests can be run with a specific Python version by running `./test --version
<version>` - for example, `./test --version 3.4.3`.
The VM mounts the Dotbot directory in `/dotbot` as read-only (you can make
edits on your host machine). You can run the test suite by `cd /dotbot/test`
and then running `./test`. Selected tests can be run by passing paths to the
tests as arguments, e.g. `./test tests/create.bash tests/defaults.bash`.
To debug tests, you can prepend the line `DEBUG=true` as the first line to any
individual test (a `.bash` file inside `test/tests`). This will enable printing
stdout/stderr.
When finished with testing, it is good to shut down the virtual machine by
running `vagrant halt`.

4
test/Vagrantfile vendored
View File

@ -1,8 +1,8 @@
Vagrant.configure(2) do |config|
config.vm.box = 'debian/buster64'
config.vm.box = 'ubuntu/bionic64'
# sync by copying for isolation
config.vm.synced_folder "..", "/dotbot", type: "rsync"
config.vm.synced_folder "..", "/dotbot", mount_options: ["ro"]
# disable default synced folder
config.vm.synced_folder ".", "/vagrant", disabled: true

View File

@ -1,6 +1,3 @@
MAXRETRY=5
TIMEOUT=1
red() {
if [ -t 1 ]; then
printf "\033[31m%s\033[0m\n" "$*"
@ -26,52 +23,35 @@ yellow() {
}
check_prereqs() {
if ! (vagrant ssh -c 'exit') >/dev/null 2>&1; then
>&2 echo "vagrant vm must be running."
return 1
check_env() {
if [[ "$(whoami)" != "vagrant" && ( "${TRAVIS}" != true || "${CI}" != true ) ]]; then
die "tests must be run inside Travis or Vagrant"
fi
}
until_success() {
local timeout=${TIMEOUT}
local attempt=0
while [ $attempt -lt $MAXRETRY ]; do
if ($@) >/dev/null 2>&1; then
return 0
fi
sleep $timeout
timeout=$((timeout * 2))
attempt=$((attempt + 1))
done
return 1
}
wait_for_vagrant() {
until_success vagrant ssh -c 'exit'
}
cleanup() {
vagrant ssh -c "
find . -not \\( \
(
if [ "$(whoami)" == "vagrant" ]; then
cd $HOME
find . -not \( \
-path './.pyenv' -o \
-path './.pyenv/*' -o \
-path './.bashrc' -o \
-path './.profile' -o \
-path './.ssh' -o \
-path './.ssh/*' \
\\) -delete" >/dev/null 2>&1
\) -delete >/dev/null 2>&1
else
find ~ -mindepth 1 -newermt "${date_stamp}" \
-not \( -path ~ -o -path "${BASEDIR}/*" \
-o -path ~/dotfiles \) \
-exec rm -rf {} +
fi
) || true
}
initialize() {
echo "initializing."
if ! vagrant ssh -c "pyenv local ${2}" >/dev/null 2>&1; then
if ! vagrant ssh -c "pyenv install -s ${2} && pyenv local ${2}" >/dev/null 2>&1; then
die "could not install python ${2}"
fi
fi
vagrant rsync >/dev/null 2>&1
tests_run=0
tests_passed=0
tests_failed=0
@ -96,8 +76,7 @@ run_test() {
tests_run=$((tests_run + 1))
printf '[%d/%d] (%s)\n' "${tests_run}" "${tests_total}" "${1}"
cleanup
vagrant ssh -c "pyenv local ${2}" >/dev/null 2>&1
if vagrant ssh -c "cd /dotbot/test/tests && bash ${1}" 2>/dev/null; then
if (cd "${BASEDIR}/test/tests" && DOTBOT_TEST=true bash "${1}"); then
pass
else
fail

View File

@ -1,37 +1,21 @@
#!/usr/bin/env bash
set -e
BASEDIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
cd "${BASEDIR}"
export BASEDIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
cd "${BASEDIR}/test"
. "./driver-lib.bash"
date_stamp="$(date --rfc-3339=ns)"
start="$(date +%s)"
check_prereqs || die "prerequisites unsatsfied."
# command line options
while [[ $# > 1 ]]
do
key="${1}"
case $key in
-v|--version)
VERSION="${2}"
shift && shift
;;
*)
# unknown option
break
;;
esac
done
VERSION="${VERSION:-3.6.4}"
check_env
declare -a tests=()
if [ $# -eq 0 ]; then
while read file; do
tests+=("${file}")
done < <(find tests -type f -name '*.bash')
done < <(find tests -type f -name '*.bash' | sort)
else
tests=("$@")
fi

View File

@ -1,6 +1,5 @@
DEBUG=${DEBUG:-false}
USE_VAGRANT=${USE_VAGRANT:-true}
DOTBOT_EXEC=${DOTBOT_EXEC:-"python /dotbot/bin/dotbot"}
DOTBOT_EXEC="${BASEDIR}/bin/dotbot"
DOTFILES="/home/$(whoami)/dotfiles"
INSTALL_CONF='install.conf.yaml'
INSTALL_CONF_JSON='install.conf.json'
@ -29,17 +28,15 @@ test_expect_failure() {
fi
}
check_vm() {
if [ "$(whoami)" != "vagrant" ]; then
>&2 echo "test can't run outside vm!"
check_env() {
if [ "${DOTBOT_TEST}" != "true" ]; then
>&2 echo "test must be run by test driver"
exit 1
fi
}
initialize() {
if ${USE_VAGRANT}; then
check_vm
fi
check_env
echo "${test_description}"
mkdir -p "${DOTFILES}"
cd

View File

@ -1,81 +0,0 @@
#!/usr/bin/env bash
set -e
# For debug only:
# export DEBUG=true
# set -x
# set -v
export BASEDIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
# Prevent execution outside of Travis CI builds
if [[ "${TRAVIS}" != true || "${CI}" != true ]]; then
echo "Error: `basename "$0"` should only be used on Travis"
exit 2
fi
# Travis runs do not rely on Vagrant
export USE_VAGRANT=false
export DOTBOT_EXEC="${BASEDIR}/bin/dotbot"
cd "${BASEDIR}"
. "test/driver-lib.bash"
travis_initialize() {
echo "initializing."
tests_run=0
tests_passed=0
tests_failed=0
tests_total="${1}"
local plural="" && [ "${tests_total}" -gt 1 ] && plural="s"
printf -- "running %d test%s...\n\n" "${tests_total}" "${plural}"
}
travis_cleanup() {
# Remove all dotfiles installed since the start, ignoring the main
# dotfiles directory, and the dotbot source directory
find ~ -mindepth 1 -newermt "${date_stamp}" \
-not \( -path ~ -o -path "${BASEDIR}/*" \
-o -path ~/dotfiles \) \
-exec rm -rf {} +
}
travis_run_test() {
tests_run=$((tests_run + 1))
printf '[%d/%d] (%s)\n' "${tests_run}" "${tests_total}" "${1}"
cd ${BASEDIR}/test/tests
if bash ${1} ; then
pass
else
fail
fi
travis_cleanup || die "unable to clean up system."
}
date_stamp="$(date --rfc-3339=ns)"
start="$(date +%s)"
declare -a tests=()
if [ $# -eq 0 ]; then
while read file; do
tests+=("${file}")
done < <(find ${BASEDIR}/test/tests -type f -name '*.bash')
else
tests=("$@")
fi
travis_initialize "${#tests[@]}"
for file in "${tests[@]}"; do
travis_run_test "$(basename "${file}")"
done
if report; then
ret=0
else
ret=1
fi
echo "(tests run in $(($(date +%s) - start)) seconds)"
exit ${ret}

View File

@ -1,18 +1,16 @@
test_description='can find python executable with different names'
. '../test-lib.bash'
if ${USE_VAGRANT}; then
DOTBOT_EXEC="/dotbot/bin/dotbot" # revert to calling it as a shell script
fi
# the test machine needs to have a binary named `python`
test_expect_success 'setup' '
mkdir ~/tmp_bin &&
(
IFS=:
for p in $PATH; do
find $p -maxdepth 1 -mindepth 1 -exec sh -c \
if [ -d $p ]; then
find $p -maxdepth 1 -mindepth 1 -exec sh -c \
'"'"'ln -sf {} $HOME/tmp_bin/$(basename {})'"'"' \;
fi
done
) &&
rm -f ~/tmp_bin/python &&

View File

@ -4,11 +4,7 @@ test_description='install shim works'
test_expect_success 'setup' '
cd ${DOTFILES}
git init
if ${USE_VAGRANT}; then
git submodule add /dotbot dotbot
else
git submodule add ${BASEDIR} dotbot
fi
git submodule add ${BASEDIR} dotbot
cp ./dotbot/tools/git-submodule/install .
echo "pear" > ${DOTFILES}/foo
'
@ -18,9 +14,6 @@ cat > ${DOTFILES}/install.conf.yaml <<EOF
- link:
~/.foo: foo
EOF
if ! ${USE_VAGRANT}; then
sed -i "" "1 s/sh$/python/" ${DOTFILES}/dotbot/bin/dotbot
fi
${DOTFILES}/install
'