Unify Vagrant and Travis-CI tests
This patch makes the tests (including the test driver) run entirely inside Vagrant, which avoids calling the very slow `vagrant` driver many times for running the tests. On my machine, `./test` runs in 22 seconds, down from hundreds of seconds prior to this patch. This also has the nice side effect of matching how the Travis CI tests were run, so there's no need for a separate `test_travis` anymore.
This commit is contained in:
Anish Athalye
parent
1e1885c45a
commit
6d24613b0b
9 changed files with 49 additions and 169 deletions
|
|
@ -13,4 +13,4 @@ python:
|
|||
sudo: false
|
||||
|
||||
script:
|
||||
- ./test/test_travis
|
||||
- ./test/test
|
||||
|
|
|
|||
|
|
@ -36,14 +36,24 @@ git submodule update --init --recursive
|
|||
Running the Tests
|
||||
-----------------
|
||||
|
||||
Before running the tests, the virtual machine must be running. It can be
|
||||
started by running `vagrant up`.
|
||||
Before running the tests, you must SSH into the VM. Start it with `vagrant up`
|
||||
and SSH in with `vagrant ssh`. All following commands must be run inside the
|
||||
VM.
|
||||
|
||||
The test suite can be run by running `./test`. Selected tests can be run by
|
||||
passing paths to the tests as arguments to `./test`.
|
||||
First, you must install a version of Python to test against, using `pyenv
|
||||
install -s {version}`. You can choose any version you like, e.g. `3.8.1`. It
|
||||
isn't particularly important to test against all supported versions of Python
|
||||
in the VM, because they will be tested by CI. Once you've installed a specific
|
||||
version of Python, activate it with `pyenv global {version}`.
|
||||
|
||||
Tests can be run with a specific Python version by running `./test --version
|
||||
<version>` - for example, `./test --version 3.4.3`.
|
||||
The VM mounts the Dotbot directory in `/dotbot` as read-only (you can make
|
||||
edits on your host machine). You can run the test suite by `cd /dotbot/test`
|
||||
and then running `./test`. Selected tests can be run by passing paths to the
|
||||
tests as arguments, e.g. `./test tests/create.bash tests/defaults.bash`.
|
||||
|
||||
To debug tests, you can prepend the line `DEBUG=true` as the first line to any
|
||||
individual test (a `.bash` file inside `test/tests`). This will enable printing
|
||||
stdout/stderr.
|
||||
|
||||
When finished with testing, it is good to shut down the virtual machine by
|
||||
running `vagrant halt`.
|
||||
|
|
|
|||
4
test/Vagrantfile
vendored
4
test/Vagrantfile
vendored
|
|
@ -1,8 +1,8 @@
|
|||
Vagrant.configure(2) do |config|
|
||||
config.vm.box = 'debian/buster64'
|
||||
config.vm.box = 'ubuntu/bionic64'
|
||||
|
||||
# sync by copying for isolation
|
||||
config.vm.synced_folder "..", "/dotbot", type: "rsync"
|
||||
config.vm.synced_folder "..", "/dotbot", mount_options: ["ro"]
|
||||
|
||||
# disable default synced folder
|
||||
config.vm.synced_folder ".", "/vagrant", disabled: true
|
||||
|
|
|
|||
|
|
@ -1,6 +1,3 @@
|
|||
MAXRETRY=5
|
||||
TIMEOUT=1
|
||||
|
||||
red() {
|
||||
if [ -t 1 ]; then
|
||||
printf "\033[31m%s\033[0m\n" "$*"
|
||||
|
|
@ -26,52 +23,35 @@ yellow() {
|
|||
}
|
||||
|
||||
|
||||
check_prereqs() {
|
||||
if ! (vagrant ssh -c 'exit') >/dev/null 2>&1; then
|
||||
>&2 echo "vagrant vm must be running."
|
||||
return 1
|
||||
check_env() {
|
||||
if [[ "$(whoami)" != "vagrant" && ( "${TRAVIS}" != true || "${CI}" != true ) ]]; then
|
||||
die "tests must be run inside Travis or Vagrant"
|
||||
fi
|
||||
}
|
||||
|
||||
until_success() {
|
||||
local timeout=${TIMEOUT}
|
||||
local attempt=0
|
||||
while [ $attempt -lt $MAXRETRY ]; do
|
||||
if ($@) >/dev/null 2>&1; then
|
||||
return 0
|
||||
fi
|
||||
sleep $timeout
|
||||
timeout=$((timeout * 2))
|
||||
attempt=$((attempt + 1))
|
||||
done
|
||||
|
||||
return 1
|
||||
}
|
||||
|
||||
wait_for_vagrant() {
|
||||
until_success vagrant ssh -c 'exit'
|
||||
}
|
||||
|
||||
cleanup() {
|
||||
vagrant ssh -c "
|
||||
find . -not \\( \
|
||||
(
|
||||
if [ "$(whoami)" == "vagrant" ]; then
|
||||
cd $HOME
|
||||
find . -not \( \
|
||||
-path './.pyenv' -o \
|
||||
-path './.pyenv/*' -o \
|
||||
-path './.bashrc' -o \
|
||||
-path './.profile' -o \
|
||||
-path './.ssh' -o \
|
||||
-path './.ssh/*' \
|
||||
\\) -delete" >/dev/null 2>&1
|
||||
\) -delete >/dev/null 2>&1
|
||||
else
|
||||
find ~ -mindepth 1 -newermt "${date_stamp}" \
|
||||
-not \( -path ~ -o -path "${BASEDIR}/*" \
|
||||
-o -path ~/dotfiles \) \
|
||||
-exec rm -rf {} +
|
||||
fi
|
||||
) || true
|
||||
}
|
||||
|
||||
initialize() {
|
||||
echo "initializing."
|
||||
if ! vagrant ssh -c "pyenv local ${2}" >/dev/null 2>&1; then
|
||||
if ! vagrant ssh -c "pyenv install -s ${2} && pyenv local ${2}" >/dev/null 2>&1; then
|
||||
die "could not install python ${2}"
|
||||
fi
|
||||
fi
|
||||
vagrant rsync >/dev/null 2>&1
|
||||
tests_run=0
|
||||
tests_passed=0
|
||||
tests_failed=0
|
||||
|
|
@ -96,8 +76,7 @@ run_test() {
|
|||
tests_run=$((tests_run + 1))
|
||||
printf '[%d/%d] (%s)\n' "${tests_run}" "${tests_total}" "${1}"
|
||||
cleanup
|
||||
vagrant ssh -c "pyenv local ${2}" >/dev/null 2>&1
|
||||
if vagrant ssh -c "cd /dotbot/test/tests && bash ${1}" 2>/dev/null; then
|
||||
if (cd "${BASEDIR}/test/tests" && DOTBOT_TEST=true bash "${1}"); then
|
||||
pass
|
||||
else
|
||||
fail
|
||||
|
|
|
|||
26
test/test
26
test/test
|
|
@ -1,37 +1,21 @@
|
|||
#!/usr/bin/env bash
|
||||
set -e
|
||||
|
||||
BASEDIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
cd "${BASEDIR}"
|
||||
export BASEDIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
cd "${BASEDIR}/test"
|
||||
. "./driver-lib.bash"
|
||||
|
||||
date_stamp="$(date --rfc-3339=ns)"
|
||||
start="$(date +%s)"
|
||||
|
||||
check_prereqs || die "prerequisites unsatsfied."
|
||||
|
||||
# command line options
|
||||
while [[ $# > 1 ]]
|
||||
do
|
||||
key="${1}"
|
||||
case $key in
|
||||
-v|--version)
|
||||
VERSION="${2}"
|
||||
shift && shift
|
||||
;;
|
||||
*)
|
||||
# unknown option
|
||||
break
|
||||
;;
|
||||
esac
|
||||
done
|
||||
VERSION="${VERSION:-3.6.4}"
|
||||
check_env
|
||||
|
||||
declare -a tests=()
|
||||
|
||||
if [ $# -eq 0 ]; then
|
||||
while read file; do
|
||||
tests+=("${file}")
|
||||
done < <(find tests -type f -name '*.bash')
|
||||
done < <(find tests -type f -name '*.bash' | sort)
|
||||
else
|
||||
tests=("$@")
|
||||
fi
|
||||
|
|
|
|||
|
|
@ -1,6 +1,5 @@
|
|||
DEBUG=${DEBUG:-false}
|
||||
USE_VAGRANT=${USE_VAGRANT:-true}
|
||||
DOTBOT_EXEC=${DOTBOT_EXEC:-"python /dotbot/bin/dotbot"}
|
||||
DOTBOT_EXEC="${BASEDIR}/bin/dotbot"
|
||||
DOTFILES="/home/$(whoami)/dotfiles"
|
||||
INSTALL_CONF='install.conf.yaml'
|
||||
INSTALL_CONF_JSON='install.conf.json'
|
||||
|
|
@ -29,17 +28,15 @@ test_expect_failure() {
|
|||
fi
|
||||
}
|
||||
|
||||
check_vm() {
|
||||
if [ "$(whoami)" != "vagrant" ]; then
|
||||
>&2 echo "test can't run outside vm!"
|
||||
check_env() {
|
||||
if [ "${DOTBOT_TEST}" != "true" ]; then
|
||||
>&2 echo "test must be run by test driver"
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
initialize() {
|
||||
if ${USE_VAGRANT}; then
|
||||
check_vm
|
||||
fi
|
||||
check_env
|
||||
echo "${test_description}"
|
||||
mkdir -p "${DOTFILES}"
|
||||
cd
|
||||
|
|
|
|||
|
|
@ -1,81 +0,0 @@
|
|||
#!/usr/bin/env bash
|
||||
set -e
|
||||
|
||||
# For debug only:
|
||||
# export DEBUG=true
|
||||
# set -x
|
||||
# set -v
|
||||
|
||||
export BASEDIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
|
||||
# Prevent execution outside of Travis CI builds
|
||||
if [[ "${TRAVIS}" != true || "${CI}" != true ]]; then
|
||||
echo "Error: `basename "$0"` should only be used on Travis"
|
||||
exit 2
|
||||
fi
|
||||
|
||||
# Travis runs do not rely on Vagrant
|
||||
export USE_VAGRANT=false
|
||||
export DOTBOT_EXEC="${BASEDIR}/bin/dotbot"
|
||||
|
||||
cd "${BASEDIR}"
|
||||
. "test/driver-lib.bash"
|
||||
|
||||
travis_initialize() {
|
||||
echo "initializing."
|
||||
tests_run=0
|
||||
tests_passed=0
|
||||
tests_failed=0
|
||||
tests_total="${1}"
|
||||
local plural="" && [ "${tests_total}" -gt 1 ] && plural="s"
|
||||
printf -- "running %d test%s...\n\n" "${tests_total}" "${plural}"
|
||||
}
|
||||
|
||||
travis_cleanup() {
|
||||
# Remove all dotfiles installed since the start, ignoring the main
|
||||
# dotfiles directory, and the dotbot source directory
|
||||
find ~ -mindepth 1 -newermt "${date_stamp}" \
|
||||
-not \( -path ~ -o -path "${BASEDIR}/*" \
|
||||
-o -path ~/dotfiles \) \
|
||||
-exec rm -rf {} +
|
||||
}
|
||||
|
||||
travis_run_test() {
|
||||
tests_run=$((tests_run + 1))
|
||||
printf '[%d/%d] (%s)\n' "${tests_run}" "${tests_total}" "${1}"
|
||||
cd ${BASEDIR}/test/tests
|
||||
if bash ${1} ; then
|
||||
pass
|
||||
else
|
||||
fail
|
||||
fi
|
||||
travis_cleanup || die "unable to clean up system."
|
||||
}
|
||||
|
||||
date_stamp="$(date --rfc-3339=ns)"
|
||||
start="$(date +%s)"
|
||||
|
||||
declare -a tests=()
|
||||
|
||||
if [ $# -eq 0 ]; then
|
||||
while read file; do
|
||||
tests+=("${file}")
|
||||
done < <(find ${BASEDIR}/test/tests -type f -name '*.bash')
|
||||
else
|
||||
tests=("$@")
|
||||
fi
|
||||
|
||||
travis_initialize "${#tests[@]}"
|
||||
|
||||
for file in "${tests[@]}"; do
|
||||
travis_run_test "$(basename "${file}")"
|
||||
done
|
||||
|
||||
if report; then
|
||||
ret=0
|
||||
else
|
||||
ret=1
|
||||
fi
|
||||
|
||||
echo "(tests run in $(($(date +%s) - start)) seconds)"
|
||||
exit ${ret}
|
||||
|
|
@ -1,18 +1,16 @@
|
|||
test_description='can find python executable with different names'
|
||||
. '../test-lib.bash'
|
||||
|
||||
if ${USE_VAGRANT}; then
|
||||
DOTBOT_EXEC="/dotbot/bin/dotbot" # revert to calling it as a shell script
|
||||
fi
|
||||
|
||||
# the test machine needs to have a binary named `python`
|
||||
test_expect_success 'setup' '
|
||||
mkdir ~/tmp_bin &&
|
||||
(
|
||||
IFS=:
|
||||
for p in $PATH; do
|
||||
find $p -maxdepth 1 -mindepth 1 -exec sh -c \
|
||||
if [ -d $p ]; then
|
||||
find $p -maxdepth 1 -mindepth 1 -exec sh -c \
|
||||
'"'"'ln -sf {} $HOME/tmp_bin/$(basename {})'"'"' \;
|
||||
fi
|
||||
done
|
||||
) &&
|
||||
rm -f ~/tmp_bin/python &&
|
||||
|
|
|
|||
|
|
@ -4,11 +4,7 @@ test_description='install shim works'
|
|||
test_expect_success 'setup' '
|
||||
cd ${DOTFILES}
|
||||
git init
|
||||
if ${USE_VAGRANT}; then
|
||||
git submodule add /dotbot dotbot
|
||||
else
|
||||
git submodule add ${BASEDIR} dotbot
|
||||
fi
|
||||
git submodule add ${BASEDIR} dotbot
|
||||
cp ./dotbot/tools/git-submodule/install .
|
||||
echo "pear" > ${DOTFILES}/foo
|
||||
'
|
||||
|
|
@ -18,9 +14,6 @@ cat > ${DOTFILES}/install.conf.yaml <<EOF
|
|||
- link:
|
||||
~/.foo: foo
|
||||
EOF
|
||||
if ! ${USE_VAGRANT}; then
|
||||
sed -i "" "1 s/sh$/python/" ${DOTFILES}/dotbot/bin/dotbot
|
||||
fi
|
||||
${DOTFILES}/install
|
||||
'
|
||||
|
||||
|
|
|
|||
Loading…
Reference in a new issue