diff --git a/.travis.yml b/.travis.yml index 2e19a14..f81b161 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,10 +6,11 @@ python: - "3.5" - "3.6" - "3.7" + - "3.8" - "nightly" - "pypy3" sudo: false script: - - ./test/test_travis + - ./test/test diff --git a/LICENSE.md b/LICENSE.md index 922999c..89a93af 100644 --- a/LICENSE.md +++ b/LICENSE.md @@ -1,7 +1,7 @@ The MIT License (MIT) ===================== -**Copyright (c) 2014-2019 Anish Athalye (me@anishathalye.com)** +**Copyright (c) 2014-2020 Anish Athalye (me@anishathalye.com)** Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in diff --git a/README.md b/README.md index 9b6c5a2..79c70c3 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# Dotbot [![Build Status](https://travis-ci.org/anishathalye/dotbot.svg?branch=master)](https://travis-ci.org/anishathalye/dotbot) +# Dotbot [![Build Status](https://travis-ci.com/anishathalye/dotbot.svg?branch=master)](https://travis-ci.com/anishathalye/dotbot) Dotbot makes installing your dotfiles as easy as `git clone $url && cd dotfiles && ./install`, even on a freshly installed system! @@ -176,8 +176,10 @@ Available extended configuration parameters: | `relink` | Removes the old target if it's a symlink (default:false) | | `force` | Force removes the old target, file or folder, and forces a new link (default:false) | | `relative` | Use a relative path to the source when creating the symlink (default:false, absolute links) | +| `canonicalize-path` | Resolve any symbolic links encountered in the source to symlink to the canonical path (default:true, real paths) | | `glob` | Treat a `*` character as a wildcard, and perform link operations on all of those matches (default:false) | | `if` | Execute this in your `$SHELL` and only link if it is successful. | +| `ignore-missing` | Do not fail if the source is missing and create the link anyway (default:false) | #### Example @@ -302,7 +304,9 @@ Clean commands are specified as an array of directories to be cleaned. Clean commands support an extended configuration syntax. In this type of configuration, commands are specified as directory paths mapping to options. If the `force` option is set to `true`, dead links are removed even if they don't -point to a file inside the dotfiles directory. +point to a file inside the dotfiles directory. If `recursive` is set to `true`, +the directory is traversed recursively (not recommended for `~` because it will +be slow). #### Example @@ -310,8 +314,10 @@ point to a file inside the dotfiles directory. - clean: ['~'] - clean: - ~/.config: + ~/: force: true + ~/.config: + recursive: true ``` ### Defaults @@ -395,7 +401,7 @@ Do you have a feature request, bug report, or patch? Great! See ## License -Copyright (c) 2014-2019 Anish Athalye. Released under the MIT License. See +Copyright (c) 2014-2020 Anish Athalye. Released under the MIT License. See [LICENSE.md][license] for details. [PyPI]: https://pypi.org/project/dotbot/ diff --git a/dotbot/__init__.py b/dotbot/__init__.py index e1fafd9..45c4cf9 100644 --- a/dotbot/__init__.py +++ b/dotbot/__init__.py @@ -1,4 +1,4 @@ from .cli import main from .plugin import Plugin -__version__ = '1.16.0' +__version__ = '1.17.0' diff --git a/dotbot/cli.py b/dotbot/cli.py index eb10f90..7275c1c 100644 --- a/dotbot/cli.py +++ b/dotbot/cli.py @@ -73,10 +73,10 @@ def main(): if not isinstance(tasks, list): raise ReadingError('Configuration file must be a list of tasks') if options.base_directory: - base_directory = options.base_directory + base_directory = os.path.abspath(options.base_directory) else: # default to directory of config file - base_directory = os.path.dirname(os.path.realpath(options.config_file)) + base_directory = os.path.dirname(os.path.abspath(options.config_file)) os.chdir(base_directory) dispatcher = Dispatcher(base_directory) success = dispatcher.dispatch(tasks) diff --git a/dotbot/context.py b/dotbot/context.py index b2dbd6c..8c42d47 100644 --- a/dotbot/context.py +++ b/dotbot/context.py @@ -1,4 +1,5 @@ import copy +import os class Context(object): ''' @@ -13,8 +14,11 @@ class Context(object): def set_base_directory(self, base_directory): self._base_directory = base_directory - def base_directory(self): - return self._base_directory + def base_directory(self, canonical_path=True): + base_directory = self._base_directory + if canonical_path: + base_directory = os.path.realpath(base_directory) + return base_directory def set_defaults(self, defaults): self._defaults = defaults diff --git a/dotbot/dispatcher.py b/dotbot/dispatcher.py index 68d0d88..c65e1bb 100644 --- a/dotbot/dispatcher.py +++ b/dotbot/dispatcher.py @@ -10,8 +10,8 @@ class Dispatcher(object): self._load_plugins() def _setup_context(self, base_directory): - path = os.path.abspath(os.path.realpath( - os.path.expanduser(base_directory))) + path = os.path.abspath( + os.path.expanduser(base_directory)) if not os.path.exists(path): raise DispatchError('Nonexistent base directory') self._context = Context(path) diff --git a/dotbot/plugins/clean.py b/dotbot/plugins/clean.py index 22c975e..82d77a6 100644 --- a/dotbot/plugins/clean.py +++ b/dotbot/plugins/clean.py @@ -18,18 +18,20 @@ class Clean(dotbot.Plugin): def _process_clean(self, targets): success = True defaults = self._context.defaults().get(self._directive, {}) - force = defaults.get('force', False) for target in targets: - if isinstance(targets, dict): + force = defaults.get('force', False) + recursive = defaults.get('recursive', False) + if isinstance(targets, dict) and isinstance(targets[target], dict): force = targets[target].get('force', force) - success &= self._clean(target, force) + recursive = targets[target].get('recursive', recursive) + success &= self._clean(target, force, recursive) if success: self._log.info('All targets have been cleaned') else: self._log.error('Some targets were not successfully cleaned') return success - def _clean(self, target, force): + def _clean(self, target, force, recursive): ''' Cleans all the broken symbolic links in target if they point to a subdirectory of the base directory or if forced to clean. @@ -39,6 +41,11 @@ class Clean(dotbot.Plugin): return True for item in os.listdir(os.path.expandvars(os.path.expanduser(target))): path = os.path.join(os.path.expandvars(os.path.expanduser(target)), item) + if recursive and os.path.isdir(path): + # isdir implies not islink -- we don't want to descend into + # symlinked directories. okay to do a recursive call here + # because depth should be fairly limited + self._clean(path, force, recursive) if not os.path.exists(path) and os.path.islink(path): points_at = os.path.join(os.path.dirname(path), os.readlink(path)) if self._in_directory(path, self._context.base_directory()) or force: diff --git a/dotbot/plugins/link.py b/dotbot/plugins/link.py index 0b47ac6..82a61ce 100644 --- a/dotbot/plugins/link.py +++ b/dotbot/plugins/link.py @@ -26,19 +26,23 @@ class Link(dotbot.Plugin): for destination, source in links.items(): destination = os.path.expandvars(destination) relative = defaults.get('relative', False) + canonical_path = defaults.get('canonicalize-path', True) force = defaults.get('force', False) relink = defaults.get('relink', False) create = defaults.get('create', False) use_glob = defaults.get('glob', False) test = defaults.get('if', None) + ignore_missing = defaults.get('ignore-missing', False) if isinstance(source, dict): # extended config test = source.get('if', test) relative = source.get('relative', relative) + canonical_path = source.get('canonicalize-path', canonical_path) force = source.get('force', force) relink = source.get('relink', relink) create = source.get('create', create) use_glob = source.get('glob', use_glob) + ignore_missing = source.get('ignore-missing', ignore_missing) path = self._default_source(destination, source.get('path')) else: path = self._default_source(destination, source) @@ -49,25 +53,25 @@ class Link(dotbot.Plugin): if use_glob: self._log.debug("Globbing with path: " + str(path)) glob_results = glob.glob(path) - if len(glob_results) is 0: + if len(glob_results) == 0: self._log.warning("Globbing couldn't find anything matching " + str(path)) success = False continue glob_star_loc = path.find('*') - if glob_star_loc is -1 and destination[-1] is '/': + if glob_star_loc == -1 and destination[-1] == '/': self._log.error("Ambiguous action requested.") self._log.error("No wildcard in glob, directory use undefined: " + destination + " -> " + str(glob_results)) self._log.warning("Did you want to link the directory or into it?") success = False continue - elif glob_star_loc is -1 and len(glob_results) is 1: + elif glob_star_loc == -1 and len(glob_results) == 1: # perform a normal link operation if create: success &= self._create(destination) if force or relink: - success &= self._delete(path, destination, relative, force) - success &= self._link(path, destination, relative) + success &= self._delete(path, destination, relative, canonical_path, force) + success &= self._link(path, destination, relative, canonical_path, ignore_missing) else: self._log.lowinfo("Globs from '" + path + "': " + str(glob_results)) glob_base = path[:glob_star_loc] @@ -77,19 +81,23 @@ class Link(dotbot.Plugin): if create: success &= self._create(glob_link_destination) if force or relink: - success &= self._delete(glob_full_item, glob_link_destination, relative, force) - success &= self._link(glob_full_item, glob_link_destination, relative) + success &= self._delete(glob_full_item, glob_link_destination, relative, canonical_path, force) + success &= self._link(glob_full_item, glob_link_destination, relative, canonical_path, ignore_missing) else: if create: success &= self._create(destination) - if not self._exists(os.path.join(self._context.base_directory(), path)): + if not ignore_missing and not self._exists(os.path.join(self._context.base_directory(), path)): + # we seemingly check this twice (here and in _link) because + # if the file doesn't exist and force is True, we don't + # want to remove the original (this is tested by + # link-force-leaves-when-nonexistent.bash) success = False self._log.warning('Nonexistent source %s -> %s' % (destination, path)) continue if force or relink: - success &= self._delete(path, destination, relative, force) - success &= self._link(path, destination, relative) + success &= self._delete(path, destination, relative, canonical_path, force) + success &= self._link(path, destination, relative, canonical_path, ignore_missing) if success: self._log.info('All links have been set up') else: @@ -153,9 +161,9 @@ class Link(dotbot.Plugin): self._log.lowinfo('Creating directory %s' % parent) return success - def _delete(self, source, path, relative, force): + def _delete(self, source, path, relative, canonical_path, force): success = True - source = os.path.join(self._context.base_directory(), source) + source = os.path.join(self._context.base_directory(canonical_path=canonical_path), source) fullpath = os.path.expanduser(path) if relative: source = self._relative_path(source, fullpath) @@ -189,7 +197,7 @@ class Link(dotbot.Plugin): destination_dir = os.path.dirname(destination) return os.path.relpath(source, destination_dir) - def _link(self, source, link_name, relative): + def _link(self, source, link_name, relative, canonical_path, ignore_missing): ''' Links link_name to source. @@ -197,7 +205,8 @@ class Link(dotbot.Plugin): ''' success = False destination = os.path.expanduser(link_name) - absolute_source = os.path.join(self._context.base_directory(), source) + base_directory = self._context.base_directory(canonical_path=canonical_path) + absolute_source = os.path.join(base_directory, source) if relative: source = self._relative_path(absolute_source, destination) else: @@ -209,7 +218,7 @@ class Link(dotbot.Plugin): # we need to use absolute_source below because our cwd is the dotfiles # directory, and if source is relative, it will be relative to the # destination directory - elif not self._exists(link_name) and self._exists(absolute_source): + elif not self._exists(link_name) and (ignore_missing or self._exists(absolute_source)): try: os.symlink(source, destination) except OSError: diff --git a/lib/pyyaml b/lib/pyyaml index 0f64cbf..2f463cf 160000 --- a/lib/pyyaml +++ b/lib/pyyaml @@ -1 +1 @@ -Subproject commit 0f64cbfa54b0b22dc7b776b7b98a7cd657e84d78 +Subproject commit 2f463cf5b0e98a52bc20e348d1e69761bf263b86 diff --git a/setup.py b/setup.py index 6fe1d14..e3b2198 100644 --- a/setup.py +++ b/setup.py @@ -73,7 +73,7 @@ setup( ], install_requires=[ - 'PyYAML>=5.1.2,<6', + 'PyYAML>=5.3,<6', ], # To provide executable scripts, use entry points in preference to the diff --git a/test/README.md b/test/README.md index 993a753..9ec79f5 100644 --- a/test/README.md +++ b/test/README.md @@ -36,14 +36,24 @@ git submodule update --init --recursive Running the Tests ----------------- -Before running the tests, the virtual machine must be running. It can be -started by running `vagrant up`. +Before running the tests, you must SSH into the VM. Start it with `vagrant up` +and SSH in with `vagrant ssh`. All following commands must be run inside the +VM. -The test suite can be run by running `./test`. Selected tests can be run by -passing paths to the tests as arguments to `./test`. +First, you must install a version of Python to test against, using `pyenv +install -s {version}`. You can choose any version you like, e.g. `3.8.1`. It +isn't particularly important to test against all supported versions of Python +in the VM, because they will be tested by CI. Once you've installed a specific +version of Python, activate it with `pyenv global {version}`. -Tests can be run with a specific Python version by running `./test --version -` - for example, `./test --version 3.4.3`. +The VM mounts the Dotbot directory in `/dotbot` as read-only (you can make +edits on your host machine). You can run the test suite by `cd /dotbot/test` +and then running `./test`. Selected tests can be run by passing paths to the +tests as arguments, e.g. `./test tests/create.bash tests/defaults.bash`. + +To debug tests, you can prepend the line `DEBUG=true` as the first line to any +individual test (a `.bash` file inside `test/tests`). This will enable printing +stdout/stderr. When finished with testing, it is good to shut down the virtual machine by running `vagrant halt`. diff --git a/test/Vagrantfile b/test/Vagrantfile index 347c402..fbdfe39 100644 --- a/test/Vagrantfile +++ b/test/Vagrantfile @@ -1,8 +1,8 @@ Vagrant.configure(2) do |config| - config.vm.box = 'debian/buster64' + config.vm.box = 'ubuntu/bionic64' # sync by copying for isolation - config.vm.synced_folder "..", "/dotbot", type: "rsync" + config.vm.synced_folder "..", "/dotbot", mount_options: ["ro"] # disable default synced folder config.vm.synced_folder ".", "/vagrant", disabled: true diff --git a/test/driver-lib.bash b/test/driver-lib.bash index 02a71a5..dcbcba6 100644 --- a/test/driver-lib.bash +++ b/test/driver-lib.bash @@ -1,6 +1,3 @@ -MAXRETRY=5 -TIMEOUT=1 - red() { if [ -t 1 ]; then printf "\033[31m%s\033[0m\n" "$*" @@ -26,52 +23,35 @@ yellow() { } -check_prereqs() { - if ! (vagrant ssh -c 'exit') >/dev/null 2>&1; then - >&2 echo "vagrant vm must be running." - return 1 +check_env() { + if [[ "$(whoami)" != "vagrant" && ( "${TRAVIS}" != true || "${CI}" != true ) ]]; then + die "tests must be run inside Travis or Vagrant" fi } -until_success() { - local timeout=${TIMEOUT} - local attempt=0 - while [ $attempt -lt $MAXRETRY ]; do - if ($@) >/dev/null 2>&1; then - return 0 - fi - sleep $timeout - timeout=$((timeout * 2)) - attempt=$((attempt + 1)) - done - - return 1 -} - -wait_for_vagrant() { - until_success vagrant ssh -c 'exit' -} - cleanup() { - vagrant ssh -c " - find . -not \\( \ + ( + if [ "$(whoami)" == "vagrant" ]; then + cd $HOME + find . -not \( \ -path './.pyenv' -o \ -path './.pyenv/*' -o \ -path './.bashrc' -o \ -path './.profile' -o \ -path './.ssh' -o \ -path './.ssh/*' \ - \\) -delete" >/dev/null 2>&1 + \) -delete >/dev/null 2>&1 + else + find ~ -mindepth 1 -newermt "${date_stamp}" \ + -not \( -path ~ -o -path "${BASEDIR}/*" \ + -o -path ~/dotfiles \) \ + -exec rm -rf {} + + fi + ) || true } initialize() { echo "initializing." - if ! vagrant ssh -c "pyenv local ${2}" >/dev/null 2>&1; then - if ! vagrant ssh -c "pyenv install -s ${2} && pyenv local ${2}" >/dev/null 2>&1; then - die "could not install python ${2}" - fi - fi - vagrant rsync >/dev/null 2>&1 tests_run=0 tests_passed=0 tests_failed=0 @@ -96,8 +76,7 @@ run_test() { tests_run=$((tests_run + 1)) printf '[%d/%d] (%s)\n' "${tests_run}" "${tests_total}" "${1}" cleanup - vagrant ssh -c "pyenv local ${2}" >/dev/null 2>&1 - if vagrant ssh -c "cd /dotbot/test/tests && bash ${1}" 2>/dev/null; then + if (cd "${BASEDIR}/test/tests" && DOTBOT_TEST=true bash "${1}"); then pass else fail diff --git a/test/test b/test/test index c018c32..f944512 100755 --- a/test/test +++ b/test/test @@ -1,37 +1,21 @@ #!/usr/bin/env bash set -e -BASEDIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -cd "${BASEDIR}" +export BASEDIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +cd "${BASEDIR}/test" . "./driver-lib.bash" +date_stamp="$(date --rfc-3339=ns)" start="$(date +%s)" -check_prereqs || die "prerequisites unsatsfied." - -# command line options -while [[ $# > 1 ]] -do - key="${1}" - case $key in - -v|--version) - VERSION="${2}" - shift && shift - ;; - *) - # unknown option - break - ;; - esac -done -VERSION="${VERSION:-3.6.4}" +check_env declare -a tests=() if [ $# -eq 0 ]; then while read file; do tests+=("${file}") - done < <(find tests -type f -name '*.bash') + done < <(find tests -type f -name '*.bash' | sort) else tests=("$@") fi diff --git a/test/test-lib.bash b/test/test-lib.bash index e4d9a4e..fba9aa0 100644 --- a/test/test-lib.bash +++ b/test/test-lib.bash @@ -1,6 +1,5 @@ DEBUG=${DEBUG:-false} -USE_VAGRANT=${USE_VAGRANT:-true} -DOTBOT_EXEC=${DOTBOT_EXEC:-"python /dotbot/bin/dotbot"} +DOTBOT_EXEC="${BASEDIR}/bin/dotbot" DOTFILES="/home/$(whoami)/dotfiles" INSTALL_CONF='install.conf.yaml' INSTALL_CONF_JSON='install.conf.json' @@ -29,17 +28,15 @@ test_expect_failure() { fi } -check_vm() { - if [ "$(whoami)" != "vagrant" ]; then - >&2 echo "test can't run outside vm!" +check_env() { + if [ "${DOTBOT_TEST}" != "true" ]; then + >&2 echo "test must be run by test driver" exit 1 fi } initialize() { - if ${USE_VAGRANT}; then - check_vm - fi + check_env echo "${test_description}" mkdir -p "${DOTFILES}" cd diff --git a/test/test_travis b/test/test_travis deleted file mode 100755 index 79439e1..0000000 --- a/test/test_travis +++ /dev/null @@ -1,81 +0,0 @@ -#!/usr/bin/env bash -set -e - -# For debug only: -# export DEBUG=true -# set -x -# set -v - -export BASEDIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" - -# Prevent execution outside of Travis CI builds -if [[ "${TRAVIS}" != true || "${CI}" != true ]]; then - echo "Error: `basename "$0"` should only be used on Travis" - exit 2 -fi - -# Travis runs do not rely on Vagrant -export USE_VAGRANT=false -export DOTBOT_EXEC="${BASEDIR}/bin/dotbot" - -cd "${BASEDIR}" -. "test/driver-lib.bash" - -travis_initialize() { - echo "initializing." - tests_run=0 - tests_passed=0 - tests_failed=0 - tests_total="${1}" - local plural="" && [ "${tests_total}" -gt 1 ] && plural="s" - printf -- "running %d test%s...\n\n" "${tests_total}" "${plural}" -} - -travis_cleanup() { - # Remove all dotfiles installed since the start, ignoring the main - # dotfiles directory, and the dotbot source directory - find ~ -mindepth 1 -newermt "${date_stamp}" \ - -not \( -path ~ -o -path "${BASEDIR}/*" \ - -o -path ~/dotfiles \) \ - -exec rm -rf {} + -} - -travis_run_test() { - tests_run=$((tests_run + 1)) - printf '[%d/%d] (%s)\n' "${tests_run}" "${tests_total}" "${1}" - cd ${BASEDIR}/test/tests - if bash ${1} ; then - pass - else - fail - fi - travis_cleanup || die "unable to clean up system." -} - -date_stamp="$(date --rfc-3339=ns)" -start="$(date +%s)" - -declare -a tests=() - -if [ $# -eq 0 ]; then - while read file; do - tests+=("${file}") - done < <(find ${BASEDIR}/test/tests -type f -name '*.bash') -else - tests=("$@") -fi - -travis_initialize "${#tests[@]}" - -for file in "${tests[@]}"; do - travis_run_test "$(basename "${file}")" -done - -if report; then - ret=0 -else - ret=1 -fi - -echo "(tests run in $(($(date +%s) - start)) seconds)" -exit ${ret} diff --git a/test/tests/clean-default.bash b/test/tests/clean-default.bash new file mode 100644 index 0000000..8bb405d --- /dev/null +++ b/test/tests/clean-default.bash @@ -0,0 +1,19 @@ +test_description='clean uses default unless overridden' +. '../test-lib.bash' + +test_expect_success 'setup' ' +ln -s /nowhere ~/.g +' + +test_expect_success 'run' ' +run_dotbot < ${DOTFILES}/f && +ln -s dotfiles dotfiles-symlink +' + +test_expect_success 'run' ' +cat > "${DOTFILES}/${INSTALL_CONF}" < ${DOTFILES}/f && +ln -s dotfiles dotfiles-symlink +' + +test_expect_success 'run' ' +cat > "${DOTFILES}/${INSTALL_CONF}" < ${DOTFILES}/foo ' @@ -18,9 +14,6 @@ cat > ${DOTFILES}/install.conf.yaml <