Merge branch 'eengstrom/270-recursive-globbing'

README.md

@@ -181,16 +181,22 @@ mapped to extended configuration dictionaries.
 | `force` | Force removes the old target, file or folder, and forces a new link (default: false) |
 | `relative` | Use a relative path to the source when creating the symlink (default: false, absolute links) |
 | `canonicalize` | Resolve any symbolic links encountered in the source to symlink to the canonical path (default: true, real paths) |
-| `glob` | Treat a `*` character as a wildcard, and perform link operations on all of those matches (default: false) |
 | `if` | Execute this in your `$SHELL` and only link if it is successful. |
 | `ignore-missing` | Do not fail if the source is missing and create the link anyway (default: false) |
-| `exclude` | Array of paths to remove from glob matches. Uses same syntax as `path`. Ignored if `glob` is `false`. (default: empty, keep all matches) |
+| `glob` | Treat `path` as a glob pattern, expanding patterns referenced below, linking all *files** matched. (default: false) |
+| `exclude` | Array of glob patterns to remove from glob matches. Uses same syntax as `path`. Ignored if `glob` is `false`. (default: empty, keep all matches) |
 
-Dotbot uses [glob.glob](https://docs.python.org/3/library/glob.html#glob.glob)
-to resolve glob paths. However, due to its design, using a glob path such as
-`config/*` for example, will not match items that begin with `.`. To
-specifically capture items that begin with `.`, you will need to use a path
-like this: `config/.*`.
+When `glob: True`, Dotbot uses [glob.glob](https://docs.python.org/3/library/glob.html#glob.glob) to resolve glob paths, expanding Unix shell-style wildcards, which are **not** the same as regular expressions; Only the following are expanded:
+
+| Pattern  | Meaning                                                |
+|:---------|:-------------------------------------------------------|
+| `*`      | matches anything                                       |
+| `**`     | matches any **file**, recursively (Python >= 3.5 only) |
+| `?`      | matches any single character                           |
+| `[seq]`  | matches any character in `seq`                         |
+| `[!seq]` | matches any character not in `seq`                     |
+
+However, due to the design of `glob.glob`, using a glob pattern such as `config/*`, will **not** match items that being with `.`. To specifically capture items that being with `.`, you will need to include the `.` in the pattern, like this: `config/.*`.
 
 #### Example
 
@@ -209,6 +215,8 @@ like this: `config/.*`.
     ~/.hammerspoon:
       if: '[ `uname` = Darwin ]'
       path: hammerspoon
+    ~/.config/:
+      path: dotconf/config/**
 ```
 
 If the source location is omitted or set to `null`, Dotbot will use the

dotbot/plugins/link.py

@@ -1,4 +1,5 @@
 import os
+import sys
 import glob
 import shutil
 import dotbot
@@ -124,15 +125,35 @@ class Link(dotbot.Plugin):
         else:
             return source
 
+    def _glob(self, path):
+        '''
+        Wrap `glob.glob` in a python agnostic way, catching errors in usage.
+        '''
+        if (sys.version_info < (3, 5) and '**' in path):
+            self._log.error('Link cannot handle recursive glob ("**") for Python < version 3.5: "%s"' % path)
+            return []
+        # call glob.glob; only python >= 3.5 supports recursive globs
+        found = ( glob.glob(path)
+                    if (sys.version_info < (3, 5)) else
+                  glob.glob(path, recursive=True) )
+        # if using recursive glob (`**`), filter results to return only files:
+        if '**' in path and not path.endswith(str(os.sep)):
+          self._log.debug("Excluding directories from recursive glob: " + str(path))
+          found = [f for f in found if os.path.isfile(f)]
+        # return matched results
+        return found
+
     def _create_glob_results(self, path, exclude_paths):
-        self._log.debug("Globbing with path: " + str(path))
-        base_include = glob.glob(path)
-        to_exclude = []
-        for expath in exclude_paths:
-            self._log.debug("Excluding globs with path: " + str(expath))
-            to_exclude.extend(glob.glob(expath))
-        self._log.debug("Excluded globs from '" + path + "': " + str(to_exclude))
-        ret = set(base_include) - set(to_exclude)
+        self._log.debug("Globbing with pattern: " + str(path))
+        include = self._glob(path)
+        self._log.debug("Glob found : " + str(include))
+        # filter out any paths matching the exclude globs:
+        exclude = []
+        for expat in exclude_paths:
+            self._log.debug("Excluding globs with pattern: " + str(expat))
+            exclude.extend( self._glob(expat) )
+        self._log.debug("Excluded globs from '" + path + "': " + str(exclude))
+        ret = set(include) - set(exclude)
         return list(ret)
 
     def _is_link(self, path):

test/driver-lib.bash

@@ -39,6 +39,7 @@ initialize() {
     tests_run=0
     tests_passed=0
     tests_failed=0
+    tests_skipped=0
     tests_total="${1}"
     local plural="" && [ "${tests_total}" -gt 1 ] && plural="s"
     printf -- "running %d test%s...\n\n" "${tests_total}" "${plural}"
@@ -52,7 +53,13 @@ pass() {
 
 fail() {
     tests_failed=$((tests_failed + 1))
-    yellow "-> fail!"
+    red "-> fail!"
+    echo
+}
+
+skip() {
+    tests_skipped=$((tests_skipped + 1))
+    yellow "-> skipped."
     echo
 }
 
@@ -62,6 +69,8 @@ run_test() {
     cleanup
     if (cd "${BASEDIR}/test/tests" && HOME=~/fakehome DEBUG=${2} DOTBOT_TEST=true bash "${1}"); then
         pass
+    elif [ $? -eq 42 ]; then
+        skip
     else
         fail
     fi
@@ -72,14 +81,13 @@ report() {
     printf -- "-----------\n"
     printf -- "- %3d run\n" ${tests_run}
     printf -- "- %3d passed\n" ${tests_passed}
+    printf -- "- %3d skipped\n" ${tests_skipped}
+    printf -- "- %3d failed\n" ${tests_failed}
     if [ ${tests_failed} -gt 0 ]; then
-        printf -- "- %3d failed\n" ${tests_failed}
-        echo
-        red "==> not ok!"
+        red "==> FAIL! "
         return 1
     else
-        echo
-        green "==> all ok."
+        green "==> PASS. "
         return 0
     fi
 }

test/test-lib.bash

@@ -27,6 +27,11 @@ test_expect_failure() {
     fi
 }
 
+skip_tests() {
+  # exit with special exit code picked up by driver-lib.bash
+  exit 42
+}
+
 check_env() {
     if [ "${DOTBOT_TEST}" != "true" ]; then
         >&2 echo "test must be run by test driver"
@@ -34,6 +39,19 @@ check_env() {
     fi
 }
 
+# run comparison check on python version; args:
+#   $1 - comparison operator (e.g. '>=')
+#   $2 - version number, to be passed to python (e.g. '3', '3.5', '3.6.4')
+# status code will reflect if comparison is true/false
+# e.g. `check_python_version '>=' 3.5`
+check_python_version() {
+    check="$1"
+    version="$(echo "$2" | tr . , )"
+    # this call to just `python` will work in the Vagrant-based testing VM
+    # because `pyenv` will always create a link to the "right" version.
+    python -c "import sys; exit( not (sys.version_info ${check} (${version})) )"
+}
+
 initialize() {
     check_env
     echo "${test_description}"

test/tests/link-glob-recursive.bash

@@ -0,0 +1,46 @@
+test_description='link glob recursive'
+. '../test-lib.bash'
+
+check_python_version ">=" 3.5 \
+  || test_expect_failure 'expect-fail' '
+run_dotbot -v <<EOF
+- link:
+    ~/.config/:
+      glob: true
+      path: bogus/**
+EOF
+'
+
+# Skip remaining tests if not supported
+check_python_version ">=" 3.5 \
+  || skip_tests
+
+test_expect_success 'setup' '
+mkdir -p ${DOTFILES}/config/foo/bar &&
+echo "apple" > ${DOTFILES}/config/foo/bar/a &&
+echo "banana" > ${DOTFILES}/config/foo/bar/b &&
+echo "cherry" > ${DOTFILES}/config/foo/bar/c
+'
+
+test_expect_success 'run' '
+run_dotbot -v <<EOF
+- defaults:
+    link:
+      glob: true
+      create: true
+- link:
+    ~/.config/:
+      path: config/**
+      exclude: [config/**/b]
+EOF
+'
+
+test_expect_success 'test' '
+! readlink ~/.config/ &&
+! readlink ~/.config/foo &&
+! readlink ~/.config/foo/bar &&
+readlink ~/.config/foo/bar/a &&
+grep "apple" ~/.config/foo/bar/a &&
+test \! -e ~/.config/foo/bar/b &&
+grep "cherry" ~/.config/foo/bar/c
+'