Merge remote-tracking branch 'remotes/upstream/master'

README.md

@@ -66,6 +66,9 @@ cp dotbot/tools/hg-subrepo/install .
 touch install.conf.yaml
 ```
 
+If you are using PowerShell instead of a POSIX shell, you can use the provided
+`install.ps1` script instead of `install`.
+
 To get started, you just need to fill in the `install.conf.yaml` and Dotbot
 will take care of the rest. To help you get started we have [an
 example](#full-example) config file as well as [configuration
@@ -177,10 +180,17 @@ mapped to extended configuration dictionaries.
 | `relink` | Removes the old target if it's a symlink (default: false) |
 | `force` | Force removes the old target, file or folder, and forces a new link (default: false) |
 | `relative` | Use a relative path to the source when creating the symlink (default: false, absolute links) |
-| `canonicalize-path` | Resolve any symbolic links encountered in the source to symlink to the canonical path (default: true, real paths) |
+| `canonicalize` | Resolve any symbolic links encountered in the source to symlink to the canonical path (default: true, real paths) |
 | `glob` | Treat a `*` character as a wildcard, and perform link operations on all of those matches (default: false) |
 | `if` | Execute this in your `$SHELL` and only link if it is successful. |
 | `ignore-missing` | Do not fail if the source is missing and create the link anyway (default: false) |
+| `exclude` | Array of paths to remove from glob matches. Uses same syntax as `path`. Ignored if `glob` is `false`. (default: empty, keep all matches) |
+
+Dotbot uses [glob.glob](https://docs.python.org/3/library/glob.html#glob.glob)
+to resolve glob paths. However, due to its design, using a glob path such as
+`config/*` for example, will not match items that being with `.`. To
+specifically capture items that being with `.`, you will need to use a path
+like this: `config/.*`.
 
 #### Example
 
@@ -221,6 +231,12 @@ Explicit sources:
       glob: true
       path: config/*
       relink: true
+      exclude: [ config/Code ]
+    ~/.config/Code/User/:
+      create: true
+      glob: true
+      path: config/Code/User/*
+      relink: true
 ```
 
 Implicit sources:
@@ -237,6 +253,12 @@ Implicit sources:
       glob: true
       path: config/*
       relink: true
+      exclude: [ config/Code ]
+    ~/.config/Code/User/:
+      create: true
+      glob: true
+      path: config/Code/User/*
+      relink: true
 ```
 
 ### Create
@@ -247,15 +269,30 @@ apps, plugins, shell commands, etc.
 
 #### Format
 
-Create commands are specified as an array of directories to be created.
+Create commands are specified as an array of directories to be created. If you
+want to use the optional extended configuration, create commands are specified
+as dictionaries. For convenience, it's permissible to leave the options blank
+(null) in the dictionary syntax.
+
+| Parameter | Explanation |
+| --- | --- |
+| `mode` | The file mode to use for creating the leaf directory (default: 0777) |
+
+The `mode` parameter is treated in the same way as in Python's
+[os.mkdir](https://docs.python.org/3/library/os.html#mkdir-modebits). Its
+behavior is platform-dependent. On Unix systems, the current umask value is
+first masked out.
 
 #### Example
 
 ```yaml
 - create:
-    - ~/projects
     - ~/downloads
     - ~/.vim/undo-history
+- create:
+    ~/.ssh:
+      mode: 0700
+    ~/projects:
 ```
 
 ### Shell

dotbot/cli.py

@@ -1,7 +1,7 @@
 import os, glob
 import sys
 
-from argparse import ArgumentParser
+from argparse import ArgumentParser, RawTextHelpFormatter
 from .config import ConfigReader, ReadingError
 from .dispatcher import Dispatcher, DispatchError
 from .messenger import Messenger
@@ -11,31 +11,36 @@ from .util import module
 import dotbot
 import yaml
 
-
 def add_options(parser):
-    parser.add_argument('-Q', '--super-quiet', action='store_true', help='suppress almost all output')
+    parser.add_argument('-Q', '--super-quiet', action='store_true',
-    parser.add_argument('-q', '--quiet', action='store_true', help='suppress most output')
+        help='suppress almost all output')
-    parser.add_argument('-v', '--verbose', action='store_true', help='enable verbose output')
+    parser.add_argument('-q', '--quiet', action='store_true',
-    parser.add_argument('-d', '--base-directory', help='execute commands from within BASEDIR', metavar='BASEDIR')
+        help='suppress most output')
-    parser.add_argument('-c', '--config-file', help='run commands given in CONFIGFILE', metavar='CONFIGFILE')
+    parser.add_argument('-v', '--verbose', action='count', default=0,
-    parser.add_argument(
+        help='enable verbose output\n'
-        '-p', '--plugin', action='append', dest='plugins', default=[], help='load PLUGIN as a plugin', metavar='PLUGIN'
+             '-v: typical verbose\n'
-    )
+             '-vv: also, set shell commands stderr/stdout to true')
-    parser.add_argument('--disable-built-in-plugins', action='store_true', help='disable built-in plugins')
+    parser.add_argument('-d', '--base-directory',
-    parser.add_argument(
+        help='execute commands from within BASEDIR',
-        '--plugin-dir',
+        metavar='BASEDIR')
-        action='append',
+    parser.add_argument('-c', '--config-file',
-        dest='plugin_dirs',
+        help='run commands given in CONFIGFILE', metavar='CONFIGFILE')
-        default=[],
+    parser.add_argument('-p', '--plugin', action='append', dest='plugins', default=[],
-        metavar='PLUGIN_DIR',
+        help='load PLUGIN as a plugin', metavar='PLUGIN')
-        help='load all plugins in PLUGIN_DIR',
+    parser.add_argument('--disable-built-in-plugins',
-    )
+        action='store_true', help='disable built-in plugins')
-    parser.add_argument('--only', nargs='+', help='only run specified directives', metavar='DIRECTIVE')
+    parser.add_argument('--plugin-dir', action='append', dest='plugin_dirs', default=[],
-    parser.add_argument('--except', nargs='+', dest='skip', help='skip specified directives', metavar='DIRECTIVE')
+        metavar='PLUGIN_DIR', help='load all plugins in PLUGIN_DIR')
-    parser.add_argument('--force-color', dest='force_color', action='store_true', help='force color output')
+    parser.add_argument('--only', nargs='+',
-    parser.add_argument('--no-color', dest='no_color', action='store_true', help='disable color output')
+            help='only run specified directives', metavar='DIRECTIVE')
-    parser.add_argument('--version', action='store_true', help="show program's version number and exit")
+    parser.add_argument('--except', nargs='+', dest='skip',
-
+            help='skip specified directives', metavar='DIRECTIVE')
+    parser.add_argument('--force-color', dest='force_color', action='store_true',
+        help='force color output')
+    parser.add_argument('--no-color', dest='no_color', action='store_true',
+        help='disable color output')
+    parser.add_argument('--version', action='store_true',
+        help='show program\'s version number and exit')
 
 def read_config(config_file):
     reader = ConfigReader(config_file)
@@ -45,20 +50,20 @@ def read_config(config_file):
 def main(additional_args=None):
     log = Messenger()
     try:
-        parser = ArgumentParser()
+        parser = ArgumentParser(formatter_class=RawTextHelpFormatter)
         add_options(parser)
         options = parser.parse_args()
         if additional_args is not None:
             print("got explicit arguments")
             options = parser.parse_args(additional_args)
         if options.version:
-            print("Dotbot version %s (yaml: %s)" % (dotbot.__version__, yaml.__version__))
+            print('Dotbot version %s (yaml: %s)' % (dotbot.__version__, yaml.__version__))
             exit(0)
         if options.super_quiet:
             log.set_level(Level.WARNING)
         if options.quiet:
             log.set_level(Level.INFO)
-        if options.verbose:
+        if options.verbose > 0:
             log.set_level(Level.DEBUG)
 
         if options.force_color and options.no_color:
@@ -76,38 +81,38 @@ def main(additional_args=None):
             from .plugins import Clean, Create, Link, Shell
         plugin_paths = []
         for directory in plugin_directories:
-            for plugin_path in glob.glob(os.path.join(directory, "*.py")):
+          for plugin_path in glob.glob(os.path.join(directory, '*.py')):
-                plugin_paths.append(plugin_path)
+            plugin_paths.append(plugin_path)
         for plugin_path in options.plugins:
             plugin_paths.append(plugin_path)
         for plugin_path in plugin_paths:
             abspath = os.path.abspath(plugin_path)
             module.load(abspath)
         if not options.config_file:
-            log.error("No configuration file specified")
+            log.error('No configuration file specified')
             exit(1)
         # read tasks from config file
         tasks = read_config(options.config_file)
         if tasks is None:
-            log.warning("Configuration file is empty, no work to do")
+            log.warning('Configuration file is empty, no work to do')
             tasks = []
         if not isinstance(tasks, list):
-            raise ReadingError("Configuration file must be a list of tasks")
+            raise ReadingError('Configuration file must be a list of tasks')
         if options.base_directory:
             base_directory = os.path.abspath(options.base_directory)
         else:
             # default to directory of config file
             base_directory = os.path.dirname(os.path.abspath(options.config_file))
         os.chdir(base_directory)
-        dispatcher = Dispatcher(base_directory, only=options.only, skip=options.skip)
+        dispatcher = Dispatcher(base_directory, only=options.only, skip=options.skip, options=options)
         success = dispatcher.dispatch(tasks)
         if success:
-            log.info("\n==> All tasks executed successfully")
+            log.info('\n==> All tasks executed successfully')
         else:
-            raise DispatchError("\n==> Some tasks were not executed successfully")
+            raise DispatchError('\n==> Some tasks were not executed successfully')
     except (ReadingError, DispatchError) as e:
-        log.error("%s" % e)
+        log.error('%s' % e)
         exit(1)
     except KeyboardInterrupt:
-        log.error("\n==> Operation aborted")
+        log.error('\n==> Operation aborted')
         exit(1)

dotbot/context.py

@@ -1,5 +1,6 @@
 import copy
 import os
+from argparse import Namespace
 
 
 class Context(object):
@@ -7,9 +8,10 @@ class Context(object):
     Contextual data and information for plugins.
     """
 
-    def __init__(self, base_directory):
+    def __init__(self, base_directory, options=Namespace()):
         self._base_directory = base_directory
         self._defaults = {}
+        self._options = options
         pass
 
     def set_base_directory(self, base_directory):
@@ -26,3 +28,6 @@ class Context(object):
 
     def defaults(self):
         return copy.deepcopy(self._defaults)
+
+    def options(self):
+        return copy.deepcopy(self._options)

dotbot/dispatcher.py

@@ -1,4 +1,5 @@
 import os
+from argparse import Namespace
 from .plugin import Plugin
 from .messenger import Messenger
 from .context import Context
@@ -6,18 +7,18 @@ import traceback
 
 
 class Dispatcher(object):
-    def __init__(self, base_directory, only=None, skip=None):
+    def __init__(self, base_directory, only=None, skip=None, options=Namespace()):
         self._log = Messenger()
-        self._setup_context(base_directory)
+        self._setup_context(base_directory, options)
         self._load_plugins()
         self._only = only
         self._skip = skip
 
-    def _setup_context(self, base_directory):
+    def _setup_context(self, base_directory, options):
         path = os.path.abspath(os.path.expanduser(base_directory))
         if not os.path.exists(path):
-            raise DispatchError("Nonexistent base directory")
+            raise DispatchError('Nonexistent base directory')
-        self._context = Context(path)
+        self._context = Context(path, options)
 
     def dispatch(self, tasks):
         success = True
@@ -32,15 +33,12 @@ class Dispatcher(object):
                     self._log.info("Skipping action %s" % action)
                     continue
                 handled = False
-                # print("\tcurrent action", action)
+                if action == 'defaults':
-                if action == "defaults":
                     self._context.set_defaults(task[action])  # replace, not update
                     handled = True
                     # keep going, let other plugins handle this if they want
                 for plugin in self._plugins:
-
                     if plugin.can_handle(action):
-                        # print("Action:", action)
                         try:
                             success &= plugin.handle(action, task[action])
                             handled = True
@@ -51,7 +49,7 @@ class Dispatcher(object):
                             self._log.debug(err)
                 if not handled:
                     success = False
-                    self._log.error('Action "%s" not handled' % action)
+                    self._log.error('Action %s not handled' % action)
         return success
 
     def _load_plugins(self):

dotbot/plugins/create.py

@@ -3,48 +3,54 @@ import dotbot
 
 
 class Create(dotbot.Plugin):
-    """
+    '''
     Create empty paths.
-    """
+    '''
 
-    _directive = "create"
+    _directive = 'create'
 
     def can_handle(self, directive):
         return directive == self._directive
 
     def handle(self, directive, data):
         if directive != self._directive:
-            raise ValueError("Create cannot handle directive %s" % directive)
+            raise ValueError('Create cannot handle directive %s' % directive)
         return self._process_paths(data)
 
     def _process_paths(self, paths):
         success = True
-        for path in paths:
+        defaults = self._context.defaults().get('create', {})
-            path = os.path.normpath(os.path.expandvars(os.path.expanduser(path)))
+        for key in paths:
-            success &= self._create(path)
+            path = os.path.normpath(os.path.expandvars(os.path.expanduser(key)))
+            mode = defaults.get('mode', 0o777)  # same as the default for os.makedirs
+            if isinstance(paths, dict):
+                options = paths[key]
+                if options:
+                    mode = options.get('mode', mode)
+            success &= self._create(path, mode)
         if success:
-            self._log.info("All paths have been set up")
+            self._log.info('All paths have been set up')
         else:
-            self._log.error("Some paths were not successfully set up")
+            self._log.error('Some paths were not successfully set up')
         return success
 
     def _exists(self, path):
-        """
+        '''
         Returns true if the path exists.
-        """
+        '''
         path = os.path.expanduser(path)
         return os.path.exists(path)
 
-    def _create(self, path):
+    def _create(self, path, mode):
         success = True
         if not self._exists(path):
-            self._log.debug("Trying to create path %s" % path)
+            self._log.debug('Trying to create path %s with mode %o' % (path, mode))
             try:
-                self._log.lowinfo("Creating path %s" % path)
+                self._log.lowinfo('Creating path %s' % path)
-                os.makedirs(path)
+                os.makedirs(path, mode)
             except OSError:
-                self._log.warning("Failed to create path %s" % path)
+                self._log.warning('Failed to create path %s' % path)
                 success = False
         else:
-            self._log.lowinfo("Path exists %s" % path)
+            self._log.lowinfo('Path exists %s' % path)
         return success

dotbot/plugins/link.py

@@ -7,39 +7,41 @@ import subprocess
 
 
 class Link(dotbot.Plugin):
-    """
+    '''
     Symbolically links dotfiles.
-    """
+    '''
 
-    _directive = "link"
+    _directive = 'link'
 
     def can_handle(self, directive):
         return directive == self._directive
 
     def handle(self, directive, data):
         if directive != self._directive:
-            raise ValueError("Link cannot handle directive %s" % directive)
+            raise ValueError('Link cannot handle directive %s' % directive)
         return self._process_links(data)
 
     def _get_default_flags(self):
         """Get flags for process links from default file."""
         defaults = self._context.defaults().get("link", {})
         relative = defaults.get("relative", False)
-        canonical_path = defaults.get("canonicalize-path", True)
+        canonical_path = defaults.get("canonicalize", defaults.get("canonicalize-path", True))
         force = defaults.get("force", False)
         relink = defaults.get("relink", False)
         create = defaults.get("create", False)
         use_glob = defaults.get("glob", False)
         test = defaults.get("if", None)
         ignore_missing = defaults.get("ignore-missing", False)
-        return relative, canonical_path, force, relink, create, use_glob, test, ignore_missing
+        exclude_paths = defaults.get('exclude', [])
+        return relative, canonical_path, force, relink, create, use_glob, test, ignore_missing, exclude_paths
 
 
     def _process_links(self, links_dict):
         # print("symlinking\n\t", links)
         success = True
         (relative_default, canonical_path_default, force_flag_default, relink_flag_default,
-         create_dir_flag_default, use_glob_default, shell_command_default, ignore_missing_default) = self._get_default_flags()
+         create_dir_flag_default, use_glob_default, shell_command_default,
+         ignore_missing_default, exclude_paths_default) = self._get_default_flags()
 
         for destination, source_dict in links_dict.items():
             destination = os.path.expandvars(destination)
@@ -49,36 +51,38 @@ class Link(dotbot.Plugin):
                 # extended config
                 shell_command = source_dict.get("if", shell_command_default)
                 relative = source_dict.get("relative", relative_default)
-                canonical_path = source_dict.get("canonicalize-path", canonical_path_default)
+                # support old "canonicalize-path" key for compatibility
+                canonical_path = source_dict.get("canonicalize", source_dict.get(
+                    "canonicalize-path", canonical_path_default))
                 force_flag = source_dict.get("force", force_flag_default)
                 relink_flag = source_dict.get("relink", relink_flag_default)
                 create_dir_flag = source_dict.get("create", create_dir_flag_default)
                 use_glob = source_dict.get("glob", use_glob_default)
                 ignore_missing = source_dict.get("ignore-missing", ignore_missing_default)
+                exclude_paths = source_dict.get("exclude", exclude_paths_default)
 
             else:
                 path = self._default_source(destination, source_dict)
 
                 (shell_command, relative, canonical_path, force_flag, relink_flag,
-                create_dir_flag, use_glob, ignore_missing) = (shell_command_default, relative_default, canonical_path_default, force_flag_default, relink_flag_default,
+                create_dir_flag, use_glob, ignore_missing, exclude_paths) = (shell_command_default,
-                create_dir_flag_default, use_glob_default, ignore_missing_default)
+                                                               relative_default, canonical_path_default, force_flag_default, relink_flag_default,
+                create_dir_flag_default, use_glob_default, ignore_missing_default, exclude_paths_default)
             if shell_command is not None and not self._test_success(shell_command):
                 self._log.lowinfo("Skipping %s" % destination)
                 continue
             path = os.path.expandvars(os.path.expanduser(path))
             if use_glob:
-                self._log.debug("Globbing with path: " + str(path))
+                glob_results = self._create_glob_results(path, exclude_paths)
-                glob_results = glob.glob(path)
                 if len(glob_results) == 0:
                     self._log.warning("Globbing couldn't find anything matching " + str(path))
                     success = False
                     continue
-                glob_star_loc = path.find("*")
+                glob_star_loc = path.find('*')
-                if glob_star_loc == -1 and destination[-1] == "/":
+                if glob_star_loc == -1 and destination[-1] == '/':
                     self._log.error("Ambiguous action requested.")
-                    self._log.error(
+                    self._log.error("No wildcard in glob, directory use undefined: " +
-                        "No wildcard in glob, directory use undefined: " + destination + " -> " + str(glob_results)
+                        destination + " -> " + str(glob_results))
-                    )
                     self._log.warning("Did you want to link the directory or into it?")
                     success = False
                     continue
@@ -92,8 +96,10 @@ class Link(dotbot.Plugin):
                 else:
                     self._log.lowinfo("Globs from '" + path + "': " + str(glob_results))
                     glob_base = path[:glob_star_loc]
+                    if glob_base.endswith('/.') or glob_base == '.':
+                        glob_base = path[:glob_star_loc - 1]
                     for glob_full_item in glob_results:
-                        glob_item = glob_full_item[len(glob_base) :]
+                        glob_item = glob_full_item[len(glob_base):]
                         glob_link_destination = os.path.join(destination, glob_item)
                         if create_dir_flag:
                             success &= self._create_dir(glob_link_destination)
@@ -115,15 +121,16 @@ class Link(dotbot.Plugin):
                     # want to remove the original (this is tested by
                     # link-force-leaves-when-nonexistent.bash)
                     success = False
-                    self._log.warning("Nonexistent source %s -> %s" % (destination, path))
+                    self._log.warning('Nonexistent source %s -> %s' %
+                        (destination, path))
                     continue
                 if force_flag or relink_flag:
                     success &= self._delete(path, destination, relative, canonical_path, force_flag)
                 success &= self._link(path, destination, relative, canonical_path, ignore_missing)
         if success:
-            self._log.info("All links have been set up")
+            self._log.info('All links have been set up')
         else:
-            self._log.error("Some links were not successfully set up")
+            self._log.error('Some links were not successfully set up')
         return success
 
     def _test_success(self, command):
@@ -135,25 +142,36 @@ class Link(dotbot.Plugin):
     def _default_source(self, destination, source):
         if source is None:
             basename = os.path.basename(destination)
-            if basename.startswith("."):
+            if basename.startswith('.'):
                 return basename[1:]
             else:
                 return basename
         else:
             return source
 
+    def _create_glob_results(self, path, exclude_paths):
+        self._log.debug("Globbing with path: " + str(path))
+        base_include = glob.glob(path)
+        to_exclude = []
+        for expath in exclude_paths:
+            self._log.debug("Excluding globs with path: " + str(expath))
+            to_exclude.extend(glob.glob(expath))
+        self._log.debug("Excluded globs from '" + path + "': " + str(to_exclude))
+        ret = set(base_include) - set(to_exclude)
+        return list(ret)
+
     def _is_link(self, path):
-        """
+        '''
         Returns true if the path is a symbolic link.
-        """
+        '''
         return os.path.islink(os.path.expanduser(path))
 
     def _get_link_destination(self, path):
-        """
+        '''
         Returns the destination of the symbolic link. Truncates the  \\?\ start to a path if it
         is present. This is an identifier which allows >255 character file name links to work.
         Since this function is for the point of comparison, it is okay to truncate
-        """
+        '''
         # path = os.path.normpath(path)
         path = os.path.expanduser(path)
         try:
@@ -173,9 +191,9 @@ class Link(dotbot.Plugin):
             return read_link
 
     def _exists(self, path):
-        """
+        '''
         Returns true if the path exists. Returns false if contains dangling symbolic links.
-        """
+        '''
         path = os.path.expanduser(path)
         return os.path.exists(path)
 
@@ -188,10 +206,10 @@ class Link(dotbot.Plugin):
             try:
                 os.makedirs(parent)
             except OSError:
-                self._log.warning("Failed to create directory %s" % parent)
+                self._log.warning('Failed to create directory %s' % parent)
                 success = False
             else:
-                self._log.lowinfo("Creating directory %s" % parent)
+                self._log.lowinfo('Creating directory %s' % parent)
         return success
 
     def _delete(self, source, path, relative, canonical_path, force):
@@ -216,29 +234,29 @@ class Link(dotbot.Plugin):
                         os.remove(fullpath)
                         removed = True
             except OSError:
-                self._log.warning("Failed to remove %s" % path)
+                self._log.warning('Failed to remove %s' % path)
                 success = False
             else:
                 if removed:
-                    self._log.lowinfo("Removing %s" % path)
+                    self._log.lowinfo('Removing %s' % path)
         return success
 
     def _relative_path(self, source, destination):
-        """
+        '''
         Returns the relative path to get to the source file from the
         destination file.
-        """
+        '''
         destination_dir = os.path.dirname(destination)
         return os.path.relpath(source, destination_dir)
 
     def _link(self, dotfile_source, target_path_to_link_at, relative_path, canonical_path, ignore_missing):
-        """
+        '''
         Links link_name to source.
         :param target_path_to_link_at is the file path where we are putting a symlink back to
         dotfile_source
 
         Returns true if successfully linked files.
-        """
+        '''
         success_flag = False
         destination = os.path.normpath(os.path.expanduser(target_path_to_link_at))
         base_directory = self._context.base_directory(canonical_path=canonical_path)

dotbot/plugins/shell.py

@@ -5,35 +5,38 @@ import dotbot.util
 
 
 class Shell(dotbot.Plugin):
-    """
+    '''
     Run arbitrary shell commands.
-    """
+    '''
 
-    _directive = "shell"
+    _directive = 'shell'
+    _has_shown_override_message = False
 
     def can_handle(self, directive):
         return directive == self._directive
 
     def handle(self, directive, data):
         if directive != self._directive:
-            raise ValueError("Shell cannot handle directive %s" % directive)
+            raise ValueError('Shell cannot handle directive %s' %
+                directive)
         return self._process_commands(data)
 
     def _process_commands(self, data):
         success = True
-        defaults = self._context.defaults().get("shell", {})
+        defaults = self._context.defaults().get('shell', {})
+        options = self._get_option_overrides()
         for item in data:
-            stdin = defaults.get("stdin", False)
+            stdin = defaults.get('stdin', False)
-            stdout = defaults.get("stdout", False)
+            stdout = defaults.get('stdout', False)
-            stderr = defaults.get("stderr", False)
+            stderr = defaults.get('stderr', False)
-            quiet = defaults.get("quiet", False)
+            quiet = defaults.get('quiet', False)
             if isinstance(item, dict):
-                cmd = item["command"]
+                cmd = item['command']
-                msg = item.get("description", None)
+                msg = item.get('description', None)
-                stdin = item.get("stdin", stdin)
+                stdin = item.get('stdin', stdin)
-                stdout = item.get("stdout", stdout)
+                stdout = item.get('stdout', stdout)
-                stderr = item.get("stderr", stderr)
+                stderr = item.get('stderr', stderr)
-                quiet = item.get("quiet", quiet)
+                quiet = item.get('quiet', quiet)
             elif isinstance(item, list):
                 cmd = item[0]
                 msg = item[1] if len(item) > 1 else None
@@ -43,17 +46,34 @@ class Shell(dotbot.Plugin):
             if msg is None:
                 self._log.lowinfo(cmd)
             elif quiet:
-                self._log.lowinfo("%s" % msg)
+                self._log.lowinfo('%s' % msg)
             else:
-                self._log.lowinfo("%s [%s]" % (msg, cmd))
+                self._log.lowinfo('%s [%s]' % (msg, cmd))
+            stdout = options.get('stdout', stdout)
+            stderr = options.get('stderr', stderr)
             ret = dotbot.util.shell_command(
-                cmd, cwd=self._context.base_directory(), enable_stdin=stdin, enable_stdout=stdout, enable_stderr=stderr
+                cmd,
+                cwd=self._context.base_directory(),
+                enable_stdin=stdin,
+                enable_stdout=stdout,
+                enable_stderr=stderr
             )
             if ret != 0:
                 success = False
-                self._log.warning("Command [%s] failed" % cmd)
+                self._log.warning('Command [%s] failed' % cmd)
         if success:
-            self._log.info("All commands have been executed")
+            self._log.info('All commands have been executed')
         else:
-            self._log.error("Some commands were not successfully executed")
+            self._log.error('Some commands were not successfully executed')
         return success
+
+    def _get_option_overrides(self):
+        ret = {}
+        options = self._context.options()
+        if options.verbose > 1:
+            ret['stderr'] = True
+            ret['stdout'] = True
+            if not self._has_shown_override_message:
+                self._log.debug("Shell: Found cli option to force show stderr and stdout.")
+                self._has_shown_override_message = True
+        return ret

test/README.md

@@ -51,8 +51,8 @@ edits on your host machine). You can run the test suite by `cd /dotbot/test`
 and then running `./test`. Selected tests can be run by passing paths to the
 tests as arguments, e.g. `./test tests/create.bash tests/defaults.bash`.
 
-To debug tests, you can prepend the line `DEBUG=true` as the first line to any
+To debug tests, you can run the test driver with the `--debug` (or `-d` short
-individual test (a `.bash` file inside `test/tests`). This will enable printing
+form) flag, e.g. `./test --debug tests/link-if.bash`. This will enable printing
 stdout/stderr.
 
 When finished with testing, it is good to shut down the virtual machine by

test/driver-lib.bash

@@ -60,7 +60,7 @@ run_test() {
     tests_run=$((tests_run + 1))
     printf '[%d/%d] (%s)\n' "${tests_run}" "${tests_total}" "${1}"
     cleanup
-    if (cd "${BASEDIR}/test/tests" && HOME=~/fakehome DOTBOT_TEST=true bash "${1}"); then
+    if (cd "${BASEDIR}/test/tests" && HOME=~/fakehome DEBUG=${2} DOTBOT_TEST=true bash "${1}"); then
         pass
     else
         fail

test/test

@@ -10,6 +10,23 @@ start="$(date +%s)"
 
 check_env
 
+# parse flags; must come before positional arguments
+POSITIONAL=()
+DEBUG=false
+while [[ $# -gt 0 ]]; do
+    case $1 in
+        -d|--debug)
+            DEBUG=true
+            shift
+            ;;
+        *)
+            POSITIONAL+=("$1")
+            shift
+            ;;
+    esac
+done
+set -- "${POSITIONAL[@]}" # restore positional arguments
+
 declare -a tests=()
 
 if [ $# -eq 0 ]; then
@@ -20,10 +37,10 @@ else
     tests=("$@")
 fi
 
-initialize "${#tests[@]}" "${VERSION}"
+initialize "${#tests[@]}"
 
 for file in "${tests[@]}"; do
-    run_test "$(basename "${file}")" "${VERSION}"
+    run_test "$(basename "${file}")" "${DEBUG}"
 done
 
 if report; then

test/test-lib.bash

@@ -1,4 +1,3 @@
-DEBUG=${DEBUG:-false}
 DOTBOT_EXEC="${BASEDIR}/bin/dotbot"
 DOTFILES="${HOME}/dotfiles"
 INSTALL_CONF='install.conf.yaml'

test/tests/create-mode.bash

@@ -0,0 +1,26 @@
+test_description='create mode'
+. '../test-lib.bash'
+
+test_expect_success 'run' '
+run_dotbot -v <<EOF
+- defaults:
+    create:
+      mode: 0755
+- create:
+    - ~/downloads
+    - ~/.vim/undo-history
+- create:
+    ~/.ssh:
+      mode: 0700
+    ~/projects:
+EOF
+'
+
+test_expect_success 'test' '
+[ -d ~/downloads ] &&
+[ -d ~/.vim/undo-history ] &&
+[ -d ~/.ssh ] &&
+[ -d ~/projects ] &&
+[ "$(stat -c %a ~/.ssh)" = "700" ] &&
+[ "$(stat -c %a ~/downloads)" = "755" ]
+'

test/tests/except.bash

@@ -24,6 +24,7 @@ run_dotbot --except shell <<EOF
   - echo "pear" > ~/z
 - link:
     ~/x: x
+EOF
 '
 
 test_expect_success 'test' '

test/tests/link-glob-exclude.bash

@@ -0,0 +1,123 @@
+test_description='link glob exclude'
+. '../test-lib.bash'
+
+test_expect_success 'setup 1' '
+mkdir -p ${DOTFILES}/config/{foo,bar,baz} &&
+echo "apple" > ${DOTFILES}/config/foo/a &&
+echo "banana" > ${DOTFILES}/config/bar/b &&
+echo "cherry" > ${DOTFILES}/config/bar/c &&
+echo "donut" > ${DOTFILES}/config/baz/d
+'
+
+test_expect_success 'run 1' '
+run_dotbot -v <<EOF
+- defaults:
+    link:
+      glob: true
+      create: true
+- link:
+    ~/.config/:
+      path: config/*
+      exclude: [config/baz]
+EOF
+'
+
+test_expect_success 'test 1' '
+! readlink ~/.config/ &&
+readlink ~/.config/foo &&
+! readlink ~/.config/baz &&
+grep "apple" ~/.config/foo/a &&
+grep "banana" ~/.config/bar/b &&
+grep "cherry" ~/.config/bar/c
+'
+
+test_expect_success 'setup 2' '
+rm -rf ~/.config &&
+mkdir ${DOTFILES}/config/baz/buzz &&
+echo "egg" > ${DOTFILES}/config/baz/buzz/e
+'
+
+test_expect_success 'run 2' '
+run_dotbot -v <<EOF
+- defaults:
+    link:
+      glob: true
+      create: true
+- link:
+    ~/.config/:
+      path: config/*/*
+      exclude: [config/baz/*]
+EOF
+'
+
+test_expect_success 'test 2' '
+! readlink ~/.config/ &&
+! readlink ~/.config/foo &&
+[ ! -d ~/.config/baz ] &&
+readlink ~/.config/foo/a &&
+grep "apple" ~/.config/foo/a &&
+grep "banana" ~/.config/bar/b &&
+grep "cherry" ~/.config/bar/c
+'
+
+test_expect_success 'setup 3' '
+rm -rf ~/.config &&
+mkdir ${DOTFILES}/config/baz/bizz &&
+echo "grape" > ${DOTFILES}/config/baz/bizz/g
+'
+
+test_expect_success 'run 3' '
+run_dotbot -v <<EOF
+- defaults:
+    link:
+      glob: true
+      create: true
+- link:
+    ~/.config/:
+      path: config/*/*
+      exclude: [config/baz/buzz]
+EOF
+'
+
+test_expect_success 'test 3' '
+! readlink ~/.config/ &&
+! readlink ~/.config/foo &&
+readlink ~/.config/foo/a &&
+! readlink ~/.config/baz/buzz &&
+readlink ~/.config/baz/bizz &&
+grep "apple" ~/.config/foo/a &&
+grep "banana" ~/.config/bar/b &&
+grep "cherry" ~/.config/bar/c &&
+grep "donut" ~/.config/baz/d &&
+grep "grape" ~/.config/baz/bizz/g
+'
+
+test_expect_success 'setup 4' '
+rm -rf ~/.config &&
+mkdir ${DOTFILES}/config/fiz &&
+echo "fig" > ${DOTFILES}/config/fiz/f
+'
+
+test_expect_success 'run 4' '
+run_dotbot -v <<EOF
+- defaults:
+    link:
+      glob: true
+      create: true
+- link:
+    ~/.config/:
+      path: config/*/*
+      exclude: [config/baz/*, config/fiz/*]
+EOF
+'
+
+test_expect_success 'test 4' '
+! readlink ~/.config/ &&
+! readlink ~/.config/foo &&
+[ ! -d ~/.config/baz ] &&
+[ ! -d ~/.config/fiz ] &&
+readlink ~/.config/foo/a &&
+grep "apple" ~/.config/foo/a &&
+grep "banana" ~/.config/bar/b &&
+grep "cherry" ~/.config/bar/c
+'

test/tests/link-glob-multi-star.bash

@@ -1,4 +1,4 @@
-test_description='link glob'
+test_description='link glob multi star'
 . '../test-lib.bash'
 
 test_expect_success 'setup' '

test/tests/link-glob.bash

@@ -45,3 +45,49 @@ grep "apple" ~/bin/a &&
 grep "banana" ~/bin/b &&
 grep "cherry" ~/bin/c
 '
+
+test_expect_success 'setup 3' '
+rm -rf ~/bin &&
+echo "dot_apple" > ${DOTFILES}/bin/.a &&
+echo "dot_banana" > ${DOTFILES}/bin/.b &&
+echo "dot_cherry" > ${DOTFILES}/bin/.c
+'
+
+test_expect_success 'run 3' '
+run_dotbot -v <<EOF
+- defaults:
+    link:
+      glob: true
+      create: true
+- link:
+    ~/bin/: bin/.*
+EOF
+'
+
+test_expect_success 'test 3' '
+grep "dot_apple" ~/bin/.a &&
+grep "dot_banana" ~/bin/.b &&
+grep "dot_cherry" ~/bin/.c
+'
+
+test_expect_success 'setup 4' '
+rm -rf ~/bin &&
+echo "dot_apple" > ${DOTFILES}/.a &&
+echo "dot_banana" > ${DOTFILES}/.b &&
+echo "dot_cherry" > ${DOTFILES}/.c
+'
+
+test_expect_success 'run 4' '
+run_dotbot -v <<EOF
+- link:
+    "~":
+      path: .*
+      glob: true
+EOF
+'
+
+test_expect_success 'test 4' '
+grep "dot_apple" ~/.a &&
+grep "dot_banana" ~/.b &&
+grep "dot_cherry" ~/.c
+'

test/tests/link-no-canonicalize.bash

@@ -3,6 +3,7 @@ test_description='linking path canonicalization can be disabled'
 
 test_expect_success 'setup' '
 echo "apple" > ${DOTFILES}/f &&
+echo "grape" > ${DOTFILES}/g &&
 ln -s dotfiles dotfiles-symlink
 '
 
@@ -21,3 +22,19 @@ ${DOTBOT_EXEC} -c ./dotfiles-symlink/${INSTALL_CONF}
 test_expect_success 'test' '
 [ "$(readlink ~/.f | cut -d/ -f5-)" = "dotfiles-symlink/f" ]
 '
+
+test_expect_success 'run 2' '
+cat > "${DOTFILES}/${INSTALL_CONF}" <<EOF
+- defaults:
+    link:
+      canonicalize: false
+- link:
+    ~/.g:
+      path: g
+EOF
+${DOTBOT_EXEC} -c ./dotfiles-symlink/${INSTALL_CONF}
+'
+
+test_expect_success 'test' '
+[ "$(readlink ~/.g | cut -d/ -f5-)" = "dotfiles-symlink/g" ]
+'

test/tests/only-defaults.bash

@@ -14,6 +14,7 @@ run_dotbot --only link <<EOF
   - echo "pear" > ~/z
 - link:
     ~/d/x: x
+EOF
 '
 
 test_expect_success 'test' '

test/tests/only.bash

@@ -24,6 +24,7 @@ run_dotbot --only link <<EOF
   - echo "pear" > ~/z
 - link:
     ~/x: x
+EOF
 '
 
 test_expect_success 'test' '

test/tests/plugin.bash

@@ -1,7 +1,7 @@
 test_description='plugin loading works'
 . '../test-lib.bash'
 
-test_expect_success 'setup' '
+test_expect_success 'setup 1' '
 cat > ${DOTFILES}/test.py <<EOF
 import dotbot
 import os.path
@@ -17,12 +17,48 @@ class Test(dotbot.Plugin):
 EOF
 '
 
-test_expect_success 'run' '
+test_expect_success 'run 1' '
 run_dotbot --plugin ${DOTFILES}/test.py <<EOF
 - test: ~
 EOF
 '
 
-test_expect_success 'test' '
+test_expect_success 'test 1' '
+grep "it works" ~/flag
+'
+
+test_expect_success 'setup 2' '
+rm ${DOTFILES}/test.py;
+cat > ${DOTFILES}/test.py <<EOF
+import dotbot
+import os.path
+
+class Test(dotbot.Plugin):
+    def can_handle(self, directive):
+        return directive == "test"
+
+    def handle(self, directive, data):
+        self._log.debug("Attempting to get options from Context")
+        options = self._context.options()
+        if len(options.plugins) != 1:
+            self._log.debug("Context.options.plugins length is %i, expected 1" % len(options.plugins))
+            return False
+        if not options.plugins[0].endswith("test.py"):
+            self._log.debug("Context.options.plugins[0] is %s, expected end with test.py" % options.plugins[0])
+            return False
+
+        with open(os.path.expanduser("~/flag"), "w") as f:
+            f.write("it works")
+        return True
+EOF
+'
+
+test_expect_success 'run 2' '
+run_dotbot --plugin ${DOTFILES}/test.py <<EOF
+- test: ~
+EOF
+'
+
+test_expect_success 'test 2' '
 grep "it works" ~/flag
 '

test/tests/shell-cli-override-config.bash

@@ -0,0 +1,79 @@
+test_description='cli options can override config file'
+. '../test-lib.bash'
+
+test_expect_success 'run 1' '
+(run_dotbot -vv | (grep "^apple")) <<EOF
+- shell:
+  -
+    command: echo apple
+EOF
+'
+
+test_expect_success 'run 2' '
+(run_dotbot -vv | (grep "^apple")) <<EOF
+- shell:
+  -
+    command: echo apple
+    stdout: false
+EOF
+'
+
+test_expect_success 'run 3' '
+(run_dotbot -vv | (grep "^apple")) <<EOF
+- defaults:
+    shell:
+      stdout: false
+- shell:
+  - command: echo apple
+EOF
+'
+
+# Control to make sure stderr redirection is working as expected
+test_expect_failure 'run 4' '
+(run_dotbot -vv | (grep "^apple")) <<EOF
+- shell:
+  - command: echo apple >&2
+EOF
+'
+
+test_expect_success 'run 5' '
+(run_dotbot -vv 2>&1 | (grep "^apple")) <<EOF
+- shell:
+  - command: echo apple >&2
+EOF
+'
+
+test_expect_success 'run 6' '
+(run_dotbot -vv 2>&1 | (grep "^apple")) <<EOF
+- shell:
+  -
+    command: echo apple >&2
+    stdout: false
+EOF
+'
+
+test_expect_success 'run 7' '
+(run_dotbot -vv 2>&1 | (grep "^apple")) <<EOF
+- defaults:
+    shell:
+      stdout: false
+- shell:
+  - command: echo apple >&2
+EOF
+'
+
+# Make sure that we must use verbose level 2
+# This preserves backwards compatability
+test_expect_failure 'run 8' '
+(run_dotbot -v | (grep "^apple")) <<EOF
+- shell:
+  - command: echo apple
+EOF
+'
+
+test_expect_failure 'run 9' '
+(run_dotbot -v | (grep "^apple")) <<EOF
+- shell:
+  - command: echo apple >&2
+EOF
+'

tools/git-submodule/install.ps1

@@ -0,0 +1,22 @@
+$ErrorActionPreference = "Stop"
+
+$CONFIG = "install.conf.yaml"
+$DOTBOT_DIR = "dotbot"
+
+$DOTBOT_BIN = "bin/dotbot"
+$BASEDIR = $PSScriptRoot
+
+Set-Location $BASEDIR
+git -C $DOTBOT_DIR submodule sync --quiet --recursive
+git submodule update --init --recursive $DOTBOT_DIR
+
+foreach ($PYTHON in ('python', 'python3', 'python2')) {
+    # Python redirects to Microsoft Store in Windows 10 when not installed
+    if (& { $ErrorActionPreference = "SilentlyContinue"
+            ![string]::IsNullOrEmpty((&$PYTHON -V))
+            $ErrorActionPreference = "Stop" }) {
+        &$PYTHON $(Join-Path $BASEDIR -ChildPath $DOTBOT_DIR | Join-Path -ChildPath $DOTBOT_BIN) -d $BASEDIR -c $CONFIG $Args
+        return
+    }
+}
+Write-Error "Error: Cannot find Python."

tools/hg-subrepo/install.ps1

@@ -0,0 +1,21 @@
+$ErrorActionPreference = "Stop"
+
+$CONFIG = "install.conf.yaml"
+$DOTBOT_DIR = "dotbot"
+
+$DOTBOT_BIN = "bin/dotbot"
+$BASEDIR = $PSScriptRoot
+
+Set-Location $BASEDIR
+
+Set-Location $DOTBOT_DIR && git submodule update --init --recursive
+foreach ($PYTHON in ('python', 'python3', 'python2')) {
+    # Python redirects to Microsoft Store in Windows 10 when not installed
+    if (& { $ErrorActionPreference = "SilentlyContinue"
+            ![string]::IsNullOrEmpty((&$PYTHON -V))
+            $ErrorActionPreference = "Stop" }) {
+        &$PYTHON $(Join-Path $BASEDIR -ChildPath $DOTBOT_DIR | Join-Path -ChildPath $DOTBOT_BIN) -d $BASEDIR -c $CONFIG $Args
+        return
+    }
+}
+Write-Error "Error: Cannot find Python."