Create an upgrade command

This command will assist users with migration from 1.x.x to 2.0.0.

test/conftest.py

@@ -84,6 +84,7 @@ def supported_commands():
         'introspect',
         'list',
         'perms',
+        'upgrade',
         'version',
         ]
 

test/test_unit_issue_legacy_path_warning.py

@@ -10,25 +10,30 @@ import pytest
         'encrypt',
         'files.gpg',
         'bootstrap',
-        'hooks',
+        'hooks/pre_command',
+        'hooks/post_command',
         ],
     )
-def test_legacy_warning(tmpdir, runner, yadm, legacy_path):
+@pytest.mark.parametrize(
+    'upgrade', [True, False], ids=['upgrade', 'no-upgrade'])
+def test_legacy_warning(tmpdir, runner, yadm, upgrade, legacy_path):
     """Use issue_legacy_path_warning"""
     home = tmpdir.mkdir('home')
 
     if legacy_path:
-        home.mkdir(f'.yadm').mkdir(legacy_path)
+        home.mkdir(f'.yadm').ensure(legacy_path)
 
+    main_args = 'MAIN_ARGS=("upgrade")' if upgrade else ''
     script = f"""
         HOME={home}
         YADM_TEST=1 source {yadm}
+        {main_args}
         issue_legacy_path_warning
     """
     run = runner(command=['bash'], inp=script)
     assert run.success
     assert run.err == ''
-    if legacy_path:
+    if legacy_path and not upgrade:
         assert 'Legacy configuration paths have been detected' in run.out
     else:
         assert run.out.rstrip() == ''

test/test_unit_upgrade.py

@@ -0,0 +1,101 @@
+"""Unit tests: upgrade"""
+import pytest
+
+LEGACY_PATHS = [
+    'config',
+    'encrypt',
+    'files.gpg',
+    'bootstrap',
+    'hooks/pre_command',
+    'hooks/post_command',
+]
+
+# used:
+# YADM_COMPATIBILITY
+# YADM_DIR
+# YADM_LEGACY_DIR
+# GIT_PROGRAM
+@pytest.mark.parametrize('condition', ['compat', 'equal', 'existing_repo'])
+def test_upgrade_errors(tmpdir, runner, yadm, condition):
+    """Test upgrade() error conditions"""
+
+    compatibility = 'YADM_COMPATIBILITY=1' if condition == 'compat' else ''
+
+    home = tmpdir.mkdir('home')
+    yadm_dir = home.join('.config/yadm')
+    legacy_dir = home.join('.yadm')
+    if condition == 'equal':
+        legacy_dir = yadm_dir
+    if condition == 'existing_repo':
+        yadm_dir.ensure_dir('repo.git')
+        legacy_dir.ensure_dir('repo.git')
+
+    script = f"""
+        YADM_TEST=1 source {yadm}
+        {compatibility}
+        YADM_DIR="{yadm_dir}"
+        YADM_REPO="{yadm_dir}/repo.git"
+        YADM_LEGACY_DIR="{legacy_dir}"
+        upgrade
+    """
+    run = runner(command=['bash'], inp=script)
+    assert run.failure
+    assert run.err == ''
+    assert 'Unable to upgrade' in run.out
+    if condition == 'compat':
+        assert 'YADM_COMPATIBILITY' in run.out
+    if condition == 'equal':
+        assert 'has been resolved as' in run.out
+    if condition == 'existing_repo':
+        assert 'already exists' in run.out
+
+
+@pytest.mark.parametrize('condition', ['no-paths', 'untracked', 'tracked'])
+def test_upgrade(tmpdir, runner, yadm, condition):
+    """Test upgrade()
+
+    When testing the condition of git-tracked data, "echo" will be used as a
+    mock for git. echo will return true, simulating a positive result from "git
+    ls-files". Also echo will report the parameters for "git mv".
+    """
+    home = tmpdir.mkdir('home')
+    yadm_dir = home.join('.config/yadm')
+    legacy_dir = home.join('.yadm')
+
+    if condition != 'no-paths':
+        legacy_dir.join('repo.git/config').write('test-repo', ensure=True)
+        for lpath in LEGACY_PATHS:
+            legacy_dir.join(lpath).write(lpath, ensure=True)
+
+    git = 'echo' if condition == 'tracked' else 'git'
+
+    script = f"""
+        YADM_TEST=1 source {yadm}
+        YADM_DIR="{yadm_dir}"
+        YADM_REPO="{yadm_dir}/repo.git"
+        YADM_LEGACY_DIR="{legacy_dir}"
+        GIT_PROGRAM="{git}"
+        upgrade
+    """
+    run = runner(command=['bash'], inp=script)
+    assert run.success
+    assert run.err == ''
+    if condition == 'no-paths':
+        assert 'Upgrade is not necessary' in run.out
+    else:
+        for lpath in LEGACY_PATHS + ['repo.git']:
+            expected = (
+                f'Moving {legacy_dir.join(lpath)} '
+                f'to {yadm_dir.join(lpath)}')
+            assert expected in run.out
+        if condition == 'untracked':
+            assert 'test-repo' in yadm_dir.join('repo.git/config').read()
+            for lpath in LEGACY_PATHS:
+                assert lpath in yadm_dir.join(lpath).read()
+        elif condition == 'tracked':
+            for lpath in LEGACY_PATHS:
+                expected = (
+                    f'mv {legacy_dir.join(lpath)} '
+                    f'{yadm_dir.join(lpath)}')
+                assert expected in run.out
+            assert 'files tracked by yadm have been renamed' in run.out

yadm

@@ -72,7 +72,7 @@ function main() {
 
   # parse command line arguments
   local retval=0
-  internal_commands="^(alt|bootstrap|clean|clone|config|decrypt|encrypt|enter|help|init|introspect|list|perms|version)$"
+  internal_commands="^(alt|bootstrap|clean|clone|config|decrypt|encrypt|enter|help|init|introspect|list|perms|upgrade|version)$"
   if [ -z "$*" ] ; then
     # no argumnts will result in help()
     help
@@ -505,13 +505,12 @@ function alt_future_linking() {
     filename="${alt_filenames[$index]}"
     target="${alt_targets[$index]}"
     template_cmd="${alt_template_cmds[$index]}"
-    basedir=${filename%/*}
     if [ -n "$template_cmd" ]; then
       # a template is defined, process the template
       debug "Creating $filename from template $target"
       [ -n "$loud" ] && echo "Creating $filename from template $target"
       # ensure the destination path exists
-      [ -e "$basedir" ] || mkdir -p "$basedir"
+      assert_parent "$filename"
       # remove any existing symlink before processing template
       [ -L "$filename" ] && rm -f "$filename"
       "$template_cmd" "$target" "$filename"
@@ -520,7 +519,7 @@ function alt_future_linking() {
       debug "Linking $target to $filename"
       [ -n "$loud" ] && echo "Linking $target to $filename"
       # ensure the destination path exists
-      [ -e "$basedir" ] || mkdir -p "$basedir"
+      assert_parent "$filename"
       if [ "$do_copy" -eq 1 ]; then
         # remove any existing symlink before copying
         [ -L "$filename" ] && rm -f "$filename"
@@ -975,6 +974,7 @@ init
 introspect
 list
 perms
+upgrade
 version
 EOF
 }
@@ -1062,6 +1062,66 @@ function perms() {
 
 }
 
+function upgrade() {
+
+  local actions_performed
+  actions_performed=0
+  local repo_updates
+  repo_updates=0
+
+  [ "$YADM_COMPATIBILITY" = "1" ] && \
+    error_out "Unable to upgrade. YADM_COMPATIBILITY is set to '1'."
+
+  [ "$YADM_DIR" = "$YADM_LEGACY_DIR" ] && \
+    error_out "Unable to upgrade. yadm dir has been resolved as '$YADM_LEGACY_DIR'."
+
+  # handle legacy repo
+  if [ -d "$YADM_LEGACY_DIR/repo.git" ]; then
+    # legacy repo detected, it must be moved to YADM_REPO
+    if [ -e "$YADM_REPO" ]; then
+      error_out "Unable to upgrade. '$YADM_REPO' already exists. Refusing to overwrite it."
+    else
+      actions_performed=1
+      echo "Moving $YADM_LEGACY_DIR/repo.git to $YADM_REPO"
+      assert_parent "$YADM_REPO"
+      mv "$YADM_LEGACY_DIR/repo.git" "$YADM_REPO"
+    fi
+  fi
+
+  # handle other legacy paths
+  for legacy_path in                      \
+    "$YADM_LEGACY_DIR/config"             \
+    "$YADM_LEGACY_DIR/encrypt"            \
+    "$YADM_LEGACY_DIR/files.gpg"          \
+    "$YADM_LEGACY_DIR/bootstrap"          \
+    "$YADM_LEGACY_DIR"/hooks/{pre,post}_* \
+  ;                                       \
+  do
+    if [ -e "$legacy_path" ]; then
+      new_filename=${legacy_path#$YADM_LEGACY_DIR/}
+      new_filename="$YADM_DIR/$new_filename"
+      actions_performed=1
+      echo "Moving $legacy_path to $new_filename"
+      assert_parent "$new_filename"
+      # test to see if path is "tracked" in repo, if so 'git mv' must be used
+      if GIT_DIR="$YADM_REPO" "$GIT_PROGRAM" ls-files --error-unmatch "$legacy_path" >/dev/null 2>&1; then
+        GIT_DIR="$YADM_REPO" "$GIT_PROGRAM" mv "$legacy_path" "$new_filename" && repo_updates=1
+      else
+        mv -i "$legacy_path" "$new_filename"
+      fi
+    fi
+  done
+
+  [ "$actions_performed" -eq 0 ] && \
+    echo "No legacy paths found. Upgrade is not necessary"
+
+  [ "$repo_updates" -eq 1 ] && \
+    echo "Some files tracked by yadm have been renamed. This changes should probably be commited now."
+
+  exit 0
+
+}
+
 function version() {
 
   echo "yadm $VERSION"
@@ -1110,9 +1170,8 @@ function exclude_encrypted() {
   fi
 
   if [ "${exclude_header}${encrypt_data}" != "$managed" ]; then
-    basedir=${exclude_path%/*}
-    [ -e "$basedir" ] || mkdir -p "$basedir" # assert path
     debug "Updating ${exclude_path}"
+    assert_parent "$exclude_path"
     printf "%s" "${unmanaged}${exclude_header}${encrypt_data}" > "$exclude_path"
   fi
 
@@ -1221,6 +1280,9 @@ function set_yadm_dir() {
 
 function issue_legacy_path_warning() {
 
+  # no warnings during upgrade
+  [[ "${MAIN_ARGS[*]}" =~ upgrade ]] && return
+
   # no warnings if YADM_DIR is resolved as the leacy path
   [ "$YADM_DIR" = "$YADM_LEGACY_DIR" ] && return
 
@@ -1231,14 +1293,14 @@ function issue_legacy_path_warning() {
   local legacy_found
   legacy_found=()
   # this is ordered by importance
-  for legacy_path in                   \
+  for legacy_path in                            \
-    "$YADM_LEGACY_DIR/$YADM_REPO"      \
+    "$YADM_LEGACY_DIR/$YADM_REPO"               \
-    "$YADM_LEGACY_DIR/$YADM_CONFIG"    \
+    "$YADM_LEGACY_DIR/$YADM_CONFIG"             \
-    "$YADM_LEGACY_DIR/$YADM_ENCRYPT"   \
+    "$YADM_LEGACY_DIR/$YADM_ENCRYPT"            \
-    "$YADM_LEGACY_DIR/$YADM_ARCHIVE"   \
+    "$YADM_LEGACY_DIR/$YADM_ARCHIVE"            \
-    "$YADM_LEGACY_DIR/$YADM_BOOTSTRAP" \
+    "$YADM_LEGACY_DIR/$YADM_BOOTSTRAP"          \
-    "$YADM_LEGACY_DIR/$YADM_HOOKS"     \
+    "$YADM_LEGACY_DIR/$YADM_HOOKS"/{pre,post}_* \
-  ;                                    \
+  ;                                             \
   do
     [ -e "$legacy_path" ] && legacy_found+=("$legacy_path")
   done
@@ -1258,14 +1320,15 @@ function issue_legacy_path_warning() {
   Beginning with version 2.0.0, yadm uses the XDG Base Directory Specification
   to find its configurations. Read more about this change here:
 
-    https://yadm.io/docs/xdg_config_home
+    https://yadm.io/docs/upgrade_from_1.x.x
 
   In your environment, the configuration directory has been resolved to:
 
     $YADM_DIR
 
   To remove this warning do one of the following:
-    * Move yadm configurations to the directory listed above. (RECOMMENDED)
+    * Run "yadm upgrade" to move the yadm data to the new directory. (RECOMMENDED)
+    * Manually move yadm configurations to the directory listed above.
     * Specify your preferred yadm directory with -Y each execution.
     * Define an environment variable "YADM_COMPATIBILITY=1" to run in version 1
       compatibility mode. (DEPRECATED)
@@ -1421,6 +1484,11 @@ function assert_private_dirs() {
   done
 }
 
+function assert_parent() {
+  basedir=${1%/*}
+  [ -e "$basedir" ] || mkdir -p "$basedir"
+}
+
 function display_private_perms() {
   when="$1"
   for private_dir in .ssh .gnupg; do

yadm.1

@@ -56,6 +56,8 @@ list
 
 .BR yadm " perms
 
+.BR yadm " upgrade
+
 .BR yadm " introspect
 .I category
 
@@ -255,6 +257,22 @@ configuration
 .I yadm.auto-perms
 to "false".
 .TP
+.B upgrade
+Version 2 of yadm uses a different directory for storing your configurations.
+When you start to use version 2 for the first time, you may see warnings about
+moving your data to this new directory.
+The easiest way to accomplish this is by running "yadm upgrade".
+This command will start by moving your yadm repo to the new path.
+Next it will move any configuration data to the new path.
+If the configurations are tracked within your yadm repo, this command will
+"stage" the renaming of those files in the repo's index.
+After running "yadm upgrade", you should run "yadm status" to review changes
+which have been staged, and commit them to your repository.
+
+You can read
+https://yadm.io/docs/upgrade_from_1.x.x
+for more information.
+.TP
 .B version
 Print the version of yadm.
 
@@ -262,10 +280,14 @@ Print the version of yadm.
 
 Beginning with version 2.0.0, yadm introduced a couple major changes which may
 require you to adjust your configurations.
+See the
+.B upgrade
+command for help making those adjustments.
 
 First, yadm now uses the "XDG Base Directory Specification" to find its
-configurations. You can read https://yadm.io/docs/xdg_config_home for more
+configurations. You can read
-information.
+https://yadm.io/docs/upgrade_from_1.x.x
+for more information.
 
 Second, the naming conventions for alternate files have been changed.
 You can read https://yadm.io/docs/alternates for more information.