Rework clone

Instead of doing work to find the default branch just to be able to
set up the repository before doing a fetch, do a "git clone" and let
git handle it.

Use -c core.sharedrepository=0600 to get the same result as
--shared=0600 passed to init.

Use --separate-git-dir to get the git directory in $YADM_REPO. Use a
temporary dir as work tree and remove it right after the clone is
done.

When the clone is done, iterate over all missing files in $YADM_WORK
and perform a checkout. If local files exists that differ compared
with the cloned ones the local files are left intact and the user is
instructed to deal with the conflicts.

test/test_clone.py

@@ -58,8 +58,8 @@ def test_clone(
     if not good_remote:
         # clone should fail
         assert run.failure
-        assert run.out != ''
+        assert run.out == ''
-        assert 'Unable to fetch origin' in run.err
+        assert 'Unable to clone the repository' in run.err
         assert not paths.repo.exists()
     elif repo_exists and not force:
         # can't overwrite data
@@ -76,8 +76,7 @@ def test_clone(
         # ensure conflicts are handled properly
         if conflicts:
             assert 'NOTE' in run.out
-            assert 'Merging origin/master failed' in run.out
+            assert 'Local files with content that differs' in run.out
-            assert 'Conflicts preserved' in run.out
 
         # confirm correct Git origin
         run = runner(
@@ -89,23 +88,16 @@ def test_clone(
 
         # ensure conflicts are really preserved
         if conflicts:
-            # test to see if the work tree is actually "clean"
+            # test that the conflicts are preserved in the work tree
             run = runner(
                 command=yadm_cmd('status', '-uno', '--porcelain'),
                 cwd=paths.work)
             assert run.success
             assert run.err == ''
-            assert run.out == '', 'worktree has unexpected changes'
+            assert str(ds1.tracked[0].path) in run.out
 
-            # test to see if the conflicts are stashed
+            # verify content of the conflicts
-            run = runner(command=yadm_cmd('stash', 'list'), cwd=paths.work)
+            run = runner(command=yadm_cmd('diff'), cwd=paths.work)
-            assert run.success
-            assert run.err == ''
-            assert 'Conflicts preserved' in run.out, 'conflicts not stashed'
-
-            # verify content of the stashed conflicts
-            run = runner(
-                command=yadm_cmd('stash', 'show', '-p'), cwd=paths.work)
             assert run.success
             assert run.err == ''
             assert '\n+conflict' in run.out, 'conflicts not stashed'
@@ -242,20 +234,20 @@ def test_clone_perms(
             f'initial private dir perms drwxrwxrwx.+.{private_type}',
             run.out)
         assert re.search(
-            f'pre-merge private dir perms drwxrwxrwx.+.{private_type}',
+            f'pre-checkout private dir perms drwxrwxrwx.+.{private_type}',
             run.out)
         assert re.search(
-            f'post-merge private dir perms drwxrwxrwx.+.{private_type}',
+            f'post-checkout private dir perms drwxrwxrwx.+.{private_type}',
             run.out)
     else:
         # private directories which are created, should be done prior to
-        # merging, and with secure permissions.
+        # checkout, and with secure permissions.
         assert 'initial private dir perms' not in run.out
         assert re.search(
-            f'pre-merge private dir perms drwx------.+.{private_type}',
+            f'pre-checkout private dir perms drwx------.+.{private_type}',
             run.out)
         assert re.search(
-            f'post-merge private dir perms drwx------.+.{private_type}',
+            f'post-checkout private dir perms drwx------.+.{private_type}',
             run.out)
 
     # standard perms still apply afterwards unless disabled with auto.perms
@@ -297,8 +289,8 @@ def test_alternate_branch(runner, paths, yadm_cmd, repo_config, branch):
 
     if branch == 'invalid':
         assert run.failure
-        assert 'ERROR: Clone failed' in run.err
+        assert 'ERROR: Unable to clone the repository' in run.err
-        assert f"'origin/{branch}' does not exist in {remote_url}" in run.err
+        assert f"Remote branch {branch} not found in upstream" in run.err
     else:
         assert successful_clone(run, paths, repo_config)
 
@@ -321,7 +313,6 @@ def test_alternate_branch(runner, paths, yadm_cmd, repo_config, branch):
 def successful_clone(run, paths, repo_config, expected_code=0):
     """Assert clone is successful"""
     assert run.code == expected_code
-    assert 'Initialized' in run.out
     assert oct(paths.repo.stat().mode).endswith('00'), 'Repo is not secured'
     assert repo_config('core.bare') == 'false'
     assert repo_config('status.showUntrackedFiles') == 'no'
@@ -342,10 +333,11 @@ def remote(paths, ds1_repo_copy):
 
 def test_no_repo(runner, yadm_cmd, ):
     """Test cloning without specifying a repo"""
-    run = runner(command=yadm_cmd('clone'))
+    run = runner(command=yadm_cmd('clone', '-f'))
     assert run.failure
     assert run.out == ''
-    assert 'ERROR: No repository provided' in run.err
+    assert 'ERROR: Unable to clone the repository' in run.err
+    assert 'repository \'repo.git\' does not exist' in run.err
 
 
 def verify_head(paths, branch):

test/test_default_remote_branch.py

@@ -1,27 +0,0 @@
-"""Unit tests: _default_remote_branch()"""
-import pytest
-
-
-@pytest.mark.parametrize('condition', ['found', 'missing'])
-def test(runner, paths, condition):
-    """Test _default_remote_branch()"""
-    test_branch = 'test/branch'
-    output = f'ref: refs/heads/{test_branch}\\tHEAD\\n'
-    if condition == 'missing':
-        output = 'output that is missing ref'
-    script = f"""
-        YADM_TEST=1 source {paths.pgm}
-        function git() {{
-          printf '{output}';
-          printf 'mock stderr\\n' 1>&2
-        }}
-        _default_remote_branch URL
-    """
-    print(condition)
-    run = runner(command=['bash'], inp=script)
-    assert run.success
-    assert run.err == ''
-    if condition == 'found':
-        assert run.out.strip() == test_branch
-    else:
-        assert run.out.strip() == 'master'

test/test_unit_is_valid_branch_name.py

@@ -1,40 +0,0 @@
-"""Unit tests: is_valid_branch_name"""
-import pytest
-
-# Git branches do not allow:
-#  * path component that begins with "."
-#  * double dot
-#  * "~", "^", ":", "\", space
-#  * end with a "/"
-#  * end with ".lock"
-
-
-@pytest.mark.parametrize(
-    'branch, expected', [
-        ('master', 'valid'),
-        ('path/branch', 'valid'),
-        ('path/.branch', 'invalid'),
-        ('path..branch', 'invalid'),
-        ('path~branch', 'invalid'),
-        ('path^branch', 'invalid'),
-        ('path:branch', 'invalid'),
-        ('path\\branch', 'invalid'),
-        ('path branch', 'invalid'),
-        ('path/branch/', 'invalid'),
-        ('branch.lock', 'invalid'),
-    ])
-def test_is_valid_branch_name(runner, yadm, branch, expected):
-    """Test function is_valid_branch_name()"""
-
-    script = f"""
-        YADM_TEST=1 source {yadm}
-        if is_valid_branch_name "{branch}"; then
-            echo valid
-        else
-            echo invalid
-        fi
-    """
-    run = runner(command=['bash'], inp=script)
-    assert run.success
-    assert run.err == ''
-    assert run.out.strip() == expected

yadm

@@ -127,7 +127,7 @@ function main() {
           ;;
           -l) # used by decrypt()
             DO_LIST="YES"
-            [ "$YADM_COMMAND" = "config" ] && YADM_ARGS+=("$1")
+            [[ "$YADM_COMMAND" =~ ^(clone|config)$ ]] && YADM_ARGS+=("$1")
           ;;
           -w) # used by init() and clone()
             if [[ ! "$2" =~ ^/ ]] ; then
@@ -705,84 +705,73 @@ function clean() {
 
 }
 
-function _default_remote_branch() {
-  local ls_remote
-  ls_remote=$("$GIT_PROGRAM" ls-remote -q --symref "$1" 2>/dev/null)
-  match="^ref:[[:blank:]]+refs/heads/([^[:blank:]]+)"
-  if [[ "$ls_remote" =~ $match ]] ; then
-    echo "${BASH_REMATCH[1]}"
-  else
-    echo master
-  fi
-}
-
 function clone() {
 
   DO_BOOTSTRAP=1
-  local branch=
+  local -a args
-
+  local -i do_checkout=1
-  local repo_url=
   while [[ $# -gt 0 ]] ; do
-    key="$1"
+    case "$1" in
-    case $key in
-      -b)
-        if ! is_valid_branch_name "$2"; then
-          error_out "You must provide a branch name when using '-b'"
-        fi
-        branch="$2"
-        shift
-      ;;
       --bootstrap) # force bootstrap, without prompt
         DO_BOOTSTRAP=2
       ;;
       --no-bootstrap) # prevent bootstrap, without prompt
         DO_BOOTSTRAP=3
       ;;
-      *) # use first found argument as the URL
+      --checkout)
-        [ -z "$repo_url" ] && repo_url="$1"
+        do_checkout=1
+      ;;
+      -n|--no-checkout)
+        do_checkout=0
+      ;;
+      --bare|--mirror|--recurse-submodules*|--recursive|--separate-git-dir=*)
+        # ignore arguments without separate parameter
+      ;;
+      --separate-git-dir)
+        # ignore arguments with separate parameter
+        shift
+      ;;
+      *)
+        args+=("$1")
       ;;
     esac
     shift
   done
 
-  [ -z "$repo_url" ] && error_out "No repository provided"
-
-  [ -z "$branch" ] && branch=$(_default_remote_branch "$repo_url")
-
   [ -n "$DEBUG" ] && display_private_perms "initial"
 
-  # shellcheck disable=SC2119
+  # safety check, don't attempt to clone when the repo is already present
-  # clone will begin with a bare repo
+  [ -d "$YADM_REPO" ] && [ -z "$FORCE" ] &&
-  init
+    error_out "Git repo already exists. [$YADM_REPO]\nUse '-f' if you want to force it to be overwritten."
 
-  # configure local HEAD with the correct branch
+  # remove existing if forcing the clone to happen anyway
-  printf 'ref: refs/heads/%s\n' "$branch" > "${YADM_REPO}/HEAD"
+  [ -d "$YADM_REPO" ] && {
-
+    debug "Removing existing repo prior to clone"
-  # add the specified remote, and configure the repo to track origin/$branch
-  debug "Adding remote to new repo"
-  "$GIT_PROGRAM" remote add origin "$repo_url"
-  debug "Configuring new repo to track origin/${branch}"
-  "$GIT_PROGRAM" config "branch.${branch}.remote" origin
-  "$GIT_PROGRAM" config "branch.${branch}.merge" "refs/heads/${branch}"
-
-  # fetch / merge (and possibly fallback to reset)
-  debug "Doing an initial fetch of the origin"
-  "$GIT_PROGRAM" fetch origin || {
-    debug "Removing repo after failed clone"
     rm -rf "$YADM_REPO"
-    error_out "Unable to fetch origin $repo_url"
   }
-  debug "Verifying '${branch}' is a valid branch to merge"
+
-  [ -f "${YADM_REPO}/refs/remotes/origin/${branch}" ] || {
+  local wc
+  wc="$(mktemp -d)" || error_out "Unable to create temporary directory"
+
+  # first clone without checkout
+  debug "Doing an initial clone of the repository"
+  (cd "$wc" &&
+       "$GIT_PROGRAM" -c core.sharedrepository=0600 clone --no-checkout \
+                      --separate-git-dir="$YADM_REPO" "${args[@]}" repo.git) || {
       debug "Removing repo after failed clone"
-    rm -rf "$YADM_REPO"
+      rm -rf "$YADM_REPO" "$wc"
-    error_out "Clone failed, 'origin/${branch}' does not exist in $repo_url"
+      error_out "Unable to clone the repository"
   }
+  rm -rf "$wc"
+  configure_repo
+
+  # then reset the index as the --no-checkout flag makes the index empty
+  "$GIT_PROGRAM" reset --quiet -- .
 
   if [ "$YADM_WORK" = "$HOME" ]; then
     debug "Determining if repo tracks private directories"
     for private_dir in $(private_dirs all); do
-      found_log=$("$GIT_PROGRAM" log -n 1 "origin/${branch}" -- "$private_dir" 2>/dev/null)
+      found_log=$("$GIT_PROGRAM" log -n 1 -- "$private_dir" 2>/dev/null)
       if [ -n "$found_log" ]; then
         debug "Private directory $private_dir is tracked by repo"
         assert_private_dirs "$private_dir"
@@ -790,51 +779,33 @@ function clone() {
     done
   fi
 
-  [ -n "$DEBUG" ] && display_private_perms "pre-merge"
+  # finally check out (unless instructed not to) all files that don't exist in $YADM_WORK
-  debug "Doing an initial merge of origin/${branch}"
+  if [[ $do_checkout -ne 0 ]]; then
-  "$GIT_PROGRAM" merge "origin/${branch}" || {
+      [ -n "$DEBUG" ] && display_private_perms "pre-checkout"
-    debug "Merge failed, doing a reset and stashing conflicts."
+
-    "$GIT_PROGRAM" reset "origin/${branch}"
+      cd_work "Clone" || return
-    if cd "$YADM_WORK"; then # necessary because of a bug in Git
+
-      "$GIT_PROGRAM" -c user.name='yadm clone' -c user.email='yadm' stash save Conflicts preserved from yadm clone command 2>&1
+      "$GIT_PROGRAM" ls-files --deleted | while IFS= read -r file; do
+          "$GIT_PROGRAM" checkout -- ":/$file"
+      done
+
+      if [ -n "$("$GIT_PROGRAM" ls-files --modified)" ]; then
           cat <<EOF
 **NOTE**
-  Merging origin/${branch} failed.
+  Local files with content that differs from the ones just
+  cloned were found in $YADM_WORK. They have been left
+  unmodified.
 
-  As a result, yadm did 'reset origin/${branch}', and then
+  Please review and resolve any differences appropriately.
-  stashed the conflicting data.
-
-  This likely happened because you had files in \$HOME
-  which conflicted with files tracked by origin/${branch}.
-
-  You can review the stashed conflicts with the
-  command 'yadm stash show -p' from within your
-  \$HOME directory. If you want to restore the
-  stashed data, you can run 'yadm stash apply' or
-  'yadm stash pop' and then handle the conflicts
-  in another way.
-EOF
-    else
-      # skip auto_bootstrap if conflicts could not be stashed
-      DO_BOOTSTRAP=0
-      cat <<EOF
-**NOTE**
-  Merging origin/${branch} failed.
-  yadm did 'reset origin/${branch}' instead.
-
-  yadm did not stash these conflicts beacuse it was unable
-  to change to the $YADM_WORK directory.
-
-  Please review and resolve any differences appropriately
   If you know what you're doing, and want to overwrite the
-  tracked files, consider 'yadm reset --hard origin/${branch}'
+  tracked files, consider 'yadm checkout "$YADM_WORK"'.
 EOF
       fi
-  }
 
-  [ -n "$DEBUG" ] && display_private_perms "post-merge"
+      [ -n "$DEBUG" ] && display_private_perms "post-checkout"
 
       CHANGES_POSSIBLE=1
+  fi
 
 }
 
@@ -1459,18 +1430,6 @@ function exclude_encrypted() {
 
 }
 
-function is_valid_branch_name() {
-  # Git branches do not allow:
-  #  * path component that begins with "."
-  #  * double dot
-  #  * "~", "^", ":", "\", space
-  #  * end with a "/"
-  #  * end with ".lock"
-  pattern='(\/\.|\.\.|[~^:\\ ]|\/$|\.lock$)'
-  [[ "$1" =~ $pattern ]] && return 1
-  return 0
-}
-
 function query_distro() {
   distro=""
   if command -v "$LSB_RELEASE_PROGRAM" &> /dev/null; then