Run tests directly on github runner instead of in the docker image

As this makes it possible to run the tests on different systems. Initially
ubuntu (20.04 and 24.04) and macOs (13 and 15).

.github/workflows/test.yml

@@ -1,13 +1,98 @@
 ---
 name: Tests
+
 on:  # yamllint disable-line rule:truthy
   - push
   - pull_request
   - workflow_dispatch
+
+env:
+  SC_VER: "0.10.0"
+  ESH_VER: "0.3.2"
+
 jobs:
   Tests:
-    runs-on: ubuntu-latest
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os:
+          - ubuntu-20.04
+          - ubuntu-24.04
+          - macos-13
+          - macos-15
     steps:
-      - uses: actions/checkout@v2
-      - name: Tests
-        run: make test
+      - uses: actions/checkout@v4
+
+      - name: Install dependencies on Linux
+        if: ${{ runner.os == 'Linux' }}
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y expect
+          if [ "${{ matrix.os }}" != "ubuntu-20.04" ]; then
+            sudo apt-get install -y j2cli
+          fi
+
+      - name: Install dependencies on macOS
+        if: ${{ runner.os == 'macOS' }}
+        run: |
+          command -v expect || brew install expect
+
+      - name: Prepare tools directory
+        run: |
+          mkdir "$RUNNER_TEMP/tools"
+          echo "$RUNNER_TEMP/tools" >> "$GITHUB_PATH"
+
+      - name: Install shellcheck
+        run: |
+          if [ "$RUNNER_OS" = "macOS" ]; then
+            OS=darwin
+          else
+            OS=linux
+          fi
+
+          if [ "$RUNNER_ARCH" = "ARM64" ]; then
+            ARCH=aarch64
+          else
+            ARCH=x86_64
+          fi
+
+          cd "$RUNNER_TEMP"
+
+          BASE_URL="https://github.com/koalaman/shellcheck/releases/download"
+          SC="v$SC_VER/shellcheck-v$SC_VER.$OS.$ARCH.tar.xz"
+          curl -L "$BASE_URL/$SC" | tar Jx shellcheck-v$SC_VER/shellcheck
+          mv shellcheck-v$SC_VER/shellcheck tools
+
+      - name: Install esh
+        run: |
+          cd "$RUNNER_TEMP/tools"
+
+          BASE_URL="https://github.com/jirutka/esh/raw/refs/tags"
+          curl -L -o esh "$BASE_URL/v$ESH_VER/esh"
+          chmod +x esh
+
+      - name: Add old yadm versions  # to test upgrades
+        run: |
+          for version in 1.12.0 2.5.0; do
+            git fetch origin $version:refs/tags/$version
+            git cat-file blob $version:yadm > "$RUNNER_TEMP/tools/yadm-$version"
+            chmod +x "$RUNNER_TEMP/tools/yadm-$version"
+          done
+
+      - name: Set up Python 3.11
+        if: ${{ runner.os == 'macOS' || matrix.os == 'ubuntu-20.04' }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: 3.11
+
+      - name: Install Python dependencies
+        run: |
+          python -m pip install --upgrade pip
+          python -m pip install -r test/requirements.txt
+
+      - name: Run tests
+        run: |
+          git config --global user.email test@yadm.io
+          git config --global user.name "Yadm Test"
+          pytest -v --color=yes --basetemp="$RUNNER_TEMP/pytest"

test/test_unit_parse_encrypt.py

@@ -105,7 +105,8 @@ def create_test_encrypt_data(paths):
     edata += "dirwild*\n"
     paths.work.join("dirwildcard/file1").write("", ensure=True)
     paths.work.join("dirwildcard/file2").write("", ensure=True)
-    expected.add("dirwildcard")
+    expected.add("dirwildcard/file1")
+    expected.add("dirwildcard/file2")
 
     # excludes
     edata += "exclude*\n"
@@ -186,9 +187,7 @@ def run_parse_encrypt(runner, paths, skip_parse=False, twice=False):
         YADM_WORK={paths.work}
         export YADM_WORK
         {parse_cmd}
-        export ENCRYPT_INCLUDE_FILES
-        export PARSE_ENCRYPT_SHORT
-        env
+        echo PARSE_ENCRYPT_SHORT=$PARSE_ENCRYPT_SHORT
         echo EIF_COUNT:${{#ENCRYPT_INCLUDE_FILES[@]}}
         for value in "${{ENCRYPT_INCLUDE_FILES[@]}}"; do
             echo "EIF:$value"

yadm

@@ -1940,65 +1940,30 @@ function parse_encrypt() {
   fi
 
   ENCRYPT_INCLUDE_FILES=()
-  ENCRYPT_EXCLUDE_FILES=()
-  FINAL_INCLUDE=()
 
   [ -f "$YADM_ENCRYPT" ] || return
 
   cd_work "Parsing encrypt" || return
 
-  # setting globstar to allow ** in encrypt patterns
-  # (only supported on Bash >= 4)
-  local unset_globstar
-  if ! shopt globstar &> /dev/null; then
-    unset_globstar=1
-  fi
-  shopt -s globstar &> /dev/null
+  local -a exclude
+  local -a include
 
-  exclude_pattern="^!(.+)"
-  # parse both included/excluded
-  while IFS='' read -r line || [ -n "$line" ]; do
-    if [[ ! $line =~ ^# && ! $line =~ ^[[:blank:]]*$ ]] ; then
-      local IFS=$'\n'
-      for pattern in $line; do
-        if [[ "$pattern" =~ $exclude_pattern ]]; then
-          for ex_file in ${BASH_REMATCH[1]}; do
-            if [ -e "$ex_file" ]; then
-              ENCRYPT_EXCLUDE_FILES+=("$ex_file")
-            fi
-          done
-        else
-          for in_file in $pattern; do
-            if [ -e "$in_file" ]; then
-              ENCRYPT_INCLUDE_FILES+=("$in_file")
-            fi
-          done
-        fi
-      done
-    fi
+  while IFS= read -r pattern; do
+      if [[ $pattern =~ ^# || $pattern =~ ^[[:blank:]]*$ ]]; then
+          continue
+      fi
+      if [[ $pattern =~ ^!(.*)$ ]]; then
+          exclude+=("--exclude=${pattern:1}")
+      else
+          include+=("$pattern")
+      fi
   done < "$YADM_ENCRYPT"
 
-  # remove excludes from the includes
-  #(SC2068 is disabled because in this case, we desire globbing)
-  #shellcheck disable=SC2068
-  for included in "${ENCRYPT_INCLUDE_FILES[@]}"; do
-    skip=
-    #shellcheck disable=SC2068
-    for ex_file in ${ENCRYPT_EXCLUDE_FILES[@]}; do
-      [ "$included" == "$ex_file" ] && { skip=1; break; }
-    done
-    [ -n "$skip" ] || FINAL_INCLUDE+=("$included")
-  done
-
-  # sort the encrypted files
-  #shellcheck disable=SC2207
-  IFS=$'\n' ENCRYPT_INCLUDE_FILES=($(LC_ALL=C sort <<<"${FINAL_INCLUDE[*]}"))
-  unset IFS
-
-  if [ "$unset_globstar" = "1" ]; then
-    shopt -u globstar &> /dev/null
+  if [[ ${#include} -gt 0 ]]; then
+      while IFS= read -r filename; do
+          ENCRYPT_INCLUDE_FILES+=("${filename%/}")
+      done <<< "$("$GIT_PROGRAM" ls-files --others --directory "${exclude[@]}" -- "${include[@]}")"
   fi
-
 }
 
 function builtin_dirname() {

yadm.1

@@ -750,7 +750,8 @@ gpg is used by default, but openssl can be configured with the
 .I yadm.cipher
 configuration.
 
-To use this feature, a list of patterns must be created and saved as
+To use this feature, a list of patterns (one per line) must be created and
+saved as
 .IR $HOME/.config/yadm/encrypt .
 This list of patterns should be relative to the configured
 .IR work-tree \ (usually\  $HOME ).
@@ -761,11 +762,10 @@ For example:
     .gnupg/*.gpg
 .RE
 
-Standard filename expansions (*, ?, [) are supported.
-If you have Bash version 4, you may use "**" to match all subdirectories.
-Other shell expansions like brace and tilde are not supported.
-Spaces in paths are supported, and should not be quoted.
-If a directory is specified, its contents will be included, but not recursively.
+Standard filename expansions (*, ?, [) are supported. Two consecutive asterisks
+"**" can be used to match all subdirectories.  Other shell expansions like
+brace and tilde are not supported.  Spaces in paths are supported, and should
+not be quoted.  If a directory is specified, its contents will be included.
 Paths beginning with a "!" will be excluded.
 
 The