Run tests directly on github runner instead of in the docker image

As this makes it possible to run the tests on different systems. Initially
ubuntu (20.04 and 24.04) and macOs (13 and 15).

.github/workflows/test.yml

@@ -1,13 +1,98 @@
 ---
 name: Tests
+
 on:  # yamllint disable-line rule:truthy
   - push
   - pull_request
   - workflow_dispatch
+
+env:
+  SC_VER: "0.10.0"
+  ESH_VER: "0.3.2"
+
 jobs:
   Tests:
-    runs-on: ubuntu-latest
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os:
+          - ubuntu-20.04
+          - ubuntu-24.04
+          - macos-13
+          - macos-15
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
-      - name: Tests
+
-        run: make test
+      - name: Install dependencies on Linux
+        if: ${{ runner.os == 'Linux' }}
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y expect
+          if [ "${{ matrix.os }}" != "ubuntu-20.04" ]; then
+            sudo apt-get install -y j2cli
+          fi
+
+      - name: Install dependencies on macOS
+        if: ${{ runner.os == 'macOS' }}
+        run: |
+          command -v expect || brew install expect
+
+      - name: Prepare tools directory
+        run: |
+          mkdir "$RUNNER_TEMP/tools"
+          echo "$RUNNER_TEMP/tools" >> "$GITHUB_PATH"
+
+      - name: Install shellcheck
+        run: |
+          if [ "$RUNNER_OS" = "macOS" ]; then
+            OS=darwin
+          else
+            OS=linux
+          fi
+
+          if [ "$RUNNER_ARCH" = "ARM64" ]; then
+            ARCH=aarch64
+          else
+            ARCH=x86_64
+          fi
+
+          cd "$RUNNER_TEMP"
+
+          BASE_URL="https://github.com/koalaman/shellcheck/releases/download"
+          SC="v$SC_VER/shellcheck-v$SC_VER.$OS.$ARCH.tar.xz"
+          curl -L "$BASE_URL/$SC" | tar Jx shellcheck-v$SC_VER/shellcheck
+          mv shellcheck-v$SC_VER/shellcheck tools
+
+      - name: Install esh
+        run: |
+          cd "$RUNNER_TEMP/tools"
+
+          BASE_URL="https://github.com/jirutka/esh/raw/refs/tags"
+          curl -L -o esh "$BASE_URL/v$ESH_VER/esh"
+          chmod +x esh
+
+      - name: Add old yadm versions  # to test upgrades
+        run: |
+          for version in 1.12.0 2.5.0; do
+            git fetch origin $version:refs/tags/$version
+            git cat-file blob $version:yadm > "$RUNNER_TEMP/tools/yadm-$version"
+            chmod +x "$RUNNER_TEMP/tools/yadm-$version"
+          done
+
+      - name: Set up Python 3.11
+        if: ${{ runner.os == 'macOS' || matrix.os == 'ubuntu-20.04' }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: 3.11
+
+      - name: Install Python dependencies
+        run: |
+          python -m pip install --upgrade pip
+          python -m pip install -r test/requirements.txt
+
+      - name: Run tests
+        run: |
+          git config --global user.email test@yadm.io
+          git config --global user.name "Yadm Test"
+          pytest -v --color=yes --basetemp="$RUNNER_TEMP/pytest"

test/test_unit_parse_encrypt.py

@@ -105,7 +105,8 @@ def create_test_encrypt_data(paths):
     edata += "dirwild*\n"
     paths.work.join("dirwildcard/file1").write("", ensure=True)
     paths.work.join("dirwildcard/file2").write("", ensure=True)
-    expected.add("dirwildcard")
+    expected.add("dirwildcard/file1")
+    expected.add("dirwildcard/file2")
 
     # excludes
     edata += "exclude*\n"
@@ -186,9 +187,7 @@ def run_parse_encrypt(runner, paths, skip_parse=False, twice=False):
         YADM_WORK={paths.work}
         export YADM_WORK
         {parse_cmd}
-        export ENCRYPT_INCLUDE_FILES
+        echo PARSE_ENCRYPT_SHORT=$PARSE_ENCRYPT_SHORT
-        export PARSE_ENCRYPT_SHORT
-        env
         echo EIF_COUNT:${{#ENCRYPT_INCLUDE_FILES[@]}}
         for value in "${{ENCRYPT_INCLUDE_FILES[@]}}"; do
             echo "EIF:$value"

yadm

@@ -1940,65 +1940,30 @@ function parse_encrypt() {
   fi
 
   ENCRYPT_INCLUDE_FILES=()
-  ENCRYPT_EXCLUDE_FILES=()
-  FINAL_INCLUDE=()
 
   [ -f "$YADM_ENCRYPT" ] || return
 
   cd_work "Parsing encrypt" || return
 
-  # setting globstar to allow ** in encrypt patterns
+  local -a exclude
-  # (only supported on Bash >= 4)
+  local -a include
-  local unset_globstar
-  if ! shopt globstar &> /dev/null; then
-    unset_globstar=1
-  fi
-  shopt -s globstar &> /dev/null
 
-  exclude_pattern="^!(.+)"
+  while IFS= read -r pattern; do
-  # parse both included/excluded
+      if [[ $pattern =~ ^# || $pattern =~ ^[[:blank:]]*$ ]]; then
-  while IFS='' read -r line || [ -n "$line" ]; do
+          continue
-    if [[ ! $line =~ ^# && ! $line =~ ^[[:blank:]]*$ ]] ; then
+      fi
-      local IFS=$'\n'
+      if [[ $pattern =~ ^!(.*)$ ]]; then
-      for pattern in $line; do
+          exclude+=("--exclude=${pattern:1}")
-        if [[ "$pattern" =~ $exclude_pattern ]]; then
+      else
-          for ex_file in ${BASH_REMATCH[1]}; do
+          include+=("$pattern")
-            if [ -e "$ex_file" ]; then
+      fi
-              ENCRYPT_EXCLUDE_FILES+=("$ex_file")
-            fi
-          done
-        else
-          for in_file in $pattern; do
-            if [ -e "$in_file" ]; then
-              ENCRYPT_INCLUDE_FILES+=("$in_file")
-            fi
-          done
-        fi
-      done
-    fi
   done < "$YADM_ENCRYPT"
 
-  # remove excludes from the includes
+  if [[ ${#include} -gt 0 ]]; then
-  #(SC2068 is disabled because in this case, we desire globbing)
+      while IFS= read -r filename; do
-  #shellcheck disable=SC2068
+          ENCRYPT_INCLUDE_FILES+=("${filename%/}")
-  for included in "${ENCRYPT_INCLUDE_FILES[@]}"; do
+      done <<< "$("$GIT_PROGRAM" ls-files --others --directory "${exclude[@]}" -- "${include[@]}")"
-    skip=
-    #shellcheck disable=SC2068
-    for ex_file in ${ENCRYPT_EXCLUDE_FILES[@]}; do
-      [ "$included" == "$ex_file" ] && { skip=1; break; }
-    done
-    [ -n "$skip" ] || FINAL_INCLUDE+=("$included")
-  done
-
-  # sort the encrypted files
-  #shellcheck disable=SC2207
-  IFS=$'\n' ENCRYPT_INCLUDE_FILES=($(LC_ALL=C sort <<<"${FINAL_INCLUDE[*]}"))
-  unset IFS
-
-  if [ "$unset_globstar" = "1" ]; then
-    shopt -u globstar &> /dev/null
   fi
-
 }
 
 function builtin_dirname() {

yadm.1

@@ -750,7 +750,8 @@ gpg is used by default, but openssl can be configured with the
 .I yadm.cipher
 configuration.
 
-To use this feature, a list of patterns must be created and saved as
+To use this feature, a list of patterns (one per line) must be created and
+saved as
 .IR $HOME/.config/yadm/encrypt .
 This list of patterns should be relative to the configured
 .IR work-tree \ (usually\  $HOME ).
@@ -761,11 +762,10 @@ For example:
     .gnupg/*.gpg
 .RE
 
-Standard filename expansions (*, ?, [) are supported.
+Standard filename expansions (*, ?, [) are supported. Two consecutive asterisks
-If you have Bash version 4, you may use "**" to match all subdirectories.
+"**" can be used to match all subdirectories.  Other shell expansions like
-Other shell expansions like brace and tilde are not supported.
+brace and tilde are not supported.  Spaces in paths are supported, and should
-Spaces in paths are supported, and should not be quoted.
+not be quoted.  If a directory is specified, its contents will be included.
-If a directory is specified, its contents will be included, but not recursively.
 Paths beginning with a "!" will be excluded.
 
 The