Test running tests directly on action runner

To be able to test on different systems.

.github/workflows/test.yml

@@ -1,13 +1,98 @@
 ---
 name: Tests
+
 on:  # yamllint disable-line rule:truthy
   - push
   - pull_request
   - workflow_dispatch
+
+env:
+  SC_VER: "0.10.0"
+  ESH_VER: "0.3.2"
+
 jobs:
   Tests:
-    runs-on: ubuntu-latest
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os:
+          - ubuntu-20.04
+          - ubuntu-24.04
+          - macos-13
+          - macos-15
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
-      - name: Tests
+
-        run: make test
+      - name: Install dependencies on Linux
+        if: ${{ runner.os == 'Linux' }}
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y expect
+          if [ "${{ matrix.os }}" != "ubuntu-20.04" ]; then
+            sudo apt-get install -y j2cli
+          fi
+
+      - name: Install dependencies on macOS
+        if: ${{ runner.os == 'macOS' }}
+        run: |
+          brew install expect
+
+      - name: Prepare tools directory
+        run: |
+          mkdir "$RUNNER_TEMP/tools"
+          echo "$RUNNER_TEMP/tools" >> "$GITHUB_PATH"
+
+      - name: Install shellcheck
+        run: |
+          if [ "$RUNNER_OS" = "macOS" ]; then
+            OS=darwin
+          else
+            OS=linux
+          fi
+
+          if [ "$RUNNER_ARCH" = "ARM64" ]; then
+            ARCH=aarch64
+          else
+            ARCH=x86_64
+          fi
+
+          cd "$RUNNER_TEMP"
+
+          BASE_URL="https://github.com/koalaman/shellcheck/releases/download"
+          SC="v$SC_VER/shellcheck-v$SC_VER.$OS.$ARCH.tar.xz"
+          curl -L "$BASE_URL/$SC" | tar Jx shellcheck-v$SC_VER/shellcheck
+          mv shellcheck-v$SC_VER/shellcheck tools
+
+      - name: Install esh
+        run: |
+          cd "$RUNNER_TEMP/tools"
+
+          BASE_URL="https://github.com/jirutka/esh/raw/refs/tags"
+          curl -L -o esh "$BASE_URL/v$ESH_VER/esh"
+          chmod +x esh
+
+      - name: Add old yadm versions  # to test upgrades
+        run: |
+          for version in 1.12.0 2.5.0; do
+            git fetch origin $version:refs/tags/$version
+            git cat-file blob $version:yadm > "$RUNNER_TEMP/tools/yadm-$version"
+            chmod +x "$RUNNER_TEMP/tools/yadm-$version"
+          done
+
+      - name: Set up Python 3.11
+        if: ${{ runner.os == 'macOS' }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: 3.11
+
+      - name: Install Python dependencies
+        run: |
+          python -m pip install --upgrade pip
+          python -m pip install -r test/requirements.txt
+
+      - name: Run tests
+        run: |
+          git config --global user.email test@yadm.io
+          git config --global user.name "Yadm Test"
+          pytest -v --color=yes --basetemp="$RUNNER_TEMP/pytest"

.gitignore

@@ -5,3 +5,4 @@
 .testyadm
 _site
 testenv
+__pycache__/

Makefile

@@ -176,7 +176,7 @@ man-ps:
 	@groff -man -Tps ./yadm.1 > yadm.ps
 
 yadm.md: yadm.1
-	@groff -man -Tutf8 -Z ./yadm.1 | grotty -c | col -bx | sed 's/^[A-Z]/## &/g' | sed '/yadm(1)/d' > yadm.md
+	@groff -man -Tutf8 -Z ./yadm.1 | grotty -c | col -bx | sed 's/^[A-Z]/## &/g' | sed '/YADM(1)/d' > yadm.md
 
 .PHONY: contrib
 contrib: SHELL = /bin/bash

test/test_syntax.py

@@ -1,6 +1,7 @@
 """Syntax checks"""
 
 import os
+import shutil
 
 import pytest
 
@@ -77,7 +78,11 @@ def test_yamllint(pytestconfig, runner, yamllint_version):
 
 def test_man(runner):
     """Check for warnings from man"""
-    run = runner(command=["man.REAL", "--warnings", "./yadm.1"])
+    if shutil.which("mandoc"):
+        command = ["mandoc", "-T", "lint"]
+    else:
+        command = ["groff", "-ww", "-z"]
+    run = runner(command=command + ["-man", "./yadm.1"])
     assert run.success
+    assert run.out == ""
     assert run.err == ""
-    assert "yadm - Yet Another Dotfiles Manager" in run.out

test/test_unit_configure_paths.py

@@ -89,7 +89,9 @@ def run_test(runner, paths, args, expected_matches, cwd=None):
         XDG_DATA_HOME=
         HOME="{HOME}" set_yadm_dirs
         configure_paths
-        declare -p | grep -E '(YADM|GIT)_'
+        for var in "${{!YADM_@}}" "${{!GIT_@}}"; do
+          echo "$var=\\"${{!var}}\\""
+        done
     """
     run = runner(command=["bash"], inp=script, cwd=cwd)
     assert run.success

test/test_unit_parse_encrypt.py

@@ -1,5 +1,7 @@
 """Unit tests: parse_encrypt"""
 
+import sys
+
 import pytest
 
 
@@ -123,16 +125,17 @@ def create_test_encrypt_data(paths):
     expected.add("ex ex/file4")
     expected.add("ex ex/file6.text")
 
-    # double star
+    # double star (not supported on bash 3 which is the default on macOS)
-    edata += "doublestar/**/file*\n"
+    if sys.platform != "darwin":
-    edata += "!**/file3\n"
+        edata += "doublestar/**/file*\n"
-    paths.work.join("doublestar/a/b/file1").write("", ensure=True)
+        edata += "!**/file3\n"
-    paths.work.join("doublestar/c/d/file2").write("", ensure=True)
+        paths.work.join("doublestar/a/b/file1").write("", ensure=True)
-    paths.work.join("doublestar/e/f/file3").write("", ensure=True)
+        paths.work.join("doublestar/c/d/file2").write("", ensure=True)
-    paths.work.join("doublestar/g/h/nomatch").write("", ensure=True)
+        paths.work.join("doublestar/e/f/file3").write("", ensure=True)
-    expected.add("doublestar/a/b/file1")
+        paths.work.join("doublestar/g/h/nomatch").write("", ensure=True)
-    expected.add("doublestar/c/d/file2")
+        expected.add("doublestar/a/b/file1")
-    # doublestar/e/f/file3 is excluded
+        expected.add("doublestar/c/d/file2")
+        # doublestar/e/f/file3 is excluded
 
     return edata, expected
 

test/test_upgrade.py

@@ -19,13 +19,15 @@ import pytest
     ],
 )
 @pytest.mark.parametrize("submodule", [False, True], ids=["no submodule", "with submodules"])
-def test_upgrade(tmpdir, runner, versions, submodule):
+def test_upgrade(tmpdir, runner, paths, versions, submodule):
     """Upgrade tests"""
     # pylint: disable=too-many-statements
     home = tmpdir.mkdir("HOME")
     env = {"HOME": str(home)}
     runner(["git", "config", "--global", "init.defaultBranch", "master"], env=env)
     runner(["git", "config", "--global", "protocol.file.allow", "always"], env=env)
+    runner(["git", "config", "--global", "user.email", "test@yadm.io"], env=env)
+    runner(["git", "config", "--global", "user.name", "Yadm Test"], env=env)
 
     if submodule:
         ext_repo = tmpdir.mkdir("ext_repo")
@@ -39,7 +41,7 @@ def test_upgrade(tmpdir, runner, versions, submodule):
     os.environ.pop("XDG_DATA_HOME", None)
 
     def run_version(version, *args, check_stderr=True):
-        yadm = f"yadm-{version}" if version else "/yadm/yadm"
+        yadm = f"yadm-{version}" if version else paths.pgm
         run = runner([yadm, *args], shell=True, cwd=str(home), env=env)
         assert run.success
         if check_stderr:

yadm.1

@@ -1,5 +1,5 @@
 .\" vim: set spell so=8:
-.TH yadm 1 "8 November 2024" "3.3.0"
+.TH YADM 1 "November 8, 2024" "3.3.0"
 
 .SH NAME
 
@@ -15,55 +15,55 @@ yadm \- Yet Another Dotfiles Manager
 .I git-command-or-alias
 .RI [ options ]
 
-.B yadm
+.B yadm init
-init
+.RB [ \-f ]
-.RB [ -f ]
+.RB [ \-w
-.RB [ -w
 .IR dir ]
 
-.B yadm
+.B yadm clone
-.RI clone " url
+.I url
-.RB [ -f ]
+.RB [ \-f ]
-.RB [ -w
+.RB [ \-w
 .IR dir ]
-.RB [ -b
+.RB [ \-b
 .IR branch ]
-.RB [ --bootstrap ]
+.RB [ \-\-bootstrap ]
-.RB [ --no-bootstrap ]
+.RB [ \-\-no\-bootstrap ]
 
-.B yadm
+.B yadm config
-.RI config " name
+.I name
 .RI [ value ]
 
-.B yadm
+.B yadm config
-config
+.RB [ \-e ]
-.RB [ -e ]
 
-.B yadm
+.B yadm list
-list
+.RB [ \-a ]
-.RB [ -a ]
 
-.BR yadm " bootstrap
+.B yadm bootstrap
 
-.BR yadm " encrypt
+.B yadm encrypt
 
-.BR yadm " decrypt
+.B yadm decrypt
-.RB [ -l ]
+.RB [ \-l ]
 
-.BR yadm " alt
+.B yadm alt
 
-.BR yadm " perms
+.B yadm perms
 
-.BR yadm " enter [ command ]
+.B yadm enter
+.RI [ command ]
 
-.BR yadm " git-crypt [ options ]
+.B yadm git\-crypt
+.RI [ options ]
 
-.BR yadm " transcrypt [ options ]
+.B yadm transcrypt
+.RI [ options ]
 
-.BR yadm " upgrade
+.B yadm upgrade
-.RB [ -f ]
+.RB [ \-f ]
 
-.BR yadm " introspect
+.B yadm introspect
 .I category
 
 .SH DESCRIPTION
@@ -83,8 +83,7 @@ Any command not internally handled by yadm is passed through to
 .BR git (1).
 Git commands or aliases are invoked with the yadm managed repository.
 The working directory for Git commands will be the configured
-.IR work-tree " (usually
+.IR work-tree \ (usually\  $HOME ).
-.IR $HOME ).
 
 Dotfiles are managed by using standard
 .B git
@@ -95,7 +94,7 @@ commands;
 .IR pull ,
 etc.
 
-.RI The " config
+.RI The\  config
 command is not passed directly through.
 Instead use the
 .I gitconfig
@@ -114,7 +113,7 @@ Execute
 .I $HOME/.config/yadm/bootstrap
 if it exists.
 .TP
-.BI clone " url
+.BI clone \ url
 Clone a remote repository for tracking dotfiles.
 After the contents of the remote repository have been fetched, a "check out" of
 the remote HEAD branch is attempted.
@@ -130,12 +129,12 @@ By default,
 will be used as the
 .IR work-tree ,
 but this can be overridden with the
-.BR -w " option.
+.BR -w \ option.
 yadm can be forced to overwrite an existing repository by providing the
-.BR -f " option.
+.BR -f \ option.
 If you want to use a branch other than the remote HEAD branch
 you can specify it using the
-.BR -b " option.
+.BR -b \ option.
 By default yadm will ask the user if the bootstrap program should be run (if it
 exists). The options
 .BR --bootstrap " or " --no-bootstrap
@@ -153,8 +152,7 @@ See the CONFIGURATION section for more details.
 Decrypt all files stored in
 .IR $HOME/.local/share/yadm/archive .
 Files decrypted will be relative to the configured
-.IR work-tree " (usually
+.IR work-tree \ (usually\  $HOME ).
-.IR $HOME ).
 Using the
 .B -l
 option will list the files stored without extracting them.
@@ -191,12 +189,12 @@ Emacs Tramp and Magit can manage files by using this configuration:
 With this config, use (magit-status "/yadm::").
 .RE
 .TP
-.BI git-crypt " options
+.BI git-crypt \ options
 If git-crypt is installed, this command allows you to pass options directly to
 git-crypt, with the environment configured to use the yadm repository.
 
-git-crypt enables transparent encryption and decryption of files in a git repository.
+git-crypt enables transparent encryption and decryption of files in a git
-You can read
+repository. You can read
 https://github.com/AGWA/git-crypt
 for details.
 .TP
@@ -232,17 +230,17 @@ By default,
 will be used as the
 .IR work-tree ,
 but this can be overridden with the
-.BR -w " option.
+.BR -w \ option.
 yadm can be forced to overwrite an existing repository by providing the
-.BR -f " option.
+.BR -f \ option.
 .TP
 .B list
 Print a list of files managed by yadm.
-.RB The " -a
+.RB The \ -a
 option will cause all managed files to be listed.
 Otherwise, the list will only include files from the current directory or below.
 .TP
-.BI introspect " category
+.BI introspect \ category
 Report internal yadm data. Supported categories are
 .IR commands ,
 .IR configs ,
@@ -259,12 +257,12 @@ configuration
 .I yadm.auto-perms
 to "false".
 .TP
-.BI transcrypt " options
+.BI transcrypt \ options
 If transcrypt is installed, this command allows you to pass options directly to
 transcrypt, with the environment configured to use the yadm repository.
 
-transcrypt enables transparent encryption and decryption of files in a git repository.
+transcrypt enables transparent encryption and decryption of files in a git
-You can read
+repository. You can read
 https://github.com/elasticdog/transcrypt
 for details.
 .TP
@@ -377,7 +375,8 @@ manually to update permissions.
 This feature is enabled by default.
 .TP
 .B yadm.auto-private-dirs
-Disable the automatic creating of private directories described in the section PERMISSIONS.
+Disable the automatic creating of private directories described in the section
+PERMISSIONS.
 .TP
 .B yadm.cipher
 Configure which encryption system is used by the encrypt/decrypt commands.
@@ -426,7 +425,7 @@ Disable the permission changes to
 .IR $HOME/.ssh/* .
 This feature is enabled by default.
 
-.RE
+.LP
 The following five "local" configurations are not stored in the
 .IR $HOME/.config/yadm/config,
 they are stored in the local repository.
@@ -477,22 +476,22 @@ be omitted. Most attributes can be abbreviated as a single letter.
 These are the supported attributes, in the order of the weighted precedence:
 
 .TP
-.BR template , " t
+.BR template ,\  t
 Valid when the value matches a supported template processor.
 See the TEMPLATES section for more details.
 .TP
-.BR user ,  " u
+.BR user ,\  u
 Valid if the value matches the current user.
 Current user is calculated by running
 .BR "id -u -n" .
 .TP
-.BR hostname , " h
+.BR hostname ,\  h
 Valid if the value matches the short hostname.
 Hostname is calculated by running
 .BR "uname -n" ,
 and trimming off any domain.
 .TP
-.BR class , " c
+.BR class ,\  c
 Valid if the value matches the
 .B local.class
 configuration.
@@ -501,24 +500,24 @@ Class must be manually set using
 See the CONFIGURATION section for more details about setting
 .BR local.class .
 .TP
-.BR distro , " d
+.BR distro ,\  d
 Valid if the value matches the distro.
 Distro is calculated by running
 .B "lsb_release -si"
 or by inspecting the ID from
 .BR "/etc/os-release" .
 .TP
-.BR distro_family , " f
+.BR distro_family ,\  f
 Valid if the value matches the distro family.
 Distro family is calculated by inspecting the ID_LIKE line from
 .BR "/etc/os-release" .
 .TP
-.BR os , " o
+.BR os ,\  o
 Valid if the value matches the OS.
 OS is calculated by running
 .BR "uname -s" .
 .TP
-.BR arch , " a
+.BR arch ,\  a
 Valid if the value matches the architecture.
 Architecture is calculated by running
 .BR "uname -m" .
@@ -526,12 +525,12 @@ Architecture is calculated by running
 .B default
 Valid when no other alternate is valid.
 .TP
-.BR extension , " e
+.BR extension ,\  e
 A special "condition" that doesn't affect the selection process. Its purpose is
 instead to allow the alternate file to end with a certain extension to
 e.g. make editors highlight the content properly.
-.LP
 
+.LP
 .BR NOTE :
 The OS for "Windows Subsystem for Linux" is reported as "WSL", even
 though uname identifies as "Linux".
@@ -577,7 +576,8 @@ which looks like this:
 
 .IR $HOME/path/example.txt " -> " $HOME/path/example.txt##os.Darwin
 
-Since the hostname doesn't match any of the managed files, the more generic version is chosen.
+Since the hostname doesn't match any of the managed files, the more generic
+version is chosen.
 
 If running on a Linux server named "host4", the link will be:
 
@@ -652,9 +652,10 @@ To use the j2cli Jinja template processor, specify the value of "j2"  or
 "j2cli".
 .TP
 .B envtpl
-To use the envtpl Jinja template processor, specify the value of "j2" or "envtpl".
+To use the envtpl Jinja template processor, specify the value of "j2" or
-.LP
+"envtpl".
 
+.LP
 .BR NOTE :
 Specifying "j2" as the processor will attempt to use j2cli or envtpl, whichever
 is available.
@@ -752,8 +753,7 @@ configuration.
 To use this feature, a list of patterns must be created and saved as
 .IR $HOME/.config/yadm/encrypt .
 This list of patterns should be relative to the configured
-.IR work-tree " (usually
+.IR work-tree \ (usually\  $HOME ).
-.IR $HOME ).
 For example:
 
 .RS
@@ -770,11 +770,12 @@ Paths beginning with a "!" will be excluded.
 
 The
 .B yadm encrypt
-command will find all files matching the patterns, and prompt for a password. Once a
+command will find all files matching the patterns, and prompt for a
-password has confirmed, the matching files will be encrypted and saved as
+password. Once a password has confirmed, the matching files will be encrypted
+and saved as
 .IR $HOME/.local/share/yadm/archive .
-The "encrypt" and "archive" files should be added to the yadm repository so they are
+The "encrypt" and "archive" files should be added to the yadm repository so
-available across multiple systems.
+they are available across multiple systems.
 
 To decrypt these files later, or on another system run
 .B yadm decrypt
@@ -817,7 +818,6 @@ repository. See the following web sites for more information:
 - https://github.com/elasticdog/transcrypt
 
 - https://github.com/AGWA/git-crypt
-.LP
 
 .SH PERMISSIONS
 
@@ -826,7 +826,7 @@ dependent upon the user's umask. Because of this, yadm will automatically
 update the permissions of some file paths. The "group" and "others" permissions
 will be removed from the following files:
 
-.RI - " $HOME/.local/share/yadm/archive
+.RI -\  $HOME/.local/share/yadm/archive
 
 - All files matching patterns in
 .I $HOME/.config/yadm/encrypt