Merge 80ca4a993a into 30fa6f08a4

* Update base image to Ubuntu 24.10. This uses a python version where j2cli no
  longer works when installed using pip so use the version from Ubuntu instead
  which has been patched to work.

* Update shellcheck, pylint, pytest, isort, flake8, black and yamllint to the
  latest versions. This closes #502.

* Use a longer expect timeout to fix tests failing when gpg is killed due to
  this timeout.

* Explicitly flush gpg-agent's cached passwords to fix failing tests with
  latest gnupg. Also clean up after tests to avoid having gpg-agents running
  after the test (e.g. when running tests directly without docker).

Makefile

@@ -1,5 +1,5 @@
 PYTESTS = $(wildcard test/test_*.py)
-IMAGE = docker.io/yadm/testbed:2023-07-12
+IMAGE = docker.io/yadm/testbed:2024-11-11
 OCI = docker
 
 .PHONY: all

pyproject.toml

@@ -7,6 +7,7 @@ markers = [
 
 [tool.pylint.design]
 max-args = 14
+max-positional-arguments = 10
 max-locals = 28
 max-attributes = 8
 max-statements = 65

test/Dockerfile

@@ -1,8 +1,7 @@
-FROM ubuntu:23.04
+FROM ubuntu:24.10
-MAINTAINER Tim Byrne <sultan@locehilios.com>
 
 # Shellcheck and esh versions
-ARG SC_VER=0.9.0
+ARG SC_VER=0.10.0
 ARG ESH_VER=0.3.2
 
 # Install prerequisites and configure UTF-8 locale
@@ -14,6 +13,7 @@ RUN \
     expect \
     git \
     gnupg \
+    j2cli \
     locales \
     lsb-release \
     make \
@@ -39,10 +39,9 @@ RUN cd /opt \
   && rm -f shellcheck-v$SC_VER.linux.x86_64.tar.xz \
   && ln -s /opt/shellcheck-v$SC_VER/shellcheck /usr/local/bin
 
-# Upgrade pip3 and install requirements
+# Install requirements
 COPY test/requirements.txt /tmp/requirements.txt
-RUN python3 -m pip install --break-system-packages --upgrade pip setuptools \
+RUN python3 -m pip install --break-system-packages -r /tmp/requirements.txt \
-  && python3 -m pip install --break-system-packages --upgrade -r /tmp/requirements.txt \
   && rm -f /tmp/requirements
 
 # Install esh

test/conftest.py

@@ -9,7 +9,6 @@ import pwd
 import shutil
 from subprocess import PIPE, Popen
 
-import py
 import pytest
 
 
@@ -26,37 +25,37 @@ def pytest_addoption(parser):
 @pytest.fixture(scope="session")
 def shellcheck_version():
     """Version of shellcheck supported"""
-    return "0.9.0"
+    return "0.10.0"
 
 
 @pytest.fixture(scope="session")
 def pylint_version():
     """Version of pylint supported"""
-    return "2.17.0"
+    return "3.3.1"
 
 
 @pytest.fixture(scope="session")
 def isort_version():
     """Version of isort supported"""
-    return "5.12.0"
+    return "5.13.2"
 
 
 @pytest.fixture(scope="session")
 def flake8_version():
     """Version of flake8 supported"""
-    return "6.0.0"
+    return "7.1.1"
 
 
 @pytest.fixture(scope="session")
 def black_version():
     """Version of black supported"""
-    return "23.1.0"
+    return "24.10.0"
 
 
 @pytest.fixture(scope="session")
 def yamllint_version():
     """Version of yamllint supported"""
-    return "1.30.0"
+    return "1.35.1"
 
 
 @pytest.fixture(scope="session")
@@ -246,7 +245,7 @@ class Runner:
         if not expect:
             return
         cmdline = " ".join([f'"{w}"' for w in self.command])
-        expect_script = f"set timeout 2\nspawn {cmdline}\n"
+        expect_script = f"set timeout 5\nspawn {cmdline}\n"
         for question, answer in expect:
             expect_script += "expect {\n" f'"{question}" {{send "{answer}\\r"}}\n' "timeout {close;exit 128}\n" "}\n"
         expect_script += "expect eof\n" "foreach {pid spawnid os_error_flag value} [wait] break\n" "exit $value"
@@ -575,17 +574,21 @@ def ds1(ds1_work_copy, paths, ds1_dset):
 def gnupg(tmpdir_factory, runner):
     """Location of GNUPGHOME"""
 
-    def register_gpg_password(password):
-        """Publish a new GPG mock password"""
-        py.path.local("/tmp/mock-password").write(password)
-
     home = tmpdir_factory.mktemp("gnupghome")
     home.chmod(0o700)
     conf = home.join("gpg.conf")
     conf.write("no-secmem-warning\n")
     conf.chmod(0o600)
     agentconf = home.join("gpg-agent.conf")
-    agentconf.write(f'pinentry-program {os.path.abspath("test/pinentry-mock")}\n' "max-cache-ttl 0\n")
+    agentconf.write(
+        f"""\
+pinentry-program {os.path.abspath("test/pinentry-mock")}
+max-cache-ttl 0
+browser-socket none
+extra-socket none
+disable-scdaemon
+"""
+    )
     agentconf.chmod(0o600)
     data = collections.namedtuple("GNUPG", ["home", "pw"])
     env = os.environ.copy()
@@ -594,4 +597,12 @@ def gnupg(tmpdir_factory, runner):
     # this pre-populates std files in the GNUPGHOME
     runner(["gpg", "-k"], env=env)
 
-    return data(home, register_gpg_password)
+    def register_gpg_password(password):
+        """Publish a new GPG mock password and flush cached passwords"""
+        home.join("mock-password").write(password)
+        runner(["gpgconf", "--reload", "gpg-agent"], env=env)
+
+    yield data(home, register_gpg_password)
+
+    runner(["gpgconf", "--kill", "gpg-agent"], env=env)
+    runner(["gpgconf", "--remove-socketdir", "gpg-agent"], env=env)

test/pinentry-mock

@@ -6,10 +6,9 @@
 echo "OK Pleased to meet you"
 while read -r line; do
   if [[ $line =~ GETPIN ]]; then
-    password="$(cat /tmp/mock-password 2>/dev/null)"
+    password="$(cat "$GNUPGHOME/mock-password" 2>/dev/null)"
     if [ -n "$password" ]; then
-      echo -n "D "
+      echo "D $password"
-      echo "$password"
       echo "OK";
     else
       echo "CANCEL";

test/requirements.txt

@@ -1,8 +1,8 @@
-black==23.1.0
+black==24.10.0
 envtpl
-flake8==6.0.0
+flake8==7.1.1
-isort==5.12.0
+isort==5.13.2
 j2cli
-pylint==2.17.0
+pylint==3.3.1
-pytest==7.2.2
+pytest==8.3.3
-yamllint==1.30.0
+yamllint==1.35.1

test/test_alt.py

@@ -1,4 +1,5 @@
 """Test alt"""
+
 import os
 import string
 

test/test_encryption.py

@@ -2,7 +2,6 @@
 
 import os
 import shlex
-import time
 
 import pytest
 
@@ -219,7 +218,6 @@ def test_symmetric_decrypt(runner, yadm_cmd, paths, decrypt_targets, gnupg, doli
 
     if bad_phrase:
         gnupg.pw("")
-        time.sleep(1)  # allow gpg-agent cache to expire
     else:
         gnupg.pw(PASSPHRASE)
 

test/test_help.py

@@ -1,4 +1,5 @@
 """Test help"""
+
 import pytest
 
 

test/test_list.py

@@ -20,7 +20,8 @@ def test_list(runner, yadm_cmd, paths, ds1, location):
         run_dir = paths.work
     elif location == "outside":
         run_dir = paths.work.join("..")
-    elif location == "subdir":
+    else:
+        assert location == "subdir"
         # first directory with tracked data
         run_dir = paths.work.join(ds1.tracked_dirs[0])
     with run_dir.as_cwd():

test/test_unit_choose_template_cmd.py

@@ -1,4 +1,5 @@
 """Unit tests: choose_template_cmd"""
+
 import pytest
 
 
@@ -19,7 +20,7 @@ def test_kind_default(runner, yadm, awk, label):
 
     script = f"""
         YADM_TEST=1 source {yadm}
-        function awk_available {{ { awk_avail}; }}
+        function awk_available {{ {awk_avail}; }}
         template="$(choose_template_cmd "{label}")"
         echo "TEMPLATE:$template"
     """
@@ -50,8 +51,8 @@ def test_kind_j2cli_envtpl(runner, yadm, envtpl, j2cli, label):
 
     script = f"""
         YADM_TEST=1 source {yadm}
-        function envtpl_available {{ { envtpl_avail}; }}
+        function envtpl_available {{ {envtpl_avail}; }}
-        function j2cli_available {{ { j2cli_avail}; }}
+        function j2cli_available {{ {j2cli_avail}; }}
         template="$(choose_template_cmd "{label}")"
         echo "TEMPLATE:$template"
     """

test/test_unit_copy_perms.py

@@ -1,4 +1,5 @@
 """Unit tests: copy_perms"""
+
 import os
 
 import pytest

test/test_unit_exclude_encrypted.py

@@ -1,4 +1,5 @@
 """Unit tests: exclude_encrypted"""
+
 import pytest
 
 

test/test_unit_issue_legacy_path_warning.py

@@ -1,4 +1,5 @@
 """Unit tests: issue_legacy_path_warning"""
+
 import pytest
 
 

test/test_unit_private_dirs.py

@@ -1,4 +1,5 @@
 """Unit tests: private_dirs"""
+
 import pytest
 
 

test/test_unit_query_distro.py

@@ -1,4 +1,5 @@
 """Unit tests: query_distro"""
+
 import pytest
 
 

test/test_unit_query_distro_family.py

@@ -1,4 +1,5 @@
 """Unit tests: query_distro_family"""
+
 import pytest
 
 

test/test_unit_record_score.py

@@ -1,4 +1,5 @@
 """Unit tests: record_score"""
+
 import pytest
 
 INIT_VARS = """

test/test_unit_relative_path.py

@@ -1,4 +1,5 @@
 """Unit tests: relative_path"""
+
 import pytest
 
 

test/test_unit_remove_stale_links.py

@@ -1,4 +1,5 @@
 """Unit tests: remove_stale_links"""
+
 import os
 
 import pytest

test/test_unit_report_invalid_alts.py

@@ -1,4 +1,5 @@
 """Unit tests: report_invalid_alts"""
+
 import pytest
 
 

test/test_unit_score_file.py

@@ -1,4 +1,5 @@
 """Unit tests: score_file"""
+
 import pytest
 
 CONDITION = {

test/test_unit_set_local_alt_values.py

@@ -1,4 +1,5 @@
 """Unit tests: set_local_alt_values"""
+
 import pytest
 import utils
 

test/test_unit_set_yadm_dir.py

@@ -1,4 +1,5 @@
 """Unit tests: set_yadm_dirs"""
+
 import pytest
 
 

test/test_unit_template_esh.py

@@ -1,4 +1,5 @@
 """Unit tests: template_esh"""
+
 import os
 
 FILE_MODE = 0o754

test/test_unit_template_j2.py

@@ -1,4 +1,5 @@
 """Unit tests: template_j2cli & template_envtpl"""
+
 import os
 
 import pytest

test/test_unit_upgrade.py

@@ -1,4 +1,5 @@
 """Unit tests: upgrade"""
+
 import pytest
 
 
@@ -62,11 +63,10 @@ def test_upgrade(tmpdir, runner, yadm, condition):
             function git() {{
                 echo "$@"
                 if [[ "$*" = *"submodule status" ]]; then
-                    { 'echo " 1234567 mymodule (1.0)"'
+                    {'echo " 1234567 mymodule (1.0)"' if condition == 'submodules' else ':'}
-                        if condition == 'submodules' else ':' }
                 fi
                 if [[ "$*" = *ls-files* ]]; then
-                    return { 1 if condition == 'untracked' else 0 }
+                    return {1 if condition == 'untracked' else 0}
                 fi
                 return 0
             }}

yadm

@@ -36,6 +36,7 @@ YADM_ENCRYPT="encrypt"
 YADM_BOOTSTRAP="bootstrap"
 YADM_HOOKS="hooks"
 YADM_ALT="alt"
+YADM_CHECK_CONDITION="check_condition"
 
 # these are the default paths relative to YADM_DATA
 YADM_REPO="repo.git"
@@ -175,6 +176,7 @@ function score_file() {
   fi
 
   score=0
+  declare -gA external_cache
   IFS=',' read -ra fields <<< "$conditions"
   for field in "${fields[@]}"; do
     label=${field%%.*}
@@ -238,6 +240,22 @@ function score_file() {
         score=0
         return
       fi
+    elif [[ "$label" =~ ^(x|external)$ ]]; then
+        if [[ -z "${external_cache[x$value]:-}" ]]; then
+            if [[ -x "$YADM_CHECK_CONDITION" ]]; then
+                "$YADM_CHECK_CONDITION" "$value"
+                external_cache[x$value]=$?
+            else
+                debug "No program found to check condition for $src"
+                [ -n "$loud" ] && echo "No program found to check condition for $src"
+            fi
+        fi
+        if [[ ${external_cache[x$value]:-1} -eq 0 ]]; then
+            score=$((score + 32))
+        else
+            score=0
+            return
+        fi
     # templates
     elif [[ "$label" =~ ^(t|template|yadm)$ ]]; then
       score=0
@@ -1720,6 +1738,7 @@ function configure_paths() {
   YADM_BOOTSTRAP="$YADM_DIR/$YADM_BOOTSTRAP"
   YADM_HOOKS="$YADM_DIR/$YADM_HOOKS"
   YADM_ALT="$YADM_DIR/$YADM_ALT"
+  YADM_CHECK_CONDITION="$YADM_DIR/$YADM_CHECK_CONDITION"
 
   # change paths to be relative to YADM_DATA
   YADM_REPO="$YADM_DATA/$YADM_REPO"