From 7c1192ae2e9b9866a7e1ef7bb3d00dd5bd975e3a Mon Sep 17 00:00:00 2001 From: Tim Byrne Date: Tue, 29 Dec 2020 11:10:18 -0600 Subject: [PATCH] Support upgrading from version 1 repos * yadm upgrade now updates legacy paths created by yadm 1 * Warnings are issued for invalid alternates again --- test/test_unit_report_invalid_alts.py | 30 +++++++ test/test_unit_upgrade.py | 12 +++ yadm | 111 +++++++++++++++++++++++--- 3 files changed, 141 insertions(+), 12 deletions(-) create mode 100644 test/test_unit_report_invalid_alts.py diff --git a/test/test_unit_report_invalid_alts.py b/test/test_unit_report_invalid_alts.py new file mode 100644 index 0000000..8730d61 --- /dev/null +++ b/test/test_unit_report_invalid_alts.py @@ -0,0 +1,30 @@ +"""Unit tests: report_invalid_alts""" +import pytest + + +@pytest.mark.parametrize('valid', [True, False], ids=['valid', 'no_valid']) +@pytest.mark.parametrize('previous', [True, False], ids=['prev', 'no_prev']) +def test_report_invalid_alts(runner, yadm, valid, previous): + """Use report_invalid_alts""" + + lwi = '' + alts = 'INVALID_ALT=()' + if previous: + lwi = 'LEGACY_WARNING_ISSUED=1' + if not valid: + alts = 'INVALID_ALT=("file##invalid")' + + script = f""" + YADM_TEST=1 source {yadm} + {lwi} + {alts} + report_invalid_alts + """ + run = runner(command=['bash'], inp=script) + assert run.success + assert run.out == '' + if not valid and not previous: + assert 'WARNING' in run.err + assert 'file##invalid' in run.err + else: + assert run.err == '' diff --git a/test/test_unit_upgrade.py b/test/test_unit_upgrade.py index 309438d..fa86e05 100644 --- a/test/test_unit_upgrade.py +++ b/test/test_unit_upgrade.py @@ -46,13 +46,17 @@ def test_upgrade(tmpdir, runner, yadm, condition): mock for git. echo will return true, simulating a positive result from "git ls-files". Also echo will report the parameters for "git mv". """ + legacy_paths = ('config', 'encrypt', 'bootstrap', 'hooks/pre_cmd') home = tmpdir.mkdir('home') yadm_dir = home.join('.config/yadm') yadm_data = home.join('.local/share/yadm') + yadm_legacy = home.join('.yadm') if condition != 'no-paths': yadm_dir.join('repo.git/config').write('test-repo', ensure=True) yadm_dir.join('files.gpg').write('files.gpg', ensure=True) + for path in legacy_paths: + yadm_legacy.join(path).write(path, ensure=True) mock_git = "" if condition in ['tracked', 'submodules']: @@ -68,6 +72,7 @@ def test_upgrade(tmpdir, runner, yadm, condition): script = f""" YADM_TEST=1 source {yadm} + YADM_LEGACY_DIR="{yadm_legacy}" YADM_DIR="{yadm_dir}" YADM_DATA="{yadm_data}" YADM_REPO="{yadm_data}/repo.git" @@ -89,9 +94,16 @@ def test_upgrade(tmpdir, runner, yadm, condition): f'Moving {yadm_dir.join(lpath)} ' f'to {yadm_data.join(npath)}') assert expected in run.out + for path in legacy_paths: + expected = ( + f'Moving {yadm_legacy.join(path)} ' + f'to {yadm_dir.join(path)}') + assert expected in run.out if condition == 'untracked': assert 'test-repo' in yadm_data.join('repo.git/config').read() assert 'files.gpg' in yadm_data.join('archive').read() + for path in legacy_paths: + assert path in yadm_dir.join(path).read() elif condition in ['tracked', 'submodules']: expected = ( f'mv {yadm_dir.join("files.gpg")} ' diff --git a/yadm b/yadm index 621d4f2..45646e4 100755 --- a/yadm +++ b/yadm @@ -25,6 +25,8 @@ VERSION=2.5.0 YADM_WORK="$HOME" YADM_DIR= YADM_DATA= + +YADM_LEGACY_DIR="${HOME}/.yadm" YADM_LEGACY_ARCHIVE="files.gpg" # these are the default paths relative to YADM_DIR @@ -58,6 +60,9 @@ OPERATING_SYSTEM="Unknown" ENCRYPT_INCLUDE_FILES="unparsed" +LEGACY_WARNING_ISSUED=0 +INVALID_ALT=() + GPG_OPTS=() OPENSSL_OPTS=() @@ -234,6 +239,9 @@ function score_file() { return 0 # unsupported values else + if [[ "${src##*/}" =~ .\#\#. ]]; then + INVALID_ALT+=("$src") + fi score=0 return fi @@ -540,9 +548,41 @@ function alt() { alt_linking remove_stale_links + report_invalid_alts } +function report_invalid_alts() { + [ "$LEGACY_WARNING_ISSUED" = "1" ] && return + [ "${#INVALID_ALT[@]}" = "0" ] && return + local path_list + for invalid in "${INVALID_ALT[@]}"; do + path_list="$path_list * $invalid"$'\n' + done + cat <&2 + +**WARNING** + Invalid alternates have been detected. + + Beginning with version 2.0.0, yadm uses a new naming convention for alternate + files. Read more about this change here: + + https://yadm.io/docs/upgrade_from_1 + + Or to learn more about alternates in general, read: + + https://yadm.io/docs/alternates + + To rename the invalid alternates run: + + yadm mv + + Invalid alternates detected: +${path_list} +*********** +EOF +} + function remove_stale_links() { # review alternate candidates for stale links # if a possible alt IS linked, but it's source is not part of alt_linked, @@ -1262,35 +1302,69 @@ function upgrade() { [[ -n "${YADM_OVERRIDE_REPO}${YADM_OVERRIDE_ARCHIVE}" || "$YADM_DATA" = "$YADM_DIR" ]] && \ error_out "Unable to upgrade. Paths have been overridden with command line options" + # choose a legacy repo, the version 2 location will be favored + local LEGACY_REPO= + [ -d "$YADM_LEGACY_DIR/repo.git" ] && LEGACY_REPO="$YADM_LEGACY_DIR/repo.git" + [ -d "$YADM_DIR/repo.git" ] && LEGACY_REPO="$YADM_DIR/repo.git" + # handle legacy repo - if [ -d "$YADM_DIR/repo.git" ]; then + if [ -d "$LEGACY_REPO" ]; then + # choose # legacy repo detected, it must be moved to YADM_REPO if [ -e "$YADM_REPO" ]; then error_out "Unable to upgrade. '$YADM_REPO' already exists. Refusing to overwrite it." else actions_performed=1 repo_moved=1 - echo "Moving $YADM_DIR/repo.git to $YADM_REPO" + echo "Moving $LEGACY_REPO to $YADM_REPO" assert_parent "$YADM_REPO" - mv "$YADM_DIR/repo.git" "$YADM_REPO" + mv "$LEGACY_REPO" "$YADM_REPO" fi fi GIT_DIR="$YADM_REPO" export GIT_DIR + # choose a legacy archive, the version 2 location will be favored + local LEGACY_ARCHIVE= + [ -e "$YADM_LEGACY_DIR/$YADM_LEGACY_ARCHIVE" ] && LEGACY_ARCHIVE="$YADM_LEGACY_DIR/$YADM_LEGACY_ARCHIVE" + [ -e "$YADM_DIR/$YADM_LEGACY_ARCHIVE" ] && LEGACY_ARCHIVE="$YADM_DIR/$YADM_LEGACY_ARCHIVE" + # handle legacy archive - if [ -e "$YADM_DIR/$YADM_LEGACY_ARCHIVE" ]; then + if [ -e "$LEGACY_ARCHIVE" ]; then actions_performed=1 - echo "Moving $YADM_DIR/$YADM_LEGACY_ARCHIVE to $YADM_ARCHIVE" + echo "Moving $LEGACY_ARCHIVE to $YADM_ARCHIVE" assert_parent "$YADM_ARCHIVE" # test to see if path is "tracked" in repo, if so 'git mv' must be used - if "$GIT_PROGRAM" ls-files --error-unmatch "$YADM_DIR/$YADM_LEGACY_ARCHIVE" &> /dev/null; then - "$GIT_PROGRAM" mv "$YADM_DIR/$YADM_LEGACY_ARCHIVE" "$YADM_ARCHIVE" && repo_updates=1 + if "$GIT_PROGRAM" ls-files --error-unmatch "$LEGACY_ARCHIVE" &> /dev/null; then + "$GIT_PROGRAM" mv "$LEGACY_ARCHIVE" "$YADM_ARCHIVE" && repo_updates=1 else - mv -i "$YADM_DIR/$YADM_LEGACY_ARCHIVE" "$YADM_ARCHIVE" + mv -i "$LEGACY_ARCHIVE" "$YADM_ARCHIVE" fi fi + # handle any remaining version 1 paths + for legacy_path in \ + "$YADM_LEGACY_DIR/config" \ + "$YADM_LEGACY_DIR/encrypt" \ + "$YADM_LEGACY_DIR/bootstrap" \ + "$YADM_LEGACY_DIR"/hooks/{pre,post}_* \ + ; + do + if [ -e "$legacy_path" ]; then + new_filename=${legacy_path#$YADM_LEGACY_DIR/} + new_filename="$YADM_DIR/$new_filename" + actions_performed=1 + echo "Moving $legacy_path to $new_filename" + assert_parent "$new_filename" + # test to see if path is "tracked" in repo, if so 'git mv' must be used + if "$GIT_PROGRAM" ls-files --error-unmatch "$legacy_path" &> /dev/null; then + "$GIT_PROGRAM" mv "$legacy_path" "$new_filename" && repo_updates=1 + else + mv -i "$legacy_path" "$new_filename" + fi + fi + done + # handle submodules, which need to be reinitialized if [ "$repo_moved" -ne 0 ]; then cd_work "Upgrade submodules" @@ -1489,15 +1563,24 @@ function issue_legacy_path_warning() { # no warnings during upgrade [[ "${MAIN_ARGS[*]}" =~ upgrade ]] && return + # no warnings if YADM_DIR is resolved as the leacy path + [ "$YADM_DIR" = "$YADM_LEGACY_DIR" ] && return + # no warnings if overrides have been provided [[ -n "${YADM_OVERRIDE_REPO}${YADM_OVERRIDE_ARCHIVE}" || "$YADM_DATA" = "$YADM_DIR" ]] && return # test for legacy paths local legacy_found=() # this is ordered by importance - for legacy_path in \ - "$YADM_DIR/$YADM_REPO" \ - "$YADM_DIR/$YADM_LEGACY_ARCHIVE" \ + for legacy_path in \ + "$YADM_DIR/$YADM_REPO" \ + "$YADM_DIR/$YADM_LEGACY_ARCHIVE" \ + "$YADM_LEGACY_DIR/$YADM_REPO" \ + "$YADM_LEGACY_DIR/$YADM_BOOTSTRAP" \ + "$YADM_LEGACY_DIR/$YADM_CONFIG" \ + "$YADM_LEGACY_DIR/$YADM_ENCRYPT" \ + "$YADM_LEGACY_DIR/$YADM_HOOKS"/{pre,post}_* \ + "$YADM_LEGACY_DIR/$YADM_LEGACY_ARCHIVE" \ ; do [ -e "$legacy_path" ] && legacy_found+=("$legacy_path") @@ -1516,9 +1599,10 @@ function issue_legacy_path_warning() { Legacy paths have been detected. With version 3.0.0, yadm uses the XDG Base Directory Specification - to find its configurations and data. Read more about this change here: + to find its configurations and data. Read more about these changes here: https://yadm.io/docs/upgrade_from_2 + https://yadm.io/docs/upgrade_from_1 In your environment, the data directory has been resolved to: @@ -1533,6 +1617,9 @@ function issue_legacy_path_warning() { ${path_list} *********** EOF + +LEGACY_WARNING_ISSUED=1 + } function configure_paths() {