From adfedf4b6452a33841e23469668fe1ad357d9da4 Mon Sep 17 00:00:00 2001 From: Ross Smith II Date: Sat, 3 Mar 2018 20:02:45 -0800 Subject: [PATCH 001/137] Add Mingw/Msys support --- yadm | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/yadm b/yadm index e55a287..3ee88f5 100755 --- a/yadm +++ b/yadm @@ -809,11 +809,12 @@ function set_operating_system() { fi case "$OPERATING_SYSTEM" in - CYGWIN*) + CYGWIN*|MINGW*|MSYS*) git_version=$(git --version 2>/dev/null) if [[ "$git_version" =~ windows ]] ; then USE_CYGPATH=1 fi + OPERATING_SYSTEM=$(uname -o) ;; *) ;; From 315ad0873e843b1cfa7f2a9933e15dde798cba24 Mon Sep 17 00:00:00 2001 From: Ross Smith II Date: Sun, 4 Mar 2018 06:34:08 -0800 Subject: [PATCH 002/137] Fix OS check to match (Cygwin|Msys)* --- yadm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/yadm b/yadm index 3ee88f5..6f6f8b9 100755 --- a/yadm +++ b/yadm @@ -169,7 +169,7 @@ function alt() { #; decide if a copy should be done instead of a symbolic link local do_copy=0 - if [[ $OPERATING_SYSTEM == CYGWIN* ]] ; then + if [[ $OPERATING_SYSTEM =~ (Cygwin|Msys) ]] ; then if [[ $(config --bool yadm.cygwin-copy) == "true" ]] ; then do_copy=1 fi From 54f7cbcebe835b098a23998a16eb11f4d643eeb1 Mon Sep 17 00:00:00 2001 From: Ross Smith II Date: Mon, 5 Mar 2018 07:02:46 -0800 Subject: [PATCH 003/137] Fix OS name match for Cygwin/Msys --- yadm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/yadm b/yadm index 6f6f8b9..c391306 100755 --- a/yadm +++ b/yadm @@ -169,7 +169,7 @@ function alt() { #; decide if a copy should be done instead of a symbolic link local do_copy=0 - if [[ $OPERATING_SYSTEM =~ (Cygwin|Msys) ]] ; then + if [[ $OPERATING_SYSTEM =~ ^(CYGWIN|MSYS) ]] ; then if [[ $(config --bool yadm.cygwin-copy) == "true" ]] ; then do_copy=1 fi From 0b9f5379098f1510b10473b27446afc6c1e9c9dc Mon Sep 17 00:00:00 2001 From: Brayden Banks Date: Mon, 11 Jun 2018 19:31:10 -0700 Subject: [PATCH 004/137] Allow for more complex Jinja templates By calling envtpl with a filename, Jinja can perform includes, which is useful for more general machine configuration. --- yadm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/yadm b/yadm index e55a287..7c9dcbb 100755 --- a/yadm +++ b/yadm @@ -223,7 +223,7 @@ function alt() { YADM_HOSTNAME="$local_host" \ YADM_USER="$local_user" \ YADM_DISTRO=$(query_distro) \ - "$ENVTPL_PROGRAM" < "$tracked_file" > "$real_file" + "$ENVTPL_PROGRAM" --keep-template "$tracked_file" -o "$real_file" else debug "envtpl not available, not creating $real_file from template $tracked_file" [ -n "$loud" ] && echo "envtpl not available, not creating $real_file from template $tracked_file" From e7f9616b393e9127d3fcedaa35efc2f86855e5b3 Mon Sep 17 00:00:00 2001 From: Tim Byrne Date: Wed, 11 Jul 2018 07:50:42 -0500 Subject: [PATCH 005/137] Rewrite testing system (#119) The new test system is written with py.test. These tests are more comprehensive, run faster by an order of magnitude, and are far more maintainable. The tests themselves conform to PEP8. --- .gitignore | 2 + Dockerfile | 17 +- Makefile | 15 + docker-compose.yml | 7 + pylintrc | 11 + pytest.ini | 3 + test/conftest.py | 559 ++++++++++++++++++++++++++ test/pylintrc | 1 + test/test_alt.py | 345 ++++++++++++++++ test/test_assert_private_dirs.py | 106 +++++ test/test_bootstrap.py | 31 ++ test/test_clean.py | 11 + test/test_clone.py | 274 +++++++++++++ test/test_config.py | 139 +++++++ test/test_cygwin_copy.py | 59 +++ test/test_encryption.py | 392 ++++++++++++++++++ test/test_enter.py | 85 ++++ test/test_git.py | 58 +++ test/test_help.py | 17 + test/test_hooks.py | 90 +++++ test/test_init.py | 78 ++++ test/test_introspect.py | 46 +++ test/test_jinja.py | 186 +++++++++ test/test_list.py | 47 +++ test/test_perms.py | 111 +++++ test/test_syntax.py | 41 ++ test/test_unit_bootstrap_available.py | 33 ++ test/test_unit_configure_paths.py | 80 ++++ test/test_unit_parse_encrypt.py | 175 ++++++++ test/test_unit_query_distro.py | 26 ++ test/test_unit_set_os.py | 36 ++ test/test_unit_x_program.py | 46 +++ test/test_version.py | 35 ++ test/utils.py | 59 +++ 34 files changed, 3218 insertions(+), 3 deletions(-) create mode 100644 docker-compose.yml create mode 100644 pylintrc create mode 100644 pytest.ini create mode 100644 test/conftest.py create mode 120000 test/pylintrc create mode 100644 test/test_alt.py create mode 100644 test/test_assert_private_dirs.py create mode 100644 test/test_bootstrap.py create mode 100644 test/test_clean.py create mode 100644 test/test_clone.py create mode 100644 test/test_config.py create mode 100644 test/test_cygwin_copy.py create mode 100644 test/test_encryption.py create mode 100644 test/test_enter.py create mode 100644 test/test_git.py create mode 100644 test/test_help.py create mode 100644 test/test_hooks.py create mode 100644 test/test_init.py create mode 100644 test/test_introspect.py create mode 100644 test/test_jinja.py create mode 100644 test/test_list.py create mode 100644 test/test_perms.py create mode 100644 test/test_syntax.py create mode 100644 test/test_unit_bootstrap_available.py create mode 100644 test/test_unit_configure_paths.py create mode 100644 test/test_unit_parse_encrypt.py create mode 100644 test/test_unit_query_distro.py create mode 100644 test/test_unit_set_os.py create mode 100644 test/test_unit_x_program.py create mode 100644 test/test_version.py create mode 100644 test/utils.py diff --git a/.gitignore b/.gitignore index 7aa998b..af9e6f5 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,6 @@ .DS_Store +.env .jekyll-metadata +.pytest_cache .sass-cache _site diff --git a/Dockerfile b/Dockerfile index 74700c4..ee29bf2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,9 +1,20 @@ -FROM ubuntu:yakkety +FROM ubuntu:18.04 MAINTAINER Tim Byrne +# No input during build +ENV DEBIAN_FRONTEND noninteractive + +# UTF8 locale +RUN apt-get update && apt-get install -y locales +RUN locale-gen en_US.UTF-8 +ENV LANG='en_US.UTF-8' LANGUAGE='en_US:en' LC_ALL='en_US.UTF-8' + +# Convenience settings for the testbed's root account +RUN echo 'set -o vi' >> /root/.bashrc + # Install prerequisites -RUN apt-get update && apt-get install -y git gnupg1 make shellcheck bats expect curl python-pip lsb-release -RUN pip install envtpl +RUN apt-get update && apt-get install -y git gnupg1 make shellcheck=0.4.6-1 bats expect curl python3-pip lsb-release +RUN pip3 install envtpl pytest==3.6.4 pylint==1.9.2 flake8==3.5.0 # Force GNUPG version 1 at path /usr/bin/gpg RUN ln -fs /usr/bin/gpg1 /usr/bin/gpg diff --git a/Makefile b/Makefile index f544a1d..691b168 100644 --- a/Makefile +++ b/Makefile @@ -23,6 +23,11 @@ test: bats shellcheck parallel: ls test/*bats | time parallel -q -P0 -- docker run --rm -v "$$PWD:/yadm:ro" yadm/testbed bash -c 'bats {}' +.PHONY: pytest +pytest: + @echo Running all pytest tests + @pytest -v + .PHONY: bats bats: @echo Running all bats tests @@ -58,3 +63,13 @@ man: .PHONY: wide wide: man ./yadm.1 + +.PHONY: sync-clock +sync-clock: + docker run --rm --privileged alpine hwclock -s + +.PHONY: .env +.env: + virtualenv --python=python3 .env + .env/bin/pip3 install --upgrade pip setuptools + .env/bin/pip3 install --upgrade pytest pylint==1.9.2 flake8==3.5.0 diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..4fe0b86 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,7 @@ +--- +version: '3' +services: + testbed: + volumes: + - .:/yadm:ro + image: yadm/testbed:latest diff --git a/pylintrc b/pylintrc new file mode 100644 index 0000000..2ae18b6 --- /dev/null +++ b/pylintrc @@ -0,0 +1,11 @@ +[BASIC] +good-names=pytestmark + +[DESIGN] +max-args=14 +max-locals=26 +max-attributes=8 +max-statements=65 + +[MESSAGES CONTROL] +disable=redefined-outer-name diff --git a/pytest.ini b/pytest.ini new file mode 100644 index 0000000..c7fc1db --- /dev/null +++ b/pytest.ini @@ -0,0 +1,3 @@ +[pytest] +cache_dir = /tmp +addopts = -ra diff --git a/test/conftest.py b/test/conftest.py new file mode 100644 index 0000000..4fefabe --- /dev/null +++ b/test/conftest.py @@ -0,0 +1,559 @@ +"""Global tests configuration and fixtures""" + +import collections +import copy +import distutils.dir_util # pylint: disable=no-name-in-module,import-error +import os +import platform +import pwd +from subprocess import Popen, PIPE +import pytest + + +@pytest.fixture(scope='session') +def shellcheck_version(): + """Version of shellcheck supported""" + return '0.4.6' + + +@pytest.fixture(scope='session') +def pylint_version(): + """Version of pylint supported""" + return '1.9.2' + + +@pytest.fixture(scope='session') +def flake8_version(): + """Version of flake8 supported""" + return '3.5.0' + + +@pytest.fixture(scope='session') +def tst_user(): + """Test session's user id""" + return pwd.getpwuid(os.getuid()).pw_name + + +@pytest.fixture(scope='session') +def tst_host(): + """Test session's short hostname value""" + return platform.node().split('.')[0] + + +@pytest.fixture(scope='session') +def tst_distro(runner): + """Test session's distro""" + distro = '' + try: + run = runner(command=['lsb_release', '-si'], report=False) + distro = run.out.strip() + except BaseException: + pass + return distro + + +@pytest.fixture(scope='session') +def tst_sys(): + """Test session's uname value""" + return platform.system() + + +@pytest.fixture(scope='session') +def cygwin_sys(): + """CYGWIN uname id""" + return 'CYGWIN_NT-6.1-WOW64' + + +@pytest.fixture(scope='session') +def supported_commands(): + """List of supported commands + + This list should be updated every time yadm learns a new command. + """ + return [ + 'alt', + 'bootstrap', + 'clean', + 'clone', + 'config', + 'decrypt', + 'encrypt', + 'enter', + 'gitconfig', + 'help', + 'init', + 'introspect', + 'list', + 'perms', + 'version', + ] + + +@pytest.fixture(scope='session') +def supported_configs(): + """List of supported config options + + This list should be updated every time yadm learns a new config. + """ + return [ + 'local.class', + 'local.hostname', + 'local.os', + 'local.user', + 'yadm.auto-alt', + 'yadm.auto-perms', + 'yadm.auto-private-dirs', + 'yadm.cygwin-copy', + 'yadm.git-program', + 'yadm.gpg-perms', + 'yadm.gpg-program', + 'yadm.gpg-recipient', + 'yadm.ssh-perms', + ] + + +@pytest.fixture(scope='session') +def supported_switches(): + """List of supported switches + + This list should be updated every time yadm learns a new switch. + """ + return [ + '--yadm-archive', + '--yadm-bootstrap', + '--yadm-config', + '--yadm-dir', + '--yadm-encrypt', + '--yadm-repo', + '-Y', + ] + + +@pytest.fixture(scope='session') +def supported_local_configs(supported_configs): + """List of supported local config options""" + return [c for c in supported_configs if c.startswith('local.')] + + +class Runner(object): + """Class for running commands + + Within yadm tests, this object should be used when running commands that + require: + + * Acting on the status code + * Parsing the output of the command + * Passing input to the command + + Other instances of simply running commands should use os.system(). + """ + + def __init__( + self, + command, + inp=None, + shell=False, + cwd=None, + env=None, + expect=None, + report=True): + if shell: + self.command = ' '.join([str(cmd) for cmd in command]) + else: + self.command = command + self.inp = inp + self.wrap(expect) + process = Popen( + self.command, + stdin=PIPE, + stdout=PIPE, + stderr=PIPE, + shell=shell, + cwd=cwd, + env=env, + ) + input_bytes = self.inp + if self.inp: + input_bytes = self.inp.encode() + (out_bstream, err_bstream) = process.communicate(input=input_bytes) + self.out = out_bstream.decode() + self.err = err_bstream.decode() + self.code = process.wait() + self.success = self.code == 0 + self.failure = self.code != 0 + if report: + self.report() + + def __repr__(self): + return f'Runner({self.command})' + + def report(self): + """Print code/stdout/stderr""" + print(f'{self}') + print(f' RUN: code:{self.code}') + if self.inp: + print(f' RUN: input:\n{self.inp}') + print(f' RUN: stdout:\n{self.out}') + print(f' RUN: stderr:\n{self.err}') + + def wrap(self, expect): + """Wrap command with expect""" + if not expect: + return + cmdline = ' '.join([f'"{w}"' for w in self.command]) + expect_script = f'set timeout 2\nspawn {cmdline}\n' + for question, answer in expect: + expect_script += ( + 'expect {\n' + f'"{question}" {{send "{answer}\\r"}}\n' + 'timeout {close;exit 128}\n' + '}\n') + expect_script += ( + 'expect eof\n' + 'foreach {pid spawnid os_error_flag value} [wait] break\n' + 'exit $value') + self.inp = expect_script + print(f'EXPECT:{expect_script}') + self.command = ['expect'] + + +@pytest.fixture(scope='session') +def runner(): + """Class for running commands""" + return Runner + + +@pytest.fixture(scope='session') +def config_git(): + """Configure global git configuration, if missing""" + os.system( + 'git config user.name || ' + 'git config --global user.name "test"') + os.system( + 'git config user.email || ' + 'git config --global user.email "test@test.test"') + return None + + +@pytest.fixture() +def repo_config(runner, paths): + """Function to query a yadm repo configuration value""" + + def query_func(key): + """Query a yadm repo configuration value""" + run = runner( + command=('git', 'config', '--local', key), + env={'GIT_DIR': paths.repo}, + report=False, + ) + return run.out.rstrip() + + return query_func + + +@pytest.fixture(scope='session') +def yadm(): + """Path to yadm program to be tested""" + full_path = os.path.realpath('yadm') + assert os.path.isfile(full_path), "yadm program file isn't present" + return full_path + + +@pytest.fixture() +def paths(tmpdir, yadm): + """Function scoped test paths""" + dir_root = tmpdir.mkdir('root') + dir_work = dir_root.mkdir('work') + dir_yadm = dir_root.mkdir('yadm') + dir_repo = dir_yadm.mkdir('repo.git') + dir_hooks = dir_yadm.mkdir('hooks') + dir_remote = dir_root.mkdir('remote') + file_archive = dir_yadm.join('files.gpg') + file_bootstrap = dir_yadm.join('bootstrap') + file_config = dir_yadm.join('config') + file_encrypt = dir_yadm.join('encrypt') + paths = collections.namedtuple( + 'Paths', [ + 'pgm', + 'root', + 'work', + 'yadm', + 'repo', + 'hooks', + 'remote', + 'archive', + 'bootstrap', + 'config', + 'encrypt', + ]) + return paths( + yadm, + dir_root, + dir_work, + dir_yadm, + dir_repo, + dir_hooks, + dir_remote, + file_archive, + file_bootstrap, + file_config, + file_encrypt, + ) + + +@pytest.fixture() +def yadm_y(paths): + """Generate custom command_list function""" + def command_list(*args): + """Produce params for running yadm with -Y""" + return [paths.pgm, '-Y', str(paths.yadm)] + list(args) + return command_list + + +class DataFile(object): + """Datafile object""" + + def __init__(self, path, tracked=True, private=False): + self.__path = path + self.__parent = None + self.__tracked = tracked + self.__private = private + + @property + def path(self): + """Path property""" + return self.__path + + @property + def relative(self): + """Relative path property""" + if self.__parent: + return self.__parent.join(self.path) + raise BaseException('Unable to provide relative path, no parent') + + @property + def tracked(self): + """Tracked property""" + return self.__tracked + + @property + def private(self): + """Private property""" + return self.__private + + def relative_to(self, parent): + """Update all relative paths to this py.path""" + self.__parent = parent + return + + +class DataSet(object): + """Dataset object""" + + def __init__(self): + self.__files = list() + self.__dirs = list() + self.__tracked_dirs = list() + self.__private_dirs = list() + self.__relpath = None + + def __repr__(self): + return ( + f'[DS with {len(self)} files; ' + f'{len(self.tracked)} tracked, ' + f'{len(self.private)} private]' + ) + + def __iter__(self): + return iter(self.__files) + + def __len__(self): + return len(self.__files) + + def __contains__(self, datafile): + if [f for f in self.__files if f.path == datafile]: + return True + if datafile in self.__files: + return True + return False + + @property + def files(self): + """List of DataFiles in DataSet""" + return list(self.__files) + + @property + def tracked(self): + """List of tracked DataFiles in DataSet""" + return [f for f in self.__files if f.tracked] + + @property + def private(self): + """List of private DataFiles in DataSet""" + return [f for f in self.__files if f.private] + + @property + def dirs(self): + """List of directories in DataSet""" + return list(self.__dirs) + + @property + def plain_dirs(self): + """List of directories in DataSet not starting with '.'""" + return [d for d in self.dirs if not d.startswith('.')] + + @property + def hidden_dirs(self): + """List of directories in DataSet starting with '.'""" + return [d for d in self.dirs if d.startswith('.')] + + @property + def tracked_dirs(self): + """List of directories in DataSet not starting with '.'""" + return [d for d in self.__tracked_dirs if not d.startswith('.')] + + @property + def private_dirs(self): + """List of directories in DataSet considered 'private'""" + return list(self.__private_dirs) + + def add_file(self, path, tracked=True, private=False): + """Add file to data set""" + if path not in self: + datafile = DataFile(path, tracked, private) + if self.__relpath: + datafile.relative_to(self.__relpath) + self.__files.append(datafile) + + dname = os.path.dirname(path) + if dname and dname not in self.__dirs: + self.__dirs.append(dname) + if tracked: + self.__tracked_dirs.append(dname) + if private: + self.__private_dirs.append(dname) + + def relative_to(self, relpath): + """Update all relative paths to this py.path""" + self.__relpath = relpath + for datafile in self.files: + datafile.relative_to(self.__relpath) + return + + +@pytest.fixture(scope='session') +def ds1_dset(tst_sys, cygwin_sys): + """Meta-data for dataset one files""" + dset = DataSet() + dset.add_file('t1') + dset.add_file('d1/t2') + dset.add_file(f'test_alt##S') + dset.add_file(f'test_alt##S.H') + dset.add_file(f'test_alt##S.H.U') + dset.add_file(f'test_alt##C.S.H.U') + dset.add_file(f'test alt/test alt##S') + dset.add_file(f'test alt/test alt##S.H') + dset.add_file(f'test alt/test alt##S.H.U') + dset.add_file(f'test alt/test alt##C.S.H.U') + dset.add_file(f'test_cygwin_copy##{tst_sys}') + dset.add_file(f'test_cygwin_copy##{cygwin_sys}') + dset.add_file('u1', tracked=False) + dset.add_file('d2/u2', tracked=False) + dset.add_file('.ssh/p1', tracked=False, private=True) + dset.add_file('.ssh/.p2', tracked=False, private=True) + dset.add_file('.gnupg/p3', tracked=False, private=True) + dset.add_file('.gnupg/.p4', tracked=False, private=True) + return dset + + +@pytest.fixture(scope='session') +def ds1_data(tmpdir_factory, config_git, ds1_dset, runner): + """A set of test data, worktree & repo""" + # pylint: disable=unused-argument + # This is ignored because + # @pytest.mark.usefixtures('config_git') + # cannot be applied to another fixture. + + data = tmpdir_factory.mktemp('ds1') + + work = data.mkdir('work') + for datafile in ds1_dset: + work.join(datafile.path).write(datafile.path, ensure=True) + + repo = data.mkdir('repo.git') + env = os.environ.copy() + env['GIT_DIR'] = str(repo) + runner( + command=['git', 'init', '--shared=0600', '--bare', str(repo)], + report=False) + runner( + command=['git', 'config', 'core.bare', 'false'], + env=env, + report=False) + runner( + command=['git', 'config', 'status.showUntrackedFiles', 'no'], + env=env, + report=False) + runner( + command=['git', 'config', 'yadm.managed', 'true'], + env=env, + report=False) + runner( + command=['git', 'config', 'core.worktree', str(work)], + env=env, + report=False) + runner( + command=['git', 'add'] + + [str(work.join(f.path)) for f in ds1_dset if f.tracked], + env=env) + runner( + command=['git', 'commit', '--allow-empty', '-m', 'Initial commit'], + env=env, + report=False) + + data = collections.namedtuple('Data', ['work', 'repo']) + return data(work, repo) + + +@pytest.fixture() +def ds1_work_copy(ds1_data, paths): + """Function scoped copy of ds1_data.work""" + distutils.dir_util.copy_tree( # pylint: disable=no-member + str(ds1_data.work), str(paths.work)) + return None + + +@pytest.fixture() +def ds1_repo_copy(runner, ds1_data, paths): + """Function scoped copy of ds1_data.repo""" + distutils.dir_util.copy_tree( # pylint: disable=no-member + str(ds1_data.repo), str(paths.repo)) + env = os.environ.copy() + env['GIT_DIR'] = str(paths.repo) + runner( + command=['git', 'config', 'core.worktree', str(paths.work)], + env=env, + report=False) + return None + + +@pytest.fixture() +def ds1_copy(ds1_work_copy, ds1_repo_copy): + """Function scoped copy of ds1_data""" + # pylint: disable=unused-argument + # This is ignored because + # @pytest.mark.usefixtures('ds1_work_copy', 'ds1_repo_copy') + # cannot be applied to another fixture. + return None + + +@pytest.fixture() +def ds1(ds1_work_copy, paths, ds1_dset): + """Function scoped ds1_dset w/paths""" + # pylint: disable=unused-argument + # This is ignored because + # @pytest.mark.usefixtures('ds1_copy') + # cannot be applied to another fixture. + dscopy = copy.deepcopy(ds1_dset) + dscopy.relative_to(copy.deepcopy(paths.work)) + return dscopy diff --git a/test/pylintrc b/test/pylintrc new file mode 120000 index 0000000..05334af --- /dev/null +++ b/test/pylintrc @@ -0,0 +1 @@ +../pylintrc \ No newline at end of file diff --git a/test/test_alt.py b/test/test_alt.py new file mode 100644 index 0000000..60b7339 --- /dev/null +++ b/test/test_alt.py @@ -0,0 +1,345 @@ +"""Test alt""" + +import os +import re +import string +import pytest +import utils + +# These test IDs are broken. During the writing of these tests, problems have +# been discovered in the way yadm orders matching files. +BROKEN_TEST_IDS = [ + 'test_wild[tracked-##C.S.H.U-C-S%-H%-U]', + 'test_wild[tracked-##C.S.H.U-C-S-H%-U]', + 'test_wild[encrypted-##C.S.H.U-C-S%-H%-U]', + 'test_wild[encrypted-##C.S.H.U-C-S-H%-U]', + ] + +PRECEDENCE = [ + '##', + '##$tst_sys', + '##$tst_sys.$tst_host', + '##$tst_sys.$tst_host.$tst_user', + '##$tst_class', + '##$tst_class.$tst_sys', + '##$tst_class.$tst_sys.$tst_host', + '##$tst_class.$tst_sys.$tst_host.$tst_user', + ] + +WILD_TEMPLATES = [ + '##$tst_class', + '##$tst_class.$tst_sys', + '##$tst_class.$tst_sys.$tst_host', + '##$tst_class.$tst_sys.$tst_host.$tst_user', + ] + +WILD_TESTED = set() + + +@pytest.mark.parametrize('precedence_index', range(len(PRECEDENCE))) +@pytest.mark.parametrize( + 'tracked, encrypt, exclude', [ + (False, False, False), + (True, False, False), + (False, True, False), + (False, True, True), + ], ids=[ + 'untracked', + 'tracked', + 'encrypted', + 'excluded', + ]) +@pytest.mark.usefixtures('ds1_copy') +def test_alt(runner, yadm_y, paths, + tst_sys, tst_host, tst_user, + tracked, encrypt, exclude, + precedence_index): + """Test alternate linking + + This test is done by iterating for the number of templates in PRECEDENCE. + With each iteration, another file is left off the list. So with each + iteration, the template with the "highest precedence" is left out. The file + using the highest precedence should be the one linked. + """ + + # set the class + tst_class = 'testclass' + utils.set_local(paths, 'class', tst_class) + + # process the templates in PRECEDENCE + precedence = list() + for template in PRECEDENCE: + precedence.append( + string.Template(template).substitute( + tst_class=tst_class, + tst_host=tst_host, + tst_sys=tst_sys, + tst_user=tst_user, + ) + ) + + # create files using a subset of files + for suffix in precedence[0:precedence_index+1]: + utils.create_alt_files(paths, suffix, tracked=tracked, + encrypt=encrypt, exclude=exclude) + + # run alt to trigger linking + run = runner(yadm_y('alt')) + assert run.success + assert run.err == '' + linked = linked_list(run.out) + + # assert the proper linking has occurred + for file_path in (utils.ALT_FILE1, utils.ALT_FILE2): + source_file = file_path + precedence[precedence_index] + if tracked or (encrypt and not exclude): + assert paths.work.join(file_path).islink() + assert paths.work.join(file_path).read() == source_file + assert str(paths.work.join(source_file)) in linked + else: + assert not paths.work.join(file_path).exists() + assert str(paths.work.join(source_file)) not in linked + + +def short_template(template): + """Translate template into something short for test IDs""" + return string.Template(template).substitute( + tst_class='C', + tst_host='H', + tst_sys='S', + tst_user='U', + ) + + +@pytest.mark.parametrize('wild_user', [True, False], ids=['U%', 'U']) +@pytest.mark.parametrize('wild_host', [True, False], ids=['H%', 'H']) +@pytest.mark.parametrize('wild_sys', [True, False], ids=['S%', 'S']) +@pytest.mark.parametrize('wild_class', [True, False], ids=['C%', 'C']) +@pytest.mark.parametrize('template', WILD_TEMPLATES, ids=short_template) +@pytest.mark.parametrize( + 'tracked, encrypt', [ + (True, False), + (False, True), + ], ids=[ + 'tracked', + 'encrypted', + ]) +@pytest.mark.usefixtures('ds1_copy') +def test_wild(request, runner, yadm_y, paths, + tst_sys, tst_host, tst_user, + tracked, encrypt, + wild_class, wild_host, wild_sys, wild_user, + template): + """Test wild linking + + These tests are done by creating permutations of the possible files using + WILD_TEMPLATES. Each case is then tested (while skipping the already tested + permutations for efficiency). + """ + + if request.node.name in BROKEN_TEST_IDS: + pytest.xfail( + 'This test is known to be broken. ' + 'This bug needs to be fixed.') + + tst_class = 'testclass' + + # determine the "wild" version of the suffix + str_class = '%' if wild_class else tst_class + str_host = '%' if wild_host else tst_host + str_sys = '%' if wild_sys else tst_sys + str_user = '%' if wild_user else tst_user + wild_suffix = string.Template(template).substitute( + tst_class=str_class, + tst_host=str_host, + tst_sys=str_sys, + tst_user=str_user, + ) + + # determine the "standard" version of the suffix + std_suffix = string.Template(template).substitute( + tst_class=tst_class, + tst_host=tst_host, + tst_sys=tst_sys, + tst_user=tst_user, + ) + + # skip over duplicate tests (this seems to be the simplest way to cover the + # permutations of tests, while skipping duplicates.) + test_key = f'{tracked}{encrypt}{wild_suffix}{std_suffix}' + if test_key in WILD_TESTED: + return + else: + WILD_TESTED.add(test_key) + + # set the class + utils.set_local(paths, 'class', tst_class) + + # create files using the wild suffix + utils.create_alt_files(paths, wild_suffix, tracked=tracked, + encrypt=encrypt, exclude=False) + + # run alt to trigger linking + run = runner(yadm_y('alt')) + assert run.success + assert run.err == '' + linked = linked_list(run.out) + + # assert the proper linking has occurred + for file_path in (utils.ALT_FILE1, utils.ALT_FILE2): + source_file = file_path + wild_suffix + assert paths.work.join(file_path).islink() + assert paths.work.join(file_path).read() == source_file + assert str(paths.work.join(source_file)) in linked + + # create files using the standard suffix + utils.create_alt_files(paths, std_suffix, tracked=tracked, + encrypt=encrypt, exclude=False) + + # run alt to trigger linking + run = runner(yadm_y('alt')) + assert run.success + assert run.err == '' + linked = linked_list(run.out) + + # assert the proper linking has occurred + for file_path in (utils.ALT_FILE1, utils.ALT_FILE2): + source_file = file_path + std_suffix + assert paths.work.join(file_path).islink() + assert paths.work.join(file_path).read() == source_file + assert str(paths.work.join(source_file)) in linked + + +@pytest.mark.usefixtures('ds1_copy') +def test_local_override(runner, yadm_y, paths, + tst_sys, tst_host, tst_user): + """Test local overrides""" + + # define local overrides + utils.set_local(paths, 'class', 'or-class') + utils.set_local(paths, 'hostname', 'or-hostname') + utils.set_local(paths, 'os', 'or-os') + utils.set_local(paths, 'user', 'or-user') + + # create files, the first would normally be the most specific version + # however, the second is the overridden version which should be preferred. + utils.create_alt_files( + paths, f'##or-class.{tst_sys}.{tst_host}.{tst_user}') + utils.create_alt_files( + paths, '##or-class.or-os.or-hostname.or-user') + + # run alt to trigger linking + run = runner(yadm_y('alt')) + assert run.success + assert run.err == '' + linked = linked_list(run.out) + + # assert the proper linking has occurred + for file_path in (utils.ALT_FILE1, utils.ALT_FILE2): + source_file = file_path + '##or-class.or-os.or-hostname.or-user' + assert paths.work.join(file_path).islink() + assert paths.work.join(file_path).read() == source_file + assert str(paths.work.join(source_file)) in linked + + +@pytest.mark.parametrize('suffix', ['AAA', 'ZZZ', 'aaa', 'zzz']) +@pytest.mark.usefixtures('ds1_copy') +def test_class_case(runner, yadm_y, paths, tst_sys, suffix): + """Test range of class cases""" + + # set the class + utils.set_local(paths, 'class', suffix) + + # create files + endings = [suffix] + if tst_sys == 'Linux': + # Only create all of these side-by-side on Linux, which is + # unquestionably case-sensitive. This would break tests on + # case-insensitive systems. + endings = ['AAA', 'ZZZ', 'aaa', 'zzz'] + for ending in endings: + utils.create_alt_files(paths, f'##{ending}') + + # run alt to trigger linking + run = runner(yadm_y('alt')) + assert run.success + assert run.err == '' + linked = linked_list(run.out) + + # assert the proper linking has occurred + for file_path in (utils.ALT_FILE1, utils.ALT_FILE2): + source_file = file_path + f'##{suffix}' + assert paths.work.join(file_path).islink() + assert paths.work.join(file_path).read() == source_file + assert str(paths.work.join(source_file)) in linked + + +@pytest.mark.parametrize('autoalt', [None, 'true', 'false']) +@pytest.mark.usefixtures('ds1_copy') +def test_auto_alt(runner, yadm_y, paths, autoalt): + """Test setting auto-alt""" + + # set the value of auto-alt + if autoalt: + os.system(' '.join(yadm_y('config', 'yadm.auto-alt', autoalt))) + + # create file + suffix = '##' + utils.create_alt_files(paths, suffix) + + # run status to possibly trigger linking + run = runner(yadm_y('status')) + assert run.success + assert run.err == '' + linked = linked_list(run.out) + + # assert the proper linking has occurred + for file_path in (utils.ALT_FILE1, utils.ALT_FILE2): + source_file = file_path + suffix + if autoalt == 'false': + assert not paths.work.join(file_path).exists() + else: + assert paths.work.join(file_path).islink() + assert paths.work.join(file_path).read() == source_file + # no linking output when run via auto-alt + assert str(paths.work.join(source_file)) not in linked + + +@pytest.mark.parametrize('delimiter', ['.', '_']) +@pytest.mark.usefixtures('ds1_copy') +def test_delimiter(runner, yadm_y, paths, + tst_sys, tst_host, tst_user, delimiter): + """Test delimiters used""" + + suffix = '##' + delimiter.join([tst_sys, tst_host, tst_user]) + + # create file + utils.create_alt_files(paths, suffix) + + # run alt to trigger linking + run = runner(yadm_y('alt')) + assert run.success + assert run.err == '' + linked = linked_list(run.out) + + # assert the proper linking has occurred + # only a delimiter of '.' is valid + for file_path in (utils.ALT_FILE1, utils.ALT_FILE2): + source_file = file_path + suffix + if delimiter == '.': + assert paths.work.join(file_path).islink() + assert paths.work.join(file_path).read() == source_file + assert str(paths.work.join(source_file)) in linked + else: + assert not paths.work.join(file_path).exists() + assert str(paths.work.join(source_file)) not in linked + + +def linked_list(output): + """Parse output, and return list of linked files""" + linked = dict() + for line in output.splitlines(): + match = re.match('Linking (.+) to (.+)$', line) + if match: + linked[match.group(2)] = match.group(1) + return linked.values() diff --git a/test/test_assert_private_dirs.py b/test/test_assert_private_dirs.py new file mode 100644 index 0000000..65cb0b7 --- /dev/null +++ b/test/test_assert_private_dirs.py @@ -0,0 +1,106 @@ +"""Test asserting private directories""" + +import os +import re +import pytest + +pytestmark = pytest.mark.usefixtures('ds1_copy') +PRIVATE_DIRS = ['.gnupg', '.ssh'] + + +def test_pdirs_missing(runner, yadm_y, paths): + """Private dirs (private dirs missing) + + When a git command is run + And private directories are missing + Create private directories prior to command + """ + + # confirm directories are missing at start + for pdir in PRIVATE_DIRS: + path = paths.work.join(pdir) + if path.exists(): + path.remove() + assert not path.exists() + + # run status + run = runner(command=yadm_y('status'), env={'DEBUG': 'yes'}) + assert run.success + assert run.err == '' + assert 'On branch master' in run.out + + # confirm directories are created + # and are protected + for pdir in PRIVATE_DIRS: + path = paths.work.join(pdir) + assert path.exists() + assert oct(path.stat().mode).endswith('00'), 'Directory is not secured' + + # confirm directories are created before command is run: + assert re.search( + r'Creating.+\.gnupg.+Creating.+\.ssh.+Running git command git status', + run.out, re.DOTALL), 'directories created before command is run' + + +def test_pdirs_missing_apd_false(runner, yadm_y, paths): + """Private dirs (private dirs missing / yadm.auto-private-dirs=false) + + When a git command is run + And private directories are missing + But auto-private-dirs is false + Do not create private dirs + """ + + # confirm directories are missing at start + for pdir in PRIVATE_DIRS: + path = paths.work.join(pdir) + if path.exists(): + path.remove() + assert not path.exists() + + # set configuration + os.system(' '.join(yadm_y( + 'config', '--bool', 'yadm.auto-private-dirs', 'false'))) + + # run status + run = runner(command=yadm_y('status')) + assert run.success + assert run.err == '' + assert 'On branch master' in run.out + + # confirm directories are STILL missing + for pdir in PRIVATE_DIRS: + assert not paths.work.join(pdir).exists() + + +def test_pdirs_exist_apd_false(runner, yadm_y, paths): + """Private dirs (private dirs exist / yadm.auto-perms=false) + + When a git command is run + And private directories exist + And yadm is configured not to auto update perms + Do not alter directories + """ + + # create permissive directories + for pdir in PRIVATE_DIRS: + path = paths.work.join(pdir) + if not path.isdir(): + path.mkdir() + path.chmod(0o777) + assert oct(path.stat().mode).endswith('77'), 'Directory is secure.' + + # set configuration + os.system(' '.join(yadm_y( + 'config', '--bool', 'yadm.auto-perms', 'false'))) + + # run status + run = runner(command=yadm_y('status')) + assert run.success + assert run.err == '' + assert 'On branch master' in run.out + + # created directories are STILL permissive + for pdir in PRIVATE_DIRS: + path = paths.work.join(pdir) + assert oct(path.stat().mode).endswith('77'), 'Directory is secure' diff --git a/test/test_bootstrap.py b/test/test_bootstrap.py new file mode 100644 index 0000000..2adbe33 --- /dev/null +++ b/test/test_bootstrap.py @@ -0,0 +1,31 @@ +"""Test bootstrap""" + +import pytest + + +@pytest.mark.parametrize( + 'exists, executable, code, expect', [ + (False, False, 1, 'Cannot execute bootstrap'), + (True, False, 1, 'is not an executable program'), + (True, True, 123, 'Bootstrap successful'), + ], ids=[ + 'missing', + 'not executable', + 'executable', + ]) +def test_bootstrap( + runner, yadm_y, paths, exists, executable, code, expect): + """Test bootstrap command""" + if exists: + paths.bootstrap.write('') + if executable: + paths.bootstrap.write( + '#!/bin/bash\n' + f'echo {expect}\n' + f'exit {code}\n' + ) + paths.bootstrap.chmod(0o775) + run = runner(command=yadm_y('bootstrap')) + assert run.code == code + assert run.err == '' + assert expect in run.out diff --git a/test/test_clean.py b/test/test_clean.py new file mode 100644 index 0000000..9a2221a --- /dev/null +++ b/test/test_clean.py @@ -0,0 +1,11 @@ +"""Test clean""" + + +def test_clean_command(runner, yadm_y): + """Run with clean command""" + run = runner(command=yadm_y('clean')) + # do nothing, this is a dangerous Git command when managing dot files + # report the command as disabled and exit as a failure + assert run.failure + assert run.err == '' + assert 'disabled' in run.out diff --git a/test/test_clone.py b/test/test_clone.py new file mode 100644 index 0000000..90febe1 --- /dev/null +++ b/test/test_clone.py @@ -0,0 +1,274 @@ +"""Test clone""" + +import os +import re +import pytest + +BOOTSTRAP_CODE = 123 +BOOTSTRAP_MSG = 'Bootstrap successful' + + +@pytest.mark.usefixtures('remote') +@pytest.mark.parametrize( + 'good_remote, repo_exists, force, conflicts', [ + (False, False, False, False), + (True, False, False, False), + (True, True, False, False), + (True, True, True, False), + (True, False, False, True), + ], ids=[ + 'bad remote', + 'simple', + 'existing repo', + '-f', + 'conflicts', + ]) +def test_clone( + runner, paths, yadm_y, repo_config, ds1, + good_remote, repo_exists, force, conflicts): + """Test basic clone operation""" + + # determine remote url + remote_url = f'file://{paths.remote}' + if not good_remote: + remote_url = 'file://bad_remote' + + old_repo = None + if repo_exists: + # put a repo in the way + paths.repo.mkdir() + old_repo = paths.repo.join('old_repo') + old_repo.write('old_repo') + + if conflicts: + ds1.tracked[0].relative.write('conflict') + assert ds1.tracked[0].relative.exists() + + # run the clone command + args = ['clone', '-w', paths.work] + if force: + args += ['-f'] + args += [remote_url] + run = runner(command=yadm_y(*args)) + + if not good_remote: + # clone should fail + assert run.failure + assert run.err != '' + assert 'Unable to fetch origin' in run.out + assert not paths.repo.exists() + elif repo_exists and not force: + # can't overwrite data + assert run.failure + assert run.err == '' + assert 'Git repo already exists' in run.out + else: + # clone should succeed, and repo should be configured properly + assert successful_clone(run, paths, repo_config) + + # ensure conflicts are handled properly + if conflicts: + assert 'NOTE' in run.out + assert 'Merging origin/master failed' in run.out + assert 'Conflicts preserved' in run.out + + # confirm correct Git origin + run = runner( + command=('git', 'remote', '-v', 'show'), + env={'GIT_DIR': paths.repo}) + assert run.success + assert run.err == '' + assert f'origin\t{remote_url}' in run.out + + # ensure conflicts are really preserved + if conflicts: + # test to see if the work tree is actually "clean" + run = runner( + command=yadm_y('status', '-uno', '--porcelain'), + cwd=paths.work) + assert run.success + assert run.err == '' + assert run.out == '', 'worktree has unexpected changes' + + # test to see if the conflicts are stashed + run = runner(command=yadm_y('stash', 'list'), cwd=paths.work) + assert run.success + assert run.err == '' + assert 'Conflicts preserved' in run.out, 'conflicts not stashed' + + # verify content of the stashed conflicts + run = runner(command=yadm_y('stash', 'show', '-p'), cwd=paths.work) + assert run.success + assert run.err == '' + assert '\n+conflict' in run.out, 'conflicts not stashed' + + # another force-related assertion + if old_repo: + if force: + assert not old_repo.exists() + else: + assert old_repo.exists() + + +@pytest.mark.usefixtures('remote') +@pytest.mark.parametrize( + 'bs_exists, bs_param, answer', [ + (False, '--bootstrap', None), + (True, '--bootstrap', None), + (True, '--no-bootstrap', None), + (True, None, 'n'), + (True, None, 'y'), + ], ids=[ + 'force, missing', + 'force, existing', + 'prevent', + 'existing, answer n', + 'existing, answer y', + ]) +def test_clone_bootstrap( + runner, paths, yadm_y, repo_config, bs_exists, bs_param, answer): + """Test bootstrap clone features""" + + # establish a bootstrap + create_bootstrap(paths, bs_exists) + + # run the clone command + args = ['clone', '-w', paths.work] + if bs_param: + args += [bs_param] + args += [f'file://{paths.remote}'] + expect = [] + if answer: + expect.append(('Would you like to execute it now', answer)) + run = runner(command=yadm_y(*args), expect=expect) + + if answer: + assert 'Would you like to execute it now' in run.out + + expected_code = 0 + if bs_exists and bs_param != '--no-bootstrap': + expected_code = BOOTSTRAP_CODE + + if answer == 'y': + expected_code = BOOTSTRAP_CODE + assert BOOTSTRAP_MSG in run.out + elif answer == 'n': + expected_code = 0 + assert BOOTSTRAP_MSG not in run.out + + assert successful_clone(run, paths, repo_config, expected_code) + + if not bs_exists: + assert BOOTSTRAP_MSG not in run.out + + +def create_bootstrap(paths, exists): + """Create bootstrap file for test""" + if exists: + paths.bootstrap.write( + '#!/bin/sh\n' + f'echo {BOOTSTRAP_MSG}\n' + f'exit {BOOTSTRAP_CODE}\n') + paths.bootstrap.chmod(0o775) + assert paths.bootstrap.exists() + else: + assert not paths.bootstrap.exists() + + +@pytest.mark.usefixtures('remote') +@pytest.mark.parametrize( + 'private_type, in_repo, in_work', [ + ('ssh', False, True), + ('gnupg', False, True), + ('ssh', True, True), + ('gnupg', True, True), + ('ssh', True, False), + ('gnupg', True, False), + ], ids=[ + 'open ssh, not tracked', + 'open gnupg, not tracked', + 'open ssh, tracked', + 'open gnupg, tracked', + 'missing ssh, tracked', + 'missing gnupg, tracked', + ]) +def test_clone_perms( + runner, yadm_y, paths, repo_config, + private_type, in_repo, in_work): + """Test clone permission-related functions""" + + # update remote repo to include private data + if in_repo: + rpath = paths.work.mkdir(f'.{private_type}').join('related') + rpath.write('related') + os.system(f'GIT_DIR="{paths.remote}" git add {rpath}') + os.system(f'GIT_DIR="{paths.remote}" git commit -m "{rpath}"') + rpath.remove() + + # ensure local private data is insecure at the start + if in_work: + pdir = paths.work.join(f'.{private_type}') + if not pdir.exists(): + pdir.mkdir() + pfile = pdir.join('existing') + pfile.write('existing') + pdir.chmod(0o777) + pfile.chmod(0o777) + else: + paths.work.remove() + paths.work.mkdir() + + run = runner( + yadm_y('clone', '-d', '-w', paths.work, f'file://{paths.remote}')) + + assert successful_clone(run, paths, repo_config) + if in_work: + # private directories which already exist, should be left as they are, + # which in this test is "insecure". + assert re.search( + f'initial private dir perms drwxrwxrwx.+.{private_type}', + run.out) + assert re.search( + f'pre-merge private dir perms drwxrwxrwx.+.{private_type}', + run.out) + assert re.search( + f'post-merge private dir perms drwxrwxrwx.+.{private_type}', + run.out) + else: + # private directories which are created, should be done prior to + # merging, and with secure permissions. + assert 'initial private dir perms' not in run.out + assert re.search( + f'pre-merge private dir perms drwx------.+.{private_type}', + run.out) + assert re.search( + f'post-merge private dir perms drwx------.+.{private_type}', + run.out) + + # standard perms still apply afterwards unless disabled with auto.perms + assert oct( + paths.work.join(f'.{private_type}').stat().mode).endswith('00'), ( + f'.{private_type} has not been secured by auto.perms') + + +def successful_clone(run, paths, repo_config, expected_code=0): + """Assert clone is successful""" + assert run.code == expected_code + assert 'Initialized' in run.out + assert oct(paths.repo.stat().mode).endswith('00'), 'Repo is not secured' + assert repo_config('core.bare') == 'false' + assert repo_config('status.showUntrackedFiles') == 'no' + assert repo_config('yadm.managed') == 'true' + return True + + +@pytest.fixture() +def remote(paths, ds1_repo_copy): + """Function scoped remote (based on ds1)""" + # pylint: disable=unused-argument + # This is ignored because + # @pytest.mark.usefixtures('ds1_remote_copy') + # cannot be applied to another fixture. + paths.remote.remove() + paths.repo.move(paths.remote) + return None diff --git a/test/test_config.py b/test/test_config.py new file mode 100644 index 0000000..4e44b1c --- /dev/null +++ b/test/test_config.py @@ -0,0 +1,139 @@ +"""Test config""" + +import os +import pytest + +TEST_SECTION = 'test' +TEST_ATTRIBUTE = 'attribute' +TEST_KEY = f'{TEST_SECTION}.{TEST_ATTRIBUTE}' +TEST_VALUE = 'testvalue' +TEST_FILE = f'[{TEST_SECTION}]\n\t{TEST_ATTRIBUTE} = {TEST_VALUE}' + + +def test_config_no_params(runner, yadm_y, supported_configs): + """No parameters + + Display instructions + Display supported configs + Exit with 0 + """ + + run = runner(yadm_y('config')) + + assert run.success + assert run.err == '' + assert 'Please read the CONFIGURATION section' in run.out + for config in supported_configs: + assert config in run.out + + +def test_config_read_missing(runner, yadm_y): + """Read missing attribute + + Display an empty value + Exit with 0 + """ + + run = runner(yadm_y('config', TEST_KEY)) + + assert run.success + assert run.err == '' + assert run.out == '' + + +def test_config_write(runner, yadm_y, paths): + """Write attribute + + Display no output + Update configuration file + Exit with 0 + """ + + run = runner(yadm_y('config', TEST_KEY, TEST_VALUE)) + + assert run.success + assert run.err == '' + assert run.out == '' + assert paths.config.read().strip() == TEST_FILE + + +def test_config_read(runner, yadm_y, paths): + """Read attribute + + Display value + Exit with 0 + """ + + paths.config.write(TEST_FILE) + run = runner(yadm_y('config', TEST_KEY)) + + assert run.success + assert run.err == '' + assert run.out.strip() == TEST_VALUE + + +def test_config_update(runner, yadm_y, paths): + """Update attribute + + Display no output + Update configuration file + Exit with 0 + """ + + paths.config.write(TEST_FILE) + + run = runner(yadm_y('config', TEST_KEY, TEST_VALUE + 'extra')) + + assert run.success + assert run.err == '' + assert run.out == '' + + assert paths.config.read().strip() == TEST_FILE + 'extra' + + +@pytest.mark.usefixtures('ds1_repo_copy') +def test_config_local_read(runner, yadm_y, paths, supported_local_configs): + """Read local attribute + + Display value from the repo config + Exit with 0 + """ + + # populate test values + for config in supported_local_configs: + os.system( + f'GIT_DIR="{paths.repo}" ' + f'git config --local "{config}" "value_of_{config}"') + + # run yadm config + for config in supported_local_configs: + run = runner(yadm_y('config', config)) + assert run.success + assert run.err == '' + assert run.out.strip() == f'value_of_{config}' + + +@pytest.mark.usefixtures('ds1_repo_copy') +def test_config_local_write(runner, yadm_y, paths, supported_local_configs): + """Write local attribute + + Display no output + Write value to the repo config + Exit with 0 + """ + + # run yadm config + for config in supported_local_configs: + run = runner(yadm_y('config', config, f'value_of_{config}')) + assert run.success + assert run.err == '' + assert run.out == '' + + # verify test values + for config in supported_local_configs: + run = runner( + command=('git', 'config', config), + env={'GIT_DIR': paths.repo}) + assert run.success + assert run.err == '' + assert run.out.strip() == f'value_of_{config}' diff --git a/test/test_cygwin_copy.py b/test/test_cygwin_copy.py new file mode 100644 index 0000000..82b08ba --- /dev/null +++ b/test/test_cygwin_copy.py @@ -0,0 +1,59 @@ +"""Test yadm.cygwin_copy""" + +import os +import pytest + + +@pytest.mark.parametrize( + 'setting, is_cygwin, expect_link, pre_existing', [ + (None, False, True, None), + (True, False, True, None), + (False, False, True, None), + (None, True, True, None), + (True, True, False, None), + (False, True, True, None), + (True, True, False, 'link'), + (True, True, False, 'file'), + ], + ids=[ + 'unset, non-cygwin', + 'true, non-cygwin', + 'false, non-cygwin', + 'unset, cygwin', + 'true, cygwin', + 'false, cygwin', + 'pre-existing symlink', + 'pre-existing file', + ]) +@pytest.mark.usefixtures('ds1_copy') +def test_cygwin_copy( + runner, yadm_y, paths, cygwin_sys, tst_sys, + setting, is_cygwin, expect_link, pre_existing): + """Test yadm.cygwin_copy""" + + if setting is not None: + os.system(' '.join(yadm_y('config', 'yadm.cygwin-copy', str(setting)))) + + expected_content = f'test_cygwin_copy##{tst_sys}' + alt_path = paths.work.join('test_cygwin_copy') + if pre_existing == 'symlink': + alt_path.mklinkto(expected_content) + elif pre_existing == 'file': + alt_path.write('wrong content') + + uname_path = paths.root.join('tmp').mkdir() + if is_cygwin: + uname = uname_path.join('uname') + uname.write(f'#!/bin/sh\necho "{cygwin_sys}"\n') + uname.chmod(0o777) + expected_content = f'test_cygwin_copy##{cygwin_sys}' + env = os.environ.copy() + env['PATH'] = ':'.join([str(uname_path), env['PATH']]) + + run = runner(yadm_y('alt'), env=env) + assert run.success + assert run.err == '' + assert 'Linking' in run.out + + assert alt_path.read() == expected_content + assert alt_path.islink() == expect_link diff --git a/test/test_encryption.py b/test/test_encryption.py new file mode 100644 index 0000000..f107ad5 --- /dev/null +++ b/test/test_encryption.py @@ -0,0 +1,392 @@ +"""Test encryption""" + +import os +import pipes +import pytest + +KEY_FILE = 'test/test_key' +KEY_FINGERPRINT = 'F8BBFC746C58945442349BCEBA54FFD04C599B1A' +KEY_NAME = 'yadm-test1' +KEY_TRUST = 'test/ownertrust.txt' +PASSPHRASE = 'ExamplePassword' + +pytestmark = pytest.mark.usefixtures('config_git') + + +def add_asymmetric_key(): + """Add asymmetric key""" + os.system(f'gpg --import {pipes.quote(KEY_FILE)}') + os.system(f'gpg --import-ownertrust < {pipes.quote(KEY_TRUST)}') + + +def remove_asymmetric_key(): + """Remove asymmetric key""" + os.system( + f'gpg --batch --yes ' + f'--delete-secret-keys {pipes.quote(KEY_FINGERPRINT)}') + os.system(f'gpg --batch --yes --delete-key {pipes.quote(KEY_FINGERPRINT)}') + + +@pytest.fixture +def asymmetric_key(): + """Fixture for asymmetric key, removed in teardown""" + add_asymmetric_key() + yield KEY_NAME + remove_asymmetric_key() + + +@pytest.fixture +def encrypt_targets(yadm_y, paths): + """Fixture for setting up data to encrypt + + This fixture: + * inits an empty repo + * creates test files in the work tree + * creates a ".yadm/encrypt" file for testing: + * standard files + * standard globs + * directories + * comments + * empty lines and lines with just space + * exclusions + * returns a list of expected encrypted files + """ + + # init empty yadm repo + os.system(' '.join(yadm_y('init', '-w', str(paths.work), '-f'))) + + expected = [] + + # standard files w/ dirs & spaces + paths.work.join('inc file1').write('inc file1') + expected.append('inc file1') + paths.encrypt.write('inc file1\n') + paths.work.join('inc dir').mkdir() + paths.work.join('inc dir/inc file2').write('inc file2') + expected.append('inc dir/inc file2') + paths.encrypt.write('inc dir/inc file2\n', mode='a') + + # standard globs w/ dirs & spaces + paths.work.join('globs file1').write('globs file1') + expected.append('globs file1') + paths.work.join('globs dir').mkdir() + paths.work.join('globs dir/globs file2').write('globs file2') + expected.append('globs dir/globs file2') + paths.encrypt.write('globs*\n', mode='a') + + # blank lines + paths.encrypt.write('\n \n\t\n', mode='a') + + # comments + paths.work.join('commentfile1').write('commentfile1') + paths.encrypt.write('#commentfile1\n', mode='a') + paths.encrypt.write(' #commentfile1\n', mode='a') + + # exclusions + paths.work.join('extest').mkdir() + paths.encrypt.write('extest/*\n', mode='a') # include within extest + paths.work.join('extest/inglob1').write('inglob1') + paths.work.join('extest/exglob1').write('exglob1') + paths.work.join('extest/exglob2').write('exglob2') + paths.encrypt.write('!extest/ex*\n', mode='a') # exclude the ex* + expected.append('extest/inglob1') # should be left with only in* + + return expected + + +@pytest.fixture(scope='session') +def decrypt_targets(tmpdir_factory, runner): + """Fixture for setting data to decrypt + + This fixture: + * creates symmetric/asymmetric encrypted archives + * creates a list of expected decrypted files + """ + + tmpdir = tmpdir_factory.mktemp('decrypt_targets') + symmetric = tmpdir.join('symmetric.tar.gz.gpg') + asymmetric = tmpdir.join('asymmetric.tar.gz.gpg') + + expected = [] + + tmpdir.join('decrypt1').write('decrypt1') + expected.append('decrypt1') + tmpdir.join('decrypt2').write('decrypt2') + expected.append('decrypt2') + tmpdir.join('subdir').mkdir() + tmpdir.join('subdir/decrypt3').write('subdir/decrypt3') + expected.append('subdir/decrypt3') + + run = runner( + ['tar', 'cvf', '-'] + + expected + + ['|', 'gpg', '--batch', '--yes', '-c'] + + ['--passphrase', pipes.quote(PASSPHRASE)] + + ['--output', pipes.quote(str(symmetric))], + cwd=tmpdir, + shell=True) + assert run.success + + add_asymmetric_key() + run = runner( + ['tar', 'cvf', '-'] + + expected + + ['|', 'gpg', '--batch', '--yes', '-e'] + + ['-r', pipes.quote(KEY_NAME)] + + ['--output', pipes.quote(str(asymmetric))], + cwd=tmpdir, + shell=True) + assert run.success + remove_asymmetric_key() + + return { + 'asymmetric': asymmetric, + 'expected': expected, + 'symmetric': symmetric, + } + + +@pytest.mark.parametrize( + 'mismatched_phrase', [False, True], + ids=['matching_phrase', 'mismatched_phrase']) +@pytest.mark.parametrize( + 'missing_encrypt', [False, True], + ids=['encrypt_exists', 'encrypt_missing']) +@pytest.mark.parametrize( + 'overwrite', [False, True], + ids=['clean', 'overwrite']) +def test_symmetric_encrypt( + runner, yadm_y, paths, encrypt_targets, + overwrite, missing_encrypt, mismatched_phrase): + """Test symmetric encryption""" + + if missing_encrypt: + paths.encrypt.remove() + + matched_phrase = PASSPHRASE + if mismatched_phrase: + matched_phrase = 'mismatched' + + if overwrite: + paths.archive.write('existing archive') + + run = runner(yadm_y('encrypt'), expect=[ + ('passphrase:', PASSPHRASE), + ('passphrase:', matched_phrase), + ]) + + if missing_encrypt or mismatched_phrase: + assert run.failure + else: + assert run.success + assert run.err == '' + + if missing_encrypt: + assert 'does not exist' in run.out + elif mismatched_phrase: + assert 'invalid passphrase' in run.out + else: + assert encrypted_data_valid(runner, paths.archive, encrypt_targets) + + +@pytest.mark.parametrize( + 'wrong_phrase', [False, True], + ids=['correct_phrase', 'wrong_phrase']) +@pytest.mark.parametrize( + 'archive_exists', [True, False], + ids=['archive_exists', 'archive_missing']) +@pytest.mark.parametrize( + 'dolist', [False, True], + ids=['decrypt', 'list']) +def test_symmetric_decrypt( + runner, yadm_y, paths, decrypt_targets, + dolist, archive_exists, wrong_phrase): + """Test decryption""" + + # init empty yadm repo + os.system(' '.join(yadm_y('init', '-w', str(paths.work), '-f'))) + + phrase = PASSPHRASE + if wrong_phrase: + phrase = 'wrong-phrase' + + if archive_exists: + decrypt_targets['symmetric'].copy(paths.archive) + + # to test overwriting + paths.work.join('decrypt1').write('pre-existing file') + + args = [] + + if dolist: + args.append('-l') + run = runner(yadm_y('decrypt') + args, expect=[('passphrase:', phrase)]) + + if archive_exists and not wrong_phrase: + assert run.success + assert run.err == '' + if dolist: + for filename in decrypt_targets['expected']: + if filename != 'decrypt1': # this one should exist + assert not paths.work.join(filename).exists() + assert filename in run.out + else: + for filename in decrypt_targets['expected']: + assert paths.work.join(filename).read() == filename + else: + assert run.failure + + +@pytest.mark.usefixtures('asymmetric_key') +@pytest.mark.parametrize( + 'ask', [False, True], + ids=['no_ask', 'ask']) +@pytest.mark.parametrize( + 'key_exists', [True, False], + ids=['key_exists', 'key_missing']) +@pytest.mark.parametrize( + 'overwrite', [False, True], + ids=['clean', 'overwrite']) +def test_asymmetric_encrypt( + runner, yadm_y, paths, encrypt_targets, + overwrite, key_exists, ask): + """Test asymmetric encryption""" + + # specify encryption recipient + if ask: + os.system(' '.join(yadm_y('config', 'yadm.gpg-recipient', 'ASK'))) + expect = [('Enter the user ID', KEY_NAME), ('Enter the user ID', '')] + else: + os.system(' '.join(yadm_y('config', 'yadm.gpg-recipient', KEY_NAME))) + expect = [] + + if overwrite: + paths.archive.write('existing archive') + + if not key_exists: + remove_asymmetric_key() + + run = runner(yadm_y('encrypt'), expect=expect) + + if key_exists: + assert run.success + assert encrypted_data_valid(runner, paths.archive, encrypt_targets) + else: + assert run.failure + assert 'Unable to write' in run.out + + if ask: + assert 'Enter the user ID' in run.out + + +@pytest.mark.usefixtures('asymmetric_key') +@pytest.mark.parametrize( + 'key_exists', [True, False], + ids=['key_exists', 'key_missing']) +@pytest.mark.parametrize( + 'dolist', [False, True], + ids=['decrypt', 'list']) +def test_asymmetric_decrypt( + runner, yadm_y, paths, decrypt_targets, + dolist, key_exists): + """Test decryption""" + + # init empty yadm repo + os.system(' '.join(yadm_y('init', '-w', str(paths.work), '-f'))) + + decrypt_targets['asymmetric'].copy(paths.archive) + + # to test overwriting + paths.work.join('decrypt1').write('pre-existing file') + + if not key_exists: + remove_asymmetric_key() + + args = [] + + if dolist: + args.append('-l') + run = runner(yadm_y('decrypt') + args) + + if key_exists: + assert run.success + if dolist: + for filename in decrypt_targets['expected']: + if filename != 'decrypt1': # this one should exist + assert not paths.work.join(filename).exists() + assert filename in run.out + else: + for filename in decrypt_targets['expected']: + assert paths.work.join(filename).read() == filename + else: + assert run.failure + assert 'Unable to extract encrypted files' in run.out + + +@pytest.mark.parametrize( + 'untracked', + [False, 'y', 'n'], + ids=['tracked', 'untracked_answer_y', 'untracked_answer_n']) +def test_offer_to_add(runner, yadm_y, paths, encrypt_targets, untracked): + """Test offer to add encrypted archive + + All the other encryption tests use an archive outside of the work tree. + However, the archive is often inside the work tree, and if it is, there + should be an offer to add it to the repo if it is not tracked. + """ + + worktree_archive = paths.work.join('worktree-archive.tar.gpg') + expect = [ + ('passphrase:', PASSPHRASE), + ('passphrase:', PASSPHRASE), + ] + + if untracked: + expect.append(('add it now', untracked)) + else: + worktree_archive.write('exists') + os.system(' '.join(yadm_y('add', str(worktree_archive)))) + + run = runner( + yadm_y('encrypt', '--yadm-archive', str(worktree_archive)), + expect=expect + ) + + assert run.success + assert run.err == '' + assert encrypted_data_valid(runner, worktree_archive, encrypt_targets) + + run = runner( + yadm_y('status', '--porcelain', '-uall', str(worktree_archive))) + assert run.success + assert run.err == '' + + if untracked == 'y': + # should be added to the index + assert f'A {worktree_archive.basename}' in run.out + elif untracked == 'n': + # should NOT be added to the index + assert f'?? {worktree_archive.basename}' in run.out + else: + # should appear modified in the index + assert f'AM {worktree_archive.basename}' in run.out + + +def encrypted_data_valid(runner, encrypted, expected): + """Verify encrypted data matches expectations""" + run = runner([ + 'gpg', + '--passphrase', pipes.quote(PASSPHRASE), + '-d', pipes.quote(str(encrypted)), + '2>/dev/null', + '|', 'tar', 't'], shell=True, report=False) + file_count = 0 + for filename in run.out.splitlines(): + if filename.endswith('/'): + continue + file_count += 1 + assert filename in expected, ( + f'Unexpected file in archive: {filename}') + assert file_count == len(expected), ( + 'Number of files in archive does not match expected') + return True diff --git a/test/test_enter.py b/test/test_enter.py new file mode 100644 index 0000000..202df32 --- /dev/null +++ b/test/test_enter.py @@ -0,0 +1,85 @@ +"""Test enter""" + +import os +import warnings +import pytest + + +@pytest.mark.parametrize( + 'shell, success', [ + ('delete', True), + ('', False), + ('/usr/bin/env', True), + ('noexec', False), + ], ids=[ + 'shell-missing', + 'shell-empty', + 'shell-env', + 'shell-noexec', + ]) +@pytest.mark.usefixtures('ds1_copy') +def test_enter(runner, yadm_y, paths, shell, success): + """Enter tests""" + env = os.environ.copy() + if shell == 'delete': + # remove shell + if 'SHELL' in env: + del env['SHELL'] + elif shell == 'noexec': + # specify a non-executable path + noexec = paths.root.join('noexec') + noexec.write('') + noexec.chmod(0o664) + env['SHELL'] = str(noexec) + else: + env['SHELL'] = shell + run = runner(command=yadm_y('enter'), env=env) + assert run.success == success + assert run.err == '' + prompt = f'yadm shell ({paths.repo})' + if success: + assert run.out.startswith('Entering yadm repo') + assert run.out.rstrip().endswith('Leaving yadm repo') + if shell == 'delete': + # When SHELL is empty (unlikely), it is attempted to be run anyway. + # This is a but which must be fixed. + warnings.warn('Unhandled bug: SHELL executed when empty', Warning) + else: + assert f'PROMPT={prompt}' in run.out + assert f'PS1={prompt}' in run.out + assert f'GIT_DIR={paths.repo}' in run.out + if not success: + assert 'does not refer to an executable' in run.out + if 'env' in shell: + assert f'GIT_DIR={paths.repo}' in run.out + assert 'PROMPT=yadm shell' in run.out + assert 'PS1=yadm shell' in run.out + + +@pytest.mark.parametrize( + 'shell, opts, path', [ + ('bash', '--norc', '\\w'), + ('csh', '-f', '%~'), + ('zsh', '-f', '%~'), + ], ids=[ + 'bash', + 'csh', + 'zsh', + ]) +@pytest.mark.usefixtures('ds1_copy') +def test_enter_shell_ops(runner, yadm_y, paths, shell, opts, path): + """Enter tests for specific shell options""" + + # Create custom shell to detect options passed + custom_shell = paths.root.join(shell) + custom_shell.write('#!/bin/sh\necho OPTS=$*\necho PROMPT=$PROMPT') + custom_shell.chmod(0o775) + + env = os.environ.copy() + env['SHELL'] = custom_shell + + run = runner(command=yadm_y('enter'), env=env) + assert run.success + assert run.err == '' + assert f'OPTS={opts}' in run.out + assert f'PROMPT=yadm shell ({paths.repo}) {path} >' in run.out diff --git a/test/test_git.py b/test/test_git.py new file mode 100644 index 0000000..427c54a --- /dev/null +++ b/test/test_git.py @@ -0,0 +1,58 @@ +"""Test git""" + +import re +import pytest + + +@pytest.mark.usefixtures('ds1_copy') +def test_git(runner, yadm_y, paths): + """Test series of passthrough git commands + + Passthru unknown commands to Git + Git command 'add' - badfile + Git command 'add' + Git command 'status' + Git command 'commit' + Git command 'log' + """ + + # passthru unknown commands to Git + run = runner(command=yadm_y('bogus')) + assert run.failure + assert "git: 'bogus' is not a git command." in run.err + assert "See 'git --help'" in run.err + assert run.out == '' + + # git command 'add' - badfile + run = runner(command=yadm_y('add', '-v', 'does_not_exist')) + assert run.code == 128 + assert "pathspec 'does_not_exist' did not match any files" in run.err + assert run.out == '' + + # git command 'add' + newfile = paths.work.join('test_git') + newfile.write('test_git') + run = runner(command=yadm_y('add', '-v', str(newfile))) + assert run.success + assert run.err == '' + assert "add 'test_git'" in run.out + + # git command 'status' + run = runner(command=yadm_y('status')) + assert run.success + assert run.err == '' + assert re.search(r'new file:\s+test_git', run.out) + + # git command 'commit' + run = runner(command=yadm_y('commit', '-m', 'Add test_git')) + assert run.success + assert run.err == '' + assert '1 file changed' in run.out + assert '1 insertion' in run.out + assert re.search(r'create mode .+ test_git', run.out) + + # git command 'log' + run = runner(command=yadm_y('log', '--oneline')) + assert run.success + assert run.err == '' + assert 'Add test_git' in run.out diff --git a/test/test_help.py b/test/test_help.py new file mode 100644 index 0000000..79a7652 --- /dev/null +++ b/test/test_help.py @@ -0,0 +1,17 @@ +"""Test help""" + + +def test_missing_command(runner, yadm_y): + """Run without any command""" + run = runner(command=yadm_y()) + assert run.failure + assert run.err == '' + assert run.out.startswith('Usage: yadm') + + +def test_help_command(runner, yadm_y): + """Run with help command""" + run = runner(command=yadm_y('help')) + assert run.failure + assert run.err == '' + assert run.out.startswith('Usage: yadm') diff --git a/test/test_hooks.py b/test/test_hooks.py new file mode 100644 index 0000000..f1df91e --- /dev/null +++ b/test/test_hooks.py @@ -0,0 +1,90 @@ +"""Test hooks""" + +import pytest + + +@pytest.mark.parametrize( + 'pre, pre_code, post, post_code', [ + (False, 0, False, 0), + (True, 0, False, 0), + (True, 5, False, 0), + (False, 0, True, 0), + (False, 0, True, 5), + (True, 0, True, 0), + (True, 5, True, 5), + ], ids=[ + 'no-hooks', + 'pre-success', + 'pre-fail', + 'post-success', + 'post-fail', + 'pre-post-success', + 'pre-post-fail', + ]) +def test_hooks( + runner, yadm_y, paths, + pre, pre_code, post, post_code): + """Test pre/post hook""" + + # generate hooks + if pre: + create_hook(paths, 'pre_version', pre_code) + if post: + create_hook(paths, 'post_version', post_code) + + # run yadm + run = runner(yadm_y('version')) + # when a pre hook fails, yadm should exit with the hook's code + assert run.code == pre_code + assert run.err == '' + + if pre: + assert 'HOOK:pre_version' in run.out + # if pre hook is missing or successful, yadm itself should exit 0 + if run.success: + if post: + assert 'HOOK:post_version' in run.out + else: + # when a pre hook fails, yadm should not run the command + assert 'version will not be run' in run.out + # when a pre hook fails, yadm should not run the post hook + assert 'HOOK:post_version' not in run.out + + +# repo fixture is needed to test the population of YADM_HOOK_WORK +@pytest.mark.usefixtures('ds1_repo_copy') +def test_hook_env(runner, yadm_y, paths): + """Test hook environment""" + + # test will be done with a non existent "git" passthru command + # which should exit with a failing code + cmd = 'passthrucmd' + + # write the hook + hook = paths.hooks.join(f'post_{cmd}') + hook.write('#!/bin/sh\nenv\n') + hook.chmod(0o755) + + run = runner(yadm_y(cmd, 'extra_args')) + + # expect passthru to fail + assert run.failure + assert f"'{cmd}' is not a git command" in run.err + + # verify hook environment + assert 'YADM_HOOK_EXIT=1\n' in run.out + assert f'YADM_HOOK_COMMAND={cmd}\n' in run.out + assert f'YADM_HOOK_FULL_COMMAND={cmd} extra_args\n' in run.out + assert f'YADM_HOOK_REPO={paths.repo}\n' in run.out + assert f'YADM_HOOK_WORK={paths.work}\n' in run.out + + +def create_hook(paths, name, code): + """Create hook""" + hook = paths.hooks.join(name) + hook.write( + '#!/bin/sh\n' + f'echo HOOK:{name}\n' + f'exit {code}\n' + ) + hook.chmod(0o755) diff --git a/test/test_init.py b/test/test_init.py new file mode 100644 index 0000000..1519b38 --- /dev/null +++ b/test/test_init.py @@ -0,0 +1,78 @@ +"""Test init""" + +import pytest + + +@pytest.mark.parametrize( + 'alt_work, repo_present, force', [ + (False, False, False), + (True, False, False), + (False, True, False), + (False, True, True), + (True, True, True), + ], ids=[ + 'simple', + '-w', + 'existing repo', + '-f', + '-w & -f', + ]) +@pytest.mark.usefixtures('ds1_work_copy') +def test_init( + runner, yadm_y, paths, repo_config, alt_work, repo_present, force): + """Test init + + Repos should have attribs: + - 0600 permissions + - not bare + - worktree = $HOME + - showUntrackedFiles = no + - yadm.managed = true + """ + + # these tests will assume this for $HOME + home = str(paths.root.mkdir('HOME')) + + # ds1_work_copy comes WITH an empty repo dir present. + old_repo = paths.repo.join('old_repo') + if repo_present: + # Let's put some data in it, so we can confirm that data is gone when + # forced to be overwritten. + old_repo.write('old repo data') + assert old_repo.isfile() + else: + paths.repo.remove() + + # command args + args = ['init'] + if alt_work: + args.extend(['-w', paths.work]) + if force: + args.append('-f') + + # run init + run = runner(yadm_y(*args), env={'HOME': home}) + assert run.err == '' + + if repo_present and not force: + assert run.failure + assert 'repo already exists' in run.out + assert old_repo.isfile(), 'Missing original repo' + else: + assert run.success + assert 'Initialized empty shared Git repository' in run.out + + if repo_present: + assert not old_repo.isfile(), 'Original repo still exists' + + if alt_work: + assert repo_config('core.worktree') == paths.work + else: + assert repo_config('core.worktree') == home + + # uniform repo assertions + assert oct(paths.repo.stat().mode).endswith('00'), ( + 'Repo is not secure') + assert repo_config('core.bare') == 'false' + assert repo_config('status.showUntrackedFiles') == 'no' + assert repo_config('yadm.managed') == 'true' diff --git a/test/test_introspect.py b/test/test_introspect.py new file mode 100644 index 0000000..9026e98 --- /dev/null +++ b/test/test_introspect.py @@ -0,0 +1,46 @@ +"""Test introspect""" + +import pytest + + +@pytest.mark.parametrize( + 'name', [ + '', + 'invalid', + 'commands', + 'configs', + 'repo', + 'switches', + ]) +def test_introspect_category( + runner, yadm_y, paths, name, + supported_commands, supported_configs, supported_switches): + """Validate introspection category""" + if name: + run = runner(command=yadm_y('introspect', name)) + else: + run = runner(command=yadm_y('introspect')) + + assert run.success + assert run.err == '' + + expected = [] + if name == 'commands': + expected = supported_commands + elif name == 'config': + expected = supported_configs + elif name == 'switches': + expected = supported_switches + + # assert values + if name in ('', 'invalid'): + assert run.out == '' + if name == 'repo': + assert run.out.rstrip() == paths.repo + + # make sure every expected value is present + for value in expected: + assert value in run.out + # make sure nothing extra is present + if expected: + assert len(run.out.split()) == len(expected) diff --git a/test/test_jinja.py b/test/test_jinja.py new file mode 100644 index 0000000..6e43f35 --- /dev/null +++ b/test/test_jinja.py @@ -0,0 +1,186 @@ +"""Test jinja""" + +import os +import re +import pytest +import utils + + +@pytest.fixture(scope='module') +def envtpl_present(runner): + """Is envtpl present and working?""" + try: + run = runner(command=['envtpl', '-h']) + if run.success: + return True + except BaseException: + pass + return False + + +@pytest.mark.usefixtures('ds1_copy') +def test_local_override(runner, yadm_y, paths, + tst_distro, envtpl_present): + """Test local overrides""" + if not envtpl_present: + pytest.skip('Unable to test without envtpl.') + + # define local overrides + utils.set_local(paths, 'class', 'or-class') + utils.set_local(paths, 'hostname', 'or-hostname') + utils.set_local(paths, 'os', 'or-os') + utils.set_local(paths, 'user', 'or-user') + + template = ( + 'j2-{{ YADM_CLASS }}-' + '{{ YADM_OS }}-{{ YADM_HOSTNAME }}-' + '{{ YADM_USER }}-{{ YADM_DISTRO }}' + ) + expected = f'j2-or-class-or-os-or-hostname-or-user-{tst_distro}' + + utils.create_alt_files(paths, '##yadm.j2', content=template) + + # run alt to trigger linking + run = runner(yadm_y('alt')) + assert run.success + assert run.err == '' + created = created_list(run.out) + + # assert the proper creation has occurred + for file_path in (utils.ALT_FILE1, utils.ALT_FILE2): + source_file = file_path + '##yadm.j2' + assert paths.work.join(file_path).isfile() + lines = paths.work.join(file_path).readlines(cr=False) + assert lines[0] == source_file + assert lines[1] == expected + assert str(paths.work.join(source_file)) in created + + +@pytest.mark.parametrize('autoalt', [None, 'true', 'false']) +@pytest.mark.usefixtures('ds1_copy') +def test_auto_alt(runner, yadm_y, paths, autoalt, tst_sys, + envtpl_present): + """Test setting auto-alt""" + + if not envtpl_present: + pytest.skip('Unable to test without envtpl.') + + # set the value of auto-alt + if autoalt: + os.system(' '.join(yadm_y('config', 'yadm.auto-alt', autoalt))) + + # create file + jinja_suffix = '##yadm.j2' + utils.create_alt_files(paths, jinja_suffix, content='{{ YADM_OS }}') + + # run status to possibly trigger linking + run = runner(yadm_y('status')) + assert run.success + assert run.err == '' + created = created_list(run.out) + + # assert the proper creation has occurred + for file_path in (utils.ALT_FILE1, utils.ALT_FILE2): + source_file = file_path + jinja_suffix + if autoalt == 'false': + assert not paths.work.join(file_path).exists() + else: + assert paths.work.join(file_path).isfile() + lines = paths.work.join(file_path).readlines(cr=False) + assert lines[0] == source_file + assert lines[1] == tst_sys + # no created output when run via auto-alt + assert str(paths.work.join(source_file)) not in created + + +@pytest.mark.usefixtures('ds1_copy') +def test_jinja_envtpl_missing(runner, paths): + """Test operation when envtpl is missing""" + + script = f""" + YADM_TEST=1 source {paths.pgm} + process_global_args -Y "{paths.yadm}" + set_operating_system + configure_paths + ENVTPL_PROGRAM='envtpl_missing' main alt + """ + + utils.create_alt_files(paths, '##yadm.j2') + + run = runner(command=['bash'], inp=script) + assert run.success + assert run.err == '' + assert f'envtpl not available, not creating' in run.out + + +@pytest.mark.parametrize( + 'tracked, encrypt, exclude', [ + (False, False, False), + (True, False, False), + (False, True, False), + (False, True, True), + ], ids=[ + 'untracked', + 'tracked', + 'encrypted', + 'excluded', + ]) +@pytest.mark.usefixtures('ds1_copy') +def test_jinja(runner, yadm_y, paths, + tst_sys, tst_host, tst_user, tst_distro, + tracked, encrypt, exclude, + envtpl_present): + """Test jinja processing""" + + if not envtpl_present: + pytest.skip('Unable to test without envtpl.') + + jinja_suffix = '##yadm.j2' + + # set the class + tst_class = 'testclass' + utils.set_local(paths, 'class', tst_class) + + template = ( + 'j2-{{ YADM_CLASS }}-' + '{{ YADM_OS }}-{{ YADM_HOSTNAME }}-' + '{{ YADM_USER }}-{{ YADM_DISTRO }}' + ) + expected = ( + f'j2-{tst_class}-' + f'{tst_sys}-{tst_host}-' + f'{tst_user}-{tst_distro}' + ) + + utils.create_alt_files(paths, jinja_suffix, content=template, + tracked=tracked, encrypt=encrypt, exclude=exclude) + + # run alt to trigger linking + run = runner(yadm_y('alt')) + assert run.success + assert run.err == '' + created = created_list(run.out) + + # assert the proper creation has occurred + for file_path in (utils.ALT_FILE1, utils.ALT_FILE2): + source_file = file_path + jinja_suffix + if tracked or (encrypt and not exclude): + assert paths.work.join(file_path).isfile() + lines = paths.work.join(file_path).readlines(cr=False) + assert lines[0] == source_file + assert lines[1] == expected + assert str(paths.work.join(source_file)) in created + else: + assert not paths.work.join(file_path).exists() + assert str(paths.work.join(source_file)) not in created + + +def created_list(output): + """Parse output, and return list of created files""" + + created = dict() + for line in output.splitlines(): + match = re.match('Creating (.+) from template (.+)$', line) + if match: + created[match.group(1)] = match.group(2) + return created.values() diff --git a/test/test_list.py b/test/test_list.py new file mode 100644 index 0000000..44a5573 --- /dev/null +++ b/test/test_list.py @@ -0,0 +1,47 @@ +"""Test list""" + +import os +import pytest + + +@pytest.mark.parametrize( + 'location', [ + 'work', + 'outside', + 'subdir', + ]) +@pytest.mark.usefixtures('ds1_copy') +def test_list(runner, yadm_y, paths, ds1, location): + """List tests""" + if location == 'work': + run_dir = paths.work + elif location == 'outside': + run_dir = paths.work.join('..') + elif location == 'subdir': + # first directory with tracked data + run_dir = paths.work.join(ds1.tracked_dirs[0]) + with run_dir.as_cwd(): + # test with '-a' + # should get all tracked files, relative to the work path + run = runner(command=yadm_y('list', '-a')) + assert run.success + assert run.err == '' + returned_files = set(run.out.splitlines()) + expected_files = set([e.path for e in ds1 if e.tracked]) + assert returned_files == expected_files + # test without '-a' + # should get all tracked files, relative to the work path unless in a + # subdir, then those should be a limited set of files, relative to the + # subdir + run = runner(command=yadm_y('list')) + assert run.success + assert run.err == '' + returned_files = set(run.out.splitlines()) + if location == 'subdir': + basepath = os.path.basename(os.getcwd()) + # only expect files within the subdir + # names should be relative to subdir + expected_files = set( + [e.path[len(basepath)+1:] for e in ds1 + if e.tracked and e.path.startswith(basepath)]) + assert returned_files == expected_files diff --git a/test/test_perms.py b/test/test_perms.py new file mode 100644 index 0000000..eb7ad8f --- /dev/null +++ b/test/test_perms.py @@ -0,0 +1,111 @@ +"""Test perms""" + +import os +import warnings +import pytest + + +@pytest.mark.parametrize('autoperms', ['notest', 'unset', 'true', 'false']) +@pytest.mark.usefixtures('ds1_copy') +def test_perms(runner, yadm_y, paths, ds1, autoperms): + """Test perms""" + # set the value of auto-perms + if autoperms != 'notest': + if autoperms != 'unset': + os.system(' '.join(yadm_y('config', 'yadm.auto-perms', autoperms))) + + # privatepaths will hold all paths that should become secured + privatepaths = [paths.work.join('.ssh'), paths.work.join('.gnupg')] + privatepaths += [paths.work.join(private.path) for private in ds1.private] + + # create an archive file + os.system(f'touch "{str(paths.archive)}"') + privatepaths.append(paths.archive) + + # create encrypted file test data + efile1 = paths.work.join('efile1') + efile1.write('efile1') + efile2 = paths.work.join('efile2') + efile2.write('efile2') + paths.encrypt.write('efile1\nefile2\n!efile1\n') + insecurepaths = [efile1] + privatepaths.append(efile2) + + # assert these paths begin unsecured + for private in privatepaths + insecurepaths: + assert not oct(private.stat().mode).endswith('00'), ( + 'Path started secured') + + cmd = 'perms' + if autoperms != 'notest': + cmd = 'status' + run = runner(yadm_y(cmd)) + assert run.success + assert run.err == '' + if cmd == 'perms': + assert run.out == '' + + # these paths should be secured if processing perms + for private in privatepaths: + if '.p2' in private.basename or '.p4' in private.basename: + # Dot files within .ssh/.gnupg are not protected. + # This is a but which must be fixed + warnings.warn('Unhandled bug: private dot files', Warning) + continue + if autoperms == 'false': + assert not oct(private.stat().mode).endswith('00'), ( + 'Path should not be secured') + else: + assert oct(private.stat().mode).endswith('00'), ( + 'Path has not been secured') + + # these paths should never be secured + for private in insecurepaths: + assert not oct(private.stat().mode).endswith('00'), ( + 'Path should not be secured') + + +@pytest.mark.parametrize('sshperms', [None, 'true', 'false']) +@pytest.mark.parametrize('gpgperms', [None, 'true', 'false']) +@pytest.mark.usefixtures('ds1_copy') +def test_perms_control(runner, yadm_y, paths, ds1, sshperms, gpgperms): + """Test fine control of perms""" + # set the value of ssh-perms + if sshperms: + os.system(' '.join(yadm_y('config', 'yadm.ssh-perms', sshperms))) + + # set the value of gpg-perms + if gpgperms: + os.system(' '.join(yadm_y('config', 'yadm.gpg-perms', gpgperms))) + + # privatepaths will hold all paths that should become secured + privatepaths = [paths.work.join('.ssh'), paths.work.join('.gnupg')] + privatepaths += [paths.work.join(private.path) for private in ds1.private] + + # assert these paths begin unsecured + for private in privatepaths: + assert not oct(private.stat().mode).endswith('00'), ( + 'Path started secured') + + run = runner(yadm_y('perms')) + assert run.success + assert run.err == '' + assert run.out == '' + + # these paths should be secured if processing perms + for private in privatepaths: + if '.p2' in private.basename or '.p4' in private.basename: + # Dot files within .ssh/.gnupg are not protected. + # This is a but which must be fixed + warnings.warn('Unhandled bug: private dot files', Warning) + continue + if ( + (sshperms == 'false' and 'ssh' in str(private)) + or + (gpgperms == 'false' and 'gnupg' in str(private)) + ): + assert not oct(private.stat().mode).endswith('00'), ( + 'Path should not be secured') + else: + assert oct(private.stat().mode).endswith('00'), ( + 'Path has not been secured') diff --git a/test/test_syntax.py b/test/test_syntax.py new file mode 100644 index 0000000..5408885 --- /dev/null +++ b/test/test_syntax.py @@ -0,0 +1,41 @@ +"""Syntax checks""" + +import os +import pytest + + +def test_yadm_syntax(runner, yadm): + """Is syntactically valid""" + run = runner(command=['bash', '-n', yadm]) + assert run.success + + +def test_shellcheck(runner, yadm, shellcheck_version): + """Passes shellcheck""" + run = runner(command=['shellcheck', '-V'], report=False) + if f'version: {shellcheck_version}' not in run.out: + pytest.skip('Unsupported shellcheck version') + run = runner(command=['shellcheck', '-s', 'bash', yadm]) + assert run.success + + +def test_pylint(runner, pylint_version): + """Passes pylint""" + run = runner(command=['pylint', '--version'], report=False) + if f'pylint {pylint_version}' not in run.out: + pytest.skip('Unsupported pylint version') + pyfiles = list() + for tfile in os.listdir('test'): + if tfile.endswith('.py'): + pyfiles.append(f'test/{tfile}') + run = runner(command=['pylint'] + pyfiles) + assert run.success + + +def test_flake8(runner, flake8_version): + """Passes flake8""" + run = runner(command=['flake8', '--version'], report=False) + if not run.out.startswith(flake8_version): + pytest.skip('Unsupported flake8 version') + run = runner(command=['flake8', 'test']) + assert run.success diff --git a/test/test_unit_bootstrap_available.py b/test/test_unit_bootstrap_available.py new file mode 100644 index 0000000..f37ac08 --- /dev/null +++ b/test/test_unit_bootstrap_available.py @@ -0,0 +1,33 @@ +"""Unit tests: bootstrap_available""" + + +def test_bootstrap_missing(runner, paths): + """Test result of bootstrap_available, when bootstrap missing""" + run_test(runner, paths, False) + + +def test_bootstrap_no_exec(runner, paths): + """Test result of bootstrap_available, when bootstrap not executable""" + paths.bootstrap.write('') + paths.bootstrap.chmod(0o644) + run_test(runner, paths, False) + + +def test_bootstrap_exec(runner, paths): + """Test result of bootstrap_available, when bootstrap executable""" + paths.bootstrap.write('') + paths.bootstrap.chmod(0o775) + run_test(runner, paths, True) + + +def run_test(runner, paths, success): + """Run bootstrap_available, and test result""" + script = f""" + YADM_TEST=1 source {paths.pgm} + YADM_BOOTSTRAP='{paths.bootstrap}' + bootstrap_available + """ + run = runner(command=['bash'], inp=script) + assert run.success == success + assert run.err == '' + assert run.out == '' diff --git a/test/test_unit_configure_paths.py b/test/test_unit_configure_paths.py new file mode 100644 index 0000000..094ff6b --- /dev/null +++ b/test/test_unit_configure_paths.py @@ -0,0 +1,80 @@ +"""Unit tests: configure_paths""" + +import pytest + +ARCHIVE = 'files.gpg' +BOOTSTRAP = 'bootstrap' +CONFIG = 'config' +ENCRYPT = 'encrypt' +HOME = '/testhome' +REPO = 'repo.git' +YDIR = '.yadm' + + +@pytest.mark.parametrize( + 'override, expect', [ + (None, {}), + ('-Y', {}), + ('--yadm-repo', {'repo': 'YADM_REPO', 'git': 'GIT_DIR'}), + ('--yadm-config', {'config': 'YADM_CONFIG'}), + ('--yadm-encrypt', {'encrypt': 'YADM_ENCRYPT'}), + ('--yadm-archive', {'archive': 'YADM_ARCHIVE'}), + ('--yadm-bootstrap', {'bootstrap': 'YADM_BOOTSTRAP'}), + ], ids=[ + 'default', + 'override yadm dir', + 'override repo', + 'override config', + 'override encrypt', + 'override archive', + 'override bootstrap', + ]) +def test_config(runner, paths, override, expect): + """Test configure_paths""" + opath = 'override' + matches = match_map() + args = [] + if override == '-Y': + matches = match_map('/' + opath) + + if override: + args = [override, '/' + opath] + for ekey in expect.keys(): + matches[ekey] = f'{expect[ekey]}="/{opath}"' + run_test( + runner, paths, + [override, opath], + ['must specify a fully qualified'], 1) + + run_test(runner, paths, args, matches.values(), 0) + + +def match_map(yadm_dir=None): + """Create a dictionary of matches, relative to yadm_dir""" + if not yadm_dir: + yadm_dir = '/'.join([HOME, YDIR]) + return { + 'yadm': f'YADM_DIR="{yadm_dir}"', + 'repo': f'YADM_REPO="{yadm_dir}/{REPO}"', + 'config': f'YADM_CONFIG="{yadm_dir}/{CONFIG}"', + 'encrypt': f'YADM_ENCRYPT="{yadm_dir}/{ENCRYPT}"', + 'archive': f'YADM_ARCHIVE="{yadm_dir}/{ARCHIVE}"', + 'bootstrap': f'YADM_BOOTSTRAP="{yadm_dir}/{BOOTSTRAP}"', + 'git': f'GIT_DIR="{yadm_dir}/{REPO}"', + } + + +def run_test(runner, paths, args, expected_matches, expected_code=0): + """Run proces global args, and run configure_paths""" + argstring = ' '.join(['"'+a+'"' for a in args]) + script = f""" + YADM_TEST=1 HOME="{HOME}" source {paths.pgm} + process_global_args {argstring} + configure_paths + declare -p | grep -E '(YADM|GIT)_' + """ + run = runner(command=['bash'], inp=script) + assert run.code == expected_code + assert run.err == '' + for match in expected_matches: + assert match in run.out diff --git a/test/test_unit_parse_encrypt.py b/test/test_unit_parse_encrypt.py new file mode 100644 index 0000000..914d907 --- /dev/null +++ b/test/test_unit_parse_encrypt.py @@ -0,0 +1,175 @@ +"""Unit tests: parse_encrypt""" + +import pytest + + +def test_not_called(runner, paths): + """Test parse_encrypt (not called)""" + run = run_parse_encrypt(runner, paths, skip_parse=True) + assert run.success + assert run.err == '' + assert 'EIF:unparsed' in run.out, 'EIF should be unparsed' + assert 'EIF_COUNT:1' in run.out, 'Only value of EIF should be unparsed' + + +def test_short_circuit(runner, paths): + """Test parse_encrypt (short-circuit)""" + run = run_parse_encrypt(runner, paths, twice=True) + assert run.success + assert run.err == '' + assert 'PARSE_ENCRYPT_SHORT=parse_encrypt() not reprocessed' in run.out, ( + 'parse_encrypt() should short-circuit') + + +@pytest.mark.parametrize( + 'encrypt', [ + ('missing'), + ('empty'), + ]) +def test_empty(runner, paths, encrypt): + """Test parse_encrypt (file missing/empty)""" + + # write encrypt file + if encrypt == 'missing': + assert not paths.encrypt.exists(), 'Encrypt should be missing' + else: + paths.encrypt.write('') + assert paths.encrypt.exists(), 'Encrypt should exist' + assert paths.encrypt.size() == 0, 'Encrypt should be empty' + + # run parse_encrypt + run = run_parse_encrypt(runner, paths) + assert run.success + assert run.err == '' + + # validate parsing result + assert 'EIF_COUNT:0' in run.out, 'EIF should be empty' + + +@pytest.mark.usefixtures('ds1_repo_copy') +def test_file_parse_encrypt(runner, paths): + """Test parse_encrypt + + Test an array of supported features of the encrypt configuration. + """ + + edata = '' + expected = set() + + # empty line + edata += '\n' + + # simple comments + edata += '# a simple comment\n' + edata += ' # a comment with leading space\n' + + # unreferenced directory + paths.work.join('unreferenced').mkdir() + + # simple files + edata += 'simple_file\n' + edata += 'simple.file\n' + paths.work.join('simple_file').write('') + paths.work.join('simple.file').write('') + paths.work.join('simple_file2').write('') + paths.work.join('simple.file2').write('') + expected.add('simple_file') + expected.add('simple.file') + + # simple files in directories + edata += 'simple_dir/simple_file\n' + paths.work.join('simple_dir/simple_file').write('', ensure=True) + paths.work.join('simple_dir/simple_file2').write('', ensure=True) + expected.add('simple_dir/simple_file') + + # paths with spaces + edata += 'with space/with space\n' + paths.work.join('with space/with space').write('', ensure=True) + paths.work.join('with space/with space2').write('', ensure=True) + expected.add('with space/with space') + + # hidden files + edata += '.hidden\n' + paths.work.join('.hidden').write('') + expected.add('.hidden') + + # hidden files in directories + edata += '.hidden_dir/.hidden_file\n' + paths.work.join('.hidden_dir/.hidden_file').write('', ensure=True) + expected.add('.hidden_dir/.hidden_file') + + # wildcards + edata += 'wild*\n' + paths.work.join('wildcard1').write('', ensure=True) + paths.work.join('wildcard2').write('', ensure=True) + expected.add('wildcard1') + expected.add('wildcard2') + + edata += 'dirwild*\n' + paths.work.join('dirwildcard/file1').write('', ensure=True) + paths.work.join('dirwildcard/file2').write('', ensure=True) + expected.add('dirwildcard') + + # excludes + edata += 'exclude*\n' + edata += 'ex ex/*\n' + paths.work.join('exclude_file1').write('') + paths.work.join('exclude_file2.ex').write('') + paths.work.join('exclude_file3.ex3').write('') + expected.add('exclude_file1') + expected.add('exclude_file3.ex3') + edata += '!*.ex\n' + edata += '!ex ex/*.txt\n' + paths.work.join('ex ex/file4').write('', ensure=True) + paths.work.join('ex ex/file5.txt').write('', ensure=True) + paths.work.join('ex ex/file6.text').write('', ensure=True) + expected.add('ex ex/file4') + expected.add('ex ex/file6.text') + + # write encrypt file + print(f'ENCRYPT:\n---\n{edata}---\n') + paths.encrypt.write(edata) + assert paths.encrypt.isfile() + + # run parse_encrypt + run = run_parse_encrypt(runner, paths) + assert run.success + assert run.err == '' + + assert f'EIF_COUNT:{len(expected)}' in run.out, 'EIF count wrong' + for expected_file in expected: + assert f'EIF:{expected_file}\n' in run.out + + +def run_parse_encrypt( + runner, paths, + skip_parse=False, + twice=False): + """Run parse_encrypt + + A count of ENCRYPT_INCLUDE_FILES will be reported as EIF_COUNT:X. All + values of ENCRYPT_INCLUDE_FILES will be reported as individual EIF:value + lines. + """ + parse_cmd = 'parse_encrypt' + if skip_parse: + parse_cmd = '' + if twice: + parse_cmd = 'parse_encrypt; parse_encrypt' + script = f""" + YADM_TEST=1 source {paths.pgm} + YADM_ENCRYPT={paths.encrypt} + export YADM_ENCRYPT + GIT_DIR={paths.repo} + export GIT_DIR + {parse_cmd} + export ENCRYPT_INCLUDE_FILES + export PARSE_ENCRYPT_SHORT + env + echo EIF_COUNT:${{#ENCRYPT_INCLUDE_FILES[@]}} + for value in "${{ENCRYPT_INCLUDE_FILES[@]}}"; do + echo "EIF:$value" + done + """ + run = runner(command=['bash'], inp=script) + return run diff --git a/test/test_unit_query_distro.py b/test/test_unit_query_distro.py new file mode 100644 index 0000000..3c53c54 --- /dev/null +++ b/test/test_unit_query_distro.py @@ -0,0 +1,26 @@ +"""Unit tests: query_distro""" + + +def test_lsb_release_present(runner, yadm, tst_distro): + """Match lsb_release -si when present""" + script = f""" + YADM_TEST=1 source {yadm} + query_distro + """ + run = runner(command=['bash'], inp=script) + assert run.success + assert run.err == '' + assert run.out.rstrip() == tst_distro + + +def test_lsb_release_missing(runner, yadm): + """Empty value when missing""" + script = f""" + YADM_TEST=1 source {yadm} + LSB_RELEASE_PROGRAM="missing_lsb_release" + query_distro + """ + run = runner(command=['bash'], inp=script) + assert run.success + assert run.err == '' + assert run.out.rstrip() == '' diff --git a/test/test_unit_set_os.py b/test/test_unit_set_os.py new file mode 100644 index 0000000..d2f2a2a --- /dev/null +++ b/test/test_unit_set_os.py @@ -0,0 +1,36 @@ +"""Unit tests: set_operating_system""" + +import pytest + + +@pytest.mark.parametrize( + 'proc_value, expected_os', [ + ('missing', 'uname'), + ('has Microsoft inside', 'WSL'), + ('another value', 'uname'), + ], ids=[ + '/proc/version missing', + '/proc/version includes MS', + '/proc/version excludes MS', + ]) +def test_set_operating_system( + runner, paths, tst_sys, proc_value, expected_os): + """Run set_operating_system and test result""" + + # Normally /proc/version (set in PROC_VERSION) is inspected to identify + # WSL. During testing, we will override that value. + proc_version = paths.root.join('proc_version') + if proc_value != 'missing': + proc_version.write(proc_value) + script = f""" + YADM_TEST=1 source {paths.pgm} + PROC_VERSION={proc_version} + set_operating_system + echo $OPERATING_SYSTEM + """ + run = runner(command=['bash'], inp=script) + assert run.success + assert run.err == '' + if expected_os == 'uname': + expected_os = tst_sys + assert run.out.rstrip() == expected_os diff --git a/test/test_unit_x_program.py b/test/test_unit_x_program.py new file mode 100644 index 0000000..3233a3d --- /dev/null +++ b/test/test_unit_x_program.py @@ -0,0 +1,46 @@ +"""Unit tests: yadm.[git,gpg]-program""" + +import os +import pytest + + +@pytest.mark.parametrize( + 'executable, success, value, match', [ + (None, True, 'program', None), + ('cat', True, 'cat', None), + ('badprogram', False, None, 'badprogram'), + ], ids=[ + 'executable missing', + 'valid alternative', + 'invalid alternative', + ]) +@pytest.mark.parametrize('program', ['git', 'gpg']) +def test_x_program( + runner, yadm_y, paths, program, executable, success, value, match): + """Set yadm.X-program, and test result of require_X""" + + # set configuration + if executable: + os.system(' '.join(yadm_y( + 'config', f'yadm.{program}-program', executable))) + + # test require_[git,gpg] + script = f""" + YADM_TEST=1 source {paths.pgm} + YADM_CONFIG="{paths.config}" + require_{program} + echo ${program.upper()}_PROGRAM + """ + run = runner(command=['bash'], inp=script) + assert run.success == success + assert run.err == '' + + # [GIT,GPG]_PROGRAM set correctly + if value == 'program': + assert run.out.rstrip() == program + elif value: + assert run.out.rstrip() == value + + # error reported about bad config + if match: + assert match in run.out diff --git a/test/test_version.py b/test/test_version.py new file mode 100644 index 0000000..023eb82 --- /dev/null +++ b/test/test_version.py @@ -0,0 +1,35 @@ +"""Test version""" + +import re +import pytest + + +@pytest.fixture(scope='module') +def expected_version(yadm): + """ + Expected semantic version number. This is taken directly out of yadm, + searching for the VERSION= string. + """ + yadm_version = re.findall( + r'VERSION=([^\n]+)', + open(yadm).read()) + if yadm_version: + return yadm_version[0] + pytest.fail(f'version not found in {yadm}') + return 'not found' + + +def test_semantic_version(expected_version): + """Version is semantic""" + # semantic version conforms to MAJOR.MINOR.PATCH + assert re.search(r'^\d+\.\d+\.\d+$', expected_version), ( + 'does not conform to MAJOR.MINOR.PATCH') + + +def test_reported_version( + runner, yadm_y, expected_version): + """Report correct version""" + run = runner(command=yadm_y('version')) + assert run.success + assert run.err == '' + assert run.out == f'yadm {expected_version}\n' diff --git a/test/utils.py b/test/utils.py new file mode 100644 index 0000000..411dde1 --- /dev/null +++ b/test/utils.py @@ -0,0 +1,59 @@ +"""Testing Utilities + +This module holds values/functions common to multiple tests. +""" + +import os + +ALT_FILE1 = 'test_alt' +ALT_FILE2 = 'test alt/test alt' + + +def set_local(paths, variable, value): + """Set local override""" + os.system( + f'GIT_DIR={str(paths.repo)} ' + f'git config --local "local.{variable}" "{value}"' + ) + + +def create_alt_files(paths, suffix, + preserve=False, tracked=True, + encrypt=False, exclude=False, + content=None): + """Create new files, and add to the repo + + This is used for testing alternate files. In each case, a suffix is + appended to two standard file paths. Particulars of the file creation and + repo handling are dependent upon the function arguments. + """ + + if not preserve: + if paths.work.join(ALT_FILE1).exists(): + paths.work.join(ALT_FILE1).remove(rec=1, ignore_errors=True) + assert not paths.work.join(ALT_FILE1).exists() + if paths.work.join(ALT_FILE2).exists(): + paths.work.join(ALT_FILE2).remove(rec=1, ignore_errors=True) + assert not paths.work.join(ALT_FILE2).exists() + + new_file1 = paths.work.join(ALT_FILE1 + suffix) + new_file1.write(ALT_FILE1 + suffix, ensure=True) + new_file2 = paths.work.join(ALT_FILE2 + suffix) + new_file2.write(ALT_FILE2 + suffix, ensure=True) + if content: + new_file1.write('\n' + content, mode='a', ensure=True) + new_file2.write('\n' + content, mode='a', ensure=True) + assert new_file1.exists() + assert new_file2.exists() + + if tracked: + for path in (new_file1, new_file2): + os.system(f'GIT_DIR={str(paths.repo)} git add "{path}"') + os.system(f'GIT_DIR={str(paths.repo)} git commit -m "Add test files"') + + if encrypt: + paths.encrypt.write(f'{ALT_FILE1 + suffix}\n', mode='a') + paths.encrypt.write(f'{ALT_FILE2 + suffix}\n', mode='a') + if exclude: + paths.encrypt.write(f'!{ALT_FILE1 + suffix}\n', mode='a') + paths.encrypt.write(f'!{ALT_FILE2 + suffix}\n', mode='a') From 0f2039f79dbf473da3c41c5ca4cd889db948cbb1 Mon Sep 17 00:00:00 2001 From: Tim Byrne Date: Mon, 25 Feb 2019 16:30:05 -0600 Subject: [PATCH 006/137] Create a flag to identify when running inside testbed --- Dockerfile | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Dockerfile b/Dockerfile index ee29bf2..622932a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -19,6 +19,9 @@ RUN pip3 install envtpl pytest==3.6.4 pylint==1.9.2 flake8==3.5.0 # Force GNUPG version 1 at path /usr/bin/gpg RUN ln -fs /usr/bin/gpg1 /usr/bin/gpg +# Create a flag to identify when running inside the yadm testbed +RUN touch /.yadmtestbed + # /yadm will be the work directory for all tests # docker commands should mount the local yadm project as /yadm WORKDIR /yadm From 23e4b38ef2c93ec32bc97f229c1e16ce0956dc69 Mon Sep 17 00:00:00 2001 From: Tim Byrne Date: Mon, 25 Feb 2019 16:32:34 -0600 Subject: [PATCH 007/137] Update Makefile * Add usage/help * Check for dependencies in testing targets * Remove bats-based targets * Change location of testenv --- .gitignore | 2 +- Makefile | 184 +++++++++++++++++++++++++++++++++++------------------ 2 files changed, 123 insertions(+), 63 deletions(-) diff --git a/.gitignore b/.gitignore index af9e6f5..aa13f8f 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,6 @@ .DS_Store -.env .jekyll-metadata .pytest_cache .sass-cache _site +testenv diff --git a/Makefile b/Makefile index 691b168..65edc57 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,126 @@ +PYTESTS = $(wildcard test/test_*.py) + .PHONY: all -all: yadm.md contrib +all: + @$(MAKE) usage | less + +# Display usage for all make targets +.PHONY: usage +usage: + @echo + @echo 'make TARGET [option=value, ...]' + @echo + @echo 'TESTING' + @echo + @echo ' make test [testargs=ARGS]' + @echo ' - Run all tests. "testargs" can specify a single string of arguments' + @echo ' for py.test.' + @echo + @echo ' make .py [testargs=ARGS]' + @echo ' - Run tests from a specific test file. "testargs" can specify a' + @echo ' single string of arguments for py.test.' + @echo + @echo ' make testhost [version=VERSION]' + @echo ' - Create an ephemeral container for doing adhoc yadm testing. The' + @echo ' HEAD revision of yadm will be used unless "version" is' + @echo ' specified. "version" can be set to any commit, branch, tag, etc.' + @echo ' The targeted "version" will be retrieved from the repo, and' + @echo ' linked into the container as a local volume.' + @echo + @echo 'LINTING' + @echo + @echo ' make testenv' + @echo ' - Create a python virtual environment with the dependencies used' + @echo " by yadm's testbed environment. Creating and activating this" + @echo ' environment might be useful if your editor does real time' + @echo ' linting of python files. After creating the virtual environment,' + @echo ' you can activate it by typing' + @echo + @echo 'MANPAGES' + @echo + @echo ' make man' + @echo ' - View yadm.1 as a standard manpage.' + @echo + @echo ' make man-wide' + @echo ' - View yadm.1 as a manpage, using all columns of your display.' + @echo + @echo ' make man-ps' + @echo ' - Create a postscript version of the manpage.' + @echo + @echo 'FILE GENERATION' + @echo + @echo ' make yadm.md' + @echo ' - Generate the markdown version of the manpage (for viewing on' + @echo ' the web).' + @echo + @echo ' make contrib' + @echo ' - Generate the CONTRIBUTORS file, from the repo history.' + @echo + @echo 'UTILITIES' + @echo + @echo ' make sync-clock' + @echo ' - Reset the hardware clock for the docker hypervisor host. This' + @echo ' can be useful for docker engine hosts which are not' + @echo ' Linux-based.' + @echo + +# Make it possible to run make specifying a py.test test file +.PHONY: $(PYTESTS) +$(PYTESTS): + @$(MAKE) test testargs="-k $@ $(testargs)" +%.py: + @$(MAKE) test testargs="-k $@ $(testargs)" + +# Run all tests with additional testargs +.PHONY: test +test: + @if [ -f /.yadmtestbed ]; then \ + cd /yadm && \ + py.test -v $(testargs); \ + else \ + if command -v "docker-compose" >/dev/null 2>&1; then \ + docker-compose run --rm testbed make test testargs="$(testargs)"; \ + else \ + echo "Sorry, this make test requires docker-compose to be installed."; \ + false; \ + fi \ + fi + +.PHONY: testhost +testhost: + @if ! command -v "docker" >/dev/null 2>&1; then \ + echo "Sorry, this make target requires docker to be installed."; \ + false; \ + fi + @version=HEAD + @rm -rf /tmp/testhost + @git show $(version):yadm > /tmp/testhost + @chmod a+x /tmp/testhost + @echo Starting testhost version=\"$$version\" + @docker run -w /root --hostname testhost --rm -it -v "/tmp/testhost:/bin/yadm:ro" yadm/testbed:latest bash -l + +.PHONY: testenv +testenv: + @echo 'Creating a local virtual environment in "testenv/"' + @echo + virtualenv --python=python3 testenv + testenv/bin/pip3 install --upgrade pip setuptools + testenv/bin/pip3 install --upgrade pytest pylint==1.9.2 flake8==3.5.0 + @echo + @echo 'To activate this test environment type:' + @echo ' source testenv/bin/activate' + +.PHONY: man +man: + @groff -man -Tascii ./yadm.1 | less + +.PHONY: man-wide +man-wide: + @man ./yadm.1 + +.PHONY: man-ps +man-ps: + @groff -man -Tps ./yadm.1 > yadm.ps yadm.md: yadm.1 @groff -man -Tascii ./yadm.1 | col -bx | sed 's/^[A-Z]/## &/g' | sed '/yadm(1)/d' > yadm.md @@ -9,67 +130,6 @@ contrib: @echo "CONTRIBUTORS\n" > CONTRIBUTORS @git shortlog -ns master gh-pages dev dev-pages | cut -f2 >> CONTRIBUTORS -.PHONY: pdf -pdf: - @groff -man -Tps ./yadm.1 > yadm.ps - @open yadm.ps - @sleep 1 - @rm yadm.ps - -.PHONY: test -test: bats shellcheck - -.PHONY: parallel -parallel: - ls test/*bats | time parallel -q -P0 -- docker run --rm -v "$$PWD:/yadm:ro" yadm/testbed bash -c 'bats {}' - -.PHONY: pytest -pytest: - @echo Running all pytest tests - @pytest -v - -.PHONY: bats -bats: - @echo Running all bats tests - @GPG_AGENT_INFO= bats test - -.PHONY: shellcheck -shellcheck: - @echo Running shellcheck - @shellcheck --version || true - @shellcheck -s bash yadm bootstrap test/*.bash completion/yadm.bash_completion - @cd test; \ - for bats_file in *bats; do \ - sed 's/^@test.*{/function test() {/' "$$bats_file" > "/tmp/$$bats_file.bash"; \ - shellcheck -s bash "/tmp/$$bats_file.bash"; \ - test_result="$$?"; \ - rm -f "/tmp/$$bats_file.bash"; \ - [ "$$test_result" -ne 0 ] && exit 1; \ - done; true - -.PHONY: testhost -testhost: - @target=HEAD - @rm -rf /tmp/testhost - @git show $(target):yadm > /tmp/testhost - @chmod a+x /tmp/testhost - @echo Starting testhost target=\"$$target\" - @docker run -w /root --hostname testhost --rm -it -v "/tmp/testhost:/bin/yadm:ro" yadm/testbed:latest bash - -.PHONY: man -man: - groff -man -Tascii ./yadm.1 | less - -.PHONY: wide -wide: - man ./yadm.1 - .PHONY: sync-clock sync-clock: docker run --rm --privileged alpine hwclock -s - -.PHONY: .env -.env: - virtualenv --python=python3 .env - .env/bin/pip3 install --upgrade pip setuptools - .env/bin/pip3 install --upgrade pytest pylint==1.9.2 flake8==3.5.0 From c3a9b621894dbcdede9af05a8ea3affb1deaf589 Mon Sep 17 00:00:00 2001 From: Tim Byrne Date: Mon, 25 Feb 2019 17:08:07 -0600 Subject: [PATCH 008/137] Remove BATS-based tests --- test/000_unit_syntax.bats | 11 - test/001_unit_paths.bats | 209 ------ test/002_unit_gpg_program.bats | 67 -- test/003_unit_git_program.bats | 67 -- test/004_unit_bootstrap_available.bats | 66 -- test/005_unit_set_operating_system.bats | 76 -- test/006_unit_query_distro.bats | 49 -- test/007_unit_parse_encrypt.bats | 318 -------- test/100_accept_version.bats | 25 - test/101_accept_help.bats | 33 - test/102_accept_clean.bats | 19 - test/103_accept_git.bats | 117 --- test/104_accept_init.bats | 178 ----- test/105_accept_clone.bats | 579 --------------- test/106_accept_config.bats | 202 ----- test/107_accept_list.bats | 93 --- test/108_accept_alt.bats | 415 ----------- test/109_accept_encryption.bats | 900 ----------------------- test/110_accept_perms.bats | 173 ----- test/111_accept_wildcard_alt.bats | 223 ------ test/112_accept_bootstrap.bats | 78 -- test/113_accept_jinja_alt.bats | 203 ----- test/114_accept_enter.bats | 66 -- test/115_accept_introspect.bats | 99 --- test/116_accept_cygwin_copy.bats | 131 ---- test/117_accept_hooks.bats | 181 ----- test/118_accept_assert_private_dirs.bats | 102 --- test/common.bash | 384 ---------- 28 files changed, 5064 deletions(-) delete mode 100644 test/000_unit_syntax.bats delete mode 100644 test/001_unit_paths.bats delete mode 100644 test/002_unit_gpg_program.bats delete mode 100644 test/003_unit_git_program.bats delete mode 100644 test/004_unit_bootstrap_available.bats delete mode 100644 test/005_unit_set_operating_system.bats delete mode 100644 test/006_unit_query_distro.bats delete mode 100644 test/007_unit_parse_encrypt.bats delete mode 100644 test/100_accept_version.bats delete mode 100644 test/101_accept_help.bats delete mode 100644 test/102_accept_clean.bats delete mode 100644 test/103_accept_git.bats delete mode 100644 test/104_accept_init.bats delete mode 100644 test/105_accept_clone.bats delete mode 100644 test/106_accept_config.bats delete mode 100644 test/107_accept_list.bats delete mode 100644 test/108_accept_alt.bats delete mode 100644 test/109_accept_encryption.bats delete mode 100644 test/110_accept_perms.bats delete mode 100644 test/111_accept_wildcard_alt.bats delete mode 100644 test/112_accept_bootstrap.bats delete mode 100644 test/113_accept_jinja_alt.bats delete mode 100644 test/114_accept_enter.bats delete mode 100644 test/115_accept_introspect.bats delete mode 100644 test/116_accept_cygwin_copy.bats delete mode 100644 test/117_accept_hooks.bats delete mode 100644 test/118_accept_assert_private_dirs.bats delete mode 100644 test/common.bash diff --git a/test/000_unit_syntax.bats b/test/000_unit_syntax.bats deleted file mode 100644 index 2bb1ee3..0000000 --- a/test/000_unit_syntax.bats +++ /dev/null @@ -1,11 +0,0 @@ -load common -load_fixtures - -@test "Syntax check" { - echo " - $T_YADM must parse correctly - " - - #; check the syntax of yadm - bash -n "$T_YADM" -} diff --git a/test/001_unit_paths.bats b/test/001_unit_paths.bats deleted file mode 100644 index bf479e8..0000000 --- a/test/001_unit_paths.bats +++ /dev/null @@ -1,209 +0,0 @@ -load common -load_fixtures - -function configuration_test() { - # shellcheck source=/dev/null - YADM_TEST=1 source "$T_YADM" - status=0 - output=$( process_global_args "$@" ) || { - status=$? - true - } - if [ "$status" == 0 ]; then - process_global_args "$@" - configure_paths - fi - - echo -e "STATUS:$status\nOUTPUT:$output" - echo "CONFIGURED PATHS:" - echo " YADM_DIR:$YADM_DIR" - echo " YADM_REPO:$YADM_REPO" - echo " YADM_CONFIG:$YADM_CONFIG" - echo " YADM_ENCRYPT:$YADM_ENCRYPT" - echo " YADM_ARCHIVE:$YADM_ARCHIVE" - echo "YADM_BOOTSTRAP:$YADM_BOOTSTRAP" - echo " GIT_DIR:$GIT_DIR" -} - -@test "Default paths" { - echo " - Default paths should be defined - YADM_DIR=$DEFAULT_YADM_DIR - YADM_REPO=$DEFAULT_YADM_DIR/$DEFAULT_REPO - YADM_CONFIG=$DEFAULT_YADM_DIR/$DEFAULT_CONFIG - YADM_ENCRYPT=$DEFAULT_YADM_DIR/$DEFAULT_ENCRYPT - YADM_ARCHIVE=$DEFAULT_YADM_DIR/$DEFAULT_ARCHIVE - YADM_BOOTSTRAP=$DEFAULT_YADM_DIR/$DEFAULT_BOOTSTRAP - GIT_DIR=$DEFAULT_YADM_DIR/$DEFAULT_REPO - " - - configuration_test - - [ "$status" == 0 ] - [ "$YADM_DIR" = "$HOME/.yadm" ] - [ "$YADM_REPO" = "$DEFAULT_YADM_DIR/$DEFAULT_REPO" ] - [ "$YADM_CONFIG" = "$DEFAULT_YADM_DIR/$DEFAULT_CONFIG" ] - [ "$YADM_ENCRYPT" = "$DEFAULT_YADM_DIR/$DEFAULT_ENCRYPT" ] - [ "$YADM_ARCHIVE" = "$DEFAULT_YADM_DIR/$DEFAULT_ARCHIVE" ] - [ "$YADM_BOOTSTRAP" = "$DEFAULT_YADM_DIR/$DEFAULT_BOOTSTRAP" ] - [ "$GIT_DIR" = "$DEFAULT_YADM_DIR/$DEFAULT_REPO" ] -} - -@test "Override YADM_DIR" { - echo " - Override YADM_DIR using -Y $T_DIR_YADM - YADM_DIR should become $T_DIR_YADM - " - - TEST_ARGS=(-Y $T_DIR_YADM) - configuration_test "${TEST_ARGS[@]}" - - [ "$status" == 0 ] - [ "$YADM_DIR" = "$T_DIR_YADM" ] - [ "$YADM_REPO" = "$T_DIR_YADM/$DEFAULT_REPO" ] - [ "$YADM_CONFIG" = "$T_DIR_YADM/$DEFAULT_CONFIG" ] - [ "$YADM_ENCRYPT" = "$T_DIR_YADM/$DEFAULT_ENCRYPT" ] - [ "$YADM_ARCHIVE" = "$T_DIR_YADM/$DEFAULT_ARCHIVE" ] - [ "$YADM_BOOTSTRAP" = "$T_DIR_YADM/$DEFAULT_BOOTSTRAP" ] - [ "$GIT_DIR" = "$T_DIR_YADM/$DEFAULT_REPO" ] -} - -@test "Override YADM_DIR (not fully-qualified)" { - echo " - Override YADM_DIR using -Y 'relative/path' - yadm should fail, and report the error - " - - TEST_ARGS=(-Y relative/path) - configuration_test "${TEST_ARGS[@]}" - - [ "$status" == 1 ] - [[ "$output" =~ must\ specify\ a\ fully\ qualified ]] -} - -@test "Override YADM_REPO" { - echo " - Override YADM_REPO using --yadm-repo /custom/repo - YADM_REPO should become /custom/repo - GIT_DIR should become /custom/repo - " - - TEST_ARGS=(--yadm-repo /custom/repo) - configuration_test "${TEST_ARGS[@]}" - - [ "$YADM_REPO" = "/custom/repo" ] - [ "$GIT_DIR" = "/custom/repo" ] -} - -@test "Override YADM_REPO (not fully qualified)" { - echo " - Override YADM_REPO using --yadm-repo relative/repo - yadm should fail, and report the error - " - - TEST_ARGS=(--yadm-repo relative/repo) - configuration_test "${TEST_ARGS[@]}" - - [ "$status" == 1 ] - [[ "$output" =~ must\ specify\ a\ fully\ qualified ]] -} - -@test "Override YADM_CONFIG" { - echo " - Override YADM_CONFIG using --yadm-config /custom/config - YADM_CONFIG should become /custom/config - " - - TEST_ARGS=(--yadm-config /custom/config) - configuration_test "${TEST_ARGS[@]}" - - [ "$YADM_CONFIG" = "/custom/config" ] -} - -@test "Override YADM_CONFIG (not fully qualified)" { - echo " - Override YADM_CONFIG using --yadm-config relative/config - yadm should fail, and report the error - " - - TEST_ARGS=(--yadm-config relative/config) - configuration_test "${TEST_ARGS[@]}" - - [ "$status" == 1 ] - [[ "$output" =~ must\ specify\ a\ fully\ qualified ]] -} - -@test "Override YADM_ENCRYPT" { - echo " - Override YADM_ENCRYPT using --yadm-encrypt /custom/encrypt - YADM_ENCRYPT should become /custom/encrypt - " - - TEST_ARGS=(--yadm-encrypt /custom/encrypt) - configuration_test "${TEST_ARGS[@]}" - - [ "$YADM_ENCRYPT" = "/custom/encrypt" ] -} - -@test "Override YADM_ENCRYPT (not fully qualified)" { - echo " - Override YADM_ENCRYPT using --yadm-encrypt relative/encrypt - yadm should fail, and report the error - " - - TEST_ARGS=(--yadm-encrypt relative/encrypt) - configuration_test "${TEST_ARGS[@]}" - - [ "$status" == 1 ] - [[ "$output" =~ must\ specify\ a\ fully\ qualified ]] -} - -@test "Override YADM_ARCHIVE" { - echo " - Override YADM_ARCHIVE using --yadm-archive /custom/archive - YADM_ARCHIVE should become /custom/archive - " - - TEST_ARGS=(--yadm-archive /custom/archive) - configuration_test "${TEST_ARGS[@]}" - - [ "$YADM_ARCHIVE" = "/custom/archive" ] -} - -@test "Override YADM_ARCHIVE (not fully qualified)" { - echo " - Override YADM_ARCHIVE using --yadm-archive relative/archive - yadm should fail, and report the error - " - - TEST_ARGS=(--yadm-archive relative/archive) - configuration_test "${TEST_ARGS[@]}" - - [ "$status" == 1 ] - [[ "$output" =~ must\ specify\ a\ fully\ qualified ]] -} - -@test "Override YADM_BOOTSTRAP" { - echo " - Override YADM_BOOTSTRAP using --yadm-bootstrap /custom/bootstrap - YADM_BOOTSTRAP should become /custom/bootstrap - " - - TEST_ARGS=(--yadm-bootstrap /custom/bootstrap) - configuration_test "${TEST_ARGS[@]}" - - [ "$YADM_BOOTSTRAP" = "/custom/bootstrap" ] -} - -@test "Override YADM_BOOTSTRAP (not fully qualified)" { - echo " - Override YADM_BOOTSTRAP using --yadm-bootstrap relative/bootstrap - yadm should fail, and report the error - " - - TEST_ARGS=(--yadm-bootstrap relative/bootstrap) - configuration_test "${TEST_ARGS[@]}" - - [ "$status" == 1 ] - [[ "$output" =~ must\ specify\ a\ fully\ qualified ]] -} diff --git a/test/002_unit_gpg_program.bats b/test/002_unit_gpg_program.bats deleted file mode 100644 index 3dc29be..0000000 --- a/test/002_unit_gpg_program.bats +++ /dev/null @@ -1,67 +0,0 @@ -load common -T_YADM_CONFIG=; # populated by load_fixtures -load_fixtures -status=;output=; # populated by bats run() - -setup() { - destroy_tmp - make_parents "$T_YADM_CONFIG" -} - -teardown() { - destroy_tmp -} - -function configuration_test() { - # shellcheck source=/dev/null - YADM_TEST=1 source "$T_YADM" - # shellcheck disable=SC2034 - YADM_CONFIG="$T_YADM_CONFIG" - status=0 - { output=$( require_gpg ) && require_gpg; } || { - status=$? - true - } - - echo -e "STATUS:$status\nGPG_PROGRAM:$GPG_PROGRAM\nOUTPUT:$output" - -} - -@test "Default gpg program" { - echo " - Default gpg program should be 'gpg' - " - - configuration_test - - [ "$status" == 0 ] - [ "$GPG_PROGRAM" = "gpg" ] -} - -@test "Override gpg program (valid program)" { - echo " - Override gpg using yadm.gpg-program - Program should be 'cat' - " - - git config --file="$T_YADM_CONFIG" "yadm.gpg-program" "cat" - - configuration_test - - [ "$status" == 0 ] - [ "$GPG_PROGRAM" = "cat" ] -} - -@test "Override gpg program (invalid program)" { - echo " - Override gpg using yadm.gpg-program - Program should be 'badprogram' - " - - git config --file="$T_YADM_CONFIG" "yadm.gpg-program" "badprogram" - - configuration_test - - [ "$status" == 1 ] - [[ "$output" =~ badprogram ]] -} diff --git a/test/003_unit_git_program.bats b/test/003_unit_git_program.bats deleted file mode 100644 index 2fe31c1..0000000 --- a/test/003_unit_git_program.bats +++ /dev/null @@ -1,67 +0,0 @@ -load common -T_YADM_CONFIG=; # populated by load_fixtures -load_fixtures -status=;output=; # populated by bats run() - -setup() { - destroy_tmp - make_parents "$T_YADM_CONFIG" -} - -teardown() { - destroy_tmp -} - -function configuration_test() { - # shellcheck source=/dev/null - YADM_TEST=1 source "$T_YADM" - # shellcheck disable=SC2034 - YADM_CONFIG="$T_YADM_CONFIG" - status=0 - { output=$( require_git ) && require_git; } || { - status=$? - true - } - - echo -e "STATUS:$status\nGIT_PROGRAM:$GIT_PROGRAM\nOUTPUT:$output" - -} - -@test "Default git program" { - echo " - Default git program should be 'git' - " - - configuration_test - - [ "$status" == 0 ] - [ "$GIT_PROGRAM" = "git" ] -} - -@test "Override git program (valid program)" { - echo " - Override git using yadm.git-program - Program should be 'cat' - " - - git config --file="$T_YADM_CONFIG" "yadm.git-program" "cat" - - configuration_test - - [ "$status" == 0 ] - [ "$GIT_PROGRAM" = "cat" ] -} - -@test "Override git program (invalid program)" { - echo " - Override git using yadm.git-program - Program should be 'badprogram' - " - - git config --file="$T_YADM_CONFIG" "yadm.git-program" "badprogram" - - configuration_test - - [ "$status" == 1 ] - [[ "$output" =~ badprogram ]] -} diff --git a/test/004_unit_bootstrap_available.bats b/test/004_unit_bootstrap_available.bats deleted file mode 100644 index fc4cc0d..0000000 --- a/test/004_unit_bootstrap_available.bats +++ /dev/null @@ -1,66 +0,0 @@ -load common -T_YADM_BOOTSTRAP=; # populated by load_fixtures -load_fixtures -status=; # populated by bats run() - -setup() { - destroy_tmp - make_parents "$T_YADM_BOOTSTRAP" -} - -teardown() { - destroy_tmp -} - -function available_test() { - # shellcheck source=/dev/null - YADM_TEST=1 source "$T_YADM" - # shellcheck disable=SC2034 - YADM_BOOTSTRAP="$T_YADM_BOOTSTRAP" - status=0 - { bootstrap_available; } || { - status=$? - true - } - - echo -e "STATUS:$status" - -} - -@test "Bootstrap missing" { - echo " - When bootstrap command is missing - return 1 - " - - available_test - [ "$status" == 1 ] - -} - -@test "Bootstrap not executable" { - echo " - When bootstrap command is not executable - return 1 - " - - touch "$T_YADM_BOOTSTRAP" - - available_test - [ "$status" == 1 ] - -} - -@test "Bootstrap executable" { - echo " - When bootstrap command is not executable - return 0 - " - - touch "$T_YADM_BOOTSTRAP" - chmod a+x "$T_YADM_BOOTSTRAP" - - available_test - [ "$status" == 0 ] - -} diff --git a/test/005_unit_set_operating_system.bats b/test/005_unit_set_operating_system.bats deleted file mode 100644 index 6bbe2c6..0000000 --- a/test/005_unit_set_operating_system.bats +++ /dev/null @@ -1,76 +0,0 @@ -load common -load_fixtures - -@test "Default OS" { - echo " - By default, the value of OPERATING_SYSTEM should be reported by uname -s - " - - # shellcheck source=/dev/null - YADM_TEST=1 source "$T_YADM" - status=0 - output=$( set_operating_system; echo "$OPERATING_SYSTEM" ) || { - status=$? - true - } - - expected=$(uname -s 2>/dev/null) - - echo "output=$output" - echo "expect=$expected" - - [ "$status" == 0 ] - [ "$output" = "$expected" ] -} - -@test "Detect no WSL" { - echo " - When /proc/version does not contain Microsoft, report uname -s - " - - echo "proc version exists" > "$BATS_TMPDIR/proc_version" - - # shellcheck source=/dev/null - YADM_TEST=1 source "$T_YADM" - # shellcheck disable=SC2034 - PROC_VERSION="$BATS_TMPDIR/proc_version" - status=0 - output=$( set_operating_system; echo "$OPERATING_SYSTEM" ) || { - status=$? - true - } - - expected=$(uname -s 2>/dev/null) - - echo "output=$output" - echo "expect=$expected" - - [ "$status" == 0 ] - [ "$output" = "$expected" ] -} - -@test "Detect WSL" { - echo " - When /proc/version contains Microsoft, report WSL - " - - echo "proc version contains Microsoft in it" > "$BATS_TMPDIR/proc_version" - - # shellcheck source=/dev/null - YADM_TEST=1 source "$T_YADM" - # shellcheck disable=SC2034 - PROC_VERSION="$BATS_TMPDIR/proc_version" - status=0 - output=$( set_operating_system; echo "$OPERATING_SYSTEM" ) || { - status=$? - true - } - - expected="WSL" - - echo "output=$output" - echo "expect=$expected" - - [ "$status" == 0 ] - [ "$output" = "$expected" ] -} diff --git a/test/006_unit_query_distro.bats b/test/006_unit_query_distro.bats deleted file mode 100644 index 639ab29..0000000 --- a/test/006_unit_query_distro.bats +++ /dev/null @@ -1,49 +0,0 @@ -load common -load_fixtures - -@test "Query distro (lsb_release present)" { - echo " - Use value of lsb_release -si - " - - #shellcheck source=/dev/null - YADM_TEST=1 source "$T_YADM" - status=0 - { output=$( query_distro ); } || { - status=$? - true - } - - expected="${T_DISTRO}" - - echo "output=$output" - echo "expect=$expected" - - [ "$status" == 0 ] - [ "$output" = "$expected" ] -} - -@test "Query distro (lsb_release missing)" { - echo " - Empty value if lsb_release is missing - " - - #shellcheck source=/dev/null - YADM_TEST=1 source "$T_YADM" - LSB_RELEASE_PROGRAM="missing_lsb_release" - echo "Using $LSB_RELEASE_PROGRAM as lsb_release" - - status=0 - { output=$( query_distro ); } || { - status=$? - true - } - - expected="" - - echo "output=$output" - echo "expect=$expected" - - [ "$status" == 0 ] - [ "$output" = "$expected" ] -} diff --git a/test/007_unit_parse_encrypt.bats b/test/007_unit_parse_encrypt.bats deleted file mode 100644 index 54ee3a9..0000000 --- a/test/007_unit_parse_encrypt.bats +++ /dev/null @@ -1,318 +0,0 @@ -load common -load_fixtures - -setup() { - # SC2153 is intentional - # shellcheck disable=SC2153 - make_parents "$T_YADM_ENCRYPT" - make_parents "$T_DIR_WORK" - make_parents "$T_DIR_REPO" - mkdir "$T_DIR_WORK" - git init --shared=0600 --bare "$T_DIR_REPO" >/dev/null 2>&1 - GIT_DIR="$T_DIR_REPO" git config core.bare 'false' - GIT_DIR="$T_DIR_REPO" git config core.worktree "$T_DIR_WORK" - GIT_DIR="$T_DIR_REPO" git config yadm.managed 'true' -} - -teardown() { - destroy_tmp -} - -function run_parse() { - # shellcheck source=/dev/null - YADM_TEST=1 source "$T_YADM" - YADM_ENCRYPT="$T_YADM_ENCRYPT" - export YADM_ENCRYPT - GIT_DIR="$T_DIR_REPO" - export GIT_DIR - - # shellcheck disable=SC2034 - - status=0 - { output=$( parse_encrypt) && parse_encrypt; } || { - status=$? - true - } - - if [ "$1" == "twice" ]; then - GIT_DIR="$T_DIR_REPO" parse_encrypt - fi - - echo -e "OUTPUT:$output\n" - echo "ENCRYPT_INCLUDE_FILES:" - echo " Size: ${#ENCRYPT_INCLUDE_FILES[@]}" - echo " Items: ${ENCRYPT_INCLUDE_FILES[*]}" - echo "EXPECT_INCLUDE:" - echo " Size: ${#EXPECT_INCLUDE[@]}" - echo " Items: ${EXPECT_INCLUDE[*]}" -} - -@test "parse_encrypt (not called)" { - echo " - parse_encrypt() is not called - Array should be 'unparsed' - " - - # shellcheck source=/dev/null - YADM_TEST=1 source "$T_YADM" - - echo "ENCRYPT_INCLUDE_FILES=$ENCRYPT_INCLUDE_FILES" - - [ "$ENCRYPT_INCLUDE_FILES" == "unparsed" ] - -} - -@test "parse_encrypt (short-circuit)" { - echo " - Parsing should not happen more than once - " - - run_parse "twice" - echo "PARSE_ENCRYPT_SHORT: $PARSE_ENCRYPT_SHORT" - - [ "$status" == 0 ] - [ "$output" == "" ] - [[ "$PARSE_ENCRYPT_SHORT" =~ not\ reprocessed ]] -} - -@test "parse_encrypt (file missing)" { - echo " - .yadm/encrypt is empty - Array should be empty - " - - EXPECT_INCLUDE=() - - run_parse - - [ "$status" == 0 ] - [ "$output" == "" ] - [ "${#ENCRYPT_INCLUDE_FILES[@]}" -eq "${#EXPECT_INCLUDE[@]}" ] - [ "${ENCRYPT_INCLUDE_FILES[*]}" == "${EXPECT_INCLUDE[*]}" ] -} - -@test "parse_encrypt (empty file)" { - echo " - .yadm/encrypt is empty - Array should be empty - " - - touch "$T_YADM_ENCRYPT" - - EXPECT_INCLUDE=() - - run_parse - - [ "$status" == 0 ] - [ "$output" == "" ] - [ "${#ENCRYPT_INCLUDE_FILES[@]}" -eq "${#EXPECT_INCLUDE[@]}" ] - [ "${ENCRYPT_INCLUDE_FILES[*]}" == "${EXPECT_INCLUDE[*]}" ] -} - -@test "parse_encrypt (files)" { - echo " - .yadm/encrypt is references present and missing files - Array should be as expected - " - - echo "file1" > "$T_DIR_WORK/file1" - echo "file3" > "$T_DIR_WORK/file3" - echo "file5" > "$T_DIR_WORK/file5" - - { echo "file1" - echo "file2" - echo "file3" - echo "file4" - echo "file5" - } > "$T_YADM_ENCRYPT" - - EXPECT_INCLUDE=("file1" "file3" "file5") - - run_parse - - [ "$status" == 0 ] - [ "$output" == "" ] - [ "${#ENCRYPT_INCLUDE_FILES[@]}" -eq "${#EXPECT_INCLUDE[@]}" ] - [ "${ENCRYPT_INCLUDE_FILES[*]}" == "${EXPECT_INCLUDE[*]}" ] -} - -@test "parse_encrypt (files and dirs)" { - echo " - .yadm/encrypt is references present and missing files - .yadm/encrypt is references present and missing dirs - Array should be as expected - " - - mkdir -p "$T_DIR_WORK/dir1" - mkdir -p "$T_DIR_WORK/dir2" - echo "file1" > "$T_DIR_WORK/file1" - echo "file2" > "$T_DIR_WORK/file2" - echo "a" > "$T_DIR_WORK/dir1/a" - echo "b" > "$T_DIR_WORK/dir1/b" - - { echo "file1" - echo "file2" - echo "file3" - echo "dir1" - echo "dir2" - echo "dir3" - } > "$T_YADM_ENCRYPT" - - EXPECT_INCLUDE=("file1" "file2" "dir1" "dir2") - - run_parse - - [ "$status" == 0 ] - [ "$output" == "" ] - [ "${#ENCRYPT_INCLUDE_FILES[@]}" -eq "${#EXPECT_INCLUDE[@]}" ] - [ "${ENCRYPT_INCLUDE_FILES[*]}" == "${EXPECT_INCLUDE[*]}" ] -} - -@test "parse_encrypt (comments/empty lines)" { - echo " - .yadm/encrypt is references present and missing files - .yadm/encrypt is references present and missing dirs - .yadm/encrypt contains comments / blank lines - Array should be as expected - " - - mkdir -p "$T_DIR_WORK/dir1" - mkdir -p "$T_DIR_WORK/dir2" - echo "file1" > "$T_DIR_WORK/file1" - echo "file2" > "$T_DIR_WORK/file2" - echo "file3" > "$T_DIR_WORK/file3" - echo "a" > "$T_DIR_WORK/dir1/a" - echo "b" > "$T_DIR_WORK/dir1/b" - - { echo "file1" - echo "file2" - echo "#file3" - echo " #file3" - echo "" - echo "dir1" - echo "dir2" - echo "dir3" - } > "$T_YADM_ENCRYPT" - - EXPECT_INCLUDE=("file1" "file2" "dir1" "dir2") - - run_parse - - [ "$status" == 0 ] - [ "$output" == "" ] - [ "${#ENCRYPT_INCLUDE_FILES[@]}" -eq "${#EXPECT_INCLUDE[@]}" ] - [ "${ENCRYPT_INCLUDE_FILES[*]}" == "${EXPECT_INCLUDE[*]}" ] -} - -@test "parse_encrypt (w/spaces)" { - echo " - .yadm/encrypt is references present and missing files - .yadm/encrypt is references present and missing dirs - .yadm/encrypt references contain spaces - Array should be as expected - " - - mkdir -p "$T_DIR_WORK/di r1" - mkdir -p "$T_DIR_WORK/dir2" - echo "file1" > "$T_DIR_WORK/file1" - echo "fi le2" > "$T_DIR_WORK/fi le2" - echo "file3" > "$T_DIR_WORK/file3" - echo "a" > "$T_DIR_WORK/di r1/a" - echo "b" > "$T_DIR_WORK/di r1/b" - - { echo "file1" - echo "fi le2" - echo "#file3" - echo " #file3" - echo "" - echo "di r1" - echo "dir2" - echo "dir3" - } > "$T_YADM_ENCRYPT" - - EXPECT_INCLUDE=("file1" "fi le2" "di r1" "dir2") - - run_parse - - [ "$status" == 0 ] - [ "$output" == "" ] - [ "${#ENCRYPT_INCLUDE_FILES[@]}" -eq "${#EXPECT_INCLUDE[@]}" ] - [ "${ENCRYPT_INCLUDE_FILES[*]}" == "${EXPECT_INCLUDE[*]}" ] -} - -@test "parse_encrypt (wildcards)" { - echo " - .yadm/encrypt contains wildcards - Array should be as expected - " - - mkdir -p "$T_DIR_WORK/di r1" - mkdir -p "$T_DIR_WORK/dir2" - echo "file1" > "$T_DIR_WORK/file1" - echo "fi le2" > "$T_DIR_WORK/fi le2" - echo "file2" > "$T_DIR_WORK/file2" - echo "file3" > "$T_DIR_WORK/file3" - echo "a" > "$T_DIR_WORK/di r1/a" - echo "b" > "$T_DIR_WORK/di r1/b" - - { echo "fi*" - echo "#file3" - echo " #file3" - echo "" - echo "#dir2" - echo "di r1" - echo "dir2" - echo "dir3" - } > "$T_YADM_ENCRYPT" - - EXPECT_INCLUDE=("fi le2" "file1" "file2" "file3" "di r1" "dir2") - - run_parse - - [ "$status" == 0 ] - [ "$output" == "" ] - [ "${#ENCRYPT_INCLUDE_FILES[@]}" -eq "${#EXPECT_INCLUDE[@]}" ] - [ "${ENCRYPT_INCLUDE_FILES[*]}" == "${EXPECT_INCLUDE[*]}" ] -} - -@test "parse_encrypt (excludes)" { - echo " - .yadm/encrypt contains exclusions - Array should be as expected - " - - mkdir -p "$T_DIR_WORK/di r1" - mkdir -p "$T_DIR_WORK/dir2" - mkdir -p "$T_DIR_WORK/dir3" - echo "file1" > "$T_DIR_WORK/file1" - echo "file1.ex" > "$T_DIR_WORK/file1.ex" - echo "fi le2" > "$T_DIR_WORK/fi le2" - echo "file3" > "$T_DIR_WORK/file3" - echo "test" > "$T_DIR_WORK/test" - echo "a.txt" > "$T_DIR_WORK/di r1/a.txt" - echo "b.txt" > "$T_DIR_WORK/di r1/b.txt" - echo "c.inc" > "$T_DIR_WORK/di r1/c.inc" - - { echo "fi*" - echo "#file3" - echo " #file3" - echo "" - echo " #test" - echo "#dir2" - echo "di r1/*" - echo "dir2" - echo "dir3" - echo "dir4" - echo "!*.ex" - echo "!di r1/*.txt" - } > "$T_YADM_ENCRYPT" - - EXPECT_INCLUDE=("fi le2" "file1" "file3" "di r1/c.inc" "dir2" "dir3") - - run_parse - - [ "$status" == 0 ] - [ "$output" == "" ] - [ "${#ENCRYPT_INCLUDE_FILES[@]}" -eq "${#EXPECT_INCLUDE[@]}" ] - [ "${ENCRYPT_INCLUDE_FILES[*]}" == "${EXPECT_INCLUDE[*]}" ] -} diff --git a/test/100_accept_version.bats b/test/100_accept_version.bats deleted file mode 100644 index 962b91f..0000000 --- a/test/100_accept_version.bats +++ /dev/null @@ -1,25 +0,0 @@ -load common -load_fixtures -status=;output=; #; populated by bats run() - -@test "Command 'version'" { - echo " - When 'version' command is provided, - Print the current version with format 'yadm x.x.x' - Exit with 0 - " - - #; run yadm with 'version' command - run "$T_YADM" version - - # shellcheck source=/dev/null - - #; load yadm variables (including VERSION) - YADM_TEST=1 source "$T_YADM" - - #; validate status and output - [ $status -eq 0 ] - [ "$output" = "yadm $VERSION" ] - version_regex="^yadm [[:digit:]]+\.[[:digit:]]+\.[[:digit:]]+$" - [[ "$output" =~ $version_regex ]] -} diff --git a/test/101_accept_help.bats b/test/101_accept_help.bats deleted file mode 100644 index 5932379..0000000 --- a/test/101_accept_help.bats +++ /dev/null @@ -1,33 +0,0 @@ -load common -load_fixtures -status=;lines=; #; populated by bats run() - -@test "Missing command" { - echo " - When no command is provided, - Produce usage instructions - Exit with 1 - " - - #; run yadm with no command - run "$T_YADM" - - #; validate status and output - [ $status -eq 1 ] - [[ "${lines[0]}" =~ ^Usage: ]] -} - -@test "Command 'help'" { - echo " - When 'help' command is provided, - Produce usage instructions - Exit with value 1 - " - - #; run yadm with 'help' command - run "$T_YADM" help - - #; validate status and output - [ $status -eq 1 ] - [[ "${lines[0]}" =~ ^Usage: ]] -} diff --git a/test/102_accept_clean.bats b/test/102_accept_clean.bats deleted file mode 100644 index 1ab0e77..0000000 --- a/test/102_accept_clean.bats +++ /dev/null @@ -1,19 +0,0 @@ -load common -load_fixtures -status=;lines=; #; populated by bats run() - -@test "Command 'clean'" { - echo " - When 'clean' command is provided, - Do nothing, this is a dangerous Git command when managing dot files - Report the command as disabled - Exit with 1 - " - - #; run yadm with 'clean' command - run "$T_YADM" clean - - #; validate status and output - [ $status -eq 1 ] - [[ "${lines[0]}" =~ disabled ]] -} diff --git a/test/103_accept_git.bats b/test/103_accept_git.bats deleted file mode 100644 index 61a5a45..0000000 --- a/test/103_accept_git.bats +++ /dev/null @@ -1,117 +0,0 @@ -load common -load_fixtures -status=;output=;lines=; #; populated by bats run() - -IN_REPO=(.bash_profile .vimrc) - -function setup_environment() { - destroy_tmp - build_repo "${IN_REPO[@]}" -} - -@test "Passthru unknown commands to Git" { - echo " - When the command 'bogus' is provided - Report bogus is not a command - Exit with 1 - " - - #; start fresh - setup_environment - - #; run bogus - run "${T_YADM_Y[@]}" bogus - - #; validate status and output - [ "$status" -eq 1 ] - [[ "$output" =~ .bogus..is.not.a.git.command ]] -} - -@test "Git command 'add' - badfile" { - echo " - When the command 'add' is provided - And the file specified does not exist - Exit with 128 - " - - #; start fresh - setup_environment - - #; define a non existig testfile - local testfile="$T_DIR_WORK/does_not_exist" - - #; run add - run "${T_YADM_Y[@]}" add -v "$testfile" - - #; validate status and output - [ "$status" -eq 128 ] - [[ "$output" =~ pathspec.+did.not.match ]] -} - -@test "Git command 'add'" { - echo " - When the command 'add' is provided - Files are added to the index - Exit with 0 - " - - #; start fresh - setup_environment - - #; create a testfile - local testfile="$T_DIR_WORK/testfile" - echo "$testfile" > "$testfile" - - #; run add - run "${T_YADM_Y[@]}" add -v "$testfile" - - #; validate status and output - [ "$status" -eq 0 ] - [ "$output" = "add 'testfile'" ] -} - -@test "Git command 'status'" { - echo " - When the command 'status' is provided - Added files are shown - Exit with 0 - " - - #; run status - run "${T_YADM_Y[@]}" status - - #; validate status and output - [ "$status" -eq 0 ] - [[ "$output" =~ new\ file:[[:space:]]+testfile ]] -} - -@test "Git command 'commit'" { - echo " - When the command 'commit' is provided - Index is commited - Exit with 0 - " - - #; run commit - run "${T_YADM_Y[@]}" commit -m 'Add testfile' - - #; validate status and output - [ "$status" -eq 0 ] - [[ "${lines[1]}" =~ 1\ file\ changed ]] - [[ "${lines[1]}" =~ 1\ insertion ]] -} - -@test "Git command 'log'" { - echo " - When the command 'log' is provided - Commits are shown - Exit with 0 - " - - #; run log - run "${T_YADM_Y[@]}" log --oneline - - #; validate status and output - [ "$status" -eq 0 ] - [[ "${lines[0]}" =~ Add\ testfile ]] -} diff --git a/test/104_accept_init.bats b/test/104_accept_init.bats deleted file mode 100644 index 1751e7a..0000000 --- a/test/104_accept_init.bats +++ /dev/null @@ -1,178 +0,0 @@ -load common -load_fixtures -status=;output=; #; populated by bats run() - -setup() { - destroy_tmp - create_worktree "$T_DIR_WORK" -} - -@test "Command 'init'" { - echo " - When 'init' command is provided, - Create new repo with attributes: - - 0600 permissions - - not bare - - worktree = \$HOME - - showUntrackedFiles = no - - yadm.managed = true - Report the repo as initialized - Exit with 0 - " - - #; run init - run "${T_YADM_Y[@]}" init - - #; validate status and output - [ $status -eq 0 ] - [[ "$output" =~ Initialized ]] - - #; validate repo attributes - test_perms "$T_DIR_REPO" "drw.--.--." - test_repo_attribute "$T_DIR_REPO" core.bare false - test_repo_attribute "$T_DIR_REPO" core.worktree "$HOME" - test_repo_attribute "$T_DIR_REPO" status.showUntrackedFiles no - test_repo_attribute "$T_DIR_REPO" yadm.managed true -} - -@test "Command 'init' -w (alternate worktree)" { - echo " - When 'init' command is provided, - and '-w' is provided, - Create new repo with attributes: - - 0600 permissions - - not bare - - worktree = \$YADM_WORK - - showUntrackedFiles = no - - yadm.managed = true - Report the repo as initialized - Exit with 0 - " - - #; run init - run "${T_YADM_Y[@]}" init -w "$T_DIR_WORK" - - #; validate status and output - [ $status -eq 0 ] - [[ "$output" =~ Initialized ]] - - #; validate repo attributes - test_perms "$T_DIR_REPO" "drw.--.--." - test_repo_attribute "$T_DIR_REPO" core.bare false - test_repo_attribute "$T_DIR_REPO" core.worktree "$T_DIR_WORK" - test_repo_attribute "$T_DIR_REPO" status.showUntrackedFiles no - test_repo_attribute "$T_DIR_REPO" yadm.managed true -} - -@test "Command 'init' (existing repo)" { - echo " - When 'init' command is provided, - and a repo already exists, - Refuse to create a new repo - Exit with 1 - " - - #; create existing repo content - mkdir -p "$T_DIR_REPO" - local testfile="$T_DIR_REPO/testfile" - touch "$testfile" - - #; run init - run "${T_YADM_Y[@]}" init - - #; validate status and output - [ $status -eq 1 ] - [[ "$output" =~ already.exists ]] - - #; verify existing repo is intact - if [ ! -e "$testfile" ]; then - echo "ERROR: existing repo has been changed" - return 1 - fi - -} - -@test "Command 'init' -f (force overwrite repo)" { - echo " - When 'init' command is provided, - and '-f' is provided - and a repo already exists, - Remove existing repo - Create new repo with attributes: - - 0600 permissions - - not bare - - worktree = \$HOME - - showUntrackedFiles = no - - yadm.managed = true - Report the repo as initialized - Exit with 0 - " - - #; create existing repo content - mkdir -p "$T_DIR_REPO" - local testfile="$T_DIR_REPO/testfile" - touch "$testfile" - - #; run init - run "${T_YADM_Y[@]}" init -f - - #; validate status and output - [ $status -eq 0 ] - [[ "$output" =~ Initialized ]] - - #; verify existing repo is gone - if [ -e "$testfile" ]; then - echo "ERROR: existing repo files remain" - return 1 - fi - - #; validate repo attributes - test_perms "$T_DIR_REPO" "drw.--.--." - test_repo_attribute "$T_DIR_REPO" core.bare false - test_repo_attribute "$T_DIR_REPO" core.worktree "$HOME" - test_repo_attribute "$T_DIR_REPO" status.showUntrackedFiles no - test_repo_attribute "$T_DIR_REPO" yadm.managed true -} - -@test "Command 'init' -f -w (force overwrite repo with alternate worktree)" { - echo " - When 'init' command is provided, - and '-f' is provided - and '-w' is provided - and a repo already exists, - Remove existing repo - Create new repo with attributes: - - 0600 permissions - - not bare - - worktree = \$YADM_WORK - - showUntrackedFiles = no - - yadm.managed = true - Report the repo as initialized - Exit with 0 - " - - #; create existing repo content - mkdir -p "$T_DIR_REPO" - local testfile="$T_DIR_REPO/testfile" - touch "$testfile" - - #; run init - run "${T_YADM_Y[@]}" init -f -w "$T_DIR_WORK" - - #; validate status and output - [ $status -eq 0 ] - [[ "$output" =~ Initialized ]] - - #; verify existing repo is gone - if [ -e "$testfile" ]; then - echo "ERROR: existing repo files remain" - return 1 - fi - - #; validate repo attributes - test_perms "$T_DIR_REPO" "drw.--.--." - test_repo_attribute "$T_DIR_REPO" core.bare false - test_repo_attribute "$T_DIR_REPO" core.worktree "$T_DIR_WORK" - test_repo_attribute "$T_DIR_REPO" status.showUntrackedFiles no - test_repo_attribute "$T_DIR_REPO" yadm.managed true -} diff --git a/test/105_accept_clone.bats b/test/105_accept_clone.bats deleted file mode 100644 index 42750b8..0000000 --- a/test/105_accept_clone.bats +++ /dev/null @@ -1,579 +0,0 @@ -load common -load_fixtures -status=;output=; #; populated by bats run() - -IN_REPO=(.bash_profile .vimrc) -T_DIR_REMOTE="$T_TMP/remote" -REMOTE_URL="file:///$T_TMP/remote" - -setup() { - destroy_tmp - build_repo "${IN_REPO[@]}" - cp -rp "$T_DIR_REPO" "$T_DIR_REMOTE" -} - -create_bootstrap() { - make_parents "$T_YADM_BOOTSTRAP" - { - echo "#!/bin/bash" - echo "echo Bootstrap successful" - echo "exit 123" - } > "$T_YADM_BOOTSTRAP" - chmod a+x "$T_YADM_BOOTSTRAP" -} - -@test "Command 'clone' (bad remote)" { - echo " - When 'clone' command is provided, - and the remote is bad, - Report error - Remove the YADM_REPO - Exit with 1 - " - - #; remove existing worktree and repo - rm -rf "$T_DIR_WORK" - mkdir -p "$T_DIR_WORK" - rm -rf "$T_DIR_REPO" - - #; run clone - run "${T_YADM_Y[@]}" clone -w "$T_DIR_WORK" "file:///bogus-repo" - - #; validate status and output - [ "$status" -eq 1 ] - [[ "$output" =~ Unable\ to\ fetch\ origin ]] - - #; confirm repo directory is removed - [ ! -d "$T_DIR_REPO" ] -} - -@test "Command 'clone'" { - echo " - When 'clone' command is provided, - Create new repo with attributes: - - 0600 permissions - - not bare - - worktree = \$YADM_WORK - - showUntrackedFiles = no - - yadm.managed = true - Report the repo as cloned - A remote named origin exists - Exit with 0 - " - - #; remove existing worktree and repo - rm -rf "$T_DIR_WORK" - mkdir -p "$T_DIR_WORK" - rm -rf "$T_DIR_REPO" - - #; run clone - run "${T_YADM_Y[@]}" clone -w "$T_DIR_WORK" "$REMOTE_URL" - - #; validate status and output - [ "$status" -eq 0 ] - [[ "$output" =~ Initialized ]] - - #; validate repo attributes - test_perms "$T_DIR_REPO" "drw.--.--." - test_repo_attribute "$T_DIR_REPO" core.bare false - test_repo_attribute "$T_DIR_REPO" core.worktree "$T_DIR_WORK" - test_repo_attribute "$T_DIR_REPO" status.showUntrackedFiles no - test_repo_attribute "$T_DIR_REPO" yadm.managed true - - #; test the remote - local remote_output - remote_output=$(GIT_DIR="$T_DIR_REPO" git remote show) - [ "$remote_output" = "origin" ] -} - -@test "Command 'clone' (existing repo)" { - echo " - When 'clone' command is provided, - and a repo already exists, - Report error - Exit with 1 - " - - #; run clone - run "${T_YADM_Y[@]}" clone -w "$T_DIR_WORK" "$REMOTE_URL" - - #; validate status and output - [ "$status" -eq 1 ] - [[ "$output" =~ Git\ repo\ already\ exists ]] -} - -@test "Command 'clone' -f (force overwrite)" { - echo " - When 'clone' command is provided, - and '-f' is provided, - and a repo already exists, - Overwrite the repo with attributes: - - 0600 permissions - - not bare - - worktree = \$YADM_WORK - - showUntrackedFiles = no - - yadm.managed = true - Report the repo as cloned - A remote named origin exists - Exit with 0 - " - - #; remove existing worktree - rm -rf "$T_DIR_WORK" - mkdir -p "$T_DIR_WORK" - - #; run clone - run "${T_YADM_Y[@]}" clone -w "$T_DIR_WORK" -f "$REMOTE_URL" - - #; validate status and output - [ "$status" -eq 0 ] - [[ "$output" =~ Initialized ]] - - #; validate repo attributes - test_perms "$T_DIR_REPO" "drw.--.--." - test_repo_attribute "$T_DIR_REPO" core.bare false - test_repo_attribute "$T_DIR_REPO" core.worktree "$T_DIR_WORK" - test_repo_attribute "$T_DIR_REPO" status.showUntrackedFiles no - test_repo_attribute "$T_DIR_REPO" yadm.managed true - - #; test the remote - local remote_output - remote_output=$(GIT_DIR="$T_DIR_REPO" git remote show) - [ "$remote_output" = "origin" ] -} - -@test "Command 'clone' (existing conflicts)" { - echo " - When 'clone' command is provided, - and '-f' is provided, - and a repo already exists, - Overwrite the repo with attributes: - - 0600 permissions - - not bare - - worktree = \$YADM_WORK - - showUntrackedFiles = no - - yadm.managed = true - Report the repo as cloned - A remote named origin exists - Exit with 0 - " - - #; remove existing repo - rm -rf "$T_DIR_REPO" - - #; cause a conflict - echo "conflict" >> "$T_DIR_WORK/.bash_profile" - - #; run clone - run "${T_YADM_Y[@]}" clone -w "$T_DIR_WORK" "$REMOTE_URL" - - #; validate status and output - [ "$status" -eq 0 ] - [[ "$output" =~ Initialized ]] - - #; validate merging note - [[ "$output" =~ Merging\ origin/master\ failed ]] - [[ "$output" =~ NOTE ]] - - #; validate repo attributes - test_perms "$T_DIR_REPO" "drw.--.--." - test_repo_attribute "$T_DIR_REPO" core.bare false - test_repo_attribute "$T_DIR_REPO" core.worktree "$T_DIR_WORK" - test_repo_attribute "$T_DIR_REPO" status.showUntrackedFiles no - test_repo_attribute "$T_DIR_REPO" yadm.managed true - - #; test the remote - local remote_output - remote_output=$(GIT_DIR="$T_DIR_REPO" git remote show) - [ "$remote_output" = "origin" ] - - #; confirm yadm repo is clean - cd "$T_DIR_WORK" ||: - clean_status=$("${T_YADM_Y[@]}" status -uno --porcelain) - echo "clean_status:'$clean_status'" - [ -z "$clean_status" ] - - #; confirm conflicts are stashed - existing_stash=$("${T_YADM_Y[@]}" stash list) - echo "existing_stash:'$existing_stash'" - [[ "$existing_stash" =~ Conflicts\ preserved ]] - - stashed_conflicts=$("${T_YADM_Y[@]}" stash show -p) - echo "stashed_conflicts:'$stashed_conflicts'" - [[ "$stashed_conflicts" =~ \+conflict ]] - -} - -@test "Command 'clone' (force bootstrap, missing)" { - echo " - When 'clone' command is provided, - with the --bootstrap parameter - and bootstrap does not exists - Create new repo with attributes: - - 0600 permissions - - not bare - - worktree = \$YADM_WORK - - showUntrackedFiles = no - - yadm.managed = true - Report the repo as cloned - A remote named origin exists - Exit with 0 - " - - #; remove existing worktree and repo - rm -rf "$T_DIR_WORK" - mkdir -p "$T_DIR_WORK" - rm -rf "$T_DIR_REPO" - - #; run clone - run "${T_YADM_Y[@]}" clone --bootstrap -w "$T_DIR_WORK" "$REMOTE_URL" - - #; validate status and output - [ "$status" -eq 0 ] - [[ "$output" =~ Initialized ]] - - #; validate repo attributes - test_perms "$T_DIR_REPO" "drw.--.--." - test_repo_attribute "$T_DIR_REPO" core.bare false - test_repo_attribute "$T_DIR_REPO" core.worktree "$T_DIR_WORK" - test_repo_attribute "$T_DIR_REPO" status.showUntrackedFiles no - test_repo_attribute "$T_DIR_REPO" yadm.managed true - - #; test the remote - local remote_output - remote_output=$(GIT_DIR="$T_DIR_REPO" git remote show) - [ "$remote_output" = "origin" ] -} - -@test "Command 'clone' (force bootstrap, existing)" { - echo " - When 'clone' command is provided, - with the --bootstrap parameter - and bootstrap exists - Create new repo with attributes: - - 0600 permissions - - not bare - - worktree = \$YADM_WORK - - showUntrackedFiles = no - - yadm.managed = true - Report the repo as cloned - A remote named origin exists - Run the bootstrap - Exit with bootstrap's exit code - " - - #; remove existing worktree and repo - rm -rf "$T_DIR_WORK" - mkdir -p "$T_DIR_WORK" - rm -rf "$T_DIR_REPO" - - #; create the bootstrap - create_bootstrap - - #; run clone - run "${T_YADM_Y[@]}" clone --bootstrap -w "$T_DIR_WORK" "$REMOTE_URL" - - #; validate status and output - [ "$status" -eq 123 ] - [[ "$output" =~ Initialized ]] - [[ "$output" =~ Bootstrap\ successful ]] - - #; validate repo attributes - test_perms "$T_DIR_REPO" "drw.--.--." - test_repo_attribute "$T_DIR_REPO" core.bare false - test_repo_attribute "$T_DIR_REPO" core.worktree "$T_DIR_WORK" - test_repo_attribute "$T_DIR_REPO" status.showUntrackedFiles no - test_repo_attribute "$T_DIR_REPO" yadm.managed true - - #; test the remote - local remote_output - remote_output=$(GIT_DIR="$T_DIR_REPO" git remote show) - [ "$remote_output" = "origin" ] -} - -@test "Command 'clone' (prevent bootstrap)" { - echo " - When 'clone' command is provided, - with the --no-bootstrap parameter - and bootstrap exists - Create new repo with attributes: - - 0600 permissions - - not bare - - worktree = \$YADM_WORK - - showUntrackedFiles = no - - yadm.managed = true - Report the repo as cloned - A remote named origin exists - Do NOT run bootstrap - Exit with 0 - " - - #; remove existing worktree and repo - rm -rf "$T_DIR_WORK" - mkdir -p "$T_DIR_WORK" - rm -rf "$T_DIR_REPO" - - #; create the bootstrap - create_bootstrap - - #; run clone - run "${T_YADM_Y[@]}" clone --no-bootstrap -w "$T_DIR_WORK" "$REMOTE_URL" - - #; validate status and output - [ "$status" -eq 0 ] - [[ $output =~ Initialized ]] - [[ ! $output =~ Bootstrap\ successful ]] - - #; validate repo attributes - test_perms "$T_DIR_REPO" "drw.--.--." - test_repo_attribute "$T_DIR_REPO" core.bare false - test_repo_attribute "$T_DIR_REPO" core.worktree "$T_DIR_WORK" - test_repo_attribute "$T_DIR_REPO" status.showUntrackedFiles no - test_repo_attribute "$T_DIR_REPO" yadm.managed true - - #; test the remote - local remote_output - remote_output=$(GIT_DIR="$T_DIR_REPO" git remote show) - [ "$remote_output" = "origin" ] -} - -@test "Command 'clone' (existing bootstrap, answer n)" { - echo " - When 'clone' command is provided, - and bootstrap exists - Create new repo with attributes: - - 0600 permissions - - not bare - - worktree = \$YADM_WORK - - showUntrackedFiles = no - - yadm.managed = true - Report the repo as cloned - A remote named origin exists - Do NOT run bootstrap - Exit with 0 - " - - #; remove existing worktree and repo - rm -rf "$T_DIR_WORK" - mkdir -p "$T_DIR_WORK" - rm -rf "$T_DIR_REPO" - - #; create the bootstrap - create_bootstrap - - #; run clone - run expect < "$T_YADM_CONFIG" - - #; run config - run "${T_YADM_Y[@]}" config "$T_KEY" - - #; validate status and output - [ $status -eq 0 ] - if [ "$output" != "$T_VALUE" ]; then - echo "ERROR: Incorrect value returned. Expected '$T_VALUE', got '$output'" - return 1 - fi -} - -@test "Command 'config' (update)" { - echo " - When 'config' command is provided, - and an attribute is provided - and the attribute is already configured - Report no output - Update configuration file - Exit with 0 - " - - #; manually load a value into the configuration - make_parents "$T_YADM_CONFIG" - echo -e "${T_EXPECTED}_with_extra_data" > "$T_YADM_CONFIG" - - #; run config - run "${T_YADM_Y[@]}" config "$T_KEY" "$T_VALUE" - - #; validate status and output - [ $status -eq 0 ] - [ "$output" = "" ] - - #; validate configuration - local config - config=$(cat "$T_YADM_CONFIG") - local expected - expected=$(echo -e "$T_EXPECTED") - if [ "$config" != "$expected" ]; then - echo "ERROR: Config does not match expected" - echo "$config" - return 1 - fi -} - -@test "Command 'config' (local read)" { - echo " - When 'config' command is provided, - and an attribute is provided - and the attribute is configured - and the attribute is local.* - Fetch the value from the repo config - Report the requested value - Exit with 0 - " - - #; write local attributes - build_repo - for loption in class os hostname user; do - GIT_DIR="$T_DIR_REPO" git config "local.$loption" "custom_$loption" - done - - #; run config - for loption in class os hostname user; do - run "${T_YADM_Y[@]}" config "local.$loption" - #; validate status and output - [ $status -eq 0 ] - if [ "$output" != "custom_$loption" ]; then - echo "ERROR: Incorrect value returned. Expected 'custom_$loption', got '$output'" - return 1 - fi - done - -} - -@test "Command 'config' (local write)" { - echo " - When 'config' command is provided, - and an attribute is provided - and a value is provided - and the attribute is local.* - Report no output - Write the value to the repo config - Exit with 0 - " - - build_repo - local expected - local linecount - expected="[local]\n" - linecount=1 - for loption in class os hostname user; do - #; update expected - expected="$expected\t$loption = custom_$loption\n" - ((linecount+=1)) - #; write local attributes - run "${T_YADM_Y[@]}" config "local.$loption" "custom_$loption" - - #; validate status and output - [ $status -eq 0 ] - [ "$output" = "" ] - done - - #; validate data - local config - config=$(tail "-$linecount" "$T_DIR_REPO/config") - expected=$(echo -ne "$expected") - if [ "$config" != "$expected" ]; then - echo "ERROR: Config does not match expected" - echo -e "$config" - echo -e "EXPECTED:\n$expected" - return 1 - fi - -} diff --git a/test/107_accept_list.bats b/test/107_accept_list.bats deleted file mode 100644 index 3cc61d3..0000000 --- a/test/107_accept_list.bats +++ /dev/null @@ -1,93 +0,0 @@ -load common -load_fixtures -status=;lines=; #; populated by bats run() - -IN_REPO=(.bash_profile .hammerspoon/init.lua .vimrc) -SUBDIR=".hammerspoon" -IN_SUBDIR=(init.lua) - -function setup() { - destroy_tmp - build_repo "${IN_REPO[@]}" -} - -@test "Command 'list' -a" { - echo " - When 'list' command is provided, - and '-a' is provided, - List tracked files - Exit with 0 - " - - #; run list -a - run "${T_YADM_Y[@]}" list -a - - #; validate status and output - [ "$status" -eq 0 ] - local line=0 - for f in "${IN_REPO[@]}"; do - [ "${lines[$line]}" = "$f" ] - ((line++)) || true - done -} - -@test "Command 'list' (outside of worktree)" { - echo " - When 'list' command is provided, - and while outside of the worktree - List tracked files - Exit with 0 - " - - #; run list - run "${T_YADM_Y[@]}" list - - #; validate status and output - [ "$status" -eq 0 ] - local line=0 - for f in "${IN_REPO[@]}"; do - [ "${lines[$line]}" = "$f" ] - ((line++)) || true - done -} - -@test "Command 'list' (in root of worktree)" { - echo " - When 'list' command is provided, - and while in root of the worktree - List tracked files - Exit with 0 - " - - #; run list - run bash -c "(cd '$T_DIR_WORK'; ${T_YADM_Y[*]} list)" - - #; validate status and output - [ "$status" -eq 0 ] - local line=0 - for f in "${IN_REPO[@]}"; do - [ "${lines[$line]}" = "$f" ] - ((line++)) || true - done -} - -@test "Command 'list' (in subdirectory of worktree)" { - echo " - When 'list' command is provided, - and while in subdirectory of the worktree - List tracked files for current directory - Exit with 0 - " - - #; run list - run bash -c "(cd '$T_DIR_WORK/$SUBDIR'; ${T_YADM_Y[*]} list)" - - #; validate status and output - [ "$status" -eq 0 ] - local line=0 - for f in "${IN_SUBDIR[@]}"; do - echo "'${lines[$line]}' = '$f'" - [ "${lines[$line]}" = "$f" ] - ((line++)) || true - done -} diff --git a/test/108_accept_alt.bats b/test/108_accept_alt.bats deleted file mode 100644 index 5ebf9c8..0000000 --- a/test/108_accept_alt.bats +++ /dev/null @@ -1,415 +0,0 @@ -load common -load_fixtures -status=;output=; #; populated by bats run() - -IN_REPO=(alt* "dir one") -export TEST_TREE_WITH_ALT=1 -EXCLUDED_NAME="excluded-base" - -function create_encrypt() { - for efile in "encrypted-base##" "encrypted-system##$T_SYS" "encrypted-host##$T_SYS.$T_HOST" "encrypted-user##$T_SYS.$T_HOST.$T_USER"; do - echo "$efile" >> "$T_YADM_ENCRYPT" - echo "$efile" >> "$T_DIR_WORK/$efile" - mkdir -p "$T_DIR_WORK/dir one/$efile" - echo "dir one/$efile/file1" >> "$T_YADM_ENCRYPT" - echo "dir one/$efile/file1" >> "$T_DIR_WORK/dir one/$efile/file1" - done - - echo "$EXCLUDED_NAME##" >> "$T_YADM_ENCRYPT" - echo "!$EXCLUDED_NAME##" >> "$T_YADM_ENCRYPT" - echo "$EXCLUDED_NAME##" >> "$T_DIR_WORK/$EXCLUDED_NAME##" -} - -setup() { - destroy_tmp - build_repo "${IN_REPO[@]}" - create_encrypt -} - -function test_alt() { - local alt_type="$1" - local test_overwrite="$2" - local auto_alt="$3" - - #; detemine test parameters - case $alt_type in - base) - link_name="alt-base" - link_match="$link_name##" - ;; - system) - link_name="alt-system" - link_match="$link_name##$T_SYS" - ;; - host) - link_name="alt-host" - link_match="$link_name##$T_SYS.$T_HOST" - ;; - user) - link_name="alt-user" - link_match="$link_name##$T_SYS.$T_HOST.$T_USER" - ;; - encrypted_base) - link_name="encrypted-base" - link_match="$link_name##" - ;; - encrypted_system) - link_name="encrypted-system" - link_match="$link_name##$T_SYS" - ;; - encrypted_host) - link_name="encrypted-host" - link_match="$link_name##$T_SYS.$T_HOST" - ;; - encrypted_user) - link_name="encrypted-user" - link_match="$link_name##$T_SYS.$T_HOST.$T_USER" - ;; - override_system) - link_name="alt-override-system" - link_match="$link_name##custom_system" - ;; - override_host) - link_name="alt-override-host" - link_match="$link_name##$T_SYS.custom_host" - ;; - override_user) - link_name="alt-override-user" - link_match="$link_name##$T_SYS.$T_HOST.custom_user" - ;; - class_aaa) - link_name="alt-system" - link_match="$link_name##aaa" - ;; - class_zzz) - link_name="alt-system" - link_match="$link_name##zzz" - ;; - class_AAA) - link_name="alt-system" - link_match="$link_name##AAA" - ;; - class_ZZZ) - link_name="alt-system" - link_match="$link_name##ZZZ" - ;; - esac - dir_link_name="dir one/${link_name}" - dir_link_match="dir one/${link_match}" - - if [ "$test_overwrite" = "true" ]; then - #; create incorrect links (to overwrite) - ln -nfs "$T_DIR_WORK/dir2/file2" "$T_DIR_WORK/$link_name" - ln -nfs "$T_DIR_WORK/dir2" "$T_DIR_WORK/$dir_link_name" - else - #; verify link doesn't already exist - if [ -L "$T_DIR_WORK/$link_name" ] || [ -L "$T_DIR_WORK/$dir_link_name" ]; then - echo "ERROR: Link already exists before running yadm" - return 1 - fi - fi - - #; configure yadm.auto_alt=false - if [ "$auto_alt" = "false" ]; then - git config --file="$T_YADM_CONFIG" yadm.auto-alt false - fi - - #; run yadm (alt or status) - if [ -z "$auto_alt" ]; then - run "${T_YADM_Y[@]}" alt - #; validate status and output - echo "TEST:Link Name:$link_name" - echo "TEST:DIR Link Name:$dir_link_name" - if [ "$status" != 0 ] || [[ ! "$output" =~ Linking.+$link_name ]] || [[ ! "$output" =~ Linking.+$dir_link_name ]]; then - echo "OUTPUT:$output" - echo "STATUS:$status" - echo "ERROR: Could not confirm status and output of alt command" - return 1; - fi - else - #; running any passed through Git command should trigger auto-alt - run "${T_YADM_Y[@]}" status - if [ -n "$auto_alt" ] && [[ "$output" =~ Linking.+$link_name ]] && [[ "$output" =~ Linking.+$dir_link_name ]]; then - echo "ERROR: Reporting of link should not happen" - return 1 - fi - fi - - if [ -L "$T_DIR_WORK/$EXCLUDED_NAME" ] ; then - echo "ERROR: Found link: $T_DIR_WORK/$EXCLUDED_NAME" - echo "ERROR: Excluded files should not be linked" - return 1 - fi - - #; validate link content - if [[ "$alt_type" =~ none ]] || [ "$auto_alt" = "false" ]; then - #; no link should be present - if [ -L "$T_DIR_WORK/$link_name" ] || [ -L "$T_DIR_WORK/$dir_link_name" ]; then - echo "ERROR: Links should not exist" - return 1 - fi - else - #; correct link should be present - local link_content - local dir_link_content - link_content=$(cat "$T_DIR_WORK/$link_name") - dir_link_content=$(cat "$T_DIR_WORK/$dir_link_name/file1") - if [ "$link_content" != "$link_match" ] || [ "$dir_link_content" != "$dir_link_match/file1" ]; then - echo "link_content: $link_content" - echo "dir_link_content: $dir_link_content" - echo "ERROR: Link content is not correct" - return 1 - fi - fi -} - -@test "Command 'alt' (select base)" { - echo " - When the command 'alt' is provided - and file matches only ## - Report the linking - Verify correct file is linked - Exit with 0 - " - - test_alt 'base' 'false' '' -} - -@test "Command 'alt' (select system)" { - echo " - When the command 'alt' is provided - and file matches only ##SYSTEM - Report the linking - Verify correct file is linked - Exit with 0 - " - - test_alt 'system' 'false' '' -} - -@test "Command 'alt' (select host)" { - echo " - When the command 'alt' is provided - and file matches only ##SYSTEM.HOST - Report the linking - Verify correct file is linked - Exit with 0 - " - - test_alt 'host' 'false' '' -} - -@test "Command 'alt' (select user)" { - echo " - When the command 'alt' is provided - and file matches only ##SYSTEM.HOST.USER - Report the linking - Verify correct file is linked - Exit with 0 - " - - test_alt 'user' 'false' '' -} - -@test "Command 'alt' (select none)" { - echo " - When the command 'alt' is provided - and no file matches - Verify there is no link - Exit with 0 - " - - test_alt 'none' 'false' '' -} - -@test "Command 'alt' (select class - aaa)" { - echo " - When the command 'alt' is provided - and file matches only ##CLASS - aaa - Report the linking - Verify correct file is linked - Exit with 0 - " - - GIT_DIR="$T_DIR_REPO" git config local.class aaa - - test_alt 'class_aaa' 'false' '' -} - -@test "Command 'alt' (select class - zzz)" { - echo " - When the command 'alt' is provided - and file matches only ##CLASS - zzz - Report the linking - Verify correct file is linked - Exit with 0 - " - - GIT_DIR="$T_DIR_REPO" git config local.class zzz - - test_alt 'class_zzz' 'false' '' -} - -@test "Command 'alt' (select class - AAA)" { - echo " - When the command 'alt' is provided - and file matches only ##CLASS - AAA - Report the linking - Verify correct file is linked - Exit with 0 - " - - GIT_DIR="$T_DIR_REPO" git config local.class AAA - - test_alt 'class_AAA' 'false' '' -} - -@test "Command 'alt' (select class - ZZZ)" { - echo " - When the command 'alt' is provided - and file matches only ##CLASS - ZZZ - Report the linking - Verify correct file is linked - Exit with 0 - " - - GIT_DIR="$T_DIR_REPO" git config local.class ZZZ - - test_alt 'class_ZZZ' 'false' '' -} - -@test "Command 'auto-alt' (enabled)" { - echo " - When a command possibly changes the repo - and auto-alt is configured true - automatically process alternates - report no linking (not loud) - verify alternate created - " - - test_alt 'base' 'false' 'true' -} - -@test "Command 'auto-alt' (disabled)" { - echo " - When a command possibly changes the repo - and auto-alt is configured false - do no linking - verify no links - " - - test_alt 'base' 'false' 'false' -} - -@test "Command 'alt' (overwrite existing link)" { - echo " - When the command 'alt' is provided - and the link exists, and is wrong - Report the linking - Verify correct file is linked - Exit with 0 - " - - test_alt 'base' 'true' '' -} - -@test "Command 'alt' (select encrypted base)" { - echo " - When the command 'alt' is provided - and encrypted file matches only ## - Report the linking - Verify correct encrypted file is linked - Exit with 0 - " - - test_alt 'encrypted_base' 'false' '' -} - -@test "Command 'alt' (select encrypted system)" { - echo " - When the command 'alt' is provided - and encrypted file matches only ##SYSTEM - Report the linking - Verify correct encrypted file is linked - Exit with 0 - " - - test_alt 'encrypted_system' 'false' '' -} - -@test "Command 'alt' (select encrypted host)" { - echo " - When the command 'alt' is provided - and encrypted file matches only ##SYSTEM.HOST - Report the linking - Verify correct encrypted file is linked - Exit with 0 - " - - test_alt 'encrypted_host' 'false' '' -} - -@test "Command 'alt' (select encrypted user)" { - echo " - When the command 'alt' is provided - and encrypted file matches only ##SYSTEM.HOST.USER - Report the linking - Verify correct encrypted file is linked - Exit with 0 - " - - test_alt 'encrypted_user' 'false' '' -} - -@test "Command 'alt' (select encrypted none)" { - echo " - When the command 'alt' is provided - and no encrypted file matches - Verify there is no link - Exit with 0 - " - - test_alt 'encrypted_none' 'false' '' -} - -@test "Command 'alt' (override-system)" { - echo " - When the command 'alt' is provided - and file matches only ##SYSTEM - after setting local.os - Report the linking - Verify correct file is linked - Exit with 0 - " - - GIT_DIR="$T_DIR_REPO" git config local.os custom_system - test_alt 'override_system' 'false' '' -} - -@test "Command 'alt' (override-host)" { - echo " - When the command 'alt' is provided - and file matches only ##SYSTEM.HOST - after setting local.hostname - Report the linking - Verify correct file is linked - Exit with 0 - " - - GIT_DIR="$T_DIR_REPO" git config local.hostname custom_host - test_alt 'override_host' 'false' '' -} - -@test "Command 'alt' (override-user)" { - echo " - When the command 'alt' is provided - and file matches only ##SYSTEM.HOST.USER - after setting local.user - Report the linking - Verify correct file is linked - Exit with 0 - " - - GIT_DIR="$T_DIR_REPO" git config local.user custom_user - test_alt 'override_user' 'false' '' -} diff --git a/test/109_accept_encryption.bats b/test/109_accept_encryption.bats deleted file mode 100644 index 439f44c..0000000 --- a/test/109_accept_encryption.bats +++ /dev/null @@ -1,900 +0,0 @@ -load common -load_fixtures -status=;output=; #; populated by bats run() - -T_PASSWD="ExamplePassword" -T_ARCHIVE_SYMMETRIC="$T_TMP/build_archive.symmetric" -T_ARCHIVE_ASYMMETRIC="$T_TMP/build_archive.asymmetric" -T_KEY_NAME="yadm-test1" -T_KEY_FINGERPRINT="F8BBFC746C58945442349BCEBA54FFD04C599B1A" -T_RECIPIENT_GOOD="[yadm]\n\tgpg-recipient = yadm-test1" -T_RECIPIENT_BAD="[yadm]\n\tgpg-recipient = invalid" -T_RECIPIENT_ASK="[yadm]\n\tgpg-recipient = ASK" - -#; use gpg1 if it's available -T_GPG_PROGRAM="gpg" -if command -v gpg1 >/dev/null 2>&1; then - T_GPG_PROGRAM="gpg1" -fi - -function import_keys() { - "$T_GPG_PROGRAM" --import "test/test_key" >/dev/null 2>&1 || true - "$T_GPG_PROGRAM" --import-ownertrust < "test/ownertrust.txt" >/dev/null 2>&1 -} - -function remove_keys() { - "$T_GPG_PROGRAM" --batch --yes --delete-secret-keys "$T_KEY_FINGERPRINT" >/dev/null 2>&1 || true - "$T_GPG_PROGRAM" --batch --yes --delete-key "$T_KEY_FINGERPRINT" >/dev/null 2>&1 || true -} - -setup() { - #; start fresh - destroy_tmp - - #; import test keys - import_keys - - #; create a worktree & repo - build_repo - - #; define a YADM_ENCRYPT - make_parents "$T_YADM_ENCRYPT" - echo -e ".ssh/*.key\n.gnupg/*.gpg" > "$T_YADM_ENCRYPT" - - #; create a YADM_ARCHIVE - ( - if cd "$T_DIR_WORK"; then - # shellcheck disable=2013 - # (globbing is desired) - for f in $(sort "$T_YADM_ENCRYPT"); do - tar rf "$T_TMP/build_archive.tar" "$f" - echo "$f" >> "$T_TMP/archived_files" - done - fi - ) - - #; encrypt YADM_ARCHIVE (symmetric) - expect </dev/null - set timeout 2; - spawn "$T_GPG_PROGRAM" --yes -c --output "$T_ARCHIVE_SYMMETRIC" "$T_TMP/build_archive.tar" - expect "passphrase:" {send "$T_PASSWD\n"} - expect "passphrase:" {send "$T_PASSWD\n"} - expect "$" - foreach {pid spawnid os_error_flag value} [wait] break -EOF - - #; encrypt YADM_ARCHIVE (asymmetric) - "$T_GPG_PROGRAM" --yes --batch -e -r "$T_KEY_NAME" --output "$T_ARCHIVE_ASYMMETRIC" "$T_TMP/build_archive.tar" - - #; configure yadm to use T_GPG_PROGRAM - git config --file="$T_YADM_CONFIG" yadm.gpg-program "$T_GPG_PROGRAM" -} - -teardown() { - remove_keys -} - -function validate_archive() { - #; inventory what's in the archive - if [ "$1" = "symmetric" ]; then - expect </dev/null - set timeout 2; - spawn bash -c "($T_GPG_PROGRAM -q -d '$T_YADM_ARCHIVE' || echo 1) | tar t | sort > $T_TMP/archive_list" - expect "passphrase:" {send "$T_PASSWD\n"} - expect "$" - foreach {pid spawnid os_error_flag value} [wait] break -EOF - else - "$T_GPG_PROGRAM" -q -d "$T_YADM_ARCHIVE" | tar t | sort > "$T_TMP/archive_list" - fi - - excluded="$2" - - #; inventory what is expected in the archive - ( - if cd "$T_DIR_WORK"; then - # shellcheck disable=2013 - # (globbing is desired) - while IFS='' read -r glob || [ -n "$glob" ]; do - if [[ ! $glob =~ ^# && ! $glob =~ ^[[:space:]]*$ ]] ; then - if [[ ! $glob =~ ^!(.+) ]] ; then - local IFS=$'\n' - for matching_file in $glob; do - if [ -e "$matching_file" ]; then - if [ "$matching_file" != "$excluded" ]; then - if [ -d "$matching_file" ]; then - echo "$matching_file/" - for subfile in "$matching_file"/*; do - echo "$subfile" - done - else - echo "$matching_file" - fi - fi - fi - done - fi - fi - done < "$T_YADM_ENCRYPT" | sort > "$T_TMP/expected_list" - fi - ) - - #; compare the archive vs expected - if ! cmp -s "$T_TMP/archive_list" "$T_TMP/expected_list"; then - echo "ERROR: Archive does not contain the correct files" - echo "Contains:" - cat "$T_TMP/archive_list" - echo "Expected:" - cat "$T_TMP/expected_list" - return 1 - fi - return 0 -} - -function validate_extraction() { - #; test each file which was archived - while IFS= read -r f; do - local contents - contents=$(cat "$T_DIR_WORK/$f") - if [ "$contents" != "$f" ]; then - echo "ERROR: Contents of $T_DIR_WORK/$f is incorrect" - return 1 - fi - done < "$T_TMP/archived_files" - return 0 -} - -@test "Command 'encrypt' (missing YADM_ENCRYPT)" { - echo " - When 'encrypt' command is provided, - and YADM_ENCRYPT does not exist - Report problem - Exit with 1 - " - - #; remove YADM_ENCRYPT - rm -f "$T_YADM_ENCRYPT" - - #; run encrypt - run "${T_YADM_Y[@]}" encrypt - - #; validate status and output - [ "$status" -eq 1 ] - [[ "$output" =~ does\ not\ exist ]] -} - -@test "Command 'encrypt' (mismatched password)" { - echo " - When 'encrypt' command is provided, - and YADM_ENCRYPT is present - and the provided passwords do not match - Report problem - Exit with 1 - " - - #; run encrypt - run expect <> "$T_YADM_ENCRYPT" - - #; run encrypt - run expect < "$T_YADM_ENCRYPT" - - #; validate the archive - validate_archive symmetric -} - -@test "Command 'encrypt' (empty lines and space lines in YADM_ENCRYPT)" { - echo " - When 'encrypt' command is provided, - and YADM_ENCRYPT is present - Create YADM_ARCHIVE - Report the archive created - Archive should be valid - Exit with 0 - " - - #; add empty lines to YADM_ARCHIVE - local original_encrypt - original_encrypt=$(cat "$T_YADM_ENCRYPT") - echo -e " \n\n \n" >> "$T_YADM_ENCRYPT" - - #; run encrypt - run expect < "$T_YADM_ENCRYPT" - - #; validate the archive - validate_archive symmetric -} - -@test "Command 'encrypt' (paths with spaces/globs in YADM_ENCRYPT)" { - echo " - When 'encrypt' command is provided, - and YADM_ENCRYPT is present - Create YADM_ARCHIVE - Report the archive created - Archive should be valid - Exit with 0 - " - - #; add paths with spaces to YADM_ARCHIVE - local original_encrypt - original_encrypt=$(cat "$T_YADM_ENCRYPT") - echo -e "space test/file*" >> "$T_YADM_ENCRYPT" - - #; run encrypt - run expect <> "$T_YADM_ENCRYPT" - echo -e "!.ssh/sec*.pub" >> "$T_YADM_ENCRYPT" - - #; run encrypt - run expect <> "$T_YADM_ENCRYPT" - - #; run encrypt - run expect < "$T_YADM_ARCHIVE" - - #; run encrypt - run expect <> "$T_DIR_WORK/$f" - done < "$T_TMP/archived_files" - - #; run decrypt - run expect < "$T_YADM_CONFIG" - - #; run encrypt - run "${T_YADM_Y[@]}" encrypt - - #; validate status and output - [ "$status" -eq 1 ] - [[ "$output" =~ public\ key\ not\ found ]] || [[ "$output" =~ No\ public\ key ]] - [[ "$output" =~ Unable\ to\ write ]] -} - - -@test "Command 'encrypt' (asymmetric)" { - echo " - When 'encrypt' command is provided, - and YADM_ENCRYPT is present - and yadm.gpg-recipient refers to a valid private key - Create YADM_ARCHIVE - Report the archive created - Archive should be valid - Exit with 0 - " - - #; manually set yadm.gpg-recipient in configuration - make_parents "$T_YADM_CONFIG" - echo -e "$T_RECIPIENT_GOOD" > "$T_YADM_CONFIG" - - #; run encrypt - run "${T_YADM_Y[@]}" encrypt - - #; validate status and output - [ "$status" -eq 0 ] - [[ "$output" =~ Wrote\ new\ file:.+$T_YADM_ARCHIVE ]] - - #; validate the archive - validate_archive asymmetric -} - -@test "Command 'encrypt' (asymmetric, overwrite)" { - echo " - When 'encrypt' command is provided, - and YADM_ENCRYPT is present - and yadm.gpg-recipient refers to a valid private key - and YADM_ARCHIVE already exists - Overwrite YADM_ARCHIVE - Report the archive created - Archive should be valid - Exit with 0 - " - - #; manually set yadm.gpg-recipient in configuration - make_parents "$T_YADM_CONFIG" - echo -e "$T_RECIPIENT_GOOD" > "$T_YADM_CONFIG" - - #; Explicitly create an invalid archive - echo "EXISTING ARCHIVE" > "$T_YADM_ARCHIVE" - - #; run encrypt - run "${T_YADM_Y[@]}" encrypt - - #; validate status and output - [ "$status" -eq 0 ] - [[ "$output" =~ Wrote\ new\ file:.+$T_YADM_ARCHIVE ]] - - #; validate the archive - validate_archive asymmetric -} - -@test "Command 'encrypt' (asymmetric, ask)" { - echo " - When 'encrypt' command is provided, - and YADM_ENCRYPT is present - and yadm.gpg-recipient is set to ASK - Ask for recipient - Create YADM_ARCHIVE - Report the archive created - Archive should be valid - Exit with 0 - " - - #; manually set yadm.gpg-recipient in configuration - make_parents "$T_YADM_CONFIG" - echo -e "$T_RECIPIENT_ASK" > "$T_YADM_CONFIG" - - #; run encrypt - run expect < "$T_YADM_CONFIG" - - #; run decrypt - run "${T_YADM_Y[@]}" decrypt - - #; validate status and output - [ "$status" -eq 1 ] - [[ "$output" =~ does\ not\ exist ]] -} - -@test "Command 'decrypt' (asymmetric, missing key)" { - echo " - When 'decrypt' command is provided, - and yadm.gpg-recipient refers to a valid private key - and YADM_ARCHIVE is present - and the private key is not present - Report problem - Exit with 1 - " - - #; manually set yadm.gpg-recipient in configuration - make_parents "$T_YADM_CONFIG" - echo -e "$T_RECIPIENT_GOOD" > "$T_YADM_CONFIG" - - #; use the asymmetric archive - cp -f "$T_ARCHIVE_ASYMMETRIC" "$T_YADM_ARCHIVE" - - #; remove the private key - remove_keys - - #; run decrypt - run "${T_YADM_Y[@]}" decrypt - - #; validate status and output - [ "$status" -eq 1 ] - [[ "$output" =~ decryption\ failed ]] - [[ "$output" =~ Unable\ to\ extract ]] -} - -@test "Command 'decrypt' -l (asymmetric, missing key)" { - echo " - When 'decrypt' command is provided, - and '-l' is provided, - and yadm.gpg-recipient refers to a valid private key - and YADM_ARCHIVE is present - and the private key is not present - Report problem - Exit with 1 - " - - #; manually set yadm.gpg-recipient in configuration - make_parents "$T_YADM_CONFIG" - echo -e "$T_RECIPIENT_GOOD" > "$T_YADM_CONFIG" - - #; use the asymmetric archive - cp -f "$T_ARCHIVE_ASYMMETRIC" "$T_YADM_ARCHIVE" - - #; remove the private key - remove_keys - - #; run decrypt - run "${T_YADM_Y[@]}" decrypt - - #; validate status and output - [ "$status" -eq 1 ] - [[ "$output" =~ decryption\ failed ]] - [[ "$output" =~ Unable\ to\ extract ]] -} - -@test "Command 'decrypt' (asymmetric)" { - echo " - When 'decrypt' command is provided, - and yadm.gpg-recipient refers to a valid private key - and YADM_ARCHIVE is present - Report the data created - Data should be valid - Exit with 0 - " - - #; manually set yadm.gpg-recipient in configuration - make_parents "$T_YADM_CONFIG" - echo -e "$T_RECIPIENT_GOOD" > "$T_YADM_CONFIG" - - #; use the asymmetric archive - cp -f "$T_ARCHIVE_ASYMMETRIC" "$T_YADM_ARCHIVE" - - #; empty the worktree - rm -rf "$T_DIR_WORK" - mkdir -p "$T_DIR_WORK" - - #; run decrypt - run "${T_YADM_Y[@]}" decrypt - - #; validate status and output - [ "$status" -eq 0 ] - [[ "$output" =~ All\ files\ decrypted ]] - - #; validate the extracted files - validate_extraction -} - -@test "Command 'decrypt' (asymmetric, overwrite)" { - echo " - When 'decrypt' command is provided, - and yadm.gpg-recipient refers to a valid private key - and YADM_ARCHIVE is present - and archived content already exists - Report the data overwritten - Data should be valid - Exit with 0 - " - - #; manually set yadm.gpg-recipient in configuration - make_parents "$T_YADM_CONFIG" - echo -e "$T_RECIPIENT_GOOD" > "$T_YADM_CONFIG" - - #; use the asymmetric archive - cp -f "$T_ARCHIVE_ASYMMETRIC" "$T_YADM_ARCHIVE" - - #; alter the values of the archived files - while IFS= read -r f; do - echo "changed" >> "$T_DIR_WORK/$f" - done < "$T_TMP/archived_files" - - #; run decrypt - run "${T_YADM_Y[@]}" decrypt - - #; validate status and output - [ "$status" -eq 0 ] - [[ "$output" =~ All\ files\ decrypted ]] - - #; validate the extracted files - validate_extraction -} - -@test "Command 'decrypt' -l (asymmetric)" { - echo " - When 'decrypt' command is provided, - and '-l' is provided, - and yadm.gpg-recipient refers to a valid private key - and YADM_ARCHIVE is present - Report the contents of YADM_ARCHIVE - Exit with 0 - " - - #; manually set yadm.gpg-recipient in configuration - make_parents "$T_YADM_CONFIG" - echo -e "$T_RECIPIENT_GOOD" > "$T_YADM_CONFIG" - - #; use the asymmetric archive - cp -f "$T_ARCHIVE_ASYMMETRIC" "$T_YADM_ARCHIVE" - - #; run decrypt - run "${T_YADM_Y[@]}" decrypt -l - - #; validate status - [ "$status" -eq 0 ] - - #; validate every file is listed in output - while IFS= read -r f; do - if [[ ! "$output" =~ $f ]]; then - echo "ERROR: Did not find '$f' in output" - return 1 - fi - done < "$T_TMP/archived_files" - -} diff --git a/test/110_accept_perms.bats b/test/110_accept_perms.bats deleted file mode 100644 index a68b1a5..0000000 --- a/test/110_accept_perms.bats +++ /dev/null @@ -1,173 +0,0 @@ -load common -load_fixtures -status=;output=; #; populated by bats run() - -setup() { - destroy_tmp - build_repo -} - -function is_restricted() { - local p - for p in "${restricted[@]}"; do [ "$p" = "$1" ] && return 0; done - return 1 -} - -function validate_perms() { - local perms="$*" - - #; determine which paths should have restricted permissions - restricted=() - local p - for p in $perms; do - case $p in - ssh) - restricted=("${restricted[@]}" $T_DIR_WORK/.ssh $T_DIR_WORK/.ssh/*) - ;; - gpg) - restricted=("${restricted[@]}" $T_DIR_WORK/.gnupg $T_DIR_WORK/.gnupg/*) - ;; - *) - restricted=("${restricted[@]}" $T_DIR_WORK/$p) - ;; - esac - done - - #; validate permissions of each path in the worktere - local testpath - while IFS= read -r -d '' testpath; do - local perm_regex="....rwxrwx" - if is_restricted "$testpath"; then - perm_regex="....------" - fi - test_perms "$testpath" "$perm_regex" || return 1 - done < <(find "$T_DIR_WORK" -print0) - -} - -@test "Command 'perms'" { - echo " - When the command 'perms' is provided - Update permissions for ssh/gpg - Verify correct permissions - Exit with 0 - " - - #; run perms - run "${T_YADM_Y[@]}" perms - - #; validate status and output - [ "$status" -eq 0 ] - [ "$output" = "" ] - - #; validate permissions - validate_perms ssh gpg -} - -@test "Command 'perms' (with encrypt)" { - echo " - When the command 'perms' is provided - And YADM_ENCRYPT is present - Update permissions for ssh/gpg/encrypt - Support comments in YADM_ENCRYPT - Verify correct permissions - Exit with 0 - " - - #; this version has a comment in it - echo -e "#.vimrc\n.tmux.conf\n.hammerspoon/*\n!.tmux.conf" > "$T_YADM_ENCRYPT" - - #; run perms - run "${T_YADM_Y[@]}" perms - - #; validate status and output - [ "$status" -eq 0 ] - [ "$output" = "" ] - - #; validate permissions - validate_perms ssh gpg ".hammerspoon/*" -} - -@test "Command 'perms' (ssh-perms=false)" { - echo " - When the command 'perms' is provided - And yadm.ssh-perms=false - Update permissions for gpg only - Verify correct permissions - Exit with 0 - " - - #; configure yadm.ssh-perms - git config --file="$T_YADM_CONFIG" "yadm.ssh-perms" "false" - - #; run perms - run "${T_YADM_Y[@]}" perms - - #; validate status and output - [ "$status" -eq 0 ] - [ "$output" = "" ] - - #; validate permissions - validate_perms gpg -} - -@test "Command 'perms' (gpg-perms=false)" { - echo " - When the command 'perms' is provided - And yadm.gpg-perms=false - Update permissions for ssh only - Verify correct permissions - Exit with 0 - " - - #; configure yadm.gpg-perms - git config --file="$T_YADM_CONFIG" "yadm.gpg-perms" "false" - - #; run perms - run "${T_YADM_Y[@]}" perms - - #; validate status and output - [ "$status" -eq 0 ] - [ "$output" = "" ] - - #; validate permissions - validate_perms ssh -} - -@test "Command 'auto-perms' (enabled)" { - echo " - When a command possibly changes the repo - Update permissions for ssh/gpg - Verify correct permissions - " - - #; run status - run "${T_YADM_Y[@]}" status - - #; validate status - [ "$status" -eq 0 ] - - #; validate permissions - validate_perms ssh gpg -} - -@test "Command 'auto-perms' (disabled)" { - echo " - When a command possibly changes the repo - And yadm.auto-perms=false - Take no action - Verify permissions are intact - " - - #; configure yadm.auto-perms - git config --file="$T_YADM_CONFIG" "yadm.auto-perms" "false" - - #; run status - run "${T_YADM_Y[@]}" status - - #; validate status - [ "$status" -eq 0 ] - - #; validate permissions - validate_perms -} diff --git a/test/111_accept_wildcard_alt.bats b/test/111_accept_wildcard_alt.bats deleted file mode 100644 index dcae81f..0000000 --- a/test/111_accept_wildcard_alt.bats +++ /dev/null @@ -1,223 +0,0 @@ -load common -load_fixtures -status=;output=; #; populated by bats run() - -IN_REPO=(wild*) -export TEST_TREE_WITH_WILD=1 - -setup() { - destroy_tmp - build_repo "${IN_REPO[@]}" -} - -function test_alt() { - local link_name="$1" - local link_match="$2" - - #; run yadm alt - run "${T_YADM_Y[@]}" alt - #; validate status and output - if [ "$status" != 0 ] || [[ ! "$output" =~ Linking.+$link_name ]]; then - echo "OUTPUT:$output" - echo "ERROR: Could not confirm status and output of alt command" - return 1; - fi - - #; correct link should be present - local link_content - link_content=$(cat "$T_DIR_WORK/$link_name") - if [ "$link_content" != "$link_match" ]; then - echo "OUTPUT:$output" - echo "ERROR: Link content is not correct" - return 1 - fi -} - -@test "Command 'alt' (wild none)" { - echo " - When the command 'alt' is provided - and file matches only ## - Report the linking - Verify correct file is linked - Exit with 0 - " - - test_alt 'wild-none' 'wild-none##' -} - -@test "Command 'alt' (wild system)" { - echo " - When the command 'alt' is provided - and file matches only ##SYSTEM - with possible wildcards - Report the linking - Verify correct file is linked - Exit with 0 - " - - for WILD_S in 'local' 'wild'; do - local s_base="wild-system-$WILD_S" - case $WILD_S in local) WILD_S="$T_SYS";; wild) WILD_S="%";; esac - local match="${s_base}##${WILD_S}" - echo test_alt "$s_base" "$match" - test_alt "$s_base" "$match" - done -} - -@test "Command 'alt' (wild class)" { - echo " - When the command 'alt' is provided - and file matches only ##CLASS - with possible wildcards - Report the linking - Verify correct file is linked - Exit with 0 - " - - GIT_DIR="$T_DIR_REPO" git config local.class set_class - - for WILD_C in 'local' 'wild'; do - local c_base="wild-class-$WILD_C" - case $WILD_C in local) WILD_C="set_class";; wild) WILD_C="%";; esac - local match="${c_base}##${WILD_C}" - echo test_alt "$c_base" "$match" - test_alt "$c_base" "$match" - done -} - -@test "Command 'alt' (wild host)" { - echo " - When the command 'alt' is provided - and file matches only ##SYSTEM.HOST - with possible wildcards - Report the linking - Verify correct file is linked - Exit with 0 - " - - for WILD_S in 'local' 'wild'; do - local s_base="wild-host-$WILD_S" - case $WILD_S in local) WILD_S="$T_SYS";; wild) WILD_S="%";; esac - for WILD_H in 'local' 'wild'; do - local h_base="${s_base}-$WILD_H" - case $WILD_H in local) WILD_H="$T_HOST";; wild) WILD_H="%";; esac - local match="${h_base}##${WILD_S}.${WILD_H}" - echo test_alt "$h_base" "$match" - test_alt "$h_base" "$match" - done - done -} - -@test "Command 'alt' (wild class-system)" { - echo " - When the command 'alt' is provided - and file matches only ##CLASS.SYSTEM - with possible wildcards - Report the linking - Verify correct file is linked - Exit with 0 - " - - GIT_DIR="$T_DIR_REPO" git config local.class set_class - - for WILD_C in 'local' 'wild'; do - local c_base="wild-class-system-$WILD_C" - case $WILD_C in local) WILD_C="set_class";; wild) WILD_C="%";; esac - for WILD_S in 'local' 'wild'; do - local s_base="${c_base}-$WILD_S" - case $WILD_S in local) WILD_S="$T_SYS";; wild) WILD_S="%";; esac - local match="${s_base}##${WILD_C}.${WILD_S}" - echo test_alt "$s_base" "$match" - test_alt "$s_base" "$match" - done - done -} - -@test "Command 'alt' (wild user)" { - echo " - When the command 'alt' is provided - and file matches only ##SYSTEM.HOST.USER - with possible wildcards - Report the linking - Verify correct file is linked - Exit with 0 - " - - for WILD_S in 'local' 'wild'; do - local s_base="wild-user-$WILD_S" - case $WILD_S in local) WILD_S="$T_SYS";; wild) WILD_S="%";; esac - for WILD_H in 'local' 'wild'; do - local h_base="${s_base}-$WILD_H" - case $WILD_H in local) WILD_H="$T_HOST";; wild) WILD_H="%";; esac - for WILD_U in 'local' 'wild'; do - local u_base="${h_base}-$WILD_U" - case $WILD_U in local) WILD_U="$T_USER";; wild) WILD_U="%";; esac - local match="${u_base}##${WILD_S}.${WILD_H}.${WILD_U}" - echo test_alt "$u_base" "$match" - test_alt "$u_base" "$match" - done - done - done -} - -@test "Command 'alt' (wild class-system-host)" { - echo " - When the command 'alt' is provided - and file matches only ##CLASS.SYSTEM.HOST - with possible wildcards - Report the linking - Verify correct file is linked - Exit with 0 - " - - GIT_DIR="$T_DIR_REPO" git config local.class set_class - - for WILD_C in 'local' 'wild'; do - local c_base="wild-class-system-host-$WILD_C" - case $WILD_C in local) WILD_C="set_class";; wild) WILD_C="%";; esac - for WILD_S in 'local' 'wild'; do - local s_base="${c_base}-$WILD_S" - case $WILD_S in local) WILD_S="$T_SYS";; wild) WILD_S="%";; esac - for WILD_H in 'local' 'wild'; do - local h_base="${s_base}-$WILD_H" - case $WILD_H in local) WILD_H="$T_HOST";; wild) WILD_H="%";; esac - local match="${h_base}##${WILD_C}.${WILD_S}.${WILD_H}" - echo test_alt "$h_base" "$match" - test_alt "$h_base" "$match" - done - done - done -} - -@test "Command 'alt' (wild class-system-host-user)" { - echo " - When the command 'alt' is provided - and file matches only ##CLASS.SYSTEM.HOST.USER - with possible wildcards - Report the linking - Verify correct file is linked - Exit with 0 - " - - GIT_DIR="$T_DIR_REPO" git config local.class set_class - - for WILD_C in 'local' 'wild'; do - local c_base="wild-class-system-host-user-$WILD_C" - case $WILD_C in local) WILD_C="set_class";; wild) WILD_C="%";; esac - for WILD_S in 'local' 'wild'; do - local s_base="${c_base}-$WILD_S" - case $WILD_S in local) WILD_S="$T_SYS";; wild) WILD_S="%";; esac - for WILD_H in 'local' 'wild'; do - local h_base="${s_base}-$WILD_H" - case $WILD_H in local) WILD_H="$T_HOST";; wild) WILD_H="%";; esac - for WILD_U in 'local' 'wild'; do - local u_base="${h_base}-$WILD_U" - case $WILD_U in local) WILD_U="$T_USER";; wild) WILD_U="%";; esac - local match="${u_base}##${WILD_C}.${WILD_S}.${WILD_H}.${WILD_U}" - echo test_alt "$u_base" "$match" - test_alt "$u_base" "$match" - done - done - done - done -} diff --git a/test/112_accept_bootstrap.bats b/test/112_accept_bootstrap.bats deleted file mode 100644 index 3f23df2..0000000 --- a/test/112_accept_bootstrap.bats +++ /dev/null @@ -1,78 +0,0 @@ -load common -load_fixtures -status=;output=; #; populated by bats run() - -setup() { - destroy_tmp - build_repo -} - -@test "Command 'bootstrap' (missing file)" { - echo " - When 'bootstrap' command is provided, - and the bootstrap file is missing - Report error - Exit with 1 - " - - #; run clone - run "${T_YADM_Y[@]}" bootstrap - echo "STATUS:$status" - echo "OUTPUT:$output" - - #; validate status and output - [[ "$output" =~ Cannot\ execute\ bootstrap ]] - [ "$status" -eq 1 ] - -} - -@test "Command 'bootstrap' (not executable)" { - echo " - When 'bootstrap' command is provided, - and the bootstrap file is present - but is not executable - Report error - Exit with 1 - " - - touch "$T_YADM_BOOTSTRAP" - - #; run clone - run "${T_YADM_Y[@]}" bootstrap - echo "STATUS:$status" - echo "OUTPUT:$output" - - #; validate status and output - [[ "$output" =~ is\ not\ an\ executable\ program ]] - [ "$status" -eq 1 ] - -} - -@test "Command 'bootstrap' (bootstrap run)" { - echo " - When 'bootstrap' command is provided, - and the bootstrap file is present - and is executable - Announce the execution - Execute bootstrap - Exit with the exit code of bootstrap - " - - { - echo "#!/bin/bash" - echo "echo Bootstrap successful" - echo "exit 123" - } > "$T_YADM_BOOTSTRAP" - chmod a+x "$T_YADM_BOOTSTRAP" - - #; run clone - run "${T_YADM_Y[@]}" bootstrap - echo "STATUS:$status" - echo "OUTPUT:$output" - - #; validate status and output - [[ "$output" =~ Executing\ $T_YADM_BOOTSTRAP ]] - [[ "$output" =~ Bootstrap\ successful ]] - [ "$status" -eq 123 ] - -} diff --git a/test/113_accept_jinja_alt.bats b/test/113_accept_jinja_alt.bats deleted file mode 100644 index 0f73af9..0000000 --- a/test/113_accept_jinja_alt.bats +++ /dev/null @@ -1,203 +0,0 @@ -load common -load_fixtures -status=;output=; #; populated by bats run() - -IN_REPO=(alt* "dir one") -export TEST_TREE_WITH_ALT=1 - - -setup() { - destroy_tmp - build_repo "${IN_REPO[@]}" - echo "excluded-encrypt##yadm.j2" > "$T_YADM_ENCRYPT" - echo "included-encrypt##yadm.j2" >> "$T_YADM_ENCRYPT" - echo "!excluded-encrypt*" >> "$T_YADM_ENCRYPT" - echo "included-encrypt" > "$T_DIR_WORK/included-encrypt##yadm.j2" - echo "excluded-encrypt" > "$T_DIR_WORK/excluded-encrypt##yadm.j2" -} - - -function test_alt() { - local alt_type="$1" - local test_overwrite="$2" - local auto_alt="$3" - - #; detemine test parameters - case $alt_type in - base) - real_name="alt-jinja" - file_content_match="-${T_SYS}-${T_HOST}-${T_USER}-${T_DISTRO}" - ;; - override_all) - real_name="alt-jinja" - file_content_match="custom_class-custom_system-custom_host-custom_user-${T_DISTRO}" - ;; - encrypt) - real_name="included-encrypt" - file_content_match="included-encrypt" - missing_name="excluded-encrypt" - ;; - esac - - if [ "$test_overwrite" = "true" ] ; then - #; create incorrect links (to overwrite) - echo "BAD_CONTENT" "$T_DIR_WORK/$real_name" - else - #; verify real file doesn't already exist - if [ -e "$T_DIR_WORK/$real_name" ] ; then - echo "ERROR: real file already exists before running yadm" - return 1 - fi - fi - - #; configure yadm.auto_alt=false - if [ "$auto_alt" = "false" ]; then - git config --file="$T_YADM_CONFIG" yadm.auto-alt false - fi - - #; run yadm (alt or status) - if [ -z "$auto_alt" ]; then - run "${T_YADM_Y[@]}" alt - #; validate status and output - if [ "$status" != 0 ] || [[ ! "$output" =~ Creating.+$real_name ]]; then - echo "OUTPUT:$output" - echo "ERROR: Could not confirm status and output of alt command" - return 1; - fi - else - #; running any passed through Git command should trigger auto-alt - run "${T_YADM_Y[@]}" status - if [ -n "$auto_alt" ] && [[ "$output" =~ Creating.+$real_name ]]; then - echo "ERROR: Reporting of jinja processing should not happen" - return 1 - fi - fi - - if [ -n "$missing_name" ] && [ -f "$T_DIR_WORK/$missing_name" ]; then - echo "ERROR: File should not have been created '$missing_name'" - return 1 - fi - - #; validate link content - if [[ "$alt_type" =~ none ]] || [ "$auto_alt" = "false" ]; then - #; no real file should be present - if [ -L "$T_DIR_WORK/$real_name" ] ; then - echo "ERROR: Real file should not exist" - return 1 - fi - else - #; correct real file should be present - local file_content - file_content=$(cat "$T_DIR_WORK/$real_name") - if [ "$file_content" != "$file_content_match" ]; then - echo "file_content: ${file_content}" - echo "expected_content: ${file_content_match}" - echo "ERROR: Link content is not correct" - return 1 - fi - fi -} - -@test "Command 'alt' (envtpl missing)" { - echo " - When the command 'alt' is provided - and file matches ##yadm.j2 - Report jinja template as unprocessed - Exit with 0 - " - - # shellcheck source=/dev/null - YADM_TEST=1 source "$T_YADM" - process_global_args -Y "$T_DIR_YADM" - set_operating_system - configure_paths - - status=0 - output=$( ENVTPL_PROGRAM='envtpl_missing' main alt ) || { - status=$? - true - } - - [ $status -eq 0 ] - [[ "$output" =~ envtpl.not.available ]] -} - -@test "Command 'alt' (select jinja)" { - echo " - When the command 'alt' is provided - and file matches ##yadm.j2 - Report jinja template processing - Verify that the correct content is written - Exit with 0 - " - - test_alt 'base' 'false' '' -} - -@test "Command 'auto-alt' (enabled)" { - echo " - When a command possibly changes the repo - and auto-alt is configured true - and file matches ##yadm.j2 - automatically process alternates - report no linking (not loud) - Verify that the correct content is written - " - - test_alt 'base' 'false' 'true' -} - -@test "Command 'auto-alt' (disabled)" { - echo " - When a command possibly changes the repo - and auto-alt is configured false - and file matches ##yadm.j2 - Report no jinja template processing - Verify no content - " - - test_alt 'base' 'false' 'false' -} - -@test "Command 'alt' (overwrite existing content)" { - echo " - When the command 'alt' is provided - and file matches ##yadm.j2 - and the real file exists, and is wrong - Report jinja template processing - Verify that the correct content is written - Exit with 0 - " - - test_alt 'base' 'true' '' -} - -@test "Command 'alt' (overwritten settings)" { - echo " - When the command 'alt' is provided - and file matches ##yadm.j2 - after setting local.* - Report jinja template processing - Verify that the correct content is written - Exit with 0 - " - - GIT_DIR="$T_DIR_REPO" git config local.os custom_system - GIT_DIR="$T_DIR_REPO" git config local.user custom_user - GIT_DIR="$T_DIR_REPO" git config local.hostname custom_host - GIT_DIR="$T_DIR_REPO" git config local.class custom_class - test_alt 'override_all' 'false' '' -} - -@test "Command 'alt' (select jinja within .yadm/encrypt)" { - echo " - When the command 'alt' is provided - and file matches ##yadm.j2 within .yadm/encrypt - and file excluded within .yadm/encrypt - Report jinja template processing - Verify that the correct content is written - Exit with 0 - " - - test_alt 'encrypt' 'false' '' -} diff --git a/test/114_accept_enter.bats b/test/114_accept_enter.bats deleted file mode 100644 index b13dc35..0000000 --- a/test/114_accept_enter.bats +++ /dev/null @@ -1,66 +0,0 @@ -load common -load_fixtures -status=;output=; #; populated by bats run() - -setup() { - build_repo -} - -@test "Command 'enter' (SHELL not set)" { - echo " - When 'enter' command is provided, - And SHELL is not set - Report error - Exit with 1 - " - - SHELL= - export SHELL - run "${T_YADM_Y[@]}" enter - - #; validate status and output - [ $status -eq 1 ] - [[ "$output" =~ does.not.refer.to.an.executable ]] -} - -@test "Command 'enter' (SHELL not executable)" { - echo " - When 'enter' command is provided, - And SHELL is not executable - Report error - Exit with 1 - " - - touch "$T_TMP/badshell" - SHELL="$T_TMP/badshell" - export SHELL - run "${T_YADM_Y[@]}" enter - - #; validate status and output - [ $status -eq 1 ] - [[ "$output" =~ does.not.refer.to.an.executable ]] -} - -@test "Command 'enter' (SHELL executable)" { - echo " - When 'enter' command is provided, - And SHELL is set - Execute SHELL command - Expose GIT variables - Set prompt variables - Announce entering/leaving shell - Exit with 0 - " - - SHELL=$(command -v env) - export SHELL - run "${T_YADM_Y[@]}" enter - - #; validate status and output - [ $status -eq 0 ] - [[ "$output" =~ GIT_DIR= ]] - [[ "$output" =~ PROMPT=yadm.shell ]] - [[ "$output" =~ PS1=yadm.shell ]] - [[ "$output" =~ Entering.yadm.repo ]] - [[ "$output" =~ Leaving.yadm.repo ]] -} diff --git a/test/115_accept_introspect.bats b/test/115_accept_introspect.bats deleted file mode 100644 index 56131f2..0000000 --- a/test/115_accept_introspect.bats +++ /dev/null @@ -1,99 +0,0 @@ -load common -load_fixtures -status=;output=; #; populated by bats run() - -function count_introspect() { - local category="$1" - local expected_status="$2" - local expected_words="$3" - local expected_regex="$4" - - run "${T_YADM_Y[@]}" introspect "$category" - local output_words - output_words=$(wc -w <<< "$output") - - if [ "$status" -ne "$expected_status" ]; then - echo "ERROR: Unexpected exit code (expected $expected_status, got $status)" - return 1; - fi - - if [ "$output_words" -ne "$expected_words" ]; then - echo "ERROR: Unexpected number of output words (expected $expected_words, got $output_words)" - return 1; - fi - - if [ -n "$expected_regex" ]; then - if [[ ! "$output" =~ $expected_regex ]]; then - echo "OUTPUT:$output" - echo "ERROR: Output does not match regex: $expected_regex" - return 1; - fi - fi - -} - -@test "Command 'introspect' (no category)" { - echo " - When 'introspect' command is provided, - And no category is provided - Produce no output - Exit with 0 - " - - count_introspect "" 0 0 -} - -@test "Command 'introspect' (invalid category)" { - echo " - When 'introspect' command is provided, - And an invalid category is provided - Produce no output - Exit with 0 - " - - count_introspect "invalid_cat" 0 0 -} - -@test "Command 'introspect' (commands)" { - echo " - When 'introspect' command is provided, - And category 'commands' is provided - Produce command list - Exit with 0 - " - - count_introspect "commands" 0 15 'version' -} - -@test "Command 'introspect' (configs)" { - echo " - When 'introspect' command is provided, - And category 'configs' is provided - Produce switch list - Exit with 0 - " - - count_introspect "configs" 0 13 'yadm\.auto-alt' -} - -@test "Command 'introspect' (repo)" { - echo " - When 'introspect' command is provided, - And category 'repo' is provided - Output repo - Exit with 0 - " - - count_introspect "repo" 0 1 "$T_DIR_REPO" -} - -@test "Command 'introspect' (switches)" { - echo " - When 'introspect' command is provided, - And category 'switches' is provided - Produce switch list - Exit with 0 - " - - count_introspect "switches" 0 7 '--yadm-dir' -} diff --git a/test/116_accept_cygwin_copy.bats b/test/116_accept_cygwin_copy.bats deleted file mode 100644 index 8d1ff04..0000000 --- a/test/116_accept_cygwin_copy.bats +++ /dev/null @@ -1,131 +0,0 @@ -load common -load_fixtures -status=;output=; #; populated by bats run() - -IN_REPO=(alt*) -export TEST_TREE_WITH_CYGWIN=1 -export SIMULATED_CYGWIN="CYGWIN_NT-6.1-WOW64" - -setup() { - destroy_tmp - build_repo "${IN_REPO[@]}" -} - -test_alt() { - local cygwin_copy="$1" - local is_cygwin="$2" - local expect_link="$3" - local preexisting_link="$4" - - case "$cygwin_copy" in - true|false) - git config --file="$T_YADM_CONFIG" "yadm.cygwin-copy" "$cygwin_copy" - ;; - esac - - if [ "$is_cygwin" = "true" ]; then - echo '#!/bin/sh' > "$T_TMP/uname" - echo "echo $SIMULATED_CYGWIN" >> "$T_TMP/uname" - chmod a+x "$T_TMP/uname" - fi - - local expected_content - expected_content="$T_DIR_WORK/alt-test##$(PATH="$T_TMP:$PATH" uname -s)" - - if [ "$preexisting_link" = 'symlink' ]; then - ln -s "$expected_content" "$T_DIR_WORK/alt-test" - elif [ "$preexisting_link" = 'file' ]; then - touch "$T_DIR_WORK/alt-test" - fi - - PATH="$T_TMP:$PATH" run "${T_YADM_Y[@]}" alt - - echo "Alt output:$output" - echo "Alt status:$status" - - if [ -L "$T_DIR_WORK/alt-test" ] && [ "$expect_link" != 'true' ] ; then - echo "ERROR: Alt should be a simple file, but isn't" - return 1 - fi - if [ ! -L "$T_DIR_WORK/alt-test" ] && [ "$expect_link" = 'true' ] ; then - echo "ERROR: Alt should use symlink, but doesn't" - return 1 - fi - - if ! diff "$T_DIR_WORK/alt-test" "$expected_content"; then - echo "ERROR: Alt contains different data than expected" - return 1 - fi -} - -@test "Option 'yadm.cygwin-copy' (unset, non-cygwin)" { - echo " - When the option 'yadm.cygwin-copy' is unset - and the OS is not CYGWIN - Verify alternate is a symlink - " - test_alt 'unset' 'false' 'true' -} - -@test "Option 'yadm.cygwin-copy' (true, non-cygwin)" { - echo " - When the option 'yadm.cygwin-copy' is true - and the OS is not CYGWIN - Verify alternate is a symlink - " - test_alt 'true' 'false' 'true' -} - -@test "Option 'yadm.cygwin-copy' (false, non-cygwin)" { - echo " - When the option 'yadm.cygwin-copy' is false - and the OS is not CYGWIN - Verify alternate is a symlink - " - test_alt 'false' 'false' 'true' -} - -@test "Option 'yadm.cygwin-copy' (unset, cygwin)" { - echo " - When the option 'yadm.cygwin-copy' is unset - and the OS is CYGWIN - Verify alternate is a symlink - " - test_alt 'unset' 'true' 'true' -} - -@test "Option 'yadm.cygwin-copy' (true, cygwin)" { - echo " - When the option 'yadm.cygwin-copy' is true - and the OS is CYGWIN - Verify alternate is a copy - " - test_alt 'true' 'true' 'false' -} - -@test "Option 'yadm.cygwin-copy' (false, cygwin)" { - echo " - When the option 'yadm.cygwin-copy' is false - and the OS is CYGWIN - Verify alternate is a symlink - " - test_alt 'false' 'true' 'true' -} - -@test "Option 'yadm.cygwin-copy' (preexisting symlink) " { - echo " - When the option 'yadm.cygwin-copy' is true - and the OS is CYGWIN - Verify alternate is a copy - " - test_alt 'true' 'true' 'false' 'symlink' -} - -@test "Option 'yadm.cygwin-copy' (preexisting file) " { - echo " - When the option 'yadm.cygwin-copy' is true - and the OS is CYGWIN - Verify alternate is a copy - " - test_alt 'true' 'true' 'false' 'file' -} diff --git a/test/117_accept_hooks.bats b/test/117_accept_hooks.bats deleted file mode 100644 index 5e8f348..0000000 --- a/test/117_accept_hooks.bats +++ /dev/null @@ -1,181 +0,0 @@ -load common -load_fixtures -status=;output=; #; populated by bats run() - -version_regex="yadm [[:digit:]]+\.[[:digit:]]+\.[[:digit:]]+" - -setup() { - destroy_tmp - build_repo - mkdir -p "$T_DIR_HOOKS" -} - -function create_hook() { - hook_name="$1" - hook_exit="$2" - hook_file="$T_DIR_HOOKS/$hook_name" - { - echo "#!/bin/sh" - echo "echo ran $hook_name" - echo "env" - echo "exit $hook_exit" - } > "$hook_file" - chmod a+x "$hook_file" -} - -@test "Hooks (no hook)" { - echo " - When no hook is present - do no not run the hook - run command - Exit with 0 - " - - #; run yadm with no command - run "${T_YADM_Y[@]}" version - - [ $status -eq 0 ] - [[ "$output" =~ $version_regex ]] -} - -@test "Hooks (successful pre hook)" { - echo " - When hook is present - run hook - run command - Exit with 0 - " - - create_hook "pre_version" "0" - - #; run yadm with no command - run "${T_YADM_Y[@]}" version - - [ $status -eq 0 ] - [[ "$output" =~ ran\ pre_version ]] - [[ "$output" =~ $version_regex ]] -} - -@test "Hooks (unsuccessful pre hook)" { - echo " - When hook is present - run hook - report hook failure - do no not run command - Exit with 13 - " - - create_hook "pre_version" "13" - - #; run yadm with no command - run "${T_YADM_Y[@]}" version - - [ $status -eq 13 ] - [[ "$output" =~ ran\ pre_version ]] - [[ "$output" =~ pre_version\ was\ not\ successful ]] - [[ ! "$output" =~ $version_regex ]] -} - -@test "Hooks (successful post hook)" { - echo " - When hook is present - run command - run hook - Exit with 0 - " - - create_hook "post_version" "0" - - #; run yadm with no command - run "${T_YADM_Y[@]}" version - - [ $status -eq 0 ] - [[ "$output" =~ $version_regex ]] - [[ "$output" =~ ran\ post_version ]] -} - -@test "Hooks (unsuccessful post hook)" { - echo " - When hook is present - run command - run hook - Exit with 0 - " - - create_hook "post_version" "13" - - #; run yadm with no command - run "${T_YADM_Y[@]}" version - - [ $status -eq 0 ] - [[ "$output" =~ $version_regex ]] - [[ "$output" =~ ran\ post_version ]] -} - -@test "Hooks (successful pre hook + post hook)" { - echo " - When hook is present - run hook - run command - run hook - Exit with 0 - " - - create_hook "pre_version" "0" - create_hook "post_version" "0" - - #; run yadm with no command - run "${T_YADM_Y[@]}" version - - [ $status -eq 0 ] - [[ "$output" =~ ran\ pre_version ]] - [[ "$output" =~ $version_regex ]] - [[ "$output" =~ ran\ post_version ]] -} - -@test "Hooks (unsuccessful pre hook + post hook)" { - echo " - When hook is present - run hook - report hook failure - do no not run command - do no not run post hook - Exit with 13 - " - - create_hook "pre_version" "13" - create_hook "post_version" "0" - - #; run yadm with no command - run "${T_YADM_Y[@]}" version - - [ $status -eq 13 ] - [[ "$output" =~ ran\ pre_version ]] - [[ "$output" =~ pre_version\ was\ not\ successful ]] - [[ ! "$output" =~ $version_regex ]] - [[ ! "$output" =~ ran\ post_version ]] -} - -@test "Hooks (environment variables)" { - echo " - When hook is present - run command - run hook - hook should have access to environment variables - Exit with 0 - " - - create_hook "post_version" "0" - - #; run yadm with no command - run "${T_YADM_Y[@]}" version extra_args - - [ $status -eq 0 ] - [[ "$output" =~ $version_regex ]] - [[ "$output" =~ ran\ post_version ]] - [[ "$output" =~ YADM_HOOK_COMMAND=version ]] - [[ "$output" =~ YADM_HOOK_EXIT=0 ]] - [[ "$output" =~ YADM_HOOK_FULL_COMMAND=version\ extra_args ]] - [[ "$output" =~ YADM_HOOK_REPO=${T_DIR_REPO} ]] - [[ "$output" =~ YADM_HOOK_WORK=${T_DIR_WORK} ]] -} diff --git a/test/118_accept_assert_private_dirs.bats b/test/118_accept_assert_private_dirs.bats deleted file mode 100644 index 151a2e0..0000000 --- a/test/118_accept_assert_private_dirs.bats +++ /dev/null @@ -1,102 +0,0 @@ -load common -load_fixtures -status=;output=; #; populated by bats run() - -IN_REPO=(.bash_profile .vimrc) - -setup() { - destroy_tmp - build_repo "${IN_REPO[@]}" - rm -rf "$T_DIR_WORK" - mkdir -p "$T_DIR_WORK" -} - -@test "Private dirs (private dirs missing)" { - echo " - When a git command is run - And private directories are missing - Create private directories prior to command - " - - #; confirm directories are missing at start - [ ! -e "$T_DIR_WORK/.gnupg" ] - [ ! -e "$T_DIR_WORK/.ssh" ] - - #; run status - export DEBUG=yes - run "${T_YADM_Y[@]}" status - - #; validate status and output - [ "$status" -eq 0 ] - [[ "$output" =~ On\ branch\ master ]] - - #; confirm private directories are created - [ -d "$T_DIR_WORK/.gnupg" ] - test_perms "$T_DIR_WORK/.gnupg" "drwx------" - [ -d "$T_DIR_WORK/.ssh" ] - test_perms "$T_DIR_WORK/.ssh" "drwx------" - - #; confirm directories are created before command is run - [[ "$output" =~ Creating.+/.gnupg/.+Creating.+/.ssh/.+Running\ git\ command\ git\ status ]] -} - -@test "Private dirs (private dirs missing / yadm.auto-private-dirs=false)" { - echo " - When a git command is run - And private directories are missing - But auto-private-dirs is false - Do not create private dirs - " - - #; confirm directories are missing at start - [ ! -e "$T_DIR_WORK/.gnupg" ] - [ ! -e "$T_DIR_WORK/.ssh" ] - - #; set configuration - run "${T_YADM_Y[@]}" config --bool "yadm.auto-private-dirs" "false" - - #; run status - run "${T_YADM_Y[@]}" status - - #; validate status and output - [ "$status" -eq 0 ] - [[ "$output" =~ On\ branch\ master ]] - - #; confirm private directories are not created - [ ! -e "$T_DIR_WORK/.gnupg" ] - [ ! -e "$T_DIR_WORK/.ssh" ] -} - -@test "Private dirs (private dirs exist / yadm.auto-perms=false)" { - echo " - When a git command is run - And private directories exist - And yadm is configured not to auto update perms - Do not alter directories - " - - #shellcheck disable=SC2174 - mkdir -m 0777 -p "$T_DIR_WORK/.gnupg" "$T_DIR_WORK/.ssh" - - #; confirm directories are preset and open - [ -d "$T_DIR_WORK/.gnupg" ] - test_perms "$T_DIR_WORK/.gnupg" "drwxrwxrwx" - [ -d "$T_DIR_WORK/.ssh" ] - test_perms "$T_DIR_WORK/.ssh" "drwxrwxrwx" - - #; set configuration - run "${T_YADM_Y[@]}" config --bool "yadm.auto-perms" "false" - - #; run status - run "${T_YADM_Y[@]}" status - - #; validate status and output - [ "$status" -eq 0 ] - [[ "$output" =~ On\ branch\ master ]] - - #; confirm directories are still preset and open - [ -d "$T_DIR_WORK/.gnupg" ] - test_perms "$T_DIR_WORK/.gnupg" "drwxrwxrwx" - [ -d "$T_DIR_WORK/.ssh" ] - test_perms "$T_DIR_WORK/.ssh" "drwxrwxrwx" -} diff --git a/test/common.bash b/test/common.bash deleted file mode 100644 index 32ba8e1..0000000 --- a/test/common.bash +++ /dev/null @@ -1,384 +0,0 @@ - -#; common fixtures -function load_fixtures() { - export DEFAULT_YADM_DIR="$HOME/.yadm" - export DEFAULT_REPO="repo.git" - export DEFAULT_CONFIG="config" - export DEFAULT_ENCRYPT="encrypt" - export DEFAULT_ARCHIVE="files.gpg" - export DEFAULT_BOOTSTRAP="bootstrap" - - export T_YADM="$PWD/yadm" - export T_TMP="$BATS_TMPDIR/ytmp" - export T_DIR_YADM="$T_TMP/.yadm" - export T_DIR_WORK="$T_TMP/yadm-work" - export T_DIR_REPO="$T_DIR_YADM/repo.git" - export T_DIR_HOOKS="$T_DIR_YADM/hooks" - export T_YADM_CONFIG="$T_DIR_YADM/config" - export T_YADM_ENCRYPT="$T_DIR_YADM/encrypt" - export T_YADM_ARCHIVE="$T_DIR_YADM/files.gpg" - export T_YADM_BOOTSTRAP="$T_DIR_YADM/bootstrap" - - export T_YADM_Y - T_YADM_Y=( "$T_YADM" -Y "$T_DIR_YADM" ) - - export T_SYS - T_SYS=$(uname -s) - export T_HOST - T_HOST=$(hostname -s) - export T_USER - T_USER=$(id -u -n) - export T_DISTRO - T_DISTRO=$(lsb_release -si 2>/dev/null || true) -} - -function configure_git() { - (git config user.name || git config --global user.name 'test') >/dev/null - (git config user.email || git config --global user.email 'test@test.test') > /dev/null -} - -function make_parents() { - local parent_dir - parent_dir=$(dirname "$@") - mkdir -p "$parent_dir" -} - -function test_perms() { - local test_path="$1" - local regex="$2" - local ls - ls=$(ls -ld "$test_path") - local perms="${ls:0:10}" - if [[ ! $perms =~ $regex ]]; then - echo "ERROR: Found permissions $perms for $test_path" - return 1 - fi - return 0 -} - -function test_repo_attribute() { - local repo_dir="$1" - local attribute="$2" - local expected="$3" - local actual - actual=$(GIT_DIR="$repo_dir" git config --local "$attribute") - if [ "$actual" != "$expected" ]; then - echo "ERROR: repo attribute $attribute set to $actual" - return 1 - fi - return 0 -} - -#; create worktree at path -function create_worktree() { - local DIR_WORKTREE="$1" - if [ -z "$DIR_WORKTREE" ]; then - echo "ERROR: create_worktree() called without a path" - return 1 - fi - - if [[ ! "$DIR_WORKTREE" =~ ^$T_TMP ]]; then - echo "ERROR: create_worktree() called with a path outside of $T_TMP" - return 1 - fi - - #; remove any existing data - rm -rf "$DIR_WORKTREE" - - #; create some standard files - if [ ! -z "$TEST_TREE_WITH_ALT" ] ; then - for f in \ - "alt-none##S" \ - "alt-none##S.H" \ - "alt-none##S.H.U" \ - "alt-base##" \ - "alt-base##S" \ - "alt-base##S.H" \ - "alt-base##S.H.U" \ - "alt-system##" \ - "alt-system##S" \ - "alt-system##S.H" \ - "alt-system##S.H.U" \ - "alt-system##$T_SYS" \ - "alt-system##AAA" \ - "alt-system##ZZZ" \ - "alt-system##aaa" \ - "alt-system##zzz" \ - "alt-host##" \ - "alt-host##S" \ - "alt-host##S.H" \ - "alt-host##S.H.U" \ - "alt-host##$T_SYS.$T_HOST" \ - "alt-host##${T_SYS}_${T_HOST}" \ - "alt-user##" \ - "alt-user##S" \ - "alt-user##S.H" \ - "alt-user##S.H.U" \ - "alt-user##$T_SYS.$T_HOST.$T_USER" \ - "alt-user##${T_SYS}_${T_HOST}_${T_USER}" \ - "alt-override-system##" \ - "alt-override-system##$T_SYS" \ - "alt-override-system##custom_system" \ - "alt-override-host##" \ - "alt-override-host##$T_SYS.$T_HOST" \ - "alt-override-host##$T_SYS.custom_host" \ - "alt-override-user##" \ - "alt-override-user##S.H.U" \ - "alt-override-user##$T_SYS.$T_HOST.custom_user" \ - "dir one/alt-none##S/file1" \ - "dir one/alt-none##S/file2" \ - "dir one/alt-none##S.H/file1" \ - "dir one/alt-none##S.H/file2" \ - "dir one/alt-none##S.H.U/file1" \ - "dir one/alt-none##S.H.U/file2" \ - "dir one/alt-base##/file1" \ - "dir one/alt-base##/file2" \ - "dir one/alt-base##S/file1" \ - "dir one/alt-base##S/file2" \ - "dir one/alt-base##S.H/file1" \ - "dir one/alt-base##S.H/file2" \ - "dir one/alt-base##S.H.U/file1" \ - "dir one/alt-base##S.H.U/file2" \ - "dir one/alt-system##/file1" \ - "dir one/alt-system##/file2" \ - "dir one/alt-system##S/file1" \ - "dir one/alt-system##S/file2" \ - "dir one/alt-system##S.H/file1" \ - "dir one/alt-system##S.H/file2" \ - "dir one/alt-system##S.H.U/file1" \ - "dir one/alt-system##S.H.U/file2" \ - "dir one/alt-system##$T_SYS/file1" \ - "dir one/alt-system##$T_SYS/file2" \ - "dir one/alt-system##AAA/file1" \ - "dir one/alt-system##AAA/file2" \ - "dir one/alt-system##ZZZ/file1" \ - "dir one/alt-system##ZZZ/file2" \ - "dir one/alt-system##aaa/file1" \ - "dir one/alt-system##aaa/file2" \ - "dir one/alt-system##zzz/file1" \ - "dir one/alt-system##zzz/file2" \ - "dir one/alt-host##/file1" \ - "dir one/alt-host##/file2" \ - "dir one/alt-host##S/file1" \ - "dir one/alt-host##S/file2" \ - "dir one/alt-host##S.H/file1" \ - "dir one/alt-host##S.H/file2" \ - "dir one/alt-host##S.H.U/file1" \ - "dir one/alt-host##S.H.U/file2" \ - "dir one/alt-host##$T_SYS.$T_HOST/file1" \ - "dir one/alt-host##$T_SYS.$T_HOST/file2" \ - "dir one/alt-host##${T_SYS}_${T_HOST}/file1" \ - "dir one/alt-host##${T_SYS}_${T_HOST}/file2" \ - "dir one/alt-user##/file1" \ - "dir one/alt-user##/file2" \ - "dir one/alt-user##S/file1" \ - "dir one/alt-user##S/file2" \ - "dir one/alt-user##S.H/file1" \ - "dir one/alt-user##S.H/file2" \ - "dir one/alt-user##S.H.U/file1" \ - "dir one/alt-user##S.H.U/file2" \ - "dir one/alt-user##$T_SYS.$T_HOST.$T_USER/file1" \ - "dir one/alt-user##$T_SYS.$T_HOST.$T_USER/file2" \ - "dir one/alt-user##${T_SYS}_${T_HOST}_${T_USER}/file1" \ - "dir one/alt-user##${T_SYS}_${T_HOST}_${T_USER}/file2" \ - "dir one/alt-override-system##/file1" \ - "dir one/alt-override-system##/file2" \ - "dir one/alt-override-system##$T_SYS/file1" \ - "dir one/alt-override-system##$T_SYS/file2" \ - "dir one/alt-override-system##custom_system/file1" \ - "dir one/alt-override-system##custom_system/file2" \ - "dir one/alt-override-host##/file1" \ - "dir one/alt-override-host##/file2" \ - "dir one/alt-override-host##$T_SYS.$T_HOST/file1" \ - "dir one/alt-override-host##$T_SYS.$T_HOST/file2" \ - "dir one/alt-override-host##$T_SYS.custom_host/file1" \ - "dir one/alt-override-host##$T_SYS.custom_host/file2" \ - "dir one/alt-override-user##/file1" \ - "dir one/alt-override-user##/file2" \ - "dir one/alt-override-user##S.H.U/file1" \ - "dir one/alt-override-user##S.H.U/file2" \ - "dir one/alt-override-user##$T_SYS.$T_HOST.custom_user/file1" \ - "dir one/alt-override-user##$T_SYS.$T_HOST.custom_user/file2" \ - "dir2/file2" \ - ; - do - make_parents "$DIR_WORKTREE/$f" - echo "$f" > "$DIR_WORKTREE/$f" - done - echo "{{ YADM_CLASS }}-{{ YADM_OS }}-{{ YADM_HOSTNAME }}-{{ YADM_USER }}-{{ YADM_DISTRO }}" > "$DIR_WORKTREE/alt-jinja##yadm.j2" - fi - - #; for some cygwin tests - if [ ! -z "$TEST_TREE_WITH_CYGWIN" ] ; then - for f in \ - "alt-test##" \ - "alt-test##$T_SYS" \ - "alt-test##$SIMULATED_CYGWIN" \ - ; - do - make_parents "$DIR_WORKTREE/$f" - echo "$f" > "$DIR_WORKTREE/$f" - done - fi - - if [ ! -z "$TEST_TREE_WITH_WILD" ] ; then - #; wildcard test data - yes this is a big mess :( - #; none - for f in "wild-none##"; do - make_parents "$DIR_WORKTREE/$f" - echo "$f" > "$DIR_WORKTREE/$f" - done - #; system - for WILD_S in 'local' 'wild' 'other'; do - local s_base="wild-system-$WILD_S" - case $WILD_S in local) WILD_S="$T_SYS";; wild) WILD_S="%";; esac - local f="${s_base}##${WILD_S}" - make_parents "$DIR_WORKTREE/$f" - echo "$f" > "$DIR_WORKTREE/$f" - done - #; system.host - for WILD_S in 'local' 'wild' 'other'; do - local s_base="wild-host-$WILD_S" - case $WILD_S in local) WILD_S="$T_SYS";; wild) WILD_S="%";; esac - for WILD_H in 'local' 'wild' 'other'; do - local h_base="${s_base}-$WILD_H" - case $WILD_H in local) WILD_H="$T_HOST";; wild) WILD_H="%";; esac - local f="${h_base}##${WILD_S}.${WILD_H}" - make_parents "$DIR_WORKTREE/$f" - echo "$f" > "$DIR_WORKTREE/$f" - done - done - #; system.host.user - for WILD_S in 'local' 'wild' 'other'; do - local s_base="wild-user-$WILD_S" - case $WILD_S in local) WILD_S="$T_SYS";; wild) WILD_S="%";; esac - for WILD_H in 'local' 'wild' 'other'; do - local h_base="${s_base}-$WILD_H" - case $WILD_H in local) WILD_H="$T_HOST";; wild) WILD_H="%";; esac - for WILD_U in 'local' 'wild' 'other'; do - local u_base="${h_base}-$WILD_U" - case $WILD_U in local) WILD_U="$T_USER";; wild) WILD_U="%";; esac - local f="${u_base}##${WILD_S}.${WILD_H}.${WILD_U}" - make_parents "$DIR_WORKTREE/$f" - echo "$f" > "$DIR_WORKTREE/$f" - done - done - done - #; class - for WILD_C in 'local' 'wild' 'other'; do - local c_base="wild-class-$WILD_C" - case $WILD_C in local) WILD_C="set_class";; wild) WILD_C="%";; esac - local f="${c_base}##${WILD_C}" - make_parents "$DIR_WORKTREE/$f" - echo "$f" > "$DIR_WORKTREE/$f" - done - #; class.system - for WILD_C in 'local' 'wild' 'other'; do - local c_base="wild-class-system-$WILD_C" - case $WILD_C in local) WILD_C="set_class";; wild) WILD_C="%";; esac - for WILD_S in 'local' 'wild' 'other'; do - local s_base="${c_base}-$WILD_S" - case $WILD_S in local) WILD_S="$T_SYS";; wild) WILD_S="%";; esac - local f="${s_base}##${WILD_C}.${WILD_S}" - make_parents "$DIR_WORKTREE/$f" - echo "$f" > "$DIR_WORKTREE/$f" - done - done - #; class.system.host - for WILD_C in 'local' 'wild' 'other'; do - local c_base="wild-class-system-host-$WILD_C" - case $WILD_C in local) WILD_C="set_class";; wild) WILD_C="%";; esac - for WILD_S in 'local' 'wild' 'other'; do - local s_base="${c_base}-$WILD_S" - case $WILD_S in local) WILD_S="$T_SYS";; wild) WILD_S="%";; esac - for WILD_H in 'local' 'wild' 'other'; do - local h_base="${s_base}-$WILD_H" - case $WILD_H in local) WILD_H="$T_HOST";; wild) WILD_H="%";; esac - local f="${h_base}##${WILD_C}.${WILD_S}.${WILD_H}" - make_parents "$DIR_WORKTREE/$f" - echo "$f" > "$DIR_WORKTREE/$f" - done - done - done - #; class.system.host.user - for WILD_C in 'local' 'wild' 'other'; do - local c_base="wild-class-system-host-user-$WILD_C" - case $WILD_C in local) WILD_C="set_class";; wild) WILD_C="%";; esac - for WILD_S in 'local' 'wild' 'other'; do - local s_base="${c_base}-$WILD_S" - case $WILD_S in local) WILD_S="$T_SYS";; wild) WILD_S="%";; esac - for WILD_H in 'local' 'wild' 'other'; do - local h_base="${s_base}-$WILD_H" - case $WILD_H in local) WILD_H="$T_HOST";; wild) WILD_H="%";; esac - for WILD_U in 'local' 'wild' 'other'; do - local u_base="${h_base}-$WILD_U" - case $WILD_U in local) WILD_U="$T_USER";; wild) WILD_U="%";; esac - local f="${u_base}##${WILD_C}.${WILD_S}.${WILD_H}.${WILD_U}" - make_parents "$DIR_WORKTREE/$f" - echo "$f" > "$DIR_WORKTREE/$f" - done - done - done - done - fi - for f in \ - .bash_profile \ - .gnupg/gpg.conf \ - .gnupg/pubring.gpg \ - .gnupg/secring.gpg \ - .hammerspoon/init.lua \ - .ssh/config \ - .ssh/secret.key \ - .ssh/secret.pub \ - .tmux.conf \ - .vimrc \ - "space test/file one" \ - "space test/file two" \ - ; - do - make_parents "$DIR_WORKTREE/$f" - echo "$f" > "$DIR_WORKTREE/$f" - done - - #; change all perms (so permission updates can be observed) - find "$DIR_WORKTREE" -exec chmod 0777 '{}' ';' - -} - -#; create a repo in T_DIR_REPO -function build_repo() { - local files_to_add=( "$@" ) - - #; create a worktree - create_worktree "$T_DIR_WORK" - - #; remove the repo if it exists - if [ -e "$T_DIR_REPO" ]; then - rm -rf "$T_DIR_REPO" - fi - - #; create the repo - git init --shared=0600 --bare "$T_DIR_REPO" >/dev/null 2>&1 - - #; standard repo config - GIT_DIR="$T_DIR_REPO" git config core.bare 'false' - GIT_DIR="$T_DIR_REPO" git config core.worktree "$T_DIR_WORK" - GIT_DIR="$T_DIR_REPO" git config status.showUntrackedFiles no - GIT_DIR="$T_DIR_REPO" git config yadm.managed 'true' - - if [ ${#files_to_add[@]} -ne 0 ]; then - for f in "${files_to_add[@]}"; do - GIT_DIR="$T_DIR_REPO" git add "$T_DIR_WORK/$f" >/dev/null - done - GIT_DIR="$T_DIR_REPO" git commit -m 'Create repo template' >/dev/null - fi - -} - -#; remove all tmp files -function destroy_tmp() { - load_fixtures - rm -rf "$T_TMP" -} - -configure_git From 7e7d199721ec216e834a63ad1b6de37e5620e536 Mon Sep 17 00:00:00 2001 From: Tim Byrne Date: Wed, 27 Feb 2019 08:11:35 -0600 Subject: [PATCH 009/137] Add code of conduct Adapted from the www.contributor-covenant.org --- .github/CODE_OF_CONDUCT.md | 77 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 77 insertions(+) create mode 100644 .github/CODE_OF_CONDUCT.md diff --git a/.github/CODE_OF_CONDUCT.md b/.github/CODE_OF_CONDUCT.md new file mode 100644 index 0000000..eee7e01 --- /dev/null +++ b/.github/CODE_OF_CONDUCT.md @@ -0,0 +1,77 @@ +# Contributor Covenant Code of Conduct + +## Our Pledge + +In the interest of fostering an open and welcoming environment, we as +contributors and maintainers pledge to making participation in our project and +our community a harassment-free experience for everyone, regardless of age, body +size, disability, ethnicity, sex characteristics, gender identity and expression, +level of experience, education, socio-economic status, nationality, personal +appearance, race, religion, or sexual identity and orientation. + +## Our Standards + +Examples of behavior that contributes to creating a positive environment +include: + +* Using welcoming and inclusive language +* Being respectful of differing viewpoints and experiences +* Gracefully accepting constructive criticism +* Focusing on what is best for the community +* Showing empathy towards other community members + +Examples of unacceptable behavior by participants include: + +* The use of sexualized language or imagery and unwelcome sexual attention or + advances +* Trolling, insulting/derogatory comments, and personal or political attacks +* Public or private harassment +* Publishing others' private information, such as a physical or electronic + address, without explicit permission +* Other conduct which could reasonably be considered inappropriate in a + professional setting + +## Our Responsibilities + +Project maintainers are responsible for clarifying the standards of acceptable +behavior and are expected to take appropriate and fair corrective action in +response to any instances of unacceptable behavior. + +Project maintainers have the right and responsibility to remove, edit, or +reject comments, commits, code, wiki edits, issues, and other contributions +that are not aligned to this Code of Conduct, or to ban temporarily or +permanently any contributor for other behaviors that they deem inappropriate, +threatening, offensive, or harmful. + +## Scope + +This Code of Conduct applies both within project spaces and in public spaces +when an individual is representing the project or its community. Examples of +representing a project or community include using an official project e-mail +address, posting via an official social media account, or acting as an appointed +representative at an online or offline event. Representation of a project may be +further defined and clarified by project maintainers. + +## Enforcement + +Instances of abusive, harassing, or otherwise unacceptable behavior may be +reported by contacting the project team at . All +complaints will be reviewed and investigated and will result in a response that +is deemed necessary and appropriate to the circumstances. The project team is +obligated to maintain confidentiality with regard to the reporter of an incident. +Further details of specific enforcement policies may be posted separately. + +Project maintainers who do not follow or enforce the Code of Conduct in good +faith may face temporary or permanent repercussions as determined by other +members of the project's leadership. + +## Attribution + +This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4, +available at https://www.contributor-covenant.org/version/1/4/code-of-conduct.html + +[homepage]: https://www.contributor-covenant.org + +For answers to common questions about this code of conduct, see +https://www.contributor-covenant.org/faq + From d32e63fc92ac3544efdc5fec5304f2eb72c07a2d Mon Sep 17 00:00:00 2001 From: Tim Byrne Date: Thu, 28 Feb 2019 22:28:32 -0600 Subject: [PATCH 010/137] Split packages onto separate lines This will make it easier to see the changes in the dependencies included in the yadm/testbed image. --- Dockerfile | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 622932a..6651deb 100644 --- a/Dockerfile +++ b/Dockerfile @@ -13,8 +13,24 @@ ENV LANG='en_US.UTF-8' LANGUAGE='en_US:en' LC_ALL='en_US.UTF-8' RUN echo 'set -o vi' >> /root/.bashrc # Install prerequisites -RUN apt-get update && apt-get install -y git gnupg1 make shellcheck=0.4.6-1 bats expect curl python3-pip lsb-release -RUN pip3 install envtpl pytest==3.6.4 pylint==1.9.2 flake8==3.5.0 +RUN \ + apt-get update && \ + apt-get install -y \ + curl \ + expect \ + git \ + gnupg1 \ + lsb-release \ + make \ + python3-pip \ + shellcheck=0.4.6-1 \ + ; +RUN pip3 install \ + envtpl \ + pytest==3.6.4 \ + pylint==1.9.2 \ + flake8==3.5.0 \ + ; # Force GNUPG version 1 at path /usr/bin/gpg RUN ln -fs /usr/bin/gpg1 /usr/bin/gpg From 27859af307fda46732209e29a09e8a4b552212d1 Mon Sep 17 00:00:00 2001 From: Tim Byrne Date: Thu, 28 Feb 2019 22:59:04 -0600 Subject: [PATCH 011/137] Add make target "scripthost" Hopefully this target will help others demonstrate problems in a reproducible way. --- Makefile | 48 ++++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 42 insertions(+), 6 deletions(-) diff --git a/Makefile b/Makefile index 65edc57..b0172f0 100644 --- a/Makefile +++ b/Makefile @@ -27,6 +27,13 @@ usage: @echo ' The targeted "version" will be retrieved from the repo, and' @echo ' linked into the container as a local volume.' @echo + @echo ' make scripthost [version=VERSION]' + @echo ' - Create an ephemeral container for demonstrating a bug. After' + @echo ' exiting the shell, a log of the commands used to illustrate the' + @echo ' problem will be written to the file "script.txt". This file can' + @echo ' be useful to developers to make a repeatable test for the' + @echo ' problem.' + @echo @echo 'LINTING' @echo @echo ' make testenv' @@ -87,17 +94,39 @@ test: fi .PHONY: testhost -testhost: - @if ! command -v "docker" >/dev/null 2>&1; then \ - echo "Sorry, this make target requires docker to be installed."; \ - false; \ - fi +testhost: require-docker @version=HEAD @rm -rf /tmp/testhost @git show $(version):yadm > /tmp/testhost @chmod a+x /tmp/testhost @echo Starting testhost version=\"$$version\" - @docker run -w /root --hostname testhost --rm -it -v "/tmp/testhost:/bin/yadm:ro" yadm/testbed:latest bash -l + @docker run \ + -w /root \ + --hostname testhost \ + --rm -it \ + -v "/tmp/testhost:/bin/yadm:ro" \ + yadm/testbed:latest \ + bash -l + +.PHONY: scripthost +scripthost: require-docker + @version=HEAD + @rm -rf /tmp/testhost + @git show $(version):yadm > /tmp/testhost + @chmod a+x /tmp/testhost + @echo Starting scripthost version=\"$$version\" \(recording script\) + @printf '' > script.gz + @docker run \ + -w /root \ + --hostname scripthost \ + --rm -it \ + -v "$$PWD/script.gz:/script.gz:rw" \ + -v "/tmp/testhost:/bin/yadm:ro" \ + yadm/testbed:latest \ + bash -c "script /tmp/script -q -c 'bash -l'; gzip < /tmp/script > /script.gz" + @echo + @echo "Script saved to $$PWD/script.gz" + .PHONY: testenv testenv: @@ -133,3 +162,10 @@ contrib: .PHONY: sync-clock sync-clock: docker run --rm --privileged alpine hwclock -s + +.PHONY: require-docker +require-docker: + @if ! command -v "docker" >/dev/null 2>&1; then \ + echo "Sorry, this make target requires docker to be installed."; \ + false; \ + fi From abcc2018943416d6e0fa3d717946206931f4ff6d Mon Sep 17 00:00:00 2001 From: Tim Byrne Date: Thu, 7 Mar 2019 08:26:17 -0600 Subject: [PATCH 012/137] Improve Makefile usage info --- Makefile | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/Makefile b/Makefile index b0172f0..b65dee0 100644 --- a/Makefile +++ b/Makefile @@ -37,27 +37,29 @@ usage: @echo 'LINTING' @echo @echo ' make testenv' - @echo ' - Create a python virtual environment with the dependencies used' - @echo " by yadm's testbed environment. Creating and activating this" - @echo ' environment might be useful if your editor does real time' + @echo ' - Create a python virtual environment with the same dependencies' + @echo " used by yadm's testbed environment. Creating and activating" + @echo ' this environment might be useful if your editor does real time' @echo ' linting of python files. After creating the virtual environment,' - @echo ' you can activate it by typing' + @echo ' you can activate it by typing:' + @echo + @echo ' source testenv/bin/activate' @echo @echo 'MANPAGES' @echo @echo ' make man' - @echo ' - View yadm.1 as a standard manpage.' + @echo ' - View yadm.1 as a standard man page.' @echo @echo ' make man-wide' - @echo ' - View yadm.1 as a manpage, using all columns of your display.' + @echo ' - View yadm.1 as a man page, using all columns of your display.' @echo @echo ' make man-ps' - @echo ' - Create a postscript version of the manpage.' + @echo ' - Create a postscript version of the man page.' @echo @echo 'FILE GENERATION' @echo @echo ' make yadm.md' - @echo ' - Generate the markdown version of the manpage (for viewing on' + @echo ' - Generate the markdown version of the man page (for viewing on' @echo ' the web).' @echo @echo ' make contrib' From b443bbede27afe4fa244e065f09781d6eabac377 Mon Sep 17 00:00:00 2001 From: Tim Byrne Date: Tue, 12 Mar 2019 22:00:25 -0500 Subject: [PATCH 013/137] Add contrib/hooks --- contrib/hooks/README.md | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 contrib/hooks/README.md diff --git a/contrib/hooks/README.md b/contrib/hooks/README.md new file mode 100644 index 0000000..551f6f0 --- /dev/null +++ b/contrib/hooks/README.md @@ -0,0 +1,14 @@ +## Contributed Hooks + +Although these [hooks][hooks-help] are available as part of the official +**yadm** source tree, they have a somewhat different status. The intention is to +keep interesting and potentially useful hooks here, building a library of +examples that might help others. + +In some cases, an experimental new feature can be build entirely with hooks, and +this is a place to share it. + +I recommend *careful review* of any code from here before using it. No +guarantees of code quality is assumed. + +[hooks-help]: https://github.com/TheLocehiliosan/yadm/blob/master/yadm.md#hooks From 19c6eb6009a93ea06248d4ef4679215c45edb3d7 Mon Sep 17 00:00:00 2001 From: Tim Byrne Date: Tue, 5 Mar 2019 23:57:53 -0600 Subject: [PATCH 014/137] Add GitHub templates --- .github/ISSUE_TEMPLATE.md | 7 +++ .github/ISSUE_TEMPLATE/BUG_REPORT.md | 68 +++++++++++++++++++++++ .github/ISSUE_TEMPLATE/FEATURE_REQUEST.md | 29 ++++++++++ .github/ISSUE_TEMPLATE/OTHER.md | 23 ++++++++ .github/ISSUE_TEMPLATE/SUPPORT.md | 36 ++++++++++++ .github/PULL_REQUEST_TEMPLATE.md | 34 ++++++++++++ 6 files changed, 197 insertions(+) create mode 100644 .github/ISSUE_TEMPLATE.md create mode 100644 .github/ISSUE_TEMPLATE/BUG_REPORT.md create mode 100644 .github/ISSUE_TEMPLATE/FEATURE_REQUEST.md create mode 100644 .github/ISSUE_TEMPLATE/OTHER.md create mode 100644 .github/ISSUE_TEMPLATE/SUPPORT.md create mode 100644 .github/PULL_REQUEST_TEMPLATE.md diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md new file mode 100644 index 0000000..29dc730 --- /dev/null +++ b/.github/ISSUE_TEMPLATE.md @@ -0,0 +1,7 @@ + diff --git a/.github/ISSUE_TEMPLATE/BUG_REPORT.md b/.github/ISSUE_TEMPLATE/BUG_REPORT.md new file mode 100644 index 0000000..705dc5a --- /dev/null +++ b/.github/ISSUE_TEMPLATE/BUG_REPORT.md @@ -0,0 +1,68 @@ +--- +name: Bug report +about: Create a report to help improve yadm +title: '' +labels: bug +assignees: '' + +--- + + +### Describe the bug + +[A clear and concise description of what the bug is.] + +### To reproduce + +Can this be reproduced with the yadm/testbed docker image: [Yes/No] + + +Steps to reproduce the behavior: + +1. Run command '....' +2. Run command '....' +3. Run command '....' +4. See error + +### Expected behavior + +[A clear and concise description of what you expected to happen.] + +### Environment + + - Operating system: [Ubuntu 18.04, yadm/testbed, etc.] + - Version yadm: [found via `yadm version`] + - Version Git: [found via `git --version`] + +### Additional context + +[Add any other context about the problem here.] diff --git a/.github/ISSUE_TEMPLATE/FEATURE_REQUEST.md b/.github/ISSUE_TEMPLATE/FEATURE_REQUEST.md new file mode 100644 index 0000000..3a211ea --- /dev/null +++ b/.github/ISSUE_TEMPLATE/FEATURE_REQUEST.md @@ -0,0 +1,29 @@ +--- +name: Feature request +about: Suggest an idea for yadm +title: '' +labels: feature +assignees: '' + +--- + + +### Is your feature request related to a problem? Please describe. + +[A clear and concise description of what the problem is. Ex. I'm always frustrated when ...] + +### Describe the solution you'd like + +[A clear and concise description of what you want to happen.] + +### Describe alternatives you've considered + +[A clear and concise description of any alternative solutions or features you've +considered. For example, have you considered using yadm "hooks" as a solution?] + +### Additional context + +[Add any other context or screenshots about the feature request here.] diff --git a/.github/ISSUE_TEMPLATE/OTHER.md b/.github/ISSUE_TEMPLATE/OTHER.md new file mode 100644 index 0000000..936a4a4 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/OTHER.md @@ -0,0 +1,23 @@ +--- +name: Other issue +about: Report issues with documentation, packaging, or something else +title: '' +labels: '' +assignees: '' + +--- + + +### This issue is about + +* [ ] Man pages or command-line usage +* [ ] Website documentation +* [ ] Packaging +* [ ] Other + +### Describe the issue + +[A clear and concise description of the issue.] diff --git a/.github/ISSUE_TEMPLATE/SUPPORT.md b/.github/ISSUE_TEMPLATE/SUPPORT.md new file mode 100644 index 0000000..22bd849 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/SUPPORT.md @@ -0,0 +1,36 @@ +--- +name: Support +about: Get help using yadm +title: '' +labels: 'question' +assignees: '' + +--- + + +### This question is about + +* [ ] Installation +* [ ] Initializing / Cloning +* [ ] Alternate files +* [ ] Jinja templates +* [ ] Encryption +* [ ] Bootstrap +* [ ] Hooks +* [ ] Other + +### Describe your question + + +[A clear and concise description of the question.] diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md new file mode 100644 index 0000000..d2f12b9 --- /dev/null +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -0,0 +1,34 @@ +### What does this PR do? + +[A clear and concise description of what this pull request accomplishes.] + +### What issues does this PR fix or reference? + + +[A list of related issues / pull requests.] + +### Previous Behavior + +[Describe the existing behavior.] + +### New Behavior + +[Describe the behavior, after this PR is applied.] + +### Have [tests][1] been written for this change? + +[Yes / No] + +### Have these commits been [signed with GnuPG][2]? + +[Yes / No] + +--- + +Please review [yadm's Contributing Guide][3] for best practices. + +[1]: https://github.com/TheLocehiliosan/yadm/blob/master/.github/CONTRIBUTING.md#test-conventions +[2]: https://help.github.com/en/articles/signing-commits +[3]: https://github.com/TheLocehiliosan/yadm/blob/master/.github/CONTRIBUTING.md From f3e4c7d140ccc749c7e42182fce438575de464c7 Mon Sep 17 00:00:00 2001 From: Tim Byrne Date: Wed, 13 Mar 2019 08:07:58 -0500 Subject: [PATCH 015/137] Add contribution guideline --- .github/CONTRIBUTING.md | 364 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 364 insertions(+) create mode 100644 .github/CONTRIBUTING.md diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md new file mode 100644 index 0000000..d836d87 --- /dev/null +++ b/.github/CONTRIBUTING.md @@ -0,0 +1,364 @@ +# Introduction + +Thank you for considering contributing to **yadm**. I develop this project in my +limited spare time, so help is very appreciated. + +All contributors must follow our [Code of Conduct][conduct]. Please make sure +you are welcoming and friendly during your interactions, and report any +unacceptable behavior to . + +Contributions can take many forms, and often don’t require writing code—maybe +something could be documented more clearly, maybe a feature could be more +helpful, maybe installation could be easier. Help is welcome in any of these +areas. + +To contribute, you can: + +* Report [bugs](#reporting-a-bug) +* Request [features/enhancements](#suggesting-a-feature-or-enhancement) +* Contribute changes to [code, tests](#contributing-code), and [documentation](#improving-documentation) +* Maintain installation [packages](#maintaining-packages) +* Help other users by [answering support questions](#answering-support-questions) + +# Reporting a bug + +Notice something amiss? You’re already helping by reporting the problem! Bugs +are tracked using GitHub issues. Here are some steps you can take to help +problems get fixed quickly and effectively: + +### Before submitting an issue + +Please take a quick look to see whether the problem has been reported already +(there’s a list of [open issues][open-issues]). You can try the search function +with some related terms for a cursory check. If you do find a previous report, +please add a comment there instead of opening a new issue. + +### Security issues + +If you have found a security vulnerability, do **NOT** open an issue. + +Any security issues should be emailed directly to . In order to +determine whether you are dealing with a security issue, ask yourself these two +questions: + +* Can I access something that's not mine, or something I shouldn't have access to? +* Can I disable something for other people? + +If the answer to either of those two questions is "yes", then you're probably +dealing with a security issue. + +### Submitting a (great) bug report + +Choose the "[Bug report][new-bug]" issue type. + +Pick a descriptive title that clearly identifies the issue. + +Describe the steps that led to the problem so that we can go through the same +sequence. A clear set of steps to reproduce the problem is key to fixing an +issue. If possible, attach a [`script.gz`](#attaching-a-scriptgz) to the bug +report. + +Describe what you had expected and how that differed from what happened, and +possibly, why. + +Include the version numbers of your operating system, of **yadm**, and of Git. + +### Attaching a script.gz + +Consider trying to reproduce the bug inside a docker container using the +[yadm/testbed][testbed] docker image. Doing so will greatly increase the +likelihood of the problem being fixed. + +The easiest way to start this container, is to clone the [TheLocehiliosan/yadm +repo][yadm-repo], and use the `scripthost` make target. _(You will need `make` +and `docker` installed.)_ + +For example: + +```text +$ git clone https://github.com/TheLocehiliosan/yadm.git +$ cd yadm +$ make scripthost version=1.12.0 +Starting scripthost version="1.12.0" (recording script) +root@scripthost:~# ### run commands which +root@scripthost:~# ### demonstrate the problem +root@scripthost:~# ### a succinct set of commands is best +root@scripthost:~# exit +logout + +Script saved to script.gz +$ +``` + +A `script.gz` like this can be useful to developers to make a repeatable test +for the problem. You can attach the `script.gz` file to an issue. Look +[here][attach-help] for help with [attaching a file][attach-help]. + +# Suggesting a feature or enhancement + +Have an idea for an improvement? Creating a feature request is a good way to +communicate it. + +### Before submitting an issue + +Please take a quick look to see whether your idea has been suggested already +(there’s a list of [open issues][open-issues]). You can try the search function +with some related terms for a cursory check. If you do find a previous feature +request, please add a comment there instead of opening a new issue. + +### Submitting a (great) feature request + +Choose the "[Feature request][new-feature]" issue type. + +Summarize your idea with a clear title. + +Describe your suggestion in as much detail as possible. + +Explain alternatives you've considered. + +# Contributing code + +Wow, thank you for considering making a contribution of code! + +### Before you begin + +Please take a quick look to see whether a similar change is already being worked +on. A similar pull request may already exist. If the change is related to an +issue, look to see if that issue has an assignee. + +Consider reaching out before you start working. It's possible developers may +have some ideas and code lying around, and might be able to give you a head +start. + +[Creating a hook][hooks-help] is an easy way to begin adding features to an +already existing **yadm** operation. If the hook works well, it could be the +basis of a **yadm** feature addition. Or it might just be a [useful +hook][contrib-hooks] for someone else. + +### Design principles + +**yadm** was created with a few core design principles in mind. Please adhere to +these principles when making changes. + +* **Single repository** + * **yadm** is designed to maintain dotfiles in a single repository. + +* **Very few dependencies** + * **yadm** should be as portable as possible. This is one of the main + reasons it has only two dependencies (Bash and Git). Features using other + dependencies should gracefully downgrade instead of breaking. For example, + encryption requires GnuPG installed, and displays that information if it + is not. + +* **Sparse configuration** + * **yadm** should require very little configuration, and come with sensible + defaults. Changes requiring users to define meta-data for all of their + dotfiles will not be accepted. + +* **Maintain dotfiles in place** + * The default treatment for tracked data should be to allow it to remain a + file, in the location it is normally kept. + +* **Leverage Git** + * Stay out of the way and let Git do what it’s good at. Git has a deep and + rich set of features for just about every use case. Staying hands off for + almost all Git operations will make **yadm** more flexible and + future-proof. + +### Repository branches and tags + +* `master` + * This branch will always represent the latest release of **yadm**. +* `#.#.#` _(tags)_ + * Every release of **yadm** will have a commit tagged with the version number. +* `develop` + * This branch should be used for the basis of every change. As changes are + accepted, they will be merged into `develop`. +* `release/*` + * These are ephemeral branches used to prepare new releases. +* `hotfix/*` + * These are ephemeral branches used to prepare a patch release, which only + includes bug fixes. +* `gh-pages` + * This branch contains the yadm.io website source. +* `dev-pages` + * This branch should be used for the basis of every website change. As + changes are accepted, they will be merged into dev-pages. +* `netlify/*` + * These branches deploy configurations to Netlify websites. Currently this + is only used to drive redirections for + [bootstrap.yadm.io](https://bootstrap.yadm.io/). + +### GitHub workflow + +1. Fork the [yadm repository][yadm-repo] on GitHub. + +2. Clone your fork locally. + + ```text + $ git clone + ``` + +3. Add the official repository (`upstream`) as a remote repository. + + ```text + $ git remote add upstream https://github.com/TheLocehiliosan/yadm.git + ``` + +4. Verify you can run the test harness. _(This will require dependencies: + `make`, `docker`, and `docker-compose`)_. + + ```text + $ make test + ``` + +5. Create a feature branch, based off the `develop` branch. + + ```text + $ git checkout -b upstream/develop + ``` + +6. Add changes to your feature branch. + +7. If your changes take a few days, be sure to occasionally pull the latest + changes from upstream, to ensure that your local branch is up-to-date. + + ```text + $ git pull --rebase upstream develop + ``` + +8. When your work is done, push your local branch to your fork. + + ```text + $ git push origin + ``` + +9. [Create a pull request][pr-help] using `develop` as the "base". + +### Code conventions + +When updating the yadm code, please follow these guidelines: + +* ShellCheck + * Bash code should pass the scrutiny of [ShellCheck][shellcheck]. The + simplest way to quickly test this is to run: + * `make test testargs='-k shellcheck'` +* Interface changes + * Any changes to **yadm**'s interface should include a commit that updates + the `yadm.1` man page. + +### Test conventions + +The test system is written in Python 3 using [pytest][pytest]. Tests should be +written for all bugs fixed and features added. To make testing portable and +uniform, tests should be performed via the [yadm/testbed][testbed] docker image. +The `Makefile` has several "make targets" for testing. Running `make` by itself +will produce a help page. + +Please follow these guidelines while writing tests: + +* Organization + * Tests should be kept in the `test/` directory. + * Every test module name should start with `test_`. + * Unit tests, which test individual functions should have names that begin + with `test_unit_`. + * Completely new features should get their own test modules, while updates + to existing features should have updated test modules. +* Efficiency + * Care should be taken to make tests run as efficiently as possible. + * Scope large, unchanging, fixtures appropriately so they do not have to be + recreated multiple times. +* Linting + * Python code must pass the scrutiny of [pylint][pylint] and + [flake8][flake8]. The simplest way to quickly test this is to run: + * `make test testargs='-k pylint\ or\ flake8'` + +### Commit conventions + +When arranging your commits, please adhere to the following conventions. + +* Commit messages + * Use the "[Tim Pope][tpope-style]" style of commit messages. Here is a + [great guide][commit-style] to writing commit messages. +* Atomic commits + * Please create only [atomic commits][atomic-commits]. +* Signed commits + * All commits must be [cryptographically signed][signing-commits]. + +# Improving documentation + +Wow, thank you for considering making documentation improvements! + +There is overlap between the content of the man page, and the information on the +website. Consider reviewing both sets of documentation, and submitting similar +changes for both to improve consistency. + +### Man page changes + +The man page documentation is contained in the file `yadm.1`. This file is +formatted using [groff man macros][groff-man]. Changes to this file can be +tested using "make targets": + +```text +$ make man +$ make man-wide +$ make man-ps +``` + +While the [markdown version of the man page][yadm-man] is generated from +`yadm.1`, please do not include changes to `yadm.md` within any pull request. +That file is only updated during software releases. + +### Website changes + +**NOTE:** A [website refactoring][refactor] is being performed soon, and it is +unlikely that website changes will be accepted until this task is completed. +Better instructions for testing and submitting website changes will be written +during that refactor. + +# Maintaining packages + +Maintaining installation packages is very important for making **yadm** +accessible to as many people as possible. Thank you for considering contributing +in this way. Please consider the following: + +* Watch releases + * GitHub allows users to "watch" a project for "releases". Doing so will + provide you with notifications when a new version of **yadm** has been + released. +* Include License + * Any package of **yadm** should include the license file from the + repository. +* Dependencies + * Be sure to include dependencies in a manner appropriate to the packaging + system being used. **yadm** won't work very well without Git. :) + +# Answering support questions + +Are you an experienced **yadm** user, with an advanced knowledge of Git? Your +expertise could be useful to someone else who is starting out or struggling with +a problem. Consider reviewing the list of [open support questions][questions] to +see if you can help. + +[atomic-commits]: https://www.google.com/search?q=atomic+commits +[attach-help]: https://help.github.com/en/articles/file-attachments-on-issues-and-pull-requests +[commit-style]: https://chris.beams.io/posts/git-commit/#seven-rules +[conduct]: CODE_OF_CONDUCT.md +[contrib-hooks]: https://github.com/TheLocehiliosan/yadm/tree/master/contrib/hooks +[flake8]: https://pypi.org/project/flake8/ +[groff-man]: https://www.gnu.org/software/groff/manual/html_node/man.html +[hooks-help]: https://github.com/TheLocehiliosan/yadm/blob/master/yadm.md#hooks +[new-bug]: https://github.com/TheLocehiliosan/yadm/issues/new?template=BUG_REPORT.md +[new-feature]: https://github.com/TheLocehiliosan/yadm/issues/new?template=FEATURE_REQUEST.md +[open-issues]: https://github.com/TheLocehiliosan/yadm/issues +[pr-help]: https://help.github.com/en/articles/creating-a-pull-request-from-a-fork +[pylint]: https://pylint.org/ +[pytest]: https://pytest.org/ +[questions]: https://github.com/TheLocehiliosan/yadm/labels/question +[refactor]: https://github.com/TheLocehiliosan/yadm/issues/146 +[shellcheck]: https://www.shellcheck.net +[signing-commits]: https://help.github.com/en/articles/signing-commits +[testbed]: https://hub.docker.com/r/yadm/testbed +[tpope-style]: https://tbaggery.com/2008/04/19/a-note-about-git-commit-messages.html +[yadm-man]: https://github.com/TheLocehiliosan/yadm/blob/master/yadm.md +[yadm-repo]: https://github.com/TheLocehiliosan/yadm From bd86b66ea88b56d73375da14c4798e47c7603a6e Mon Sep 17 00:00:00 2001 From: Tim Byrne Date: Thu, 14 Mar 2019 08:22:44 -0500 Subject: [PATCH 016/137] Update README * Add shield.io badges * Add brief description * Change website to yadm.io --- README.md | 46 ++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 42 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index c50b321..41b5f38 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,45 @@ -# yadm - Yet Another Dotfiles Manager [![Build Status](https://travis-ci.org/TheLocehiliosan/yadm.svg?branch=master)](https://travis-ci.org/TheLocehiliosan/yadm) +# yadm - Yet Another Dotfiles Manager -Features, usage, examples and installation instructions can be found on the [website](https://thelocehiliosan.github.io/yadm/). +[![Latest Version][releases-badge]][releases-link] +[![Homebrew Version][homebrew-badge]][homebrew-link] +[![Copr Version][copr-badge]][copr-link] +[![License][license-badge]][license-link]
+[![Master Update][master-date]][master-commits] +[![Develop Update][develop-date]][develop-commits] +[![Website Update][website-date]][website-commits]
+[![Master Status][master-badge]][travis-ci] +[![Develop Status][develop-badge]][travis-ci] -[https://thelocehiliosan.github.io/yadm/](https://thelocehiliosan.github.io/yadm/) +[https://yadm.io/][website-link] - +[**yadm**][website-link] is a tool for managing [dotfiles][]. + +* Based on [Git][], with full range of Git's features +* Supports system-specific alternative files +* Encryption of private data using [GnuPG][] +* Customizable initialization (bootstrapping) + +Features, usage, examples and installation instructions can be found on the +[website][website-link]. + +[Git]: https://git-scm.com/ +[GnuPG]: https://gnupg.org/ +[copr-badge]: https://img.shields.io/badge/dynamic/json.svg?label=copr&prefix=v&query=%24..version&url=https%3A%2F%2Fcopr.fedorainfracloud.org%2Fapi_2%2Fbuilds%3Fproject_id%3D7041%26limit%3D1 +[copr-link]: https://copr.fedorainfracloud.org/coprs/thelocehiliosan/yadm/ +[develop-badge]: https://img.shields.io/travis/TheLocehiliosan/yadm/develop.svg?label=develop +[develop-commits]: https://github.com/TheLocehiliosan/yadm/commits/develop +[develop-date]: https://img.shields.io/github/last-commit/TheLocehiliosan/yadm/develop.svg?label=develop +[dotfiles]: https://en.wikipedia.org/wiki/Hidden_file_and_hidden_directory +[homebrew-badge]: https://img.shields.io/homebrew/v/yadm.svg +[homebrew-link]: https://formulae.brew.sh/formula/yadm +[license-badge]: https://img.shields.io/github/license/TheLocehiliosan/yadm.svg +[license-link]: https://github.com/TheLocehiliosan/yadm/blob/master/LICENSE +[master-badge]: https://img.shields.io/travis/TheLocehiliosan/yadm/master.svg?label=master +[master-commits]: https://github.com/TheLocehiliosan/yadm/commits/master +[master-date]: https://img.shields.io/github/last-commit/TheLocehiliosan/yadm/master.svg?label=master +[releases-badge]: https://img.shields.io/github/tag/TheLocehiliosan/yadm.svg?label=latest+release +[releases-link]: https://github.com/TheLocehiliosan/yadm/releases +[travis-ci]: https://travis-ci.org/TheLocehiliosan/yadm/branches +[website-commits]: https://github.com/TheLocehiliosan/yadm/commits/gh-pages +[website-date]: https://img.shields.io/github/last-commit/TheLocehiliosan/yadm/gh-pages.svg?label=website +[website-link]: https://yadm.io/ From 402b57880d5c3c393a2a871c2ce0e65f57cdd571 Mon Sep 17 00:00:00 2001 From: Tim Byrne Date: Thu, 14 Mar 2019 18:14:10 -0500 Subject: [PATCH 017/137] Update GPLv3 LICENSE information This does NOT change the licensing. --- LICENSE | 682 +++++++++++++++++++++++++++++++++++++++++++++++++++++++- yadm | 11 +- 2 files changed, 677 insertions(+), 16 deletions(-) diff --git a/LICENSE b/LICENSE index a491495..f288702 100644 --- a/LICENSE +++ b/LICENSE @@ -1,14 +1,674 @@ -yadm - Yet Another Dotfiles Manager -Copyright (C) 2015-2017 Tim Byrne + GNU GENERAL PUBLIC LICENSE + Version 3, 29 June 2007 -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation, version 3 of the License. + Copyright (C) 2007 Free Software Foundation, Inc. + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. + Preamble -You should have received a copy of the GNU General Public License -along with this program. If not, see . + The GNU General Public License is a free, copyleft license for +software and other kinds of works. + + The licenses for most software and other practical works are designed +to take away your freedom to share and change the works. By contrast, +the GNU General Public License is intended to guarantee your freedom to +share and change all versions of a program--to make sure it remains free +software for all its users. We, the Free Software Foundation, use the +GNU General Public License for most of our software; it applies also to +any other work released this way by its authors. You can apply it to +your programs, too. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +them if you wish), that you receive source code or can get it if you +want it, that you can change the software or use pieces of it in new +free programs, and that you know you can do these things. + + To protect your rights, we need to prevent others from denying you +these rights or asking you to surrender the rights. Therefore, you have +certain responsibilities if you distribute copies of the software, or if +you modify it: responsibilities to respect the freedom of others. + + For example, if you distribute copies of such a program, whether +gratis or for a fee, you must pass on to the recipients the same +freedoms that you received. You must make sure that they, too, receive +or can get the source code. And you must show them these terms so they +know their rights. + + Developers that use the GNU GPL protect your rights with two steps: +(1) assert copyright on the software, and (2) offer you this License +giving you legal permission to copy, distribute and/or modify it. + + For the developers' and authors' protection, the GPL clearly explains +that there is no warranty for this free software. For both users' and +authors' sake, the GPL requires that modified versions be marked as +changed, so that their problems will not be attributed erroneously to +authors of previous versions. + + Some devices are designed to deny users access to install or run +modified versions of the software inside them, although the manufacturer +can do so. This is fundamentally incompatible with the aim of +protecting users' freedom to change the software. The systematic +pattern of such abuse occurs in the area of products for individuals to +use, which is precisely where it is most unacceptable. Therefore, we +have designed this version of the GPL to prohibit the practice for those +products. If such problems arise substantially in other domains, we +stand ready to extend this provision to those domains in future versions +of the GPL, as needed to protect the freedom of users. + + Finally, every program is threatened constantly by software patents. +States should not allow patents to restrict development and use of +software on general-purpose computers, but in those that do, we wish to +avoid the special danger that patents applied to a free program could +make it effectively proprietary. To prevent this, the GPL assures that +patents cannot be used to render the program non-free. + + The precise terms and conditions for copying, distribution and +modification follow. + + TERMS AND CONDITIONS + + 0. Definitions. + + "This License" refers to version 3 of the GNU General Public License. + + "Copyright" also means copyright-like laws that apply to other kinds of +works, such as semiconductor masks. + + "The Program" refers to any copyrightable work licensed under this +License. Each licensee is addressed as "you". "Licensees" and +"recipients" may be individuals or organizations. + + To "modify" a work means to copy from or adapt all or part of the work +in a fashion requiring copyright permission, other than the making of an +exact copy. The resulting work is called a "modified version" of the +earlier work or a work "based on" the earlier work. + + A "covered work" means either the unmodified Program or a work based +on the Program. + + To "propagate" a work means to do anything with it that, without +permission, would make you directly or secondarily liable for +infringement under applicable copyright law, except executing it on a +computer or modifying a private copy. Propagation includes copying, +distribution (with or without modification), making available to the +public, and in some countries other activities as well. + + To "convey" a work means any kind of propagation that enables other +parties to make or receive copies. Mere interaction with a user through +a computer network, with no transfer of a copy, is not conveying. + + An interactive user interface displays "Appropriate Legal Notices" +to the extent that it includes a convenient and prominently visible +feature that (1) displays an appropriate copyright notice, and (2) +tells the user that there is no warranty for the work (except to the +extent that warranties are provided), that licensees may convey the +work under this License, and how to view a copy of this License. If +the interface presents a list of user commands or options, such as a +menu, a prominent item in the list meets this criterion. + + 1. Source Code. + + The "source code" for a work means the preferred form of the work +for making modifications to it. "Object code" means any non-source +form of a work. + + A "Standard Interface" means an interface that either is an official +standard defined by a recognized standards body, or, in the case of +interfaces specified for a particular programming language, one that +is widely used among developers working in that language. + + The "System Libraries" of an executable work include anything, other +than the work as a whole, that (a) is included in the normal form of +packaging a Major Component, but which is not part of that Major +Component, and (b) serves only to enable use of the work with that +Major Component, or to implement a Standard Interface for which an +implementation is available to the public in source code form. A +"Major Component", in this context, means a major essential component +(kernel, window system, and so on) of the specific operating system +(if any) on which the executable work runs, or a compiler used to +produce the work, or an object code interpreter used to run it. + + The "Corresponding Source" for a work in object code form means all +the source code needed to generate, install, and (for an executable +work) run the object code and to modify the work, including scripts to +control those activities. However, it does not include the work's +System Libraries, or general-purpose tools or generally available free +programs which are used unmodified in performing those activities but +which are not part of the work. For example, Corresponding Source +includes interface definition files associated with source files for +the work, and the source code for shared libraries and dynamically +linked subprograms that the work is specifically designed to require, +such as by intimate data communication or control flow between those +subprograms and other parts of the work. + + The Corresponding Source need not include anything that users +can regenerate automatically from other parts of the Corresponding +Source. + + The Corresponding Source for a work in source code form is that +same work. + + 2. Basic Permissions. + + All rights granted under this License are granted for the term of +copyright on the Program, and are irrevocable provided the stated +conditions are met. This License explicitly affirms your unlimited +permission to run the unmodified Program. The output from running a +covered work is covered by this License only if the output, given its +content, constitutes a covered work. This License acknowledges your +rights of fair use or other equivalent, as provided by copyright law. + + You may make, run and propagate covered works that you do not +convey, without conditions so long as your license otherwise remains +in force. You may convey covered works to others for the sole purpose +of having them make modifications exclusively for you, or provide you +with facilities for running those works, provided that you comply with +the terms of this License in conveying all material for which you do +not control copyright. Those thus making or running the covered works +for you must do so exclusively on your behalf, under your direction +and control, on terms that prohibit them from making any copies of +your copyrighted material outside their relationship with you. + + Conveying under any other circumstances is permitted solely under +the conditions stated below. Sublicensing is not allowed; section 10 +makes it unnecessary. + + 3. Protecting Users' Legal Rights From Anti-Circumvention Law. + + No covered work shall be deemed part of an effective technological +measure under any applicable law fulfilling obligations under article +11 of the WIPO copyright treaty adopted on 20 December 1996, or +similar laws prohibiting or restricting circumvention of such +measures. + + When you convey a covered work, you waive any legal power to forbid +circumvention of technological measures to the extent such circumvention +is effected by exercising rights under this License with respect to +the covered work, and you disclaim any intention to limit operation or +modification of the work as a means of enforcing, against the work's +users, your or third parties' legal rights to forbid circumvention of +technological measures. + + 4. Conveying Verbatim Copies. + + You may convey verbatim copies of the Program's source code as you +receive it, in any medium, provided that you conspicuously and +appropriately publish on each copy an appropriate copyright notice; +keep intact all notices stating that this License and any +non-permissive terms added in accord with section 7 apply to the code; +keep intact all notices of the absence of any warranty; and give all +recipients a copy of this License along with the Program. + + You may charge any price or no price for each copy that you convey, +and you may offer support or warranty protection for a fee. + + 5. Conveying Modified Source Versions. + + You may convey a work based on the Program, or the modifications to +produce it from the Program, in the form of source code under the +terms of section 4, provided that you also meet all of these conditions: + + a) The work must carry prominent notices stating that you modified + it, and giving a relevant date. + + b) The work must carry prominent notices stating that it is + released under this License and any conditions added under section + 7. This requirement modifies the requirement in section 4 to + "keep intact all notices". + + c) You must license the entire work, as a whole, under this + License to anyone who comes into possession of a copy. This + License will therefore apply, along with any applicable section 7 + additional terms, to the whole of the work, and all its parts, + regardless of how they are packaged. This License gives no + permission to license the work in any other way, but it does not + invalidate such permission if you have separately received it. + + d) If the work has interactive user interfaces, each must display + Appropriate Legal Notices; however, if the Program has interactive + interfaces that do not display Appropriate Legal Notices, your + work need not make them do so. + + A compilation of a covered work with other separate and independent +works, which are not by their nature extensions of the covered work, +and which are not combined with it such as to form a larger program, +in or on a volume of a storage or distribution medium, is called an +"aggregate" if the compilation and its resulting copyright are not +used to limit the access or legal rights of the compilation's users +beyond what the individual works permit. Inclusion of a covered work +in an aggregate does not cause this License to apply to the other +parts of the aggregate. + + 6. Conveying Non-Source Forms. + + You may convey a covered work in object code form under the terms +of sections 4 and 5, provided that you also convey the +machine-readable Corresponding Source under the terms of this License, +in one of these ways: + + a) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by the + Corresponding Source fixed on a durable physical medium + customarily used for software interchange. + + b) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by a + written offer, valid for at least three years and valid for as + long as you offer spare parts or customer support for that product + model, to give anyone who possesses the object code either (1) a + copy of the Corresponding Source for all the software in the + product that is covered by this License, on a durable physical + medium customarily used for software interchange, for a price no + more than your reasonable cost of physically performing this + conveying of source, or (2) access to copy the + Corresponding Source from a network server at no charge. + + c) Convey individual copies of the object code with a copy of the + written offer to provide the Corresponding Source. This + alternative is allowed only occasionally and noncommercially, and + only if you received the object code with such an offer, in accord + with subsection 6b. + + d) Convey the object code by offering access from a designated + place (gratis or for a charge), and offer equivalent access to the + Corresponding Source in the same way through the same place at no + further charge. You need not require recipients to copy the + Corresponding Source along with the object code. If the place to + copy the object code is a network server, the Corresponding Source + may be on a different server (operated by you or a third party) + that supports equivalent copying facilities, provided you maintain + clear directions next to the object code saying where to find the + Corresponding Source. Regardless of what server hosts the + Corresponding Source, you remain obligated to ensure that it is + available for as long as needed to satisfy these requirements. + + e) Convey the object code using peer-to-peer transmission, provided + you inform other peers where the object code and Corresponding + Source of the work are being offered to the general public at no + charge under subsection 6d. + + A separable portion of the object code, whose source code is excluded +from the Corresponding Source as a System Library, need not be +included in conveying the object code work. + + A "User Product" is either (1) a "consumer product", which means any +tangible personal property which is normally used for personal, family, +or household purposes, or (2) anything designed or sold for incorporation +into a dwelling. In determining whether a product is a consumer product, +doubtful cases shall be resolved in favor of coverage. For a particular +product received by a particular user, "normally used" refers to a +typical or common use of that class of product, regardless of the status +of the particular user or of the way in which the particular user +actually uses, or expects or is expected to use, the product. A product +is a consumer product regardless of whether the product has substantial +commercial, industrial or non-consumer uses, unless such uses represent +the only significant mode of use of the product. + + "Installation Information" for a User Product means any methods, +procedures, authorization keys, or other information required to install +and execute modified versions of a covered work in that User Product from +a modified version of its Corresponding Source. The information must +suffice to ensure that the continued functioning of the modified object +code is in no case prevented or interfered with solely because +modification has been made. + + If you convey an object code work under this section in, or with, or +specifically for use in, a User Product, and the conveying occurs as +part of a transaction in which the right of possession and use of the +User Product is transferred to the recipient in perpetuity or for a +fixed term (regardless of how the transaction is characterized), the +Corresponding Source conveyed under this section must be accompanied +by the Installation Information. But this requirement does not apply +if neither you nor any third party retains the ability to install +modified object code on the User Product (for example, the work has +been installed in ROM). + + The requirement to provide Installation Information does not include a +requirement to continue to provide support service, warranty, or updates +for a work that has been modified or installed by the recipient, or for +the User Product in which it has been modified or installed. Access to a +network may be denied when the modification itself materially and +adversely affects the operation of the network or violates the rules and +protocols for communication across the network. + + Corresponding Source conveyed, and Installation Information provided, +in accord with this section must be in a format that is publicly +documented (and with an implementation available to the public in +source code form), and must require no special password or key for +unpacking, reading or copying. + + 7. Additional Terms. + + "Additional permissions" are terms that supplement the terms of this +License by making exceptions from one or more of its conditions. +Additional permissions that are applicable to the entire Program shall +be treated as though they were included in this License, to the extent +that they are valid under applicable law. If additional permissions +apply only to part of the Program, that part may be used separately +under those permissions, but the entire Program remains governed by +this License without regard to the additional permissions. + + When you convey a copy of a covered work, you may at your option +remove any additional permissions from that copy, or from any part of +it. (Additional permissions may be written to require their own +removal in certain cases when you modify the work.) You may place +additional permissions on material, added by you to a covered work, +for which you have or can give appropriate copyright permission. + + Notwithstanding any other provision of this License, for material you +add to a covered work, you may (if authorized by the copyright holders of +that material) supplement the terms of this License with terms: + + a) Disclaiming warranty or limiting liability differently from the + terms of sections 15 and 16 of this License; or + + b) Requiring preservation of specified reasonable legal notices or + author attributions in that material or in the Appropriate Legal + Notices displayed by works containing it; or + + c) Prohibiting misrepresentation of the origin of that material, or + requiring that modified versions of such material be marked in + reasonable ways as different from the original version; or + + d) Limiting the use for publicity purposes of names of licensors or + authors of the material; or + + e) Declining to grant rights under trademark law for use of some + trade names, trademarks, or service marks; or + + f) Requiring indemnification of licensors and authors of that + material by anyone who conveys the material (or modified versions of + it) with contractual assumptions of liability to the recipient, for + any liability that these contractual assumptions directly impose on + those licensors and authors. + + All other non-permissive additional terms are considered "further +restrictions" within the meaning of section 10. If the Program as you +received it, or any part of it, contains a notice stating that it is +governed by this License along with a term that is a further +restriction, you may remove that term. If a license document contains +a further restriction but permits relicensing or conveying under this +License, you may add to a covered work material governed by the terms +of that license document, provided that the further restriction does +not survive such relicensing or conveying. + + If you add terms to a covered work in accord with this section, you +must place, in the relevant source files, a statement of the +additional terms that apply to those files, or a notice indicating +where to find the applicable terms. + + Additional terms, permissive or non-permissive, may be stated in the +form of a separately written license, or stated as exceptions; +the above requirements apply either way. + + 8. Termination. + + You may not propagate or modify a covered work except as expressly +provided under this License. Any attempt otherwise to propagate or +modify it is void, and will automatically terminate your rights under +this License (including any patent licenses granted under the third +paragraph of section 11). + + However, if you cease all violation of this License, then your +license from a particular copyright holder is reinstated (a) +provisionally, unless and until the copyright holder explicitly and +finally terminates your license, and (b) permanently, if the copyright +holder fails to notify you of the violation by some reasonable means +prior to 60 days after the cessation. + + Moreover, your license from a particular copyright holder is +reinstated permanently if the copyright holder notifies you of the +violation by some reasonable means, this is the first time you have +received notice of violation of this License (for any work) from that +copyright holder, and you cure the violation prior to 30 days after +your receipt of the notice. + + Termination of your rights under this section does not terminate the +licenses of parties who have received copies or rights from you under +this License. If your rights have been terminated and not permanently +reinstated, you do not qualify to receive new licenses for the same +material under section 10. + + 9. Acceptance Not Required for Having Copies. + + You are not required to accept this License in order to receive or +run a copy of the Program. Ancillary propagation of a covered work +occurring solely as a consequence of using peer-to-peer transmission +to receive a copy likewise does not require acceptance. However, +nothing other than this License grants you permission to propagate or +modify any covered work. These actions infringe copyright if you do +not accept this License. Therefore, by modifying or propagating a +covered work, you indicate your acceptance of this License to do so. + + 10. Automatic Licensing of Downstream Recipients. + + Each time you convey a covered work, the recipient automatically +receives a license from the original licensors, to run, modify and +propagate that work, subject to this License. You are not responsible +for enforcing compliance by third parties with this License. + + An "entity transaction" is a transaction transferring control of an +organization, or substantially all assets of one, or subdividing an +organization, or merging organizations. If propagation of a covered +work results from an entity transaction, each party to that +transaction who receives a copy of the work also receives whatever +licenses to the work the party's predecessor in interest had or could +give under the previous paragraph, plus a right to possession of the +Corresponding Source of the work from the predecessor in interest, if +the predecessor has it or can get it with reasonable efforts. + + You may not impose any further restrictions on the exercise of the +rights granted or affirmed under this License. For example, you may +not impose a license fee, royalty, or other charge for exercise of +rights granted under this License, and you may not initiate litigation +(including a cross-claim or counterclaim in a lawsuit) alleging that +any patent claim is infringed by making, using, selling, offering for +sale, or importing the Program or any portion of it. + + 11. Patents. + + A "contributor" is a copyright holder who authorizes use under this +License of the Program or a work on which the Program is based. The +work thus licensed is called the contributor's "contributor version". + + A contributor's "essential patent claims" are all patent claims +owned or controlled by the contributor, whether already acquired or +hereafter acquired, that would be infringed by some manner, permitted +by this License, of making, using, or selling its contributor version, +but do not include claims that would be infringed only as a +consequence of further modification of the contributor version. For +purposes of this definition, "control" includes the right to grant +patent sublicenses in a manner consistent with the requirements of +this License. + + Each contributor grants you a non-exclusive, worldwide, royalty-free +patent license under the contributor's essential patent claims, to +make, use, sell, offer for sale, import and otherwise run, modify and +propagate the contents of its contributor version. + + In the following three paragraphs, a "patent license" is any express +agreement or commitment, however denominated, not to enforce a patent +(such as an express permission to practice a patent or covenant not to +sue for patent infringement). To "grant" such a patent license to a +party means to make such an agreement or commitment not to enforce a +patent against the party. + + If you convey a covered work, knowingly relying on a patent license, +and the Corresponding Source of the work is not available for anyone +to copy, free of charge and under the terms of this License, through a +publicly available network server or other readily accessible means, +then you must either (1) cause the Corresponding Source to be so +available, or (2) arrange to deprive yourself of the benefit of the +patent license for this particular work, or (3) arrange, in a manner +consistent with the requirements of this License, to extend the patent +license to downstream recipients. "Knowingly relying" means you have +actual knowledge that, but for the patent license, your conveying the +covered work in a country, or your recipient's use of the covered work +in a country, would infringe one or more identifiable patents in that +country that you have reason to believe are valid. + + If, pursuant to or in connection with a single transaction or +arrangement, you convey, or propagate by procuring conveyance of, a +covered work, and grant a patent license to some of the parties +receiving the covered work authorizing them to use, propagate, modify +or convey a specific copy of the covered work, then the patent license +you grant is automatically extended to all recipients of the covered +work and works based on it. + + A patent license is "discriminatory" if it does not include within +the scope of its coverage, prohibits the exercise of, or is +conditioned on the non-exercise of one or more of the rights that are +specifically granted under this License. You may not convey a covered +work if you are a party to an arrangement with a third party that is +in the business of distributing software, under which you make payment +to the third party based on the extent of your activity of conveying +the work, and under which the third party grants, to any of the +parties who would receive the covered work from you, a discriminatory +patent license (a) in connection with copies of the covered work +conveyed by you (or copies made from those copies), or (b) primarily +for and in connection with specific products or compilations that +contain the covered work, unless you entered into that arrangement, +or that patent license was granted, prior to 28 March 2007. + + Nothing in this License shall be construed as excluding or limiting +any implied license or other defenses to infringement that may +otherwise be available to you under applicable patent law. + + 12. No Surrender of Others' Freedom. + + If conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot convey a +covered work so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you may +not convey it at all. For example, if you agree to terms that obligate you +to collect a royalty for further conveying from those to whom you convey +the Program, the only way you could satisfy both those terms and this +License would be to refrain entirely from conveying the Program. + + 13. Use with the GNU Affero General Public License. + + Notwithstanding any other provision of this License, you have +permission to link or combine any covered work with a work licensed +under version 3 of the GNU Affero General Public License into a single +combined work, and to convey the resulting work. The terms of this +License will continue to apply to the part which is the covered work, +but the special requirements of the GNU Affero General Public License, +section 13, concerning interaction through a network will apply to the +combination as such. + + 14. Revised Versions of this License. + + The Free Software Foundation may publish revised and/or new versions of +the GNU General Public License from time to time. Such new versions will +be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + + Each version is given a distinguishing version number. If the +Program specifies that a certain numbered version of the GNU General +Public License "or any later version" applies to it, you have the +option of following the terms and conditions either of that numbered +version or of any later version published by the Free Software +Foundation. If the Program does not specify a version number of the +GNU General Public License, you may choose any version ever published +by the Free Software Foundation. + + If the Program specifies that a proxy can decide which future +versions of the GNU General Public License can be used, that proxy's +public statement of acceptance of a version permanently authorizes you +to choose that version for the Program. + + Later license versions may give you additional or different +permissions. However, no additional obligations are imposed on any +author or copyright holder as a result of your choosing to follow a +later version. + + 15. Disclaimer of Warranty. + + THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY +APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT +HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY +OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, +THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM +IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF +ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + + 16. Limitation of Liability. + + IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS +THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY +GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE +USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF +DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD +PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), +EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF +SUCH DAMAGES. + + 17. Interpretation of Sections 15 and 16. + + If the disclaimer of warranty and limitation of liability provided +above cannot be given local legal effect according to their terms, +reviewing courts shall apply local law that most closely approximates +an absolute waiver of all civil liability in connection with the +Program, unless a warranty or assumption of liability accompanies a +copy of the Program in return for a fee. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +state the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + +Also add information on how to contact you by electronic and paper mail. + + If the program does terminal interaction, make it output a short +notice like this when it starts in an interactive mode: + + Copyright (C) + This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'. + This is free software, and you are welcome to redistribute it + under certain conditions; type `show c' for details. + +The hypothetical commands `show w' and `show c' should show the appropriate +parts of the General Public License. Of course, your program's commands +might be different; for a GUI interface, you would use an "about box". + + You should also get your employer (if you work as a programmer) or school, +if any, to sign a "copyright disclaimer" for the program, if necessary. +For more information on this, and how to apply and follow the GNU GPL, see +. + + The GNU General Public License does not permit incorporating your program +into proprietary programs. If your program is a subroutine library, you +may consider it more useful to permit linking proprietary applications with +the library. If this is what you want to do, use the GNU Lesser General +Public License instead of this License. But first, please read +. diff --git a/yadm b/yadm index e55a287..e57551f 100755 --- a/yadm +++ b/yadm @@ -1,18 +1,19 @@ #!/bin/sh # yadm - Yet Another Dotfiles Manager -# Copyright (C) 2015-2017 Tim Byrne +# Copyright (C) 2015-2019 Tim Byrne # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by -# the Free Software Foundation, version 3 of the License. - +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. - +# # You should have received a copy of the GNU General Public License -# along with this program. If not, see . +# along with this program. If not, see . #; execute script with bash (shebang line is /bin/sh for portability) if [ -z "$BASH_VERSION" ]; then From 0ff508d04694f339e0fe67cd4a1ce9bb8ddea4e1 Mon Sep 17 00:00:00 2001 From: Tim Byrne Date: Tue, 19 Mar 2019 17:27:54 -0500 Subject: [PATCH 018/137] Reduce travis-ci `sudo` requirement `sudo` is not actually required. --- .travis.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index b38f19f..0d1e77e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,5 +1,4 @@ --- -sudo: required language: bash services: - docker From af6febc598fc8f6ab7dcd9da3ebcac1bd5fa590b Mon Sep 17 00:00:00 2001 From: Tim Byrne Date: Tue, 19 Mar 2019 17:37:15 -0500 Subject: [PATCH 019/137] Use language `minimal` `bash` is actually an alias for `minimal` in travis-ci. --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 0d1e77e..a4ea858 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,5 +1,5 @@ --- -language: bash +language: minimal services: - docker before_install: From eb7462061e6f62f87cbb6e6b8e2a33e4907e57ca Mon Sep 17 00:00:00 2001 From: Tim Byrne Date: Tue, 19 Mar 2019 19:34:17 -0500 Subject: [PATCH 020/137] Allocate a pseudo-TTY This will help color shows up in travis-ci logs. --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index a4ea858..4b4efd4 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,4 +5,4 @@ services: before_install: - docker pull yadm/testbed:latest script: - - docker run --rm -v "$PWD:/yadm:ro" yadm/testbed + - docker run -t --rm -v "$PWD:/yadm:ro" yadm/testbed From 9b1ac4b76b3d5695d9cadf2b4a5bfd24861bad85 Mon Sep 17 00:00:00 2001 From: Tim Byrne Date: Fri, 15 Mar 2019 00:10:48 -0500 Subject: [PATCH 021/137] Add shields.io badge for Arch Linux --- README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/README.md b/README.md index 41b5f38..f4d9a55 100644 --- a/README.md +++ b/README.md @@ -3,6 +3,7 @@ [![Latest Version][releases-badge]][releases-link] [![Homebrew Version][homebrew-badge]][homebrew-link] [![Copr Version][copr-badge]][copr-link] +[![Arch Version][aur-badge]][aur-link] [![License][license-badge]][license-link]
[![Master Update][master-date]][master-commits] [![Develop Update][develop-date]][develop-commits] @@ -24,6 +25,8 @@ Features, usage, examples and installation instructions can be found on the [Git]: https://git-scm.com/ [GnuPG]: https://gnupg.org/ +[aur-badge]: https://img.shields.io/aur/version/yadm-git.svg +[aur-link]: https://aur.archlinux.org/packages/yadm-git [copr-badge]: https://img.shields.io/badge/dynamic/json.svg?label=copr&prefix=v&query=%24..version&url=https%3A%2F%2Fcopr.fedorainfracloud.org%2Fapi_2%2Fbuilds%3Fproject_id%3D7041%26limit%3D1 [copr-link]: https://copr.fedorainfracloud.org/coprs/thelocehiliosan/yadm/ [develop-badge]: https://img.shields.io/travis/TheLocehiliosan/yadm/develop.svg?label=develop From ca2f93146d377ad6bd320e10b44d6b9c983bc10e Mon Sep 17 00:00:00 2001 From: Tim Byrne Date: Wed, 20 Mar 2019 02:06:50 -0500 Subject: [PATCH 022/137] Add shields.io badges for website branches --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index f4d9a55..a0d681a 100644 --- a/README.md +++ b/README.md @@ -10,6 +10,8 @@ [![Website Update][website-date]][website-commits]
[![Master Status][master-badge]][travis-ci] [![Develop Status][develop-badge]][travis-ci] +[![GH Pages Status][gh-pages-badge]][travis-ci] +[![Dev Pages Status][dev-pages-badge]][travis-ci] [https://yadm.io/][website-link] @@ -29,10 +31,12 @@ Features, usage, examples and installation instructions can be found on the [aur-link]: https://aur.archlinux.org/packages/yadm-git [copr-badge]: https://img.shields.io/badge/dynamic/json.svg?label=copr&prefix=v&query=%24..version&url=https%3A%2F%2Fcopr.fedorainfracloud.org%2Fapi_2%2Fbuilds%3Fproject_id%3D7041%26limit%3D1 [copr-link]: https://copr.fedorainfracloud.org/coprs/thelocehiliosan/yadm/ +[dev-pages-badge]: https://img.shields.io/travis/TheLocehiliosan/yadm/dev-pages.svg?label=dev-pages [develop-badge]: https://img.shields.io/travis/TheLocehiliosan/yadm/develop.svg?label=develop [develop-commits]: https://github.com/TheLocehiliosan/yadm/commits/develop [develop-date]: https://img.shields.io/github/last-commit/TheLocehiliosan/yadm/develop.svg?label=develop [dotfiles]: https://en.wikipedia.org/wiki/Hidden_file_and_hidden_directory +[gh-pages-badge]: https://img.shields.io/travis/TheLocehiliosan/yadm/gh-pages.svg?label=gh-pages [homebrew-badge]: https://img.shields.io/homebrew/v/yadm.svg [homebrew-link]: https://formulae.brew.sh/formula/yadm [license-badge]: https://img.shields.io/github/license/TheLocehiliosan/yadm.svg From 826f9bc09ea1e0b53a76517959e26a79fa55885a Mon Sep 17 00:00:00 2001 From: Tim Byrne Date: Thu, 21 Mar 2019 07:38:38 -0500 Subject: [PATCH 023/137] Validate yaml files with yamllint --- Dockerfile | 5 +++-- test/conftest.py | 6 ++++++ test/test_syntax.py | 11 +++++++++++ 3 files changed, 20 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 6651deb..b221942 100644 --- a/Dockerfile +++ b/Dockerfile @@ -27,9 +27,10 @@ RUN \ ; RUN pip3 install \ envtpl \ - pytest==3.6.4 \ - pylint==1.9.2 \ flake8==3.5.0 \ + pylint==1.9.2 \ + pytest==3.6.4 \ + yamllint==1.15.0 \ ; # Force GNUPG version 1 at path /usr/bin/gpg diff --git a/test/conftest.py b/test/conftest.py index 4fefabe..0b1f904 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -28,6 +28,12 @@ def flake8_version(): return '3.5.0' +@pytest.fixture(scope='session') +def yamllint_version(): + """Version of yamllint supported""" + return '1.15.0' + + @pytest.fixture(scope='session') def tst_user(): """Test session's user id""" diff --git a/test/test_syntax.py b/test/test_syntax.py index 5408885..5e39b3a 100644 --- a/test/test_syntax.py +++ b/test/test_syntax.py @@ -39,3 +39,14 @@ def test_flake8(runner, flake8_version): pytest.skip('Unsupported flake8 version') run = runner(command=['flake8', 'test']) assert run.success + + +def test_yamllint(runner, yamllint_version): + """Passes yamllint""" + run = runner(command=['yamllint', '--version'], report=False) + if not run.out.strip().endswith(yamllint_version): + pytest.skip('Unsupported yamllint version') + run = runner( + command=['yamllint', '-s', '$(find . -name \\*.yml)'], + shell=True) + assert run.success From 282b772ce5220380be9df2901316fe0bed9d31d0 Mon Sep 17 00:00:00 2001 From: Tim Byrne Date: Fri, 22 Mar 2019 07:39:09 -0500 Subject: [PATCH 024/137] Add yamllint to testenv Also organize pip install command so software can be added/removed easier. --- Makefile | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index b65dee0..1242bfd 100644 --- a/Makefile +++ b/Makefile @@ -136,7 +136,12 @@ testenv: @echo virtualenv --python=python3 testenv testenv/bin/pip3 install --upgrade pip setuptools - testenv/bin/pip3 install --upgrade pytest pylint==1.9.2 flake8==3.5.0 + testenv/bin/pip3 install --upgrade \ + flake8==3.5.0 \ + pylint==1.9.2 \ + pytest \ + yamllint==1.15.0 \ + ; @echo @echo 'To activate this test environment type:' @echo ' source testenv/bin/activate' From 2848ca2d1d2d1b1af4a32c22dd9473f2482c65ec Mon Sep 17 00:00:00 2001 From: Tim Byrne Date: Wed, 20 Mar 2019 17:17:40 -0500 Subject: [PATCH 025/137] Update contributing guide with website info (#146) --- .github/CONTRIBUTING.md | 55 ++++++++++++++++++++++++++--------------- 1 file changed, 35 insertions(+), 20 deletions(-) diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md index d836d87..4977e4b 100644 --- a/.github/CONTRIBUTING.md +++ b/.github/CONTRIBUTING.md @@ -66,8 +66,8 @@ Include the version numbers of your operating system, of **yadm**, and of Git. ### Attaching a script.gz Consider trying to reproduce the bug inside a docker container using the -[yadm/testbed][testbed] docker image. Doing so will greatly increase the -likelihood of the problem being fixed. +[yadm/testbed][] docker image. Doing so will greatly increase the likelihood of +the problem being fixed. The easiest way to start this container, is to clone the [TheLocehiliosan/yadm repo][yadm-repo], and use the `scripthost` make target. _(You will need `make` @@ -239,21 +239,22 @@ these principles when making changes. When updating the yadm code, please follow these guidelines: -* ShellCheck - * Bash code should pass the scrutiny of [ShellCheck][shellcheck]. The - simplest way to quickly test this is to run: - * `make test testargs='-k shellcheck'` +* Code linting + * Bash code should pass the scrutiny of [ShellCheck][shellcheck]. + * Python code must pass the scrutiny of [pylint][] and [flake8][]. + * Any YAML must pass the scrutiny of [yamllint][]. + * Running `make test_syntax.py` is an easy way to run all linters. * Interface changes * Any changes to **yadm**'s interface should include a commit that updates the `yadm.1` man page. ### Test conventions -The test system is written in Python 3 using [pytest][pytest]. Tests should be -written for all bugs fixed and features added. To make testing portable and -uniform, tests should be performed via the [yadm/testbed][testbed] docker image. -The `Makefile` has several "make targets" for testing. Running `make` by itself -will produce a help page. +The test system is written in Python 3 using [pytest][]. Tests should be written +for all bugs fixed and features added. To make testing portable and uniform, +tests should be performed via the [yadm/testbed][] docker image. The `Makefile` +has several "make targets" for testing. Running `make` by itself will produce a +help page. Please follow these guidelines while writing tests: @@ -268,10 +269,6 @@ Please follow these guidelines while writing tests: * Care should be taken to make tests run as efficiently as possible. * Scope large, unchanging, fixtures appropriately so they do not have to be recreated multiple times. -* Linting - * Python code must pass the scrutiny of [pylint][pylint] and - [flake8][flake8]. The simplest way to quickly test this is to run: - * `make test testargs='-k pylint\ or\ flake8'` ### Commit conventions @@ -311,10 +308,24 @@ That file is only updated during software releases. ### Website changes -**NOTE:** A [website refactoring][refactor] is being performed soon, and it is -unlikely that website changes will be accepted until this task is completed. -Better instructions for testing and submitting website changes will be written -during that refactor. +The yadm.io website is generated using [Jekyll][jekyll]. The bulk of the +documentation is created as an ordered collection within `_docs`. To make +website testing easy and portable, use the [yadm/jekyll][] docker image. The +`Makefile` has several "make targets" for testing. Running `make` by itself will +produce a help page. + +* `make test`: + Perform tests done by continuous integration. +* `make up`: + Start a container to locally test the website. The test website will be + hosted at http://localhost:4000/ +* `make clean`: + Remove previously built data any any Jekyll containers. + +When making website changes, be sure to adhere to [code](#code-conventions) and +[commit](#commit-conventions) conventions. Use the same [GitHub +workflow](#github-workflow) when creating a pull request. However use the +`dev-pages` branch as a base instead of `develop`. # Maintaining packages @@ -348,6 +359,8 @@ see if you can help. [flake8]: https://pypi.org/project/flake8/ [groff-man]: https://www.gnu.org/software/groff/manual/html_node/man.html [hooks-help]: https://github.com/TheLocehiliosan/yadm/blob/master/yadm.md#hooks +[html-proofer]: https://github.com/gjtorikian/html-proofer +[jekyll]: https://jekyllrb.com [new-bug]: https://github.com/TheLocehiliosan/yadm/issues/new?template=BUG_REPORT.md [new-feature]: https://github.com/TheLocehiliosan/yadm/issues/new?template=FEATURE_REQUEST.md [open-issues]: https://github.com/TheLocehiliosan/yadm/issues @@ -358,7 +371,9 @@ see if you can help. [refactor]: https://github.com/TheLocehiliosan/yadm/issues/146 [shellcheck]: https://www.shellcheck.net [signing-commits]: https://help.github.com/en/articles/signing-commits -[testbed]: https://hub.docker.com/r/yadm/testbed [tpope-style]: https://tbaggery.com/2008/04/19/a-note-about-git-commit-messages.html [yadm-man]: https://github.com/TheLocehiliosan/yadm/blob/master/yadm.md [yadm-repo]: https://github.com/TheLocehiliosan/yadm +[yadm/jekyll]: https://hub.docker.com/r/yadm/jekyll +[yadm/testbed]: https://hub.docker.com/r/yadm/testbed +[yamllint]: https://github.com/adrienverge/yamllint From 58edf313aae890c60b14ab5acbe1ff555ab31848 Mon Sep 17 00:00:00 2001 From: Tim Byrne Date: Sun, 24 Mar 2019 17:05:11 -0500 Subject: [PATCH 026/137] Process .yadm/encrypt in sorted order --- test/test_unit_parse_encrypt.py | 5 +++++ yadm | 6 +++++- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/test/test_unit_parse_encrypt.py b/test/test_unit_parse_encrypt.py index 914d907..b1a1af9 100644 --- a/test/test_unit_parse_encrypt.py +++ b/test/test_unit_parse_encrypt.py @@ -140,6 +140,10 @@ def test_file_parse_encrypt(runner, paths): for expected_file in expected: assert f'EIF:{expected_file}\n' in run.out + sorted_expectations = '\n'.join( + [f'EIF:{exp}' for exp in sorted(expected)]) + assert sorted_expectations in run.out + def run_parse_encrypt( runner, paths, @@ -162,6 +166,7 @@ def run_parse_encrypt( export YADM_ENCRYPT GIT_DIR={paths.repo} export GIT_DIR + LC_ALL=C {parse_cmd} export ENCRYPT_INCLUDE_FILES export PARSE_ENCRYPT_SHORT diff --git a/yadm b/yadm index e57551f..49d9302 100755 --- a/yadm +++ b/yadm @@ -956,7 +956,11 @@ function parse_encrypt() { done [ -n "$skip" ] || FINAL_INCLUDE+=("$included") done - ENCRYPT_INCLUDE_FILES=("${FINAL_INCLUDE[@]}") + + #; sort the encrypted files + #shellcheck disable=SC2207 + IFS=$'\n' ENCRYPT_INCLUDE_FILES=($(sort <<<"${FINAL_INCLUDE[*]}")) + unset IFS fi } From 2aa1710214c5d6d08f7f50d6fd20ce571a20ccb8 Mon Sep 17 00:00:00 2001 From: Tim Byrne Date: Sun, 24 Mar 2019 17:22:11 -0500 Subject: [PATCH 027/137] Remove superfluous `;` in comments --- yadm | 182 +++++++++++++++++++++++++++++------------------------------ 1 file changed, 91 insertions(+), 91 deletions(-) diff --git a/yadm b/yadm index 49d9302..2190614 100755 --- a/yadm +++ b/yadm @@ -15,7 +15,7 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -#; execute script with bash (shebang line is /bin/sh for portability) +# execute script with bash (shebang line is /bin/sh for portability) if [ -z "$BASH_VERSION" ]; then [ "$YADM_TEST" != 1 ] && exec bash "$0" "$@" fi @@ -44,34 +44,34 @@ OPERATING_SYSTEM="Unknown" ENCRYPT_INCLUDE_FILES="unparsed" -#; flag causing path translations with cygpath +# flag causing path translations with cygpath USE_CYGPATH=0 -#; flag when something may have changes (which prompts auto actions to be performed) +# flag when something may have changes (which prompts auto actions to be performed) CHANGES_POSSIBLE=0 -#; flag when a bootstrap should be performed after cloning -#; 0: skip auto_bootstrap, 1: ask, 2: perform bootstrap, 3: prevent bootstrap +# flag when a bootstrap should be performed after cloning +# 0: skip auto_bootstrap, 1: ask, 2: perform bootstrap, 3: prevent bootstrap DO_BOOTSTRAP=0 function main() { require_git - #; capture full command, for passing to hooks + # capture full command, for passing to hooks FULL_COMMAND="$*" - #; create the YADM_DIR if it doesn't exist yet + # create the YADM_DIR if it doesn't exist yet [ -d "$YADM_DIR" ] || mkdir -p "$YADM_DIR" - #; parse command line arguments + # parse command line arguments local retval=0 internal_commands="^(alt|bootstrap|clean|clone|config|decrypt|encrypt|enter|help|init|introspect|list|perms|version)$" if [ -z "$*" ] ; then - #; no argumnts will result in help() + # no argumnts will result in help() help elif [[ "$1" =~ $internal_commands ]] ; then - #; for internal commands, process all of the arguments + # for internal commands, process all of the arguments YADM_COMMAND="$1" YADM_ARGS=() shift @@ -79,26 +79,26 @@ function main() { while [[ $# -gt 0 ]] ; do key="$1" case $key in - -a) #; used by list() + -a) # used by list() LIST_ALL="YES" ;; - -d) #; used by all commands + -d) # used by all commands DEBUG="YES" ;; - -f) #; used by init() and clone() + -f) # used by init() and clone() FORCE="YES" ;; - -l) #; used by decrypt() + -l) # used by decrypt() DO_LIST="YES" ;; - -w) #; used by init() and clone() + -w) # used by init() and clone() if [[ ! "$2" =~ ^/ ]] ; then error_out "You must specify a fully qualified work tree" fi YADM_WORK="$2" shift ;; - *) #; any unhandled arguments + *) # any unhandled arguments YADM_ARGS+=("$1") ;; esac @@ -109,14 +109,14 @@ function main() { invoke_hook "pre" $YADM_COMMAND "${YADM_ARGS[@]}" else - #; any other commands are simply passed through to git + # any other commands are simply passed through to git HOOK_COMMAND="$1" invoke_hook "pre" git_command "$@" retval="$?" fi - #; process automatic events + # process automatic events auto_alt auto_perms auto_bootstrap @@ -125,7 +125,7 @@ function main() { } -#; ****** yadm Commands ****** +# ****** yadm Commands ****** function alt() { @@ -149,7 +149,7 @@ function alt() { local_host="$(config local.hostname)" if [ -z "$local_host" ] ; then local_host=$(hostname) - local_host=${local_host%%.*} #; trim any domain from hostname + local_host=${local_host%%.*} # trim any domain from hostname fi match_host="(%|$local_host)" @@ -159,16 +159,16 @@ function alt() { fi match_user="(%|$local_user)" - #; regex for matching "##CLASS.SYSTEM.HOSTNAME.USER" + # regex for matching "##CLASS.SYSTEM.HOSTNAME.USER" match1="^(.+)##(()|$match_system|$match_system\.$match_host|$match_system\.$match_host\.$match_user)$" match2="^(.+)##($match_class|$match_class\.$match_system|$match_class\.$match_system\.$match_host|$match_class\.$match_system\.$match_host\.$match_user)$" cd_work "Alternates" || return - #; only be noisy if the "alt" command was run directly + # only be noisy if the "alt" command was run directly [ "$YADM_COMMAND" = "alt" ] && loud="YES" - #; decide if a copy should be done instead of a symbolic link + # decide if a copy should be done instead of a symbolic link local do_copy=0 if [[ $OPERATING_SYSTEM == CYGWIN* ]] ; then if [[ $(config --bool yadm.cygwin-copy) == "true" ]] ; then @@ -176,14 +176,14 @@ function alt() { fi fi - #; loop over all "tracked" files - #; for every file which matches the above regex, create a symlink + # loop over all "tracked" files + # for every file which matches the above regex, create a symlink for match in $match1 $match2; do last_linked='' local IFS=$'\n' for tracked_file in $("$GIT_PROGRAM" ls-files | sort) "${ENCRYPT_INCLUDE_FILES[@]}"; do tracked_file="$YADM_WORK/$tracked_file" - #; process both the path, and it's parent directory + # process both the path, and it's parent directory for alt_path in "$tracked_file" "${tracked_file%/*}"; do if [ -e "$alt_path" ] ; then if [[ $alt_path =~ $match ]] ; then @@ -207,8 +207,8 @@ function alt() { done done - #; loop over all "tracked" files - #; for every file which is a *##yadm.j2 create a real file + # loop over all "tracked" files + # for every file which is a *##yadm.j2 create a real file local IFS=$'\n' local match="^(.+)##yadm\\.j2$" for tracked_file in $("$GIT_PROGRAM" ls-files | sort) "${ENCRYPT_INCLUDE_FILES[@]}"; do @@ -261,13 +261,13 @@ function clone() { while [[ $# -gt 0 ]] ; do key="$1" case $key in - --bootstrap) #; force bootstrap, without prompt + --bootstrap) # force bootstrap, without prompt DO_BOOTSTRAP=2 ;; - --no-bootstrap) #; prevent bootstrap, without prompt + --no-bootstrap) # prevent bootstrap, without prompt DO_BOOTSTRAP=3 ;; - *) #; main arguments are kept intact + *) # main arguments are kept intact clone_args+=("$1") ;; esac @@ -276,18 +276,18 @@ function clone() { [ -n "$DEBUG" ] && display_private_perms "initial" - #; clone will begin with a bare repo + # clone will begin with a bare repo local empty= init $empty - #; add the specified remote, and configure the repo to track origin/master + # add the specified remote, and configure the repo to track origin/master debug "Adding remote to new repo" "$GIT_PROGRAM" remote add origin "${clone_args[@]}" debug "Configuring new repo to track origin/master" "$GIT_PROGRAM" config branch.master.remote origin "$GIT_PROGRAM" config branch.master.merge refs/heads/master - #; fetch / merge (and possibly fallback to reset) + # fetch / merge (and possibly fallback to reset) debug "Doing an initial fetch of the origin" "$GIT_PROGRAM" fetch origin || { debug "Removing repo after failed clone" @@ -327,7 +327,7 @@ function clone() { in another way. EOF else - #; skip auto_bootstrap if conflicts could not be stashed + # skip auto_bootstrap if conflicts could not be stashed DO_BOOTSTRAP=0 cat </dev/null) archive_regex="^\?\?" if [[ $archive_status =~ $archive_regex ]] ; then @@ -485,13 +485,13 @@ function git_command() { require_repo - #; translate 'gitconfig' to 'config' -- 'config' is reserved for yadm + # translate 'gitconfig' to 'config' -- 'config' is reserved for yadm if [ "$1" = "gitconfig" ] ; then set -- "config" "${@:2}" fi - #; ensure private .ssh and .gnupg directories exist first - #; TODO: consider restricting this to only commands which modify the work-tree + # ensure private .ssh and .gnupg directories exist first + # TODO: consider restricting this to only commands which modify the work-tree auto_private_dirs=$(config --bool yadm.auto-private-dirs) if [ "$auto_private_dirs" != "false" ] ; then @@ -500,7 +500,7 @@ function git_command() { CHANGES_POSSIBLE=1 - #; pass commands through to git + # pass commands through to git debug "Running git command $GIT_PROGRAM $*" "$GIT_PROGRAM" "$@" return "$?" @@ -544,17 +544,17 @@ EOF function init() { - #; safety check, don't attempt to init when the repo is already present + # safety check, don't attempt to init when the repo is already present [ -d "$YADM_REPO" ] && [ -z "$FORCE" ] && \ error_out "Git repo already exists. [$YADM_REPO]\nUse '-f' if you want to force it to be overwritten." - #; remove existing if forcing the init to happen anyway + # remove existing if forcing the init to happen anyway [ -d "$YADM_REPO" ] && { debug "Removing existing repo prior to init" rm -rf "$YADM_REPO" } - #; init a new bare repo + # init a new bare repo debug "Init new repo" "$GIT_PROGRAM" init --shared=0600 --bare "$(mixed_path "$YADM_REPO")" "$@" configure_repo @@ -629,12 +629,12 @@ function list() { require_repo - #; process relative to YADM_WORK when --all is specified + # process relative to YADM_WORK when --all is specified if [ -n "$LIST_ALL" ] ; then cd_work "List" || return fi - #; list tracked files + # list tracked files "$GIT_PROGRAM" ls-files } @@ -643,33 +643,33 @@ function perms() { parse_encrypt - #; TODO: prevent repeats in the files changed + # TODO: prevent repeats in the files changed cd_work "Perms" || return GLOBS=() - #; include the archive created by "encrypt" + # include the archive created by "encrypt" [ -f "$YADM_ARCHIVE" ] && GLOBS+=("$YADM_ARCHIVE") - #; include all .ssh files (unless disabled) + # include all .ssh files (unless disabled) if [[ $(config --bool yadm.ssh-perms) != "false" ]] ; then GLOBS+=(".ssh" ".ssh/*") fi - #; include all gpg files (unless disabled) + # include all gpg files (unless disabled) if [[ $(config --bool yadm.gpg-perms) != "false" ]] ; then GLOBS+=(".gnupg" ".gnupg/*") fi - #; include any files we encrypt + # include any files we encrypt GLOBS+=("${ENCRYPT_INCLUDE_FILES[@]}") - #; remove group/other permissions from collected globs + # remove group/other permissions from collected globs #shellcheck disable=SC2068 #(SC2068 is disabled because in this case, we desire globbing) chmod -f go-rwx ${GLOBS[@]} >/dev/null 2>&1 - #; TODO: detect and report changing permissions in a portable way + # TODO: detect and report changing permissions in a portable way } @@ -680,7 +680,7 @@ function version() { } -#; ****** Utility Functions ****** +# ****** Utility Functions ****** function query_distro() { distro="" @@ -692,54 +692,54 @@ function query_distro() { function process_global_args() { - #; global arguments are removed before the main processing is done + # global arguments are removed before the main processing is done MAIN_ARGS=() while [[ $# -gt 0 ]] ; do key="$1" case $key in - -Y|--yadm-dir) #; override the standard YADM_DIR + -Y|--yadm-dir) # override the standard YADM_DIR if [[ ! "$2" =~ ^/ ]] ; then error_out "You must specify a fully qualified yadm directory" fi YADM_DIR="$2" shift ;; - --yadm-repo) #; override the standard YADM_REPO + --yadm-repo) # override the standard YADM_REPO if [[ ! "$2" =~ ^/ ]] ; then error_out "You must specify a fully qualified repo path" fi YADM_OVERRIDE_REPO="$2" shift ;; - --yadm-config) #; override the standard YADM_CONFIG + --yadm-config) # override the standard YADM_CONFIG if [[ ! "$2" =~ ^/ ]] ; then error_out "You must specify a fully qualified config path" fi YADM_OVERRIDE_CONFIG="$2" shift ;; - --yadm-encrypt) #; override the standard YADM_ENCRYPT + --yadm-encrypt) # override the standard YADM_ENCRYPT if [[ ! "$2" =~ ^/ ]] ; then error_out "You must specify a fully qualified encrypt path" fi YADM_OVERRIDE_ENCRYPT="$2" shift ;; - --yadm-archive) #; override the standard YADM_ARCHIVE + --yadm-archive) # override the standard YADM_ARCHIVE if [[ ! "$2" =~ ^/ ]] ; then error_out "You must specify a fully qualified archive path" fi YADM_OVERRIDE_ARCHIVE="$2" shift ;; - --yadm-bootstrap) #; override the standard YADM_BOOTSTRAP + --yadm-bootstrap) # override the standard YADM_BOOTSTRAP if [[ ! "$2" =~ ^/ ]] ; then error_out "You must specify a fully qualified bootstrap path" fi YADM_OVERRIDE_BOOTSTRAP="$2" shift ;; - *) #; main arguments are kept intact + *) # main arguments are kept intact MAIN_ARGS+=("$1") ;; esac @@ -750,14 +750,14 @@ function process_global_args() { function configure_paths() { - #; change all paths to be relative to YADM_DIR + # change all paths to be relative to YADM_DIR YADM_REPO="$YADM_DIR/$YADM_REPO" YADM_CONFIG="$YADM_DIR/$YADM_CONFIG" YADM_ENCRYPT="$YADM_DIR/$YADM_ENCRYPT" YADM_ARCHIVE="$YADM_DIR/$YADM_ARCHIVE" YADM_BOOTSTRAP="$YADM_DIR/$YADM_BOOTSTRAP" - #; independent overrides for paths + # independent overrides for paths if [ -n "$YADM_OVERRIDE_REPO" ]; then YADM_REPO="$YADM_OVERRIDE_REPO" fi @@ -774,7 +774,7 @@ function configure_paths() { YADM_BOOTSTRAP="$YADM_OVERRIDE_BOOTSTRAP" fi - #; use the yadm repo for all git operations + # use the yadm repo for all git operations GIT_DIR=$(mixed_path "$YADM_REPO") export GIT_DIR @@ -784,23 +784,23 @@ function configure_repo() { debug "Configuring new repo" - #; change bare to false (there is a working directory) + # change bare to false (there is a working directory) "$GIT_PROGRAM" config core.bare 'false' - #; set the worktree for the yadm repo + # set the worktree for the yadm repo "$GIT_PROGRAM" config core.worktree "$(mixed_path "$YADM_WORK")" - #; by default, do not show untracked files and directories + # by default, do not show untracked files and directories "$GIT_PROGRAM" config status.showUntrackedFiles no - #; possibly used later to ensure we're working on the yadm repo + # possibly used later to ensure we're working on the yadm repo "$GIT_PROGRAM" config yadm.managed 'true' } function set_operating_system() { - #; special detection of WSL (windows subsystem for linux) + # special detection of WSL (windows subsystem for linux) local proc_version proc_version=$(cat "$PROC_VERSION" 2>/dev/null) if [[ "$proc_version" =~ Microsoft ]]; then @@ -851,7 +851,7 @@ function invoke_hook() { if [ -x "$hook_command" ] ; then debug "Invoking hook: $hook_command" - #; expose some internal data to all hooks + # expose some internal data to all hooks work=$(unix_path "$("$GIT_PROGRAM" config core.worktree)") YADM_HOOK_COMMAND=$HOOK_COMMAND YADM_HOOK_EXIT=$exit_status @@ -867,7 +867,7 @@ function invoke_hook() { "$hook_command" hook_status=$? - #; failing "pre" hooks will prevent commands from being run + # failing "pre" hooks will prevent commands from being run if [ "$mode" = "pre" ] && [ "$hook_status" -ne 0 ]; then echo "Hook $hook_command was not successful" echo "$HOOK_COMMAND will not be run" @@ -922,7 +922,7 @@ function parse_encrypt() { exclude_pattern="^!(.+)" if [ -f "$YADM_ENCRYPT" ] ; then - #; parse both included/excluded + # parse both included/excluded while IFS='' read -r line || [ -n "$line" ]; do if [[ ! $line =~ ^# && ! $line =~ ^[[:space:]]*$ ]] ; then local IFS=$'\n' @@ -944,7 +944,7 @@ function parse_encrypt() { fi done < "$YADM_ENCRYPT" - #; remove excludes from the includes + # remove excludes from the includes #(SC2068 is disabled because in this case, we desire globbing) FINAL_INCLUDE=() #shellcheck disable=SC2068 @@ -957,7 +957,7 @@ function parse_encrypt() { [ -n "$skip" ] || FINAL_INCLUDE+=("$included") done - #; sort the encrypted files + # sort the encrypted files #shellcheck disable=SC2207 IFS=$'\n' ENCRYPT_INCLUDE_FILES=($(sort <<<"${FINAL_INCLUDE[*]}")) unset IFS @@ -965,11 +965,11 @@ function parse_encrypt() { } -#; ****** Auto Functions ****** +# ****** Auto Functions ****** function auto_alt() { - #; process alternates if there are possible changes + # process alternates if there are possible changes if [ "$CHANGES_POSSIBLE" = "1" ] ; then auto_alt=$(config --bool yadm.auto-alt) if [ "$auto_alt" != "false" ] ; then @@ -981,7 +981,7 @@ function auto_alt() { function auto_perms() { - #; process permissions if there are possible changes + # process permissions if there are possible changes if [ "$CHANGES_POSSIBLE" = "1" ] ; then auto_perms=$(config --bool yadm.auto-perms) if [ "$auto_perms" != "false" ] ; then @@ -1010,7 +1010,7 @@ function auto_bootstrap() { } -#; ****** Prerequisites Functions ****** +# ****** Prerequisites Functions ****** function require_archive() { [ -f "$YADM_ARCHIVE" ] || error_out "$YADM_ARCHIVE does not exist. did you forget to create it?" @@ -1061,10 +1061,10 @@ function envtpl_available() { return 1 } -#; ****** Directory tranlations ****** +# ****** Directory tranlations ****** function unix_path() { - #; for paths used by bash/yadm + # for paths used by bash/yadm if [ "$USE_CYGPATH" = "1" ] ; then cygpath -u "$1" else @@ -1072,7 +1072,7 @@ function unix_path() { fi } function mixed_path() { - #; for paths used by Git + # for paths used by Git if [ "$USE_CYGPATH" = "1" ] ; then cygpath -m "$1" else @@ -1080,7 +1080,7 @@ function mixed_path() { fi } -#; ****** echo replacements ****** +# ****** echo replacements ****** function echo() { IFS=' ' printf '%s\n' "$*" @@ -1094,7 +1094,7 @@ function echo_e() { printf '%b\n' "$*" } -#; ****** Main processing (when not unit testing) ****** +# ****** Main processing (when not unit testing) ****** if [ "$YADM_TEST" != 1 ] ; then process_global_args "$@" From fb1181c8a947e2270380930a9dcef621791d8d20 Mon Sep 17 00:00:00 2001 From: Tim Byrne Date: Thu, 4 Apr 2019 18:01:27 -0500 Subject: [PATCH 028/137] Add tests for directory alternates While this feature was added back in version 1.05, tests were never added for it. These tests have identified bugs in the directory alternates. --- pylintrc | 5 ++- test/test_alt.py | 93 +++++++++++++++++++++++++++++++++++------------- test/utils.py | 47 ++++++++++++++---------- 3 files changed, 102 insertions(+), 43 deletions(-) diff --git a/pylintrc b/pylintrc index 2ae18b6..d686a0a 100644 --- a/pylintrc +++ b/pylintrc @@ -3,9 +3,12 @@ good-names=pytestmark [DESIGN] max-args=14 -max-locals=26 +max-locals=27 max-attributes=8 max-statements=65 [MESSAGES CONTROL] disable=redefined-outer-name + +[TYPECHECK] +ignored-modules=py diff --git a/test/test_alt.py b/test/test_alt.py index 60b7339..beb3b7f 100644 --- a/test/test_alt.py +++ b/test/test_alt.py @@ -3,6 +3,7 @@ import os import re import string +import py import pytest import utils @@ -33,6 +34,8 @@ WILD_TEMPLATES = [ '##$tst_class.$tst_sys.$tst_host.$tst_user', ] +TEST_PATHS = [utils.ALT_FILE1, utils.ALT_FILE2, utils.ALT_DIR] + WILD_TESTED = set() @@ -90,12 +93,18 @@ def test_alt(runner, yadm_y, paths, linked = linked_list(run.out) # assert the proper linking has occurred - for file_path in (utils.ALT_FILE1, utils.ALT_FILE2): + for file_path in TEST_PATHS: source_file = file_path + precedence[precedence_index] if tracked or (encrypt and not exclude): assert paths.work.join(file_path).islink() - assert paths.work.join(file_path).read() == source_file - assert str(paths.work.join(source_file)) in linked + target = py.path.local(paths.work.join(file_path).readlink()) + if target.isfile(): + assert paths.work.join(file_path).read() == source_file + assert str(paths.work.join(source_file)) in linked + else: + assert paths.work.join(file_path).join( + utils.CONTAINED).read() == source_file + assert str(paths.work.join(source_file)) in linked else: assert not paths.work.join(file_path).exists() assert str(paths.work.join(source_file)) not in linked @@ -169,8 +178,7 @@ def test_wild(request, runner, yadm_y, paths, test_key = f'{tracked}{encrypt}{wild_suffix}{std_suffix}' if test_key in WILD_TESTED: return - else: - WILD_TESTED.add(test_key) + WILD_TESTED.add(test_key) # set the class utils.set_local(paths, 'class', tst_class) @@ -186,11 +194,17 @@ def test_wild(request, runner, yadm_y, paths, linked = linked_list(run.out) # assert the proper linking has occurred - for file_path in (utils.ALT_FILE1, utils.ALT_FILE2): + for file_path in TEST_PATHS: source_file = file_path + wild_suffix assert paths.work.join(file_path).islink() - assert paths.work.join(file_path).read() == source_file - assert str(paths.work.join(source_file)) in linked + target = py.path.local(paths.work.join(file_path).readlink()) + if target.isfile(): + assert paths.work.join(file_path).read() == source_file + assert str(paths.work.join(source_file)) in linked + else: + assert paths.work.join(file_path).join( + utils.CONTAINED).read() == source_file + assert str(paths.work.join(source_file)) in linked # create files using the standard suffix utils.create_alt_files(paths, std_suffix, tracked=tracked, @@ -203,11 +217,17 @@ def test_wild(request, runner, yadm_y, paths, linked = linked_list(run.out) # assert the proper linking has occurred - for file_path in (utils.ALT_FILE1, utils.ALT_FILE2): + for file_path in TEST_PATHS: source_file = file_path + std_suffix assert paths.work.join(file_path).islink() - assert paths.work.join(file_path).read() == source_file - assert str(paths.work.join(source_file)) in linked + target = py.path.local(paths.work.join(file_path).readlink()) + if target.isfile(): + assert paths.work.join(file_path).read() == source_file + assert str(paths.work.join(source_file)) in linked + else: + assert paths.work.join(file_path).join( + utils.CONTAINED).read() == source_file + assert str(paths.work.join(source_file)) in linked @pytest.mark.usefixtures('ds1_copy') @@ -235,11 +255,17 @@ def test_local_override(runner, yadm_y, paths, linked = linked_list(run.out) # assert the proper linking has occurred - for file_path in (utils.ALT_FILE1, utils.ALT_FILE2): + for file_path in TEST_PATHS: source_file = file_path + '##or-class.or-os.or-hostname.or-user' assert paths.work.join(file_path).islink() - assert paths.work.join(file_path).read() == source_file - assert str(paths.work.join(source_file)) in linked + target = py.path.local(paths.work.join(file_path).readlink()) + if target.isfile(): + assert paths.work.join(file_path).read() == source_file + assert str(paths.work.join(source_file)) in linked + else: + assert paths.work.join(file_path).join( + utils.CONTAINED).read() == source_file + assert str(paths.work.join(source_file)) in linked @pytest.mark.parametrize('suffix', ['AAA', 'ZZZ', 'aaa', 'zzz']) @@ -267,11 +293,17 @@ def test_class_case(runner, yadm_y, paths, tst_sys, suffix): linked = linked_list(run.out) # assert the proper linking has occurred - for file_path in (utils.ALT_FILE1, utils.ALT_FILE2): + for file_path in TEST_PATHS: source_file = file_path + f'##{suffix}' assert paths.work.join(file_path).islink() - assert paths.work.join(file_path).read() == source_file - assert str(paths.work.join(source_file)) in linked + target = py.path.local(paths.work.join(file_path).readlink()) + if target.isfile(): + assert paths.work.join(file_path).read() == source_file + assert str(paths.work.join(source_file)) in linked + else: + assert paths.work.join(file_path).join( + utils.CONTAINED).read() == source_file + assert str(paths.work.join(source_file)) in linked @pytest.mark.parametrize('autoalt', [None, 'true', 'false']) @@ -294,15 +326,22 @@ def test_auto_alt(runner, yadm_y, paths, autoalt): linked = linked_list(run.out) # assert the proper linking has occurred - for file_path in (utils.ALT_FILE1, utils.ALT_FILE2): + for file_path in TEST_PATHS: source_file = file_path + suffix if autoalt == 'false': assert not paths.work.join(file_path).exists() else: assert paths.work.join(file_path).islink() - assert paths.work.join(file_path).read() == source_file - # no linking output when run via auto-alt - assert str(paths.work.join(source_file)) not in linked + target = py.path.local(paths.work.join(file_path).readlink()) + if target.isfile(): + assert paths.work.join(file_path).read() == source_file + # no linking output when run via auto-alt + assert str(paths.work.join(source_file)) not in linked + else: + assert paths.work.join(file_path).join( + utils.CONTAINED).read() == source_file + # no linking output when run via auto-alt + assert str(paths.work.join(source_file)) not in linked @pytest.mark.parametrize('delimiter', ['.', '_']) @@ -324,12 +363,18 @@ def test_delimiter(runner, yadm_y, paths, # assert the proper linking has occurred # only a delimiter of '.' is valid - for file_path in (utils.ALT_FILE1, utils.ALT_FILE2): + for file_path in TEST_PATHS: source_file = file_path + suffix if delimiter == '.': assert paths.work.join(file_path).islink() - assert paths.work.join(file_path).read() == source_file - assert str(paths.work.join(source_file)) in linked + target = py.path.local(paths.work.join(file_path).readlink()) + if target.isfile(): + assert paths.work.join(file_path).read() == source_file + assert str(paths.work.join(source_file)) in linked + else: + assert paths.work.join(file_path).join( + utils.CONTAINED).read() == source_file + assert str(paths.work.join(source_file)) in linked else: assert not paths.work.join(file_path).exists() assert str(paths.work.join(source_file)) not in linked diff --git a/test/utils.py b/test/utils.py index 411dde1..3303745 100644 --- a/test/utils.py +++ b/test/utils.py @@ -7,6 +7,11 @@ import os ALT_FILE1 = 'test_alt' ALT_FILE2 = 'test alt/test alt' +ALT_DIR = 'test alt/test alt dir' + +# Directory based alternates must have a tracked contained file. +# This will be the test contained file name +CONTAINED = 'contained_file' def set_local(paths, variable, value): @@ -29,31 +34,37 @@ def create_alt_files(paths, suffix, """ if not preserve: - if paths.work.join(ALT_FILE1).exists(): - paths.work.join(ALT_FILE1).remove(rec=1, ignore_errors=True) - assert not paths.work.join(ALT_FILE1).exists() - if paths.work.join(ALT_FILE2).exists(): - paths.work.join(ALT_FILE2).remove(rec=1, ignore_errors=True) - assert not paths.work.join(ALT_FILE2).exists() + for remove_path in (ALT_FILE1, ALT_FILE2, ALT_DIR): + if paths.work.join(remove_path).exists(): + paths.work.join(remove_path).remove(rec=1, ignore_errors=True) + assert not paths.work.join(remove_path).exists() new_file1 = paths.work.join(ALT_FILE1 + suffix) new_file1.write(ALT_FILE1 + suffix, ensure=True) new_file2 = paths.work.join(ALT_FILE2 + suffix) new_file2.write(ALT_FILE2 + suffix, ensure=True) - if content: - new_file1.write('\n' + content, mode='a', ensure=True) - new_file2.write('\n' + content, mode='a', ensure=True) - assert new_file1.exists() - assert new_file2.exists() + new_dir = paths.work.join(ALT_DIR + suffix).join(CONTAINED) + new_dir.write(ALT_DIR + suffix, ensure=True) + + # Do not test directory support for jinja alternates + test_paths = [new_file1, new_file2] + test_names = [ALT_FILE1, ALT_FILE2] + if suffix != '##yadm.j2': + test_paths += [new_dir] + test_names += [ALT_DIR] + + for test_path in test_paths: + if content: + test_path.write('\n' + content, mode='a', ensure=True) + assert test_path.exists() if tracked: - for path in (new_file1, new_file2): - os.system(f'GIT_DIR={str(paths.repo)} git add "{path}"') + for track_path in test_paths: + os.system(f'GIT_DIR={str(paths.repo)} git add "{track_path}"') os.system(f'GIT_DIR={str(paths.repo)} git commit -m "Add test files"') if encrypt: - paths.encrypt.write(f'{ALT_FILE1 + suffix}\n', mode='a') - paths.encrypt.write(f'{ALT_FILE2 + suffix}\n', mode='a') - if exclude: - paths.encrypt.write(f'!{ALT_FILE1 + suffix}\n', mode='a') - paths.encrypt.write(f'!{ALT_FILE2 + suffix}\n', mode='a') + for encrypt_name in test_names: + paths.encrypt.write(f'{encrypt_name + suffix}\n', mode='a') + if exclude: + paths.encrypt.write(f'!{encrypt_name + suffix}\n', mode='a') From 0c6be5e398e9fe6eb23c6110f17c4ea233c2bc15 Mon Sep 17 00:00:00 2001 From: Tim Byrne Date: Fri, 5 Apr 2019 07:46:56 -0500 Subject: [PATCH 029/137] Fix bug with alternate linked directories Previously the tracked files were sorted, and then the files and their parent directories were considered for possible alternates. Depending on the length of directories and names of files, inconsistencies would occur because the directory separator (/) would be part of the sorting. To fix this, a unique list of tracked files and their parent directories are sorted into a single list which is processed. --- yadm | 36 +++++++++++++++++------------------- 1 file changed, 17 insertions(+), 19 deletions(-) diff --git a/yadm b/yadm index 2190614..2e762aa 100755 --- a/yadm +++ b/yadm @@ -181,29 +181,27 @@ function alt() { for match in $match1 $match2; do last_linked='' local IFS=$'\n' - for tracked_file in $("$GIT_PROGRAM" ls-files | sort) "${ENCRYPT_INCLUDE_FILES[@]}"; do - tracked_file="$YADM_WORK/$tracked_file" - # process both the path, and it's parent directory - for alt_path in "$tracked_file" "${tracked_file%/*}"; do - if [ -e "$alt_path" ] ; then - if [[ $alt_path =~ $match ]] ; then - if [ "$alt_path" != "$last_linked" ] ; then - new_link="${BASH_REMATCH[1]}" - debug "Linking $alt_path to $new_link" - [ -n "$loud" ] && echo "Linking $alt_path to $new_link" - if [ "$do_copy" -eq 1 ]; then - if [ -L "$new_link" ]; then - rm -f "$new_link" - fi - cp -f "$alt_path" "$new_link" - else - ln -nfs "$alt_path" "$new_link" + # the alt_paths looped over here are a unique sorted list of both files and their immediate parent directory + for alt_path in $(for tracked in $("$GIT_PROGRAM" ls-files); do printf "%s\n" "$tracked" "${tracked%/*}"; done | sort -u) "${ENCRYPT_INCLUDE_FILES[@]}"; do + alt_path="$YADM_WORK/$alt_path" + if [ -e "$alt_path" ] ; then + if [[ $alt_path =~ $match ]] ; then + if [ "$alt_path" != "$last_linked" ] ; then + new_link="${BASH_REMATCH[1]}" + debug "Linking $alt_path to $new_link" + [ -n "$loud" ] && echo "Linking $alt_path to $new_link" + if [ "$do_copy" -eq 1 ]; then + if [ -L "$new_link" ]; then + rm -f "$new_link" fi - last_linked="$alt_path" + cp -f "$alt_path" "$new_link" + else + ln -nfs "$alt_path" "$new_link" fi + last_linked="$alt_path" fi fi - done + fi done done From 2375a0955bc79cc31220ac60db7c4fd745454c9e Mon Sep 17 00:00:00 2001 From: Tim Byrne Date: Fri, 5 Apr 2019 08:02:42 -0500 Subject: [PATCH 030/137] Standardize sort order Prevent localizations from interfering with sorting order. --- test/test_unit_parse_encrypt.py | 1 - yadm | 6 +++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/test/test_unit_parse_encrypt.py b/test/test_unit_parse_encrypt.py index b1a1af9..7ab1d30 100644 --- a/test/test_unit_parse_encrypt.py +++ b/test/test_unit_parse_encrypt.py @@ -166,7 +166,6 @@ def run_parse_encrypt( export YADM_ENCRYPT GIT_DIR={paths.repo} export GIT_DIR - LC_ALL=C {parse_cmd} export ENCRYPT_INCLUDE_FILES export PARSE_ENCRYPT_SHORT diff --git a/yadm b/yadm index 2e762aa..1fc0333 100755 --- a/yadm +++ b/yadm @@ -182,7 +182,7 @@ function alt() { last_linked='' local IFS=$'\n' # the alt_paths looped over here are a unique sorted list of both files and their immediate parent directory - for alt_path in $(for tracked in $("$GIT_PROGRAM" ls-files); do printf "%s\n" "$tracked" "${tracked%/*}"; done | sort -u) "${ENCRYPT_INCLUDE_FILES[@]}"; do + for alt_path in $(for tracked in $("$GIT_PROGRAM" ls-files); do printf "%s\n" "$tracked" "${tracked%/*}"; done | LC_ALL=C sort -u) "${ENCRYPT_INCLUDE_FILES[@]}"; do alt_path="$YADM_WORK/$alt_path" if [ -e "$alt_path" ] ; then if [[ $alt_path =~ $match ]] ; then @@ -209,7 +209,7 @@ function alt() { # for every file which is a *##yadm.j2 create a real file local IFS=$'\n' local match="^(.+)##yadm\\.j2$" - for tracked_file in $("$GIT_PROGRAM" ls-files | sort) "${ENCRYPT_INCLUDE_FILES[@]}"; do + for tracked_file in $("$GIT_PROGRAM" ls-files | LC_ALL=C sort) "${ENCRYPT_INCLUDE_FILES[@]}"; do tracked_file="$YADM_WORK/$tracked_file" if [ -e "$tracked_file" ] ; then if [[ $tracked_file =~ $match ]] ; then @@ -957,7 +957,7 @@ function parse_encrypt() { # sort the encrypted files #shellcheck disable=SC2207 - IFS=$'\n' ENCRYPT_INCLUDE_FILES=($(sort <<<"${FINAL_INCLUDE[*]}")) + IFS=$'\n' ENCRYPT_INCLUDE_FILES=($(LC_ALL=C sort <<<"${FINAL_INCLUDE[*]}")) unset IFS fi From 093fc24b1b8ee146ab25f5e1787c58798ce80b8d Mon Sep 17 00:00:00 2001 From: Tim Byrne Date: Fri, 5 Apr 2019 08:58:59 -0500 Subject: [PATCH 031/137] Test that links are removed for invalid alternates (#65) --- test/test_alt.py | 50 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/test/test_alt.py b/test/test_alt.py index beb3b7f..10315ae 100644 --- a/test/test_alt.py +++ b/test/test_alt.py @@ -380,6 +380,56 @@ def test_delimiter(runner, yadm_y, paths, assert str(paths.work.join(source_file)) not in linked +@pytest.mark.usefixtures('ds1_copy') +def test_invalid_links_removed(runner, yadm_y, paths): + """Links to invalid alternative files are removed + + This test ensures that when an already linked alternative becomes invalid + due to a change in class, the alternate link is removed. + """ + + # set the class + tst_class = 'testclass' + utils.set_local(paths, 'class', tst_class) + + # create files which match the test class + utils.create_alt_files(paths, f'##{tst_class}') + + # run alt to trigger linking + run = runner(yadm_y('alt')) + assert run.success + assert run.err == '' + linked = linked_list(run.out) + + # assert the proper linking has occurred + for file_path in TEST_PATHS: + source_file = file_path + '##' + tst_class + assert paths.work.join(file_path).islink() + target = py.path.local(paths.work.join(file_path).readlink()) + if target.isfile(): + assert paths.work.join(file_path).read() == source_file + assert str(paths.work.join(source_file)) in linked + else: + assert paths.work.join(file_path).join( + utils.CONTAINED).read() == source_file + assert str(paths.work.join(source_file)) in linked + + # change the class so there are no valid alternates + utils.set_local(paths, 'class', 'changedclass') + + # run alt to trigger linking + run = runner(yadm_y('alt')) + assert run.success + assert run.err == '' + linked = linked_list(run.out) + + # assert the linking is removed + for file_path in TEST_PATHS: + source_file = file_path + '##' + tst_class + assert not paths.work.join(file_path).exists() + assert str(paths.work.join(source_file)) not in linked + + def linked_list(output): """Parse output, and return list of linked files""" linked = dict() From a2cc970dd64f35149819ac13a6d51601e47d3999 Mon Sep 17 00:00:00 2001 From: Tim Byrne Date: Tue, 9 Apr 2019 07:53:27 -0500 Subject: [PATCH 032/137] Remove unnecessary Git invocations `git ls-files` was being called multiple times during the processing of alternates. This is now run once, and kept in an array. --- yadm | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/yadm b/yadm index 1fc0333..9278c14 100755 --- a/yadm +++ b/yadm @@ -176,13 +176,20 @@ function alt() { fi fi + # process the files tracked by yadm once, this info is used multiple times + tracked_files=() + local IFS=$'\n' + for tracked_file in $("$GIT_PROGRAM" ls-files | LC_ALL=C sort); do + tracked_files+=("$tracked_file") + done + # loop over all "tracked" files # for every file which matches the above regex, create a symlink for match in $match1 $match2; do last_linked='' local IFS=$'\n' # the alt_paths looped over here are a unique sorted list of both files and their immediate parent directory - for alt_path in $(for tracked in $("$GIT_PROGRAM" ls-files); do printf "%s\n" "$tracked" "${tracked%/*}"; done | LC_ALL=C sort -u) "${ENCRYPT_INCLUDE_FILES[@]}"; do + for alt_path in $(for tracked in "${tracked_files[@]}"; do printf "%s\n" "$tracked" "${tracked%/*}"; done | LC_ALL=C sort -u) "${ENCRYPT_INCLUDE_FILES[@]}"; do alt_path="$YADM_WORK/$alt_path" if [ -e "$alt_path" ] ; then if [[ $alt_path =~ $match ]] ; then @@ -207,9 +214,8 @@ function alt() { # loop over all "tracked" files # for every file which is a *##yadm.j2 create a real file - local IFS=$'\n' local match="^(.+)##yadm\\.j2$" - for tracked_file in $("$GIT_PROGRAM" ls-files | LC_ALL=C sort) "${ENCRYPT_INCLUDE_FILES[@]}"; do + for tracked_file in "${tracked_files[@]}" "${ENCRYPT_INCLUDE_FILES[@]}"; do tracked_file="$YADM_WORK/$tracked_file" if [ -e "$tracked_file" ] ; then if [[ $tracked_file =~ $match ]] ; then From d245f633bfbca6598556182ebba2728ecd29298f Mon Sep 17 00:00:00 2001 From: Tim Byrne Date: Wed, 10 Apr 2019 08:43:11 -0500 Subject: [PATCH 033/137] Remove invalid linked alternates (#65) --- yadm | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/yadm b/yadm index 9278c14..427257a 100755 --- a/yadm +++ b/yadm @@ -183,6 +183,16 @@ function alt() { tracked_files+=("$tracked_file") done + # generate a list of possible alt files + possible_alts=() + local IFS=$'\n' + for possible_alt in "${tracked_files[@]}" "${ENCRYPT_INCLUDE_FILES[@]}"; do + if [[ $possible_alt =~ .\#\#. ]]; then + possible_alts+=("$YADM_WORK/${possible_alt%##*}") + fi + done + alt_linked=() + # loop over all "tracked" files # for every file which matches the above regex, create a symlink for match in $match1 $match2; do @@ -204,6 +214,7 @@ function alt() { cp -f "$alt_path" "$new_link" else ln -nfs "$alt_path" "$new_link" + alt_linked+=("$alt_path") fi last_linked="$alt_path" fi @@ -212,6 +223,24 @@ function alt() { done done + # review alternate candidates for stale links + # if a possible alt IS linked, but it's target is not part of alt_linked, + # remove it. + if readlink_available; then + for stale_candidate in "${possible_alts[@]}"; do + if [ -L "$stale_candidate" ]; then + link_target=$(readlink "$stale_candidate" 2>/dev/null) + if [ -n "$link_target" ]; then + removal=yes + for review_link in "${alt_linked[@]}"; do + [ "$link_target" = "$review_link" ] && removal=no + done + [ "$removal" = "yes" ] && rm -f "$stale_candidate" + fi + fi + done + fi + # loop over all "tracked" files # for every file which is a *##yadm.j2 create a real file local match="^(.+)##yadm\\.j2$" @@ -1064,6 +1093,10 @@ function envtpl_available() { command -v "$ENVTPL_PROGRAM" >/dev/null 2>&1 && return return 1 } +function readlink_available() { + command -v "readlink" >/dev/null 2>&1 && return + return 1 +} # ****** Directory tranlations ****** From 7bc8f02d68f04d6ed7e84dad66216591eb8700f7 Mon Sep 17 00:00:00 2001 From: Tim Byrne Date: Fri, 12 Apr 2019 07:43:18 -0500 Subject: [PATCH 034/137] Add tests for jinja includes --- test/test_jinja.py | 22 ++++++++++++++++++---- test/utils.py | 19 ++++++++++++++++++- 2 files changed, 36 insertions(+), 5 deletions(-) diff --git a/test/test_jinja.py b/test/test_jinja.py index 6e43f35..54d248a 100644 --- a/test/test_jinja.py +++ b/test/test_jinja.py @@ -13,7 +13,7 @@ def envtpl_present(runner): run = runner(command=['envtpl', '-h']) if run.success: return True - except BaseException: + except OSError: pass return False @@ -35,11 +35,20 @@ def test_local_override(runner, yadm_y, paths, 'j2-{{ YADM_CLASS }}-' '{{ YADM_OS }}-{{ YADM_HOSTNAME }}-' '{{ YADM_USER }}-{{ YADM_DISTRO }}' + '-{%- ' + f"include '{utils.INCLUDE_FILE}'" + ' -%}' + ) + expected = ( + f'j2-or-class-or-os-or-hostname-or-user-{tst_distro}' + f'-{utils.INCLUDE_CONTENT}' ) - expected = f'j2-or-class-or-os-or-hostname-or-user-{tst_distro}' - utils.create_alt_files(paths, '##yadm.j2', content=template) + utils.create_alt_files(paths, '##yadm.j2', content=template, + includefile=True) + # os.system(f'find {paths.work}' + ' -name *j2 -ls -exec cat \'{}\' ";"') + # os.system(f'find {paths.work}') # run alt to trigger linking run = runner(yadm_y('alt')) assert run.success @@ -145,15 +154,20 @@ def test_jinja(runner, yadm_y, paths, 'j2-{{ YADM_CLASS }}-' '{{ YADM_OS }}-{{ YADM_HOSTNAME }}-' '{{ YADM_USER }}-{{ YADM_DISTRO }}' + '-{%- ' + f"include '{utils.INCLUDE_FILE}'" + ' -%}' ) expected = ( f'j2-{tst_class}-' f'{tst_sys}-{tst_host}-' f'{tst_user}-{tst_distro}' + f'-{utils.INCLUDE_CONTENT}' ) utils.create_alt_files(paths, jinja_suffix, content=template, - tracked=tracked, encrypt=encrypt, exclude=exclude) + tracked=tracked, encrypt=encrypt, exclude=exclude, + includefile=True) # run alt to trigger linking run = runner(yadm_y('alt')) diff --git a/test/utils.py b/test/utils.py index 3303745..577281c 100644 --- a/test/utils.py +++ b/test/utils.py @@ -13,6 +13,12 @@ ALT_DIR = 'test alt/test alt dir' # This will be the test contained file name CONTAINED = 'contained_file' +# These variables are used for making include files which will be processed +# within jinja templates +INCLUDE_FILE = 'inc_file' +INCLUDE_DIRS = ['', 'test alt'] +INCLUDE_CONTENT = '8780846c02e34c930d0afd127906668f' + def set_local(paths, variable, value): """Set local override""" @@ -25,7 +31,7 @@ def set_local(paths, variable, value): def create_alt_files(paths, suffix, preserve=False, tracked=True, encrypt=False, exclude=False, - content=None): + content=None, includefile=False): """Create new files, and add to the repo This is used for testing alternate files. In each case, a suffix is @@ -58,6 +64,8 @@ def create_alt_files(paths, suffix, test_path.write('\n' + content, mode='a', ensure=True) assert test_path.exists() + create_includefiles(includefile, paths, test_paths) + if tracked: for track_path in test_paths: os.system(f'GIT_DIR={str(paths.repo)} git add "{track_path}"') @@ -68,3 +76,12 @@ def create_alt_files(paths, suffix, paths.encrypt.write(f'{encrypt_name + suffix}\n', mode='a') if exclude: paths.encrypt.write(f'!{encrypt_name + suffix}\n', mode='a') + + +def create_includefiles(includefile, paths, test_paths): + """Generate files for testing jinja includes""" + if includefile: + for dpath in INCLUDE_DIRS: + incfile = paths.work.join(dpath + '/' + INCLUDE_FILE) + incfile.write(INCLUDE_CONTENT, ensure=True) + test_paths += [incfile] From d87a6502af92aa8c79da108cdfe79d031736b176 Mon Sep 17 00:00:00 2001 From: Tim Byrne Date: Fri, 12 Apr 2019 07:48:10 -0500 Subject: [PATCH 035/137] Factor out some branches in utils:create_alt_files() --- test/utils.py | 33 +++++++++++++++++++-------------- 1 file changed, 19 insertions(+), 14 deletions(-) diff --git a/test/utils.py b/test/utils.py index 577281c..0b52b37 100644 --- a/test/utils.py +++ b/test/utils.py @@ -64,24 +64,29 @@ def create_alt_files(paths, suffix, test_path.write('\n' + content, mode='a', ensure=True) assert test_path.exists() - create_includefiles(includefile, paths, test_paths) - - if tracked: - for track_path in test_paths: - os.system(f'GIT_DIR={str(paths.repo)} git add "{track_path}"') - os.system(f'GIT_DIR={str(paths.repo)} git commit -m "Add test files"') - - if encrypt: - for encrypt_name in test_names: - paths.encrypt.write(f'{encrypt_name + suffix}\n', mode='a') - if exclude: - paths.encrypt.write(f'!{encrypt_name + suffix}\n', mode='a') + _create_includefiles(includefile, paths, test_paths) + _create_tracked(tracked, test_paths, paths) + _create_encrypt(encrypt, test_names, suffix, paths, exclude) -def create_includefiles(includefile, paths, test_paths): - """Generate files for testing jinja includes""" +def _create_includefiles(includefile, paths, test_paths): if includefile: for dpath in INCLUDE_DIRS: incfile = paths.work.join(dpath + '/' + INCLUDE_FILE) incfile.write(INCLUDE_CONTENT, ensure=True) test_paths += [incfile] + + +def _create_tracked(tracked, test_paths, paths): + if tracked: + for track_path in test_paths: + os.system(f'GIT_DIR={str(paths.repo)} git add "{track_path}"') + os.system(f'GIT_DIR={str(paths.repo)} git commit -m "Add test files"') + + +def _create_encrypt(encrypt, test_names, suffix, paths, exclude): + if encrypt: + for encrypt_name in test_names: + paths.encrypt.write(f'{encrypt_name + suffix}\n', mode='a') + if exclude: + paths.encrypt.write(f'!{encrypt_name + suffix}\n', mode='a') From 6df85099999b1edac609e95c6e928e46c29263e9 Mon Sep 17 00:00:00 2001 From: Tim Byrne Date: Tue, 16 Apr 2019 08:04:53 -0500 Subject: [PATCH 036/137] Add vim to testbed image Having an editor within a testhost/scripthost is useful. --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index b221942..985fc7c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -24,6 +24,7 @@ RUN \ make \ python3-pip \ shellcheck=0.4.6-1 \ + vim \ ; RUN pip3 install \ envtpl \ From b404a87a89a98a0f5b5eeaf15b6ed479ee012d5f Mon Sep 17 00:00:00 2001 From: Daniel Wagenknecht Date: Tue, 16 Jul 2019 21:45:14 +0200 Subject: [PATCH 037/137] Replace direkt calls to git with $GIT_PROGRAM Some calls to git ignored the yadm.gpg-program configuration option and called the first git found in $PATH instead. Make them adhere to the configured git program by replacing the call with $GIT_PROGRAM. --- yadm | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/yadm b/yadm index 8c5fe4d..4da57c7 100755 --- a/yadm +++ b/yadm @@ -410,13 +410,13 @@ EOF # operate on the yadm repo's configuration file # this is always local to the machine - git config --local "$@" + $GIT_PROGRAM config --local "$@" CHANGES_POSSIBLE=1 else # operate on the yadm configuration file - git config --file="$(mixed_path "$YADM_CONFIG")" "$@" + $GIT_PROGRAM config --file="$(mixed_path "$YADM_CONFIG")" "$@" fi @@ -844,7 +844,7 @@ function set_operating_system() { case "$OPERATING_SYSTEM" in CYGWIN*) - git_version=$(git --version 2>/dev/null) + git_version=$($GIT_PROGRAM --version 2>/dev/null) if [[ "$git_version" =~ windows ]] ; then USE_CYGPATH=1 fi From a399c35e4e4e05e1a9299960d2fe463957376fbd Mon Sep 17 00:00:00 2001 From: Tim Byrne Date: Mon, 29 Jul 2019 08:00:09 -0500 Subject: [PATCH 038/137] Add quoting --- yadm | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/yadm b/yadm index 4da57c7..b703b36 100755 --- a/yadm +++ b/yadm @@ -410,13 +410,13 @@ EOF # operate on the yadm repo's configuration file # this is always local to the machine - $GIT_PROGRAM config --local "$@" + "$GIT_PROGRAM" config --local "$@" CHANGES_POSSIBLE=1 else # operate on the yadm configuration file - $GIT_PROGRAM config --file="$(mixed_path "$YADM_CONFIG")" "$@" + "$GIT_PROGRAM" config --file="$(mixed_path "$YADM_CONFIG")" "$@" fi @@ -844,7 +844,7 @@ function set_operating_system() { case "$OPERATING_SYSTEM" in CYGWIN*) - git_version=$($GIT_PROGRAM --version 2>/dev/null) + git_version="$("$GIT_PROGRAM" --version 2>/dev/null)" if [[ "$git_version" =~ windows ]] ; then USE_CYGPATH=1 fi From f0ad40376dc6170ac705564b9eb117b607662b17 Mon Sep 17 00:00:00 2001 From: Tim Byrne Date: Tue, 6 Aug 2019 08:19:01 -0500 Subject: [PATCH 039/137] Create YADM_HOOKS variable for hooks dir --- yadm | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/yadm b/yadm index b703b36..09f9d87 100755 --- a/yadm +++ b/yadm @@ -30,6 +30,7 @@ YADM_CONFIG="config" YADM_ENCRYPT="encrypt" YADM_ARCHIVE="files.gpg" YADM_BOOTSTRAP="bootstrap" +YADM_HOOKS="hooks" HOOK_COMMAND="" FULL_COMMAND="" @@ -789,6 +790,7 @@ function configure_paths() { YADM_ENCRYPT="$YADM_DIR/$YADM_ENCRYPT" YADM_ARCHIVE="$YADM_DIR/$YADM_ARCHIVE" YADM_BOOTSTRAP="$YADM_DIR/$YADM_BOOTSTRAP" + YADM_HOOKS="$YADM_DIR/$YADM_HOOKS" # independent overrides for paths if [ -n "$YADM_OVERRIDE_REPO" ]; then @@ -879,7 +881,7 @@ function invoke_hook() { mode="$1" exit_status="$2" - hook_command="$YADM_DIR/hooks/${mode}_$HOOK_COMMAND" + hook_command="${YADM_HOOKS}/${mode}_$HOOK_COMMAND" if [ -x "$hook_command" ] ; then debug "Invoking hook: $hook_command" From 48fc6b0db73469e196ad60c6affb8f5ac2ef35a0 Mon Sep 17 00:00:00 2001 From: Tim Byrne Date: Tue, 6 Aug 2019 08:19:45 -0500 Subject: [PATCH 040/137] Support XDG base directory specification https://standards.freedesktop.org/basedir-spec/basedir-spec-latest.html --- completion/yadm.zsh_completion | 2 +- test/test_unit_configure_paths.py | 3 +- test/test_unit_issue_legacy_path_warning.py | 34 ++++++++ test/test_unit_set_yadm_dir.py | 35 ++++++++ yadm | 95 +++++++++++++++++++-- yadm.1 | 34 ++++---- 6 files changed, 178 insertions(+), 25 deletions(-) create mode 100644 test/test_unit_issue_legacy_path_warning.py create mode 100644 test/test_unit_set_yadm_dir.py diff --git a/completion/yadm.zsh_completion b/completion/yadm.zsh_completion index 517d7be..fa79c01 100644 --- a/completion/yadm.zsh_completion +++ b/completion/yadm.zsh_completion @@ -7,7 +7,7 @@ _yadm(){ 'config:Configure a setting' 'list:List tracked files' 'alt:Create links for alternates' - 'bootstrap:Execute $HOME/.yadm/bootstrap' + 'bootstrap:Execute $HOME/.config/yadm/bootstrap' 'encrypt:Encrypt files' 'decrypt:Decrypt files' 'perms:Fix perms for private files' diff --git a/test/test_unit_configure_paths.py b/test/test_unit_configure_paths.py index 094ff6b..332277d 100644 --- a/test/test_unit_configure_paths.py +++ b/test/test_unit_configure_paths.py @@ -8,7 +8,7 @@ CONFIG = 'config' ENCRYPT = 'encrypt' HOME = '/testhome' REPO = 'repo.git' -YDIR = '.yadm' +YDIR = '.config/yadm' @pytest.mark.parametrize( @@ -70,6 +70,7 @@ def run_test(runner, paths, args, expected_matches, expected_code=0): script = f""" YADM_TEST=1 HOME="{HOME}" source {paths.pgm} process_global_args {argstring} + HOME="{HOME}" set_yadm_dir configure_paths declare -p | grep -E '(YADM|GIT)_' """ diff --git a/test/test_unit_issue_legacy_path_warning.py b/test/test_unit_issue_legacy_path_warning.py new file mode 100644 index 0000000..e4bed27 --- /dev/null +++ b/test/test_unit_issue_legacy_path_warning.py @@ -0,0 +1,34 @@ +"""Unit tests: issue_legacy_path_warning""" +import pytest + + +@pytest.mark.parametrize( + 'legacy_path', [ + None, + 'repo.git', + 'config', + 'encrypt', + 'files.gpg', + 'bootstrap', + 'hooks', + ], + ) +def test_legacy_warning(tmpdir, runner, yadm, legacy_path): + """Use issue_legacy_path_warning""" + home = tmpdir.mkdir('home') + + if legacy_path: + home.mkdir(f'.yadm').mkdir(legacy_path) + + script = f""" + HOME={home} + YADM_TEST=1 source {yadm} + issue_legacy_path_warning + """ + run = runner(command=['bash'], inp=script) + assert run.success + assert run.err == '' + if legacy_path: + assert 'Legacy configuration paths have been detected' in run.out + else: + assert run.out.rstrip() == '' diff --git a/test/test_unit_set_yadm_dir.py b/test/test_unit_set_yadm_dir.py new file mode 100644 index 0000000..65459f8 --- /dev/null +++ b/test/test_unit_set_yadm_dir.py @@ -0,0 +1,35 @@ +"""Unit tests: set_yadm_dir""" +import pytest + + +@pytest.mark.parametrize( + 'condition', + ['basic', 'override', 'xdg_config_home', 'legacy'], + ) +def test_set_yadm_dir(runner, yadm, condition): + """Test set_yadm_dir""" + setup = '' + if condition == 'override': + setup = 'YADM_DIR=/override' + elif condition == 'xdg_config_home': + setup = 'XDG_CONFIG_HOME=/xdg' + elif condition == 'legacy': + setup = 'YADM_COMPATIBILITY=1' + script = f""" + HOME=/testhome + YADM_TEST=1 source {yadm} + {setup} + set_yadm_dir + echo "$YADM_DIR" + """ + run = runner(command=['bash'], inp=script) + assert run.success + assert run.err == '' + if condition == 'basic': + assert run.out.rstrip() == '/testhome/.config/yadm' + elif condition == 'override': + assert run.out.rstrip() == '/override' + elif condition == 'xdg_config_home': + assert run.out.rstrip() == '/xdg/yadm' + elif condition == 'legacy': + assert run.out.rstrip() == '/testhome/.yadm' diff --git a/yadm b/yadm index 09f9d87..5e07709 100755 --- a/yadm +++ b/yadm @@ -23,8 +23,10 @@ fi VERSION=1.12.0 YADM_WORK="$HOME" -YADM_DIR="$HOME/.yadm" +YADM_DIR= +YADM_LEGACY_DIR="${HOME}/.yadm" +# these are the default paths relative to YADM_DIR YADM_REPO="repo.git" YADM_CONFIG="config" YADM_ENCRYPT="encrypt" @@ -558,16 +560,16 @@ Commands: yadm config - Configure a setting yadm list [-a] - List tracked files yadm alt - Create links for alternates - yadm bootstrap - Execute \$HOME/.yadm/bootstrap + yadm bootstrap - Execute \$HOME/.config/yadm/bootstrap yadm encrypt - Encrypt files yadm decrypt [-l] - Decrypt files yadm perms - Fix perms for private files Files: - \$HOME/.yadm/config - yadm's configuration file - \$HOME/.yadm/repo.git - yadm's Git repository - \$HOME/.yadm/encrypt - List of globs used for encrypt/decrypt - \$HOME/.yadm/files.gpg - Encrypted data stored here + \$HOME/.config/yadm/config - yadm's configuration file + \$HOME/.config/yadm/repo.git - yadm's Git repository + \$HOME/.config/yadm/encrypt - List of globs used for encrypt/decrypt + \$HOME/.config/yadm/files.gpg - Encrypted data stored here Use "man yadm" for complete documentation. EOF @@ -782,6 +784,86 @@ function process_global_args() { } +function set_yadm_dir() { + + # only resolve YADM_DIR if it hasn't been provided already + [ -n "$YADM_DIR" ] && return + + # compatibility with major version 1 ignores XDG_CONFIG_HOME + if [ "$YADM_COMPATIBILITY" = "1" ]; then + YADM_DIR="$YADM_LEGACY_DIR" + return + fi + + local base_yadm_dir + base_yadm_dir="$XDG_CONFIG_HOME" + if [[ ! "$base_yadm_dir" =~ ^/ ]] ; then + base_yadm_dir="${HOME}/.config" + fi + YADM_DIR="${base_yadm_dir}/yadm" + + issue_legacy_path_warning + +} + +function issue_legacy_path_warning() { + + # no warnings if YADM_DIR is resolved as the leacy path + [ "$YADM_DIR" = "$YADM_LEGACY_DIR" ] && return + + # no warnings if the legacy directory doesn't exist + [ ! -d "$YADM_LEGACY_DIR" ] && return + + # test for legacy paths + local legacy_found + legacy_found=() + # this is ordered by importance + for legacy_path in \ + "$YADM_LEGACY_DIR/$YADM_REPO" \ + "$YADM_LEGACY_DIR/$YADM_CONFIG" \ + "$YADM_LEGACY_DIR/$YADM_ENCRYPT" \ + "$YADM_LEGACY_DIR/$YADM_ARCHIVE" \ + "$YADM_LEGACY_DIR/$YADM_BOOTSTRAP" \ + "$YADM_LEGACY_DIR/$YADM_HOOKS" \ + ; \ + do + [ -e "$legacy_path" ] && legacy_found+=("$legacy_path") + done + + [ ${#legacy_found[@]} -eq 0 ] && return + + local path_list + for legacy_path in "${legacy_found[@]}"; do + path_list="$path_list * $legacy_path"$'\n' + done + + cat <> $HOME/.yadm/encrypt +.B echo ".ssh/*.key" >> $HOME/.config/yadm/encrypt Add a new pattern to the list of encrypted files .TP -.B yadm encrypt ; yadm add ~/.yadm/files.gpg ; yadm commit +.B yadm encrypt ; yadm add ~/.config/yadm/files.gpg ; yadm commit Commit a new set of encrypted files .SH REPORTING BUGS Report issues or create pull requests at GitHub: @@ -779,4 +779,4 @@ Tim Byrne .BR git (1), .BR gpg (1) -https://thelocehiliosan.github.io/yadm/ +https://yadm.io/ From cec87785787baa49664163f55fb6289bbc4552a3 Mon Sep 17 00:00:00 2001 From: Tim Byrne Date: Fri, 9 Aug 2019 07:48:10 -0500 Subject: [PATCH 041/137] Align continuation backslashes --- yadm | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/yadm b/yadm index 5e07709..6eabf44 100755 --- a/yadm +++ b/yadm @@ -255,10 +255,10 @@ function alt() { if envtpl_available; then debug "Creating $real_file from template $tracked_file" [ -n "$loud" ] && echo "Creating $real_file from template $tracked_file" - YADM_CLASS="$local_class" \ - YADM_OS="$local_system" \ + YADM_CLASS="$local_class" \ + YADM_OS="$local_system" \ YADM_HOSTNAME="$local_host" \ - YADM_USER="$local_user" \ + YADM_USER="$local_user" \ YADM_DISTRO=$(query_distro) \ "$ENVTPL_PROGRAM" --keep-template "$tracked_file" -o "$real_file" else From 289b8e0c6cecd2dd9fe6acfc9b3b704ab0c4585b Mon Sep 17 00:00:00 2001 From: Tim Byrne Date: Fri, 9 Aug 2019 07:52:41 -0500 Subject: [PATCH 042/137] Remove unnecessary continuation backslashes --- yadm | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/yadm b/yadm index 6eabf44..b4e56c1 100755 --- a/yadm +++ b/yadm @@ -581,7 +581,7 @@ EOF function init() { # safety check, don't attempt to init when the repo is already present - [ -d "$YADM_REPO" ] && [ -z "$FORCE" ] && \ + [ -d "$YADM_REPO" ] && [ -z "$FORCE" ] && error_out "Git repo already exists. [$YADM_REPO]\nUse '-f' if you want to force it to be overwritten." # remove existing if forcing the init to happen anyway @@ -1146,7 +1146,7 @@ function require_git() { GIT_PROGRAM="$alt_git" more_info="\nThis command has been set via the yadm.git-program configuration." fi - command -v "$GIT_PROGRAM" >/dev/null 2>&1 || \ + command -v "$GIT_PROGRAM" >/dev/null 2>&1 || error_out "This functionality requires Git to be installed, but the command '$GIT_PROGRAM' cannot be located.$more_info" } function require_gpg() { @@ -1160,7 +1160,7 @@ function require_gpg() { GPG_PROGRAM="$alt_gpg" more_info="\nThis command has been set via the yadm.gpg-program configuration." fi - command -v "$GPG_PROGRAM" >/dev/null 2>&1 || \ + command -v "$GPG_PROGRAM" >/dev/null 2>&1 || error_out "This functionality requires GPG to be installed, but the command '$GPG_PROGRAM' cannot be located.$more_info" } function require_repo() { From c29292d02b8a444f8917bb9637ee8b787ce86aac Mon Sep 17 00:00:00 2001 From: Tim Byrne Date: Fri, 16 Aug 2019 08:19:11 -0500 Subject: [PATCH 043/137] Split out processing of alt (past/future) --- pylintrc | 3 +- test/{test_alt.py => test_compat_alt.py} | 36 ++++++++++---- test/{test_jinja.py => test_compat_jinja.py} | 13 +++-- test/test_cygwin_copy.py | 9 +++- yadm | 52 ++++++++++++++------ 5 files changed, 83 insertions(+), 30 deletions(-) rename test/{test_alt.py => test_compat_alt.py} (94%) rename test/{test_jinja.py => test_compat_jinja.py} (94%) diff --git a/pylintrc b/pylintrc index d686a0a..1ced5a6 100644 --- a/pylintrc +++ b/pylintrc @@ -3,9 +3,10 @@ good-names=pytestmark [DESIGN] max-args=14 -max-locals=27 +max-locals=28 max-attributes=8 max-statements=65 +min-similarity-lines=6 [MESSAGES CONTROL] disable=redefined-outer-name diff --git a/test/test_alt.py b/test/test_compat_alt.py similarity index 94% rename from test/test_alt.py rename to test/test_compat_alt.py index 10315ae..466e7fc 100644 --- a/test/test_alt.py +++ b/test/test_compat_alt.py @@ -87,7 +87,9 @@ def test_alt(runner, yadm_y, paths, encrypt=encrypt, exclude=exclude) # run alt to trigger linking - run = runner(yadm_y('alt')) + env = os.environ.copy() + env['YADM_COMPATIBILITY'] = '1' + run = runner(yadm_y('alt'), env=env) assert run.success assert run.err == '' linked = linked_list(run.out) @@ -188,7 +190,9 @@ def test_wild(request, runner, yadm_y, paths, encrypt=encrypt, exclude=False) # run alt to trigger linking - run = runner(yadm_y('alt')) + env = os.environ.copy() + env['YADM_COMPATIBILITY'] = '1' + run = runner(yadm_y('alt'), env=env) assert run.success assert run.err == '' linked = linked_list(run.out) @@ -211,7 +215,9 @@ def test_wild(request, runner, yadm_y, paths, encrypt=encrypt, exclude=False) # run alt to trigger linking - run = runner(yadm_y('alt')) + env = os.environ.copy() + env['YADM_COMPATIBILITY'] = '1' + run = runner(yadm_y('alt'), env=env) assert run.success assert run.err == '' linked = linked_list(run.out) @@ -249,7 +255,9 @@ def test_local_override(runner, yadm_y, paths, paths, '##or-class.or-os.or-hostname.or-user') # run alt to trigger linking - run = runner(yadm_y('alt')) + env = os.environ.copy() + env['YADM_COMPATIBILITY'] = '1' + run = runner(yadm_y('alt'), env=env) assert run.success assert run.err == '' linked = linked_list(run.out) @@ -287,7 +295,9 @@ def test_class_case(runner, yadm_y, paths, tst_sys, suffix): utils.create_alt_files(paths, f'##{ending}') # run alt to trigger linking - run = runner(yadm_y('alt')) + env = os.environ.copy() + env['YADM_COMPATIBILITY'] = '1' + run = runner(yadm_y('alt'), env=env) assert run.success assert run.err == '' linked = linked_list(run.out) @@ -320,7 +330,9 @@ def test_auto_alt(runner, yadm_y, paths, autoalt): utils.create_alt_files(paths, suffix) # run status to possibly trigger linking - run = runner(yadm_y('status')) + env = os.environ.copy() + env['YADM_COMPATIBILITY'] = '1' + run = runner(yadm_y('status'), env=env) assert run.success assert run.err == '' linked = linked_list(run.out) @@ -356,7 +368,9 @@ def test_delimiter(runner, yadm_y, paths, utils.create_alt_files(paths, suffix) # run alt to trigger linking - run = runner(yadm_y('alt')) + env = os.environ.copy() + env['YADM_COMPATIBILITY'] = '1' + run = runner(yadm_y('alt'), env=env) assert run.success assert run.err == '' linked = linked_list(run.out) @@ -396,7 +410,9 @@ def test_invalid_links_removed(runner, yadm_y, paths): utils.create_alt_files(paths, f'##{tst_class}') # run alt to trigger linking - run = runner(yadm_y('alt')) + env = os.environ.copy() + env['YADM_COMPATIBILITY'] = '1' + run = runner(yadm_y('alt'), env=env) assert run.success assert run.err == '' linked = linked_list(run.out) @@ -418,7 +434,9 @@ def test_invalid_links_removed(runner, yadm_y, paths): utils.set_local(paths, 'class', 'changedclass') # run alt to trigger linking - run = runner(yadm_y('alt')) + env = os.environ.copy() + env['YADM_COMPATIBILITY'] = '1' + run = runner(yadm_y('alt'), env=env) assert run.success assert run.err == '' linked = linked_list(run.out) diff --git a/test/test_jinja.py b/test/test_compat_jinja.py similarity index 94% rename from test/test_jinja.py rename to test/test_compat_jinja.py index 54d248a..342d202 100644 --- a/test/test_jinja.py +++ b/test/test_compat_jinja.py @@ -50,7 +50,9 @@ def test_local_override(runner, yadm_y, paths, # os.system(f'find {paths.work}' + ' -name *j2 -ls -exec cat \'{}\' ";"') # os.system(f'find {paths.work}') # run alt to trigger linking - run = runner(yadm_y('alt')) + env = os.environ.copy() + env['YADM_COMPATIBILITY'] = '1' + run = runner(yadm_y('alt'), env=env) assert run.success assert run.err == '' created = created_list(run.out) @@ -83,7 +85,9 @@ def test_auto_alt(runner, yadm_y, paths, autoalt, tst_sys, utils.create_alt_files(paths, jinja_suffix, content='{{ YADM_OS }}') # run status to possibly trigger linking - run = runner(yadm_y('status')) + env = os.environ.copy() + env['YADM_COMPATIBILITY'] = '1' + run = runner(yadm_y('status'), env=env) assert run.success assert run.err == '' created = created_list(run.out) @@ -111,6 +115,7 @@ def test_jinja_envtpl_missing(runner, paths): process_global_args -Y "{paths.yadm}" set_operating_system configure_paths + YADM_COMPATIBILITY=1 ENVTPL_PROGRAM='envtpl_missing' main alt """ @@ -170,7 +175,9 @@ def test_jinja(runner, yadm_y, paths, includefile=True) # run alt to trigger linking - run = runner(yadm_y('alt')) + env = os.environ.copy() + env['YADM_COMPATIBILITY'] = '1' + run = runner(yadm_y('alt'), env=env) assert run.success assert run.err == '' created = created_list(run.out) diff --git a/test/test_cygwin_copy.py b/test/test_cygwin_copy.py index 82b08ba..92141e3 100644 --- a/test/test_cygwin_copy.py +++ b/test/test_cygwin_copy.py @@ -4,6 +4,8 @@ import os import pytest +@pytest.mark.parametrize( + 'compatibility', [True, False], ids=['compat', 'no-compat']) @pytest.mark.parametrize( 'setting, is_cygwin, expect_link, pre_existing', [ (None, False, True, None), @@ -28,7 +30,8 @@ import pytest @pytest.mark.usefixtures('ds1_copy') def test_cygwin_copy( runner, yadm_y, paths, cygwin_sys, tst_sys, - setting, is_cygwin, expect_link, pre_existing): + setting, is_cygwin, expect_link, pre_existing, + compatibility): """Test yadm.cygwin_copy""" if setting is not None: @@ -49,6 +52,10 @@ def test_cygwin_copy( expected_content = f'test_cygwin_copy##{cygwin_sys}' env = os.environ.copy() env['PATH'] = ':'.join([str(uname_path), env['PATH']]) + if compatibility: + env['YADM_COMPATIBILITY'] = '1' + else: + pytest.xfail('Alternates 2.0.0 has not been implemented.') run = runner(yadm_y('alt'), env=env) assert run.success diff --git a/yadm b/yadm index b4e56c1..0918938 100755 --- a/yadm +++ b/yadm @@ -135,31 +135,51 @@ function alt() { require_repo parse_encrypt + local local_class local_class="$(config local.class)" + + local local_system + local_system="$(config local.os)" + if [ -z "$local_system" ] ; then + local_system="$OPERATING_SYSTEM" + fi + + local local_host + local_host="$(config local.hostname)" + if [ -z "$local_host" ] ; then + local_host=$(hostname) + local_host=${local_host%%.*} # trim any domain from hostname + fi + + local local_user + local_user="$(config local.user)" + if [ -z "$local_user" ] ; then + local_user=$(id -u -n) + fi + + if [ "$YADM_COMPATIBILITY" = "1" ]; then + alt_past + else + alt_future + fi + +} + +function alt_future() { + # Future alternate processing, not implemented yet + return +} + +function alt_past() { + if [ -z "$local_class" ] ; then match_class="%" else match_class="$local_class" fi match_class="(%|$match_class)" - - local_system="$(config local.os)" - if [ -z "$local_system" ] ; then - local_system="$OPERATING_SYSTEM" - fi match_system="(%|$local_system)" - - local_host="$(config local.hostname)" - if [ -z "$local_host" ] ; then - local_host=$(hostname) - local_host=${local_host%%.*} # trim any domain from hostname - fi match_host="(%|$local_host)" - - local_user="$(config local.user)" - if [ -z "$local_user" ] ; then - local_user=$(id -u -n) - fi match_user="(%|$local_user)" # regex for matching "##CLASS.SYSTEM.HOSTNAME.USER" From e4e956fe21f18f3c720763553773a89656246f35 Mon Sep 17 00:00:00 2001 From: Tim Byrne Date: Sat, 17 Aug 2019 15:20:56 -0500 Subject: [PATCH 044/137] Split discovery of alternates test data into a testable function --- test/test_unit_set_local_alt_values.py | 62 ++++++++++++++++++++++++++ yadm | 25 +++++++---- 2 files changed, 78 insertions(+), 9 deletions(-) create mode 100644 test/test_unit_set_local_alt_values.py diff --git a/test/test_unit_set_local_alt_values.py b/test/test_unit_set_local_alt_values.py new file mode 100644 index 0000000..9d34484 --- /dev/null +++ b/test/test_unit_set_local_alt_values.py @@ -0,0 +1,62 @@ +"""Unit tests: set_local_alt_values""" +import pytest +import utils + + +@pytest.mark.parametrize( + 'override', [ + False, + 'class', + 'os', + 'hostname', + 'user', + ], + ids=[ + 'no-override', + 'override-class', + 'override-os', + 'override-hostname', + 'override-user', + ] + ) +@pytest.mark.usefixtures('ds1_copy') +def test_set_local_alt_values( + runner, yadm, paths, tst_sys, tst_host, tst_user, override): + """Use issue_legacy_path_warning""" + script = f""" + YADM_TEST=1 source {yadm} && + set_operating_system && + YADM_DIR={paths.yadm} configure_paths && + set_local_alt_values + echo "class='$local_class'" + echo "os='$local_system'" + echo "host='$local_host'" + echo "user='$local_user'" + """ + + if override: + utils.set_local(paths, override, 'override') + + run = runner(command=['bash'], inp=script) + assert run.success + assert run.err == '' + + if override == 'class': + assert "class='override'" in run.out + else: + assert "class=''" in run.out + + if override == 'os': + assert "os='override'" in run.out + else: + assert f"os='{tst_sys}'" in run.out + + if override == 'hostname': + assert f"host='override'" in run.out + else: + assert f"host='{tst_host}'" in run.out + + if override == 'user': + assert f"user='override'" in run.out + else: + assert f"user='{tst_user}'" in run.out diff --git a/yadm b/yadm index 0918938..badf2d4 100755 --- a/yadm +++ b/yadm @@ -135,34 +135,41 @@ function alt() { require_repo parse_encrypt + # gather values for processing alternates local local_class + local local_system + local local_host + local local_user + set_local_alt_values + + if [ "$YADM_COMPATIBILITY" = "1" ]; then + alt_past + else + alt_future + fi + +} + +function set_local_alt_values() { + local_class="$(config local.class)" - local local_system local_system="$(config local.os)" if [ -z "$local_system" ] ; then local_system="$OPERATING_SYSTEM" fi - local local_host local_host="$(config local.hostname)" if [ -z "$local_host" ] ; then local_host=$(hostname) local_host=${local_host%%.*} # trim any domain from hostname fi - local local_user local_user="$(config local.user)" if [ -z "$local_user" ] ; then local_user=$(id -u -n) fi - if [ "$YADM_COMPATIBILITY" = "1" ]; then - alt_past - else - alt_future - fi - } function alt_future() { From f3ae31f1c2d4c13417e1d9451e1129eeb94d513c Mon Sep 17 00:00:00 2001 From: Tim Byrne Date: Sun, 18 Aug 2019 12:51:53 -0500 Subject: [PATCH 045/137] Separate alternate linking code from other operations --- yadm | 110 +++++++++++++++++++++++++++++++---------------------------- 1 file changed, 57 insertions(+), 53 deletions(-) diff --git a/yadm b/yadm index badf2d4..145d249 100755 --- a/yadm +++ b/yadm @@ -142,10 +142,60 @@ function alt() { local local_user set_local_alt_values + # only be noisy if the "alt" command was run directly + local loud= + [ "$YADM_COMMAND" = "alt" ] && loud="YES" + + # decide if a copy should be done instead of a symbolic link + local do_copy=0 + [[ $OPERATING_SYSTEM == CYGWIN* ]] && + [ "$(config --bool yadm.cygwin-copy)" == "true" ] && + do_copy=1 + + cd_work "Alternates" || return + + # determine all tracked files + local tracked_files + tracked_files=() + local IFS=$'\n' + for tracked_file in $("$GIT_PROGRAM" ls-files | LC_ALL=C sort); do + tracked_files+=("$tracked_file") + done + + # generate data for removing stale links + local possible_alts + possible_alts=() + local IFS=$'\n' + for possible_alt in "${tracked_files[@]}" "${ENCRYPT_INCLUDE_FILES[@]}"; do + if [[ $possible_alt =~ .\#\#. ]]; then + possible_alts+=("$YADM_WORK/${possible_alt%##*}") + fi + done + local alt_linked + alt_linked=() + if [ "$YADM_COMPATIBILITY" = "1" ]; then - alt_past + alt_past_linking else - alt_future + alt_future_linking + fi + + # review alternate candidates for stale links + # if a possible alt IS linked, but it's target is not part of alt_linked, + # remove it. + if readlink_available; then + for stale_candidate in "${possible_alts[@]}"; do + if [ -L "$stale_candidate" ]; then + link_target=$(readlink "$stale_candidate" 2>/dev/null) + if [ -n "$link_target" ]; then + removal=yes + for review_link in "${alt_linked[@]}"; do + [ "$link_target" = "$review_link" ] && removal=no + done + [ "$removal" = "yes" ] && rm -f "$stale_candidate" + fi + fi + done fi } @@ -172,12 +222,14 @@ function set_local_alt_values() { } -function alt_future() { - # Future alternate processing, not implemented yet +function alt_future_linking() { + + return + } -function alt_past() { +function alt_past_linking() { if [ -z "$local_class" ] ; then match_class="%" @@ -193,36 +245,6 @@ function alt_past() { match1="^(.+)##(()|$match_system|$match_system\.$match_host|$match_system\.$match_host\.$match_user)$" match2="^(.+)##($match_class|$match_class\.$match_system|$match_class\.$match_system\.$match_host|$match_class\.$match_system\.$match_host\.$match_user)$" - cd_work "Alternates" || return - - # only be noisy if the "alt" command was run directly - [ "$YADM_COMMAND" = "alt" ] && loud="YES" - - # decide if a copy should be done instead of a symbolic link - local do_copy=0 - if [[ $OPERATING_SYSTEM == CYGWIN* ]] ; then - if [[ $(config --bool yadm.cygwin-copy) == "true" ]] ; then - do_copy=1 - fi - fi - - # process the files tracked by yadm once, this info is used multiple times - tracked_files=() - local IFS=$'\n' - for tracked_file in $("$GIT_PROGRAM" ls-files | LC_ALL=C sort); do - tracked_files+=("$tracked_file") - done - - # generate a list of possible alt files - possible_alts=() - local IFS=$'\n' - for possible_alt in "${tracked_files[@]}" "${ENCRYPT_INCLUDE_FILES[@]}"; do - if [[ $possible_alt =~ .\#\#. ]]; then - possible_alts+=("$YADM_WORK/${possible_alt%##*}") - fi - done - alt_linked=() - # loop over all "tracked" files # for every file which matches the above regex, create a symlink for match in $match1 $match2; do @@ -253,24 +275,6 @@ function alt_past() { done done - # review alternate candidates for stale links - # if a possible alt IS linked, but it's target is not part of alt_linked, - # remove it. - if readlink_available; then - for stale_candidate in "${possible_alts[@]}"; do - if [ -L "$stale_candidate" ]; then - link_target=$(readlink "$stale_candidate" 2>/dev/null) - if [ -n "$link_target" ]; then - removal=yes - for review_link in "${alt_linked[@]}"; do - [ "$link_target" = "$review_link" ] && removal=no - done - [ "$removal" = "yes" ] && rm -f "$stale_candidate" - fi - fi - done - fi - # loop over all "tracked" files # for every file which is a *##yadm.j2 create a real file local match="^(.+)##yadm\\.j2$" From cfda485b34459979c35993b8aa0044a564b52218 Mon Sep 17 00:00:00 2001 From: Tim Byrne Date: Tue, 1 Oct 2019 08:12:18 -0500 Subject: [PATCH 046/137] Implement future alternate processing --- test/test_alt.py | 193 +++++++++++++++++ test/test_compat_alt.py | 29 +-- test/test_compat_jinja.py | 18 +- test/test_unit_choose_template_cmd.py | 61 ++++++ test/test_unit_record_score.py | 114 ++++++++++ test/test_unit_record_template.py | 55 +++++ test/test_unit_score_file.py | 229 ++++++++++++++++++++ test/test_unit_set_local_alt_values.py | 15 ++ test/test_unit_template_builtin.py | 98 +++++++++ test/test_unit_template_j2.py | 99 +++++++++ test/utils.py | 19 +- yadm | 276 ++++++++++++++++++++++++- 12 files changed, 1168 insertions(+), 38 deletions(-) create mode 100644 test/test_alt.py create mode 100644 test/test_unit_choose_template_cmd.py create mode 100644 test/test_unit_record_score.py create mode 100644 test/test_unit_record_template.py create mode 100644 test/test_unit_score_file.py create mode 100644 test/test_unit_template_builtin.py create mode 100644 test/test_unit_template_j2.py diff --git a/test/test_alt.py b/test/test_alt.py new file mode 100644 index 0000000..e8a0001 --- /dev/null +++ b/test/test_alt.py @@ -0,0 +1,193 @@ +"""Test alt""" +import os +import string +import py +import pytest +import utils + +TEST_PATHS = [utils.ALT_FILE1, utils.ALT_FILE2, utils.ALT_DIR] + + +@pytest.mark.usefixtures('ds1_copy') +@pytest.mark.parametrize( + 'tracked,encrypt,exclude', [ + (False, False, False), + (True, False, False), + (False, True, False), + (False, True, True), + ], ids=['untracked', 'tracked', 'encrypted', 'excluded']) +def test_alt_source( + runner, yadm_y, paths, + tracked, encrypt, exclude): + """Test yadm alt operates on all expected sources of alternates""" + + utils.create_alt_files( + paths, '##default', tracked=tracked, encrypt=encrypt, exclude=exclude) + run = runner(yadm_y('alt')) + assert run.success + assert run.err == '' + linked = utils.parse_alt_output(run.out) + + for link_path in TEST_PATHS: + source_file = link_path + '##default' + if tracked or (encrypt and not exclude): + assert paths.work.join(link_path).islink() + target = py.path.local(paths.work.join(link_path).readlink()) + if target.isfile(): + assert paths.work.join(link_path).read() == source_file + assert str(paths.work.join(source_file)) in linked + else: + assert paths.work.join(link_path).join( + utils.CONTAINED).read() == source_file + assert str(paths.work.join(source_file)) in linked + else: + assert not paths.work.join(link_path).exists() + assert str(paths.work.join(source_file)) not in linked + + +@pytest.mark.usefixtures('ds1_copy') +@pytest.mark.parametrize('suffix', [ + '##default', + '##o.$tst_sys', '##os.$tst_sys', + '##c.$tst_class', '##class.$tst_class', + '##h.$tst_host', '##hostname.$tst_host', + '##u.$tst_user', '##user.$tst_user', + ]) +def test_alt_conditions( + runner, yadm_y, paths, + tst_sys, tst_host, tst_user, suffix): + """Test conditions supported by yadm alt""" + + # set the class + tst_class = 'testclass' + utils.set_local(paths, 'class', tst_class) + + suffix = string.Template(suffix).substitute( + tst_sys=tst_sys, + tst_class=tst_class, + tst_host=tst_host, + tst_user=tst_user, + ) + + utils.create_alt_files(paths, suffix) + run = runner(yadm_y('alt')) + assert run.success + assert run.err == '' + linked = utils.parse_alt_output(run.out) + + for link_path in TEST_PATHS: + source_file = link_path + suffix + assert paths.work.join(link_path).islink() + target = py.path.local(paths.work.join(link_path).readlink()) + if target.isfile(): + assert paths.work.join(link_path).read() == source_file + assert str(paths.work.join(source_file)) in linked + else: + assert paths.work.join(link_path).join( + utils.CONTAINED).read() == source_file + assert str(paths.work.join(source_file)) in linked + + +@pytest.mark.usefixtures('ds1_copy') +@pytest.mark.parametrize('kind', ['builtin', '', 'envtpl', 'j2cli', 'j2']) +@pytest.mark.parametrize('label', ['t', 'template', 'yadm', ]) +def test_alt_templates( + runner, yadm_y, paths, kind, label): + """Test templates supported by yadm alt""" + + suffix = f'##{label}.{kind}' + utils.create_alt_files(paths, suffix) + run = runner(yadm_y('alt')) + assert run.success + assert run.err == '' + created = utils.parse_alt_output(run.out, linked=False) + + for created_path in TEST_PATHS: + if created_path != utils.ALT_DIR: + source_file = created_path + suffix + assert paths.work.join(created_path).isfile() + assert paths.work.join(created_path).read().strip() == source_file + assert str(paths.work.join(source_file)) in created + + +@pytest.mark.usefixtures('ds1_copy') +@pytest.mark.parametrize('autoalt', [None, 'true', 'false']) +def test_auto_alt(runner, yadm_y, paths, autoalt): + """Test auto alt""" + + # set the value of auto-alt + if autoalt: + os.system(' '.join(yadm_y('config', 'yadm.auto-alt', autoalt))) + + utils.create_alt_files(paths, '##default') + run = runner(yadm_y('status')) + assert run.success + assert run.err == '' + linked = utils.parse_alt_output(run.out) + + for link_path in TEST_PATHS: + source_file = link_path + '##default' + if autoalt == 'false': + assert not paths.work.join(link_path).exists() + else: + assert paths.work.join(link_path).islink() + target = py.path.local(paths.work.join(link_path).readlink()) + if target.isfile(): + assert paths.work.join(link_path).read() == source_file + # no linking output when run via auto-alt + assert str(paths.work.join(source_file)) not in linked + else: + assert paths.work.join(link_path).join( + utils.CONTAINED).read() == source_file + # no linking output when run via auto-alt + assert str(paths.work.join(source_file)) not in linked + + +@pytest.mark.usefixtures('ds1_copy') +def test_stale_link_removal(runner, yadm_y, paths): + """Stale links to alternative files are removed + + This test ensures that when an already linked alternative becomes invalid + due to a change in class, the alternate link is removed. + """ + + # set the class + tst_class = 'testclass' + utils.set_local(paths, 'class', tst_class) + + # create files which match the test class + utils.create_alt_files(paths, f'##class.{tst_class}') + + # run alt to trigger linking + run = runner(yadm_y('alt')) + assert run.success + assert run.err == '' + linked = utils.parse_alt_output(run.out) + + # assert the proper linking has occurred + for stale_path in TEST_PATHS: + source_file = stale_path + '##class.' + tst_class + assert paths.work.join(stale_path).islink() + target = py.path.local(paths.work.join(stale_path).readlink()) + if target.isfile(): + assert paths.work.join(stale_path).read() == source_file + assert str(paths.work.join(source_file)) in linked + else: + assert paths.work.join(stale_path).join( + utils.CONTAINED).read() == source_file + assert str(paths.work.join(source_file)) in linked + + # change the class so there are no valid alternates + utils.set_local(paths, 'class', 'changedclass') + + # run alt to trigger linking + run = runner(yadm_y('alt')) + assert run.success + assert run.err == '' + linked = utils.parse_alt_output(run.out) + + # assert the linking is removed + for stale_path in TEST_PATHS: + source_file = stale_path + '##class.' + tst_class + assert not paths.work.join(stale_path).exists() + assert str(paths.work.join(source_file)) not in linked diff --git a/test/test_compat_alt.py b/test/test_compat_alt.py index 466e7fc..6321252 100644 --- a/test/test_compat_alt.py +++ b/test/test_compat_alt.py @@ -1,7 +1,6 @@ """Test alt""" import os -import re import string import py import pytest @@ -92,7 +91,7 @@ def test_alt(runner, yadm_y, paths, run = runner(yadm_y('alt'), env=env) assert run.success assert run.err == '' - linked = linked_list(run.out) + linked = utils.parse_alt_output(run.out) # assert the proper linking has occurred for file_path in TEST_PATHS: @@ -195,7 +194,7 @@ def test_wild(request, runner, yadm_y, paths, run = runner(yadm_y('alt'), env=env) assert run.success assert run.err == '' - linked = linked_list(run.out) + linked = utils.parse_alt_output(run.out) # assert the proper linking has occurred for file_path in TEST_PATHS: @@ -220,7 +219,7 @@ def test_wild(request, runner, yadm_y, paths, run = runner(yadm_y('alt'), env=env) assert run.success assert run.err == '' - linked = linked_list(run.out) + linked = utils.parse_alt_output(run.out) # assert the proper linking has occurred for file_path in TEST_PATHS: @@ -260,7 +259,7 @@ def test_local_override(runner, yadm_y, paths, run = runner(yadm_y('alt'), env=env) assert run.success assert run.err == '' - linked = linked_list(run.out) + linked = utils.parse_alt_output(run.out) # assert the proper linking has occurred for file_path in TEST_PATHS: @@ -300,7 +299,7 @@ def test_class_case(runner, yadm_y, paths, tst_sys, suffix): run = runner(yadm_y('alt'), env=env) assert run.success assert run.err == '' - linked = linked_list(run.out) + linked = utils.parse_alt_output(run.out) # assert the proper linking has occurred for file_path in TEST_PATHS: @@ -335,7 +334,7 @@ def test_auto_alt(runner, yadm_y, paths, autoalt): run = runner(yadm_y('status'), env=env) assert run.success assert run.err == '' - linked = linked_list(run.out) + linked = utils.parse_alt_output(run.out) # assert the proper linking has occurred for file_path in TEST_PATHS: @@ -373,7 +372,7 @@ def test_delimiter(runner, yadm_y, paths, run = runner(yadm_y('alt'), env=env) assert run.success assert run.err == '' - linked = linked_list(run.out) + linked = utils.parse_alt_output(run.out) # assert the proper linking has occurred # only a delimiter of '.' is valid @@ -415,7 +414,7 @@ def test_invalid_links_removed(runner, yadm_y, paths): run = runner(yadm_y('alt'), env=env) assert run.success assert run.err == '' - linked = linked_list(run.out) + linked = utils.parse_alt_output(run.out) # assert the proper linking has occurred for file_path in TEST_PATHS: @@ -439,20 +438,10 @@ def test_invalid_links_removed(runner, yadm_y, paths): run = runner(yadm_y('alt'), env=env) assert run.success assert run.err == '' - linked = linked_list(run.out) + linked = utils.parse_alt_output(run.out) # assert the linking is removed for file_path in TEST_PATHS: source_file = file_path + '##' + tst_class assert not paths.work.join(file_path).exists() assert str(paths.work.join(source_file)) not in linked - - -def linked_list(output): - """Parse output, and return list of linked files""" - linked = dict() - for line in output.splitlines(): - match = re.match('Linking (.+) to (.+)$', line) - if match: - linked[match.group(2)] = match.group(1) - return linked.values() diff --git a/test/test_compat_jinja.py b/test/test_compat_jinja.py index 342d202..7d3c1aa 100644 --- a/test/test_compat_jinja.py +++ b/test/test_compat_jinja.py @@ -1,7 +1,6 @@ """Test jinja""" import os -import re import pytest import utils @@ -55,7 +54,7 @@ def test_local_override(runner, yadm_y, paths, run = runner(yadm_y('alt'), env=env) assert run.success assert run.err == '' - created = created_list(run.out) + created = utils.parse_alt_output(run.out, linked=False) # assert the proper creation has occurred for file_path in (utils.ALT_FILE1, utils.ALT_FILE2): @@ -90,7 +89,7 @@ def test_auto_alt(runner, yadm_y, paths, autoalt, tst_sys, run = runner(yadm_y('status'), env=env) assert run.success assert run.err == '' - created = created_list(run.out) + created = utils.parse_alt_output(run.out, linked=False) # assert the proper creation has occurred for file_path in (utils.ALT_FILE1, utils.ALT_FILE2): @@ -180,7 +179,7 @@ def test_jinja(runner, yadm_y, paths, run = runner(yadm_y('alt'), env=env) assert run.success assert run.err == '' - created = created_list(run.out) + created = utils.parse_alt_output(run.out, linked=False) # assert the proper creation has occurred for file_path in (utils.ALT_FILE1, utils.ALT_FILE2): @@ -194,14 +193,3 @@ def test_jinja(runner, yadm_y, paths, else: assert not paths.work.join(file_path).exists() assert str(paths.work.join(source_file)) not in created - - -def created_list(output): - """Parse output, and return list of created files""" - - created = dict() - for line in output.splitlines(): - match = re.match('Creating (.+) from template (.+)$', line) - if match: - created[match.group(1)] = match.group(2) - return created.values() diff --git a/test/test_unit_choose_template_cmd.py b/test/test_unit_choose_template_cmd.py new file mode 100644 index 0000000..b592331 --- /dev/null +++ b/test/test_unit_choose_template_cmd.py @@ -0,0 +1,61 @@ +"""Unit tests: choose_template_cmd""" +import pytest + + +@pytest.mark.parametrize('label', ['', 'builtin', 'other']) +@pytest.mark.parametrize('awk', [True, False], ids=['awk', 'no-awk']) +def test_kind_builtin(runner, yadm, awk, label): + """Test kind: builtin""" + + expected = 'template_builtin' + awk_avail = 'true' + + if not awk: + awk_avail = 'false' + expected = '' + + if label == 'other': + expected = '' + + script = f""" + YADM_TEST=1 source {yadm} + function awk_available {{ { awk_avail}; }} + template="$(choose_template_cmd "{label}")" + echo "TEMPLATE:$template" + """ + run = runner(command=['bash'], inp=script) + assert run.success + assert run.err == '' + assert f'TEMPLATE:{expected}\n' in run.out + + +@pytest.mark.parametrize('label', ['envtpl', 'j2cli', 'j2', 'other']) +@pytest.mark.parametrize('envtpl', [True, False], ids=['envtpl', 'no-envtpl']) +@pytest.mark.parametrize('j2cli', [True, False], ids=['j2cli', 'no-j2cli']) +def test_kind_j2cli_envtpl(runner, yadm, envtpl, j2cli, label): + """Test kind: j2 (both j2cli & envtpl) + + j2cli is preferred over envtpl if available. + """ + + envtpl_avail = 'true' if envtpl else 'false' + j2cli_avail = 'true' if j2cli else 'false' + + if label in ('j2cli', 'j2') and j2cli: + expected = 'template_j2cli' + elif label in ('envtpl', 'j2') and envtpl: + expected = 'template_envtpl' + else: + expected = '' + + script = f""" + YADM_TEST=1 source {yadm} + function envtpl_available {{ { envtpl_avail}; }} + function j2cli_available {{ { j2cli_avail}; }} + template="$(choose_template_cmd "{label}")" + echo "TEMPLATE:$template" + """ + run = runner(command=['bash'], inp=script) + assert run.success + assert run.err == '' + assert f'TEMPLATE:{expected}\n' in run.out diff --git a/test/test_unit_record_score.py b/test/test_unit_record_score.py new file mode 100644 index 0000000..3bb1b25 --- /dev/null +++ b/test/test_unit_record_score.py @@ -0,0 +1,114 @@ +"""Unit tests: record_score""" +import pytest + +INIT_VARS = """ + score=0 + local_class=testclass + local_system=testsystem + local_host=testhost + local_user=testuser + alt_scores=() + alt_filenames=() + alt_targets=() + alt_template_cmds=() +""" + +REPORT_RESULTS = """ + echo "SIZE:${#alt_scores[@]}" + echo "SCORES:${alt_scores[@]}" + echo "FILENAMES:${alt_filenames[@]}" + echo "TARGETS:${alt_targets[@]}" +""" + + +def test_dont_record_zeros(runner, yadm): + """Record nothing if the score is zero""" + + script = f""" + YADM_TEST=1 source {yadm} + {INIT_VARS} + record_score "0" "testfile" "testtarget" + {REPORT_RESULTS} + """ + run = runner(command=['bash'], inp=script) + assert run.success + assert run.err == '' + assert 'SIZE:0\n' in run.out + assert 'SCORES:\n' in run.out + assert 'FILENAMES:\n' in run.out + assert 'TARGETS:\n' in run.out + + +def test_new_scores(runner, yadm): + """Test new scores""" + + script = f""" + YADM_TEST=1 source {yadm} + {INIT_VARS} + record_score "1" "file_one" "targ_one" + record_score "2" "file_two" "targ_two" + record_score "4" "file_three" "targ_three" + {REPORT_RESULTS} + """ + run = runner(command=['bash'], inp=script) + assert run.success + assert run.err == '' + assert 'SIZE:3\n' in run.out + assert 'SCORES:1 2 4\n' in run.out + assert 'FILENAMES:file_one file_two file_three\n' in run.out + assert 'TARGETS:targ_one targ_two targ_three\n' in run.out + + +@pytest.mark.parametrize('difference', ['lower', 'equal', 'higher']) +def test_existing_scores(runner, yadm, difference): + """Test existing scores""" + + expected_score = '2' + expected_target = 'existing_target' + if difference == 'lower': + score = '1' + elif difference == 'equal': + score = '2' + else: + score = '4' + expected_score = '4' + expected_target = 'new_target' + + script = f""" + YADM_TEST=1 source {yadm} + {INIT_VARS} + alt_scores=(2) + alt_filenames=("testfile") + alt_targets=("existing_target") + record_score "{score}" "testfile" "new_target" + {REPORT_RESULTS} + """ + run = runner(command=['bash'], inp=script) + assert run.success + assert run.err == '' + assert 'SIZE:1\n' in run.out + assert f'SCORES:{expected_score}\n' in run.out + assert 'FILENAMES:testfile\n' in run.out + assert f'TARGETS:{expected_target}\n' in run.out + + +def test_existing_template(runner, yadm): + """Record nothing if a template command is registered for this file""" + + script = f""" + YADM_TEST=1 source {yadm} + {INIT_VARS} + alt_scores=(1) + alt_filenames=("testfile") + alt_targets=() + alt_template_cmds=("existing_template") + record_score "2" "testfile" "new_target" + {REPORT_RESULTS} + """ + run = runner(command=['bash'], inp=script) + assert run.success + assert run.err == '' + assert 'SIZE:1\n' in run.out + assert 'SCORES:1\n' in run.out + assert 'FILENAMES:testfile\n' in run.out + assert 'TARGETS:\n' in run.out diff --git a/test/test_unit_record_template.py b/test/test_unit_record_template.py new file mode 100644 index 0000000..f2e0a83 --- /dev/null +++ b/test/test_unit_record_template.py @@ -0,0 +1,55 @@ +"""Unit tests: record_template""" + +INIT_VARS = """ + alt_filenames=() + alt_template_cmds=() + alt_targets=() +""" + +REPORT_RESULTS = """ + echo "SIZE:${#alt_filenames[@]}" + echo "FILENAMES:${alt_filenames[@]}" + echo "CMDS:${alt_template_cmds[@]}" + echo "TARGS:${alt_targets[@]}" +""" + + +def test_new_template(runner, yadm): + """Test new template""" + + script = f""" + YADM_TEST=1 source {yadm} + {INIT_VARS} + record_template "file_one" "cmd_one" "targ_one" + record_template "file_two" "cmd_two" "targ_two" + record_template "file_three" "cmd_three" "targ_three" + {REPORT_RESULTS} + """ + run = runner(command=['bash'], inp=script) + assert run.success + assert run.err == '' + assert 'SIZE:3\n' in run.out + assert 'FILENAMES:file_one file_two file_three\n' in run.out + assert 'CMDS:cmd_one cmd_two cmd_three\n' in run.out + assert 'TARGS:targ_one targ_two targ_three\n' in run.out + + +def test_existing_template(runner, yadm): + """Overwrite existing templates""" + + script = f""" + YADM_TEST=1 source {yadm} + {INIT_VARS} + alt_filenames=("testfile") + alt_template_cmds=("existing_cmd") + alt_targets=("existing_targ") + record_template "testfile" "new_cmd" "new_targ" + {REPORT_RESULTS} + """ + run = runner(command=['bash'], inp=script) + assert run.success + assert run.err == '' + assert 'SIZE:1\n' in run.out + assert 'FILENAMES:testfile\n' in run.out + assert 'CMDS:new_cmd\n' in run.out + assert 'TARGS:new_targ\n' in run.out diff --git a/test/test_unit_score_file.py b/test/test_unit_score_file.py new file mode 100644 index 0000000..122d580 --- /dev/null +++ b/test/test_unit_score_file.py @@ -0,0 +1,229 @@ +"""Unit tests: score_file""" +import pytest + +CONDITION = { + 'default': { + 'labels': ['default'], + 'modifier': 0, + }, + 'system': { + 'labels': ['o', 'os'], + 'modifier': 1, + }, + 'class': { + 'labels': ['c', 'class'], + 'modifier': 2, + }, + 'hostname': { + 'labels': ['h', 'hostname'], + 'modifier': 4, + }, + 'user': { + 'labels': ['u', 'user'], + 'modifier': 8, + }, + } +TEMPLATE_LABELS = ['t', 'template', 'yadm'] + + +def calculate_score(filename): + """Calculate the expected score""" + # pylint: disable=too-many-branches + score = 0 + + _, conditions = filename.split('##', 1) + + for condition in conditions.split(','): + label = condition + value = None + if '.' in condition: + label, value = condition.split('.', 1) + if label in CONDITION['default']['labels']: + score += 1000 + elif label in CONDITION['system']['labels']: + if value == 'testsystem': + score += 1000 + CONDITION['system']['modifier'] + else: + return 0 + elif label in CONDITION['class']['labels']: + if value == 'testclass': + score += 1000 + CONDITION['class']['modifier'] + else: + return 0 + elif label in CONDITION['hostname']['labels']: + if value == 'testhost': + score += 1000 + CONDITION['hostname']['modifier'] + else: + return 0 + elif label in CONDITION['user']['labels']: + if value == 'testuser': + score += 1000 + CONDITION['user']['modifier'] + else: + return 0 + elif label in TEMPLATE_LABELS: + return 0 + return score + + +@pytest.mark.parametrize( + 'default', ['default', None], ids=['default', 'no-default']) +@pytest.mark.parametrize( + 'system', ['system', None], ids=['system', 'no-system']) +@pytest.mark.parametrize( + 'cla', ['class', None], ids=['class', 'no-class']) +@pytest.mark.parametrize( + 'host', ['hostname', None], ids=['hostname', 'no-host']) +@pytest.mark.parametrize( + 'user', ['user', None], ids=['user', 'no-user']) +def test_score_values( + runner, yadm, default, system, cla, host, user): + """Test score results""" + # pylint: disable=too-many-branches + local_class = 'testclass' + local_system = 'testsystem' + local_host = 'testhost' + local_user = 'testuser' + filenames = {'filename##': 0} + + if default: + for filename in list(filenames): + for label in CONDITION[default]['labels']: + newfile = filename + if not newfile.endswith('##'): + newfile += ',' + newfile += label + filenames[newfile] = calculate_score(newfile) + if system: + for filename in list(filenames): + for match in [True, False]: + for label in CONDITION[system]['labels']: + newfile = filename + if not newfile.endswith('##'): + newfile += ',' + newfile += '.'.join([ + label, + local_system if match else 'badsys' + ]) + filenames[newfile] = calculate_score(newfile) + if cla: + for filename in list(filenames): + for match in [True, False]: + for label in CONDITION[cla]['labels']: + newfile = filename + if not newfile.endswith('##'): + newfile += ',' + newfile += '.'.join([ + label, + local_class if match else 'badclass' + ]) + filenames[newfile] = calculate_score(newfile) + if host: + for filename in list(filenames): + for match in [True, False]: + for label in CONDITION[host]['labels']: + newfile = filename + if not newfile.endswith('##'): + newfile += ',' + newfile += '.'.join([ + label, + local_host if match else 'badhost' + ]) + filenames[newfile] = calculate_score(newfile) + if user: + for filename in list(filenames): + for match in [True, False]: + for label in CONDITION[user]['labels']: + newfile = filename + if not newfile.endswith('##'): + newfile += ',' + newfile += '.'.join([ + label, + local_user if match else 'baduser' + ]) + filenames[newfile] = calculate_score(newfile) + + script = f""" + YADM_TEST=1 source {yadm} + score=0 + local_class={local_class} + local_system={local_system} + local_host={local_host} + local_user={local_user} + """ + expected = '' + for filename in filenames: + script += f""" + score_file "{filename}" + echo "{filename}" + echo "$score" + """ + expected += filename + '\n' + expected += str(filenames[filename]) + '\n' + run = runner(command=['bash'], inp=script) + assert run.success + assert run.err == '' + assert run.out == expected + + +def test_score_values_templates(runner, yadm): + """Test score results""" + local_class = 'testclass' + local_system = 'testsystem' + local_host = 'testhost' + local_user = 'testuser' + filenames = {'filename##': 0} + + for filename in list(filenames): + for label in TEMPLATE_LABELS: + newfile = filename + if not newfile.endswith('##'): + newfile += ',' + newfile += '.'.join([label, 'testtemplate']) + filenames[newfile] = calculate_score(newfile) + + script = f""" + YADM_TEST=1 source {yadm} + score=0 + local_class={local_class} + local_system={local_system} + local_host={local_host} + local_user={local_user} + """ + expected = '' + for filename in filenames: + script += f""" + score_file "{filename}" + echo "{filename}" + echo "$score" + """ + expected += filename + '\n' + expected += str(filenames[filename]) + '\n' + run = runner(command=['bash'], inp=script) + assert run.success + assert run.err == '' + assert run.out == expected + + +@pytest.mark.parametrize( + 'cmd_generated', + [True, False], + ids=['supported-template', 'unsupported-template']) +def test_template_recording(runner, yadm, cmd_generated): + """Template should be recorded if choose_template_cmd outputs a command""" + + mock = 'function choose_template_cmd() { return; }' + expected = '' + if cmd_generated: + mock = 'function choose_template_cmd() { echo "test_cmd"; }' + expected = 'template recorded' + + script = f""" + YADM_TEST=1 source {yadm} + function record_template() {{ echo "template recorded"; }} + {mock} + score_file "testfile##template.kind" + """ + run = runner(command=['bash'], inp=script) + assert run.success + assert run.err == '' + assert run.out.rstrip() == expected diff --git a/test/test_unit_set_local_alt_values.py b/test/test_unit_set_local_alt_values.py index 9d34484..d3d2447 100644 --- a/test/test_unit_set_local_alt_values.py +++ b/test/test_unit_set_local_alt_values.py @@ -60,3 +60,18 @@ def test_set_local_alt_values( assert f"user='override'" in run.out else: assert f"user='{tst_user}'" in run.out + + +def test_distro(runner, yadm): + """Assert that local_distro is set""" + + script = f""" + YADM_TEST=1 source {yadm} + function query_distro() {{ echo "testdistro"; }} + set_local_alt_values + echo "distro='$local_distro'" + """ + run = runner(command=['bash'], inp=script) + assert run.success + assert run.err == '' + assert run.out.strip() == "distro='testdistro'" diff --git a/test/test_unit_template_builtin.py b/test/test_unit_template_builtin.py new file mode 100644 index 0000000..ac141e4 --- /dev/null +++ b/test/test_unit_template_builtin.py @@ -0,0 +1,98 @@ +"""Unit tests: template_builtin""" + +# these values are also testing the handling of bizarre characters +LOCAL_CLASS = "builtin_Test+@-!^Class" +LOCAL_SYSTEM = "builtin_Test+@-!^System" +LOCAL_HOST = "builtin_Test+@-!^Host" +LOCAL_USER = "builtin_Test+@-!^User" +LOCAL_DISTRO = "builtin_Test+@-!^Distro" +TEMPLATE = f''' +start of template +builtin class = >YADM_CLASS< +builtin os = >YADM_OS< +builtin host = >YADM_HOSTNAME< +builtin user = >YADM_USER< +builtin distro = >YADM_DISTRO< +YADM_IF CLASS="wrongclass1" +wrong class 1 +YADM_END +YADM_IF CLASS="{LOCAL_CLASS}" +Included section for class = YADM_CLASS (YADM_CLASS repeated) +YADM_END +YADM_IF CLASS="wrongclass2" +wrong class 2 +YADM_END +YADM_IF OS="wrongos1" +wrong os 1 +YADM_END +YADM_IF OS="{LOCAL_SYSTEM}" +Included section for os = YADM_OS (YADM_OS repeated) +YADM_END +YADM_IF OS="wrongos2" +wrong os 2 +YADM_END +YADM_IF HOSTNAME="wronghost1" +wrong host 1 +YADM_END +YADM_IF HOSTNAME="{LOCAL_HOST}" +Included section for host = YADM_HOSTNAME (YADM_HOSTNAME repeated) +YADM_END +YADM_IF HOSTNAME="wronghost2" +wrong host 2 +YADM_END +YADM_IF USER="wronguser1" +wrong user 1 +YADM_END +YADM_IF USER="{LOCAL_USER}" +Included section for user = YADM_USER (YADM_USER repeated) +YADM_END +YADM_IF USER="wronguser2" +wrong user 2 +YADM_END +YADM_IF DISTRO="wrongdistro1" +wrong distro 1 +YADM_END +YADM_IF DISTRO="{LOCAL_DISTRO}" +Included section for distro = YADM_DISTRO (YADM_DISTRO repeated) +YADM_END +YADM_IF DISTRO="wrongdistro2" +wrong distro 2 +YADM_END +end of template +''' +EXPECTED = f''' +start of template +builtin class = >{LOCAL_CLASS}< +builtin os = >{LOCAL_SYSTEM}< +builtin host = >{LOCAL_HOST}< +builtin user = >{LOCAL_USER}< +builtin distro = >{LOCAL_DISTRO}< +Included section for class = {LOCAL_CLASS} ({LOCAL_CLASS} repeated) +Included section for os = {LOCAL_SYSTEM} ({LOCAL_SYSTEM} repeated) +Included section for host = {LOCAL_HOST} ({LOCAL_HOST} repeated) +Included section for user = {LOCAL_USER} ({LOCAL_USER} repeated) +Included section for distro = {LOCAL_DISTRO} ({LOCAL_DISTRO} repeated) +end of template +''' + + +def test_template_builtin(runner, yadm, tmpdir): + """Test template_builtin""" + + input_file = tmpdir.join('input') + input_file.write(TEMPLATE, ensure=True) + output_file = tmpdir.join('output') + + script = f""" + YADM_TEST=1 source {yadm} + local_class="{LOCAL_CLASS}" + local_system="{LOCAL_SYSTEM}" + local_host="{LOCAL_HOST}" + local_user="{LOCAL_USER}" + local_distro="{LOCAL_DISTRO}" + template_builtin "{input_file}" "{output_file}" + """ + run = runner(command=['bash'], inp=script) + assert run.success + assert run.err == '' + assert output_file.read() == EXPECTED diff --git a/test/test_unit_template_j2.py b/test/test_unit_template_j2.py new file mode 100644 index 0000000..e511576 --- /dev/null +++ b/test/test_unit_template_j2.py @@ -0,0 +1,99 @@ +"""Unit tests: template_j2cli & template_envtpl""" +import pytest + +LOCAL_CLASS = "j2_Test+@-!^Class" +LOCAL_SYSTEM = "j2_Test+@-!^System" +LOCAL_HOST = "j2_Test+@-!^Host" +LOCAL_USER = "j2_Test+@-!^User" +LOCAL_DISTRO = "j2_Test+@-!^Distro" +TEMPLATE = f''' +start of template +j2 class = >{{{{YADM_CLASS}}}}< +j2 os = >{{{{YADM_OS}}}}< +j2 host = >{{{{YADM_HOSTNAME}}}}< +j2 user = >{{{{YADM_USER}}}}< +j2 distro = >{{{{YADM_DISTRO}}}}< +{{%- if YADM_CLASS == "wrongclass1" %}} +wrong class 1 +{{%- endif %}} +{{%- if YADM_CLASS == "{LOCAL_CLASS}" %}} +Included section for class = {{{{YADM_CLASS}}}} ({{{{YADM_CLASS}}}} repeated) +{{%- endif %}} +{{%- if YADM_CLASS == "wrongclass2" %}} +wrong class 2 +{{%- endif %}} +{{%- if YADM_OS == "wrongos1" %}} +wrong os 1 +{{%- endif %}} +{{%- if YADM_OS == "{LOCAL_SYSTEM}" %}} +Included section for os = {{{{YADM_OS}}}} ({{{{YADM_OS}}}} repeated) +{{%- endif %}} +{{%- if YADM_OS == "wrongos2" %}} +wrong os 2 +{{%- endif %}} +{{%- if YADM_HOSTNAME == "wronghost1" %}} +wrong host 1 +{{%- endif %}} +{{%- if YADM_HOSTNAME == "{LOCAL_HOST}" %}} +Included section for host = {{{{YADM_HOSTNAME}}}} ({{{{YADM_HOSTNAME}}}} again) +{{%- endif %}} +{{%- if YADM_HOSTNAME == "wronghost2" %}} +wrong host 2 +{{%- endif %}} +{{%- if YADM_USER == "wronguser1" %}} +wrong user 1 +{{%- endif %}} +{{%- if YADM_USER == "{LOCAL_USER}" %}} +Included section for user = {{{{YADM_USER}}}} ({{{{YADM_USER}}}} repeated) +{{%- endif %}} +{{%- if YADM_USER == "wronguser2" %}} +wrong user 2 +{{%- endif %}} +{{%- if YADM_DISTRO == "wrongdistro1" %}} +wrong distro 1 +{{%- endif %}} +{{%- if YADM_DISTRO == "{LOCAL_DISTRO}" %}} +Included section for distro = {{{{YADM_DISTRO}}}} ({{{{YADM_DISTRO}}}} again) +{{%- endif %}} +{{%- if YADM_DISTRO == "wrongdistro2" %}} +wrong distro 2 +{{%- endif %}} +end of template +''' +EXPECTED = f''' +start of template +j2 class = >{LOCAL_CLASS}< +j2 os = >{LOCAL_SYSTEM}< +j2 host = >{LOCAL_HOST}< +j2 user = >{LOCAL_USER}< +j2 distro = >{LOCAL_DISTRO}< +Included section for class = {LOCAL_CLASS} ({LOCAL_CLASS} repeated) +Included section for os = {LOCAL_SYSTEM} ({LOCAL_SYSTEM} repeated) +Included section for host = {LOCAL_HOST} ({LOCAL_HOST} again) +Included section for user = {LOCAL_USER} ({LOCAL_USER} repeated) +Included section for distro = {LOCAL_DISTRO} ({LOCAL_DISTRO} again) +end of template +''' + + +@pytest.mark.parametrize('processor', ('j2cli', 'envtpl')) +def test_template_j2(runner, yadm, tmpdir, processor): + """Test processing by j2cli & envtpl""" + + input_file = tmpdir.join('input') + input_file.write(TEMPLATE, ensure=True) + output_file = tmpdir.join('output') + + script = f""" + YADM_TEST=1 source {yadm} + local_class="{LOCAL_CLASS}" + local_system="{LOCAL_SYSTEM}" + local_host="{LOCAL_HOST}" + local_user="{LOCAL_USER}" + local_distro="{LOCAL_DISTRO}" + template_{processor} "{input_file}" "{output_file}" + """ + run = runner(command=['bash'], inp=script) + assert run.success + assert run.err == '' + assert output_file.read() == EXPECTED diff --git a/test/utils.py b/test/utils.py index 0b52b37..00a0fdd 100644 --- a/test/utils.py +++ b/test/utils.py @@ -3,6 +3,7 @@ This module holds values/functions common to multiple tests. """ +import re import os ALT_FILE1 = 'test_alt' @@ -55,7 +56,7 @@ def create_alt_files(paths, suffix, # Do not test directory support for jinja alternates test_paths = [new_file1, new_file2] test_names = [ALT_FILE1, ALT_FILE2] - if suffix != '##yadm.j2': + if not re.match(r'##(t|template|yadm)\.', suffix): test_paths += [new_dir] test_names += [ALT_DIR] @@ -69,6 +70,22 @@ def create_alt_files(paths, suffix, _create_encrypt(encrypt, test_names, suffix, paths, exclude) +def parse_alt_output(output, linked=True): + """Parse output of 'alt', and return list of linked files""" + regex = r'Creating (.+) from template (.+)$' + if linked: + regex = r'Linking (.+) to (.+)$' + parsed_list = dict() + for line in output.splitlines(): + match = re.match(regex, line) + if match: + if linked: + parsed_list[match.group(2)] = match.group(1) + else: + parsed_list[match.group(1)] = match.group(2) + return parsed_list.values() + + def _create_includefiles(includefile, paths, test_paths): if includefile: for dpath in INCLUDE_DIRS: diff --git a/yadm b/yadm index 145d249..f7cab19 100755 --- a/yadm +++ b/yadm @@ -39,6 +39,8 @@ FULL_COMMAND="" GPG_PROGRAM="gpg" GIT_PROGRAM="git" +AWK_PROGRAM="awk" +J2CLI_PROGRAM="j2" ENVTPL_PROGRAM="envtpl" LSB_RELEASE_PROGRAM="lsb_release" @@ -128,6 +130,225 @@ function main() { } + +# ****** Alternate Processing ****** + +function score_file() { + target="$1" + filename="${target%%##*}" + conditions="${target#*##}" + score=0 + IFS=',' read -ra fields <<< "$conditions" + for field in "${fields[@]}"; do + label=${field%%.*} + value=${field#*.} + score=$((score + 1000)) + # default condition + if [[ "$label" =~ ^(default)$ ]]; then + score=$((score + 0)) + # variable conditions + elif [[ "$label" =~ ^(o|os)$ ]]; then + if [ "$value" = "$local_system" ]; then + score=$((score + 1)) + else + score=0 + return + fi + elif [[ "$label" =~ ^(c|class)$ ]]; then + if [ "$value" = "$local_class" ]; then + score=$((score + 2)) + else + score=0 + return + fi + elif [[ "$label" =~ ^(h|hostname)$ ]]; then + if [ "$value" = "$local_host" ]; then + score=$((score + 4)) + else + score=0 + return + fi + elif [[ "$label" =~ ^(u|user)$ ]]; then + if [ "$value" = "$local_user" ]; then + score=$((score + 8)) + else + score=0 + return + fi + # templates + elif [[ "$label" =~ ^(t|template|yadm)$ ]]; then + score=0 + cmd=$(choose_template_cmd "$value") + if [ -n "$cmd" ]; then + record_template "$filename" "$cmd" "$target" + else + debug "No supported template processor for template $target" + [ -n "$loud" ] && echo "No supported template processor for template $target" + fi + return 0 + # unsupported values + else + score=0 + return + fi + done + + record_score "$score" "$filename" "$target" +} + +function record_score() { + score="$1" + filename="$2" + target="$3" + + # record nothing if the score is zero + [ "$score" -eq 0 ] && return + + # search for the index of this filename, to see if we already are tracking it + index=-1 + for search_index in "${!alt_filenames[@]}"; do + if [ "${alt_filenames[$search_index]}" = "$filename" ]; then + index="$search_index" + break + fi + done + # if we don't find an existing index, create one by appending to the array + if [ "$index" -eq -1 ]; then + alt_filenames+=("$filename") + # set index to the last index (newly created one) + for index in "${!alt_filenames[@]}"; do :; done + # and set its initial score to zero + alt_scores[$index]=0 + fi + + # record nothing if a template command is registered for this file + [ "${alt_template_cmds[$index]+isset}" ] && return + + # record higher scoring targets + if [ "$score" -gt "${alt_scores[$index]}" ]; then + alt_scores[$index]="$score" + alt_targets[$index]="$target" + fi + +} + +function record_template() { + filename="$1" + cmd="$2" + target="$3" + + # search for the index of this filename, to see if we already are tracking it + index=-1 + for search_index in "${!alt_filenames[@]}"; do + if [ "${alt_filenames[$search_index]}" = "$filename" ]; then + index="$search_index" + break + fi + done + # if we don't find an existing index, create one by appending to the array + if [ "$index" -eq -1 ]; then + alt_filenames+=("$filename") + # set index to the last index (newly created one) + for index in "${!alt_filenames[@]}"; do :; done + fi + + # record the template command, last one wins + alt_template_cmds[$index]="$cmd" + alt_targets[$index]="$target" + +} + +function choose_template_cmd() { + kind="$1" + + if [ "$kind" = "builtin" ] || [ "$kind" = "" ] && awk_available; then + echo "template_builtin" + elif [ "$kind" = "j2cli" ] || [ "$kind" = "j2" ] && j2cli_available; then + echo "template_j2cli" + elif [ "$kind" = "envtpl" ] || [ "$kind" = "j2" ] && envtpl_available; then + echo "template_envtpl" + else + return # this "kind" of template is not supported + fi + +} + +# ****** Template Processors ****** + +function template_builtin() { + input="$1" + output="$2" + + awk_pgm=$(cat << "EOF" +# built-in template processor +BEGIN { + c["CLASS"] = class + c["OS"] = os + c["HOSTNAME"] = host + c["USER"] = user + c["DISTRO"] = distro + valid = conditions() + end = "^YADM_END$" + skip = "^YADM_(IF|END$)" +} +{ replace_vars() } # variable replacements +$0 ~ valid, $0 ~ end { if ($0 ~ skip) next } # valid conditional blocks +/^YADM_IF/, $0 ~ end { next } # invalid conditional blocks +{ print } + +function replace_vars() { + for (label in c) { + gsub(("YADM_" label), c[label]) + } +} + +function conditions() { + pattern = "^(" + for (label in c) { + value = c[label] + gsub(/[\\.^$(){}\[\]|*+?]/, "\\\\&", value) + pattern = sprintf("%sYADM_IF +%s *= *\"%s\"|", pattern, label, value) + } + sub(/\|$/,")",pattern) + return pattern +} +EOF + ) + + "$AWK_PROGRAM" \ + -v class="$local_class" \ + -v os="$local_system" \ + -v host="$local_host" \ + -v user="$local_user" \ + -v distro="$local_distro" \ + "$awk_pgm" \ + "$input" > "$output" +} + +function template_j2cli() { + input="$1" + output="$2" + + YADM_CLASS="$local_class" \ + YADM_OS="$local_system" \ + YADM_HOSTNAME="$local_host" \ + YADM_USER="$local_user" \ + YADM_DISTRO="$local_distro" \ + "$J2CLI_PROGRAM" "$input" -o "$output" +} + +function template_envtpl() { + input="$1" + output="$2" + + YADM_CLASS="$local_class" \ + YADM_OS="$local_system" \ + YADM_HOSTNAME="$local_host" \ + YADM_USER="$local_user" \ + YADM_DISTRO="$local_distro" \ + "$ENVTPL_PROGRAM" --keep-template "$input" -o "$output" +} + # ****** yadm Commands ****** function alt() { @@ -140,6 +361,7 @@ function alt() { local local_system local local_host local local_user + local local_distro set_local_alt_values # only be noisy if the "alt" command was run directly @@ -220,12 +442,54 @@ function set_local_alt_values() { local_user=$(id -u -n) fi + local_distro="$(query_distro)" + } function alt_future_linking() { + local alt_scores + local alt_filenames + local alt_targets + local alt_template_cmds + alt_scores=() + alt_filenames=() + alt_targets=() + alt_template_cmds=() - return + for alt_path in $(for tracked in "${tracked_files[@]}"; do printf "%s\n" "$tracked" "${tracked%/*}"; done | LC_ALL=C sort -u) "${ENCRYPT_INCLUDE_FILES[@]}"; do + alt_path="$YADM_WORK/$alt_path" + if [[ "$alt_path" =~ .\#\#. ]]; then + if [ -e "$alt_path" ] ; then + score_file "$alt_path" + fi + fi + done + + for index in "${!alt_filenames[@]}"; do + filename="${alt_filenames[$index]}" + target="${alt_targets[$index]}" + template_cmd="${alt_template_cmds[$index]}" + if [ -n "$template_cmd" ]; then + # a template is defined, process the template + debug "Creating $filename from template $target" + [ -n "$loud" ] && echo "Creating $filename from template $target" + "$template_cmd" "$target" "$filename" + elif [ -n "$target" ]; then + # a link target is defined, create symlink + debug "Linking $target to $filename" + [ -n "$loud" ] && echo "Linking $target to $filename" + if [ "$do_copy" -eq 1 ]; then + if [ -L "$filename" ]; then + rm -f "$filename" + fi + cp -f "$target" "$filename" + else + ln -nfs "$target" "$filename" + alt_linked+=("$target") + fi + fi + done } @@ -290,7 +554,7 @@ function alt_past_linking() { YADM_OS="$local_system" \ YADM_HOSTNAME="$local_host" \ YADM_USER="$local_user" \ - YADM_DISTRO=$(query_distro) \ + YADM_DISTRO="$local_distro" \ "$ENVTPL_PROGRAM" --keep-template "$tracked_file" -o "$real_file" else debug "envtpl not available, not creating $real_file from template $tracked_file" @@ -1204,6 +1468,14 @@ function bootstrap_available() { [ -f "$YADM_BOOTSTRAP" ] && [ -x "$YADM_BOOTSTRAP" ] && return return 1 } +function awk_available() { + command -v "$AWK_PROGRAM" >/dev/null 2>&1 && return + return 1 +} +function j2cli_available() { + command -v "$J2CLI_PROGRAM" >/dev/null 2>&1 && return + return 1 +} function envtpl_available() { command -v "$ENVTPL_PROGRAM" >/dev/null 2>&1 && return return 1 From 2508378617eeb86ade81077606eebf98dd7f00fc Mon Sep 17 00:00:00 2001 From: Tim Byrne Date: Wed, 25 Sep 2019 23:12:59 -0500 Subject: [PATCH 047/137] Upgrade yadm testbed * Update software in Dockerfile * Add j2cli * Bump supported versions of linters --- .travis.yml | 4 ++-- Dockerfile | 9 +++++---- Makefile | 12 ++++++------ docker-compose.yml | 2 +- test/conftest.py | 6 +++--- 5 files changed, 17 insertions(+), 16 deletions(-) diff --git a/.travis.yml b/.travis.yml index 4b4efd4..3c2b8ea 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,6 +3,6 @@ language: minimal services: - docker before_install: - - docker pull yadm/testbed:latest + - docker pull yadm/testbed:2019-09-25 script: - - docker run -t --rm -v "$PWD:/yadm:ro" yadm/testbed + - docker run -t --rm -v "$PWD:/yadm:ro" yadm/testbed:2019-09-25 diff --git a/Dockerfile b/Dockerfile index 985fc7c..be8ae09 100644 --- a/Dockerfile +++ b/Dockerfile @@ -28,10 +28,11 @@ RUN \ ; RUN pip3 install \ envtpl \ - flake8==3.5.0 \ - pylint==1.9.2 \ - pytest==3.6.4 \ - yamllint==1.15.0 \ + j2cli \ + flake8==3.7.8 \ + pylint==2.4.1 \ + pytest==5.1.3 \ + yamllint==1.17.0 \ ; # Force GNUPG version 1 at path /usr/bin/gpg diff --git a/Makefile b/Makefile index 1242bfd..c6dbca2 100644 --- a/Makefile +++ b/Makefile @@ -107,7 +107,7 @@ testhost: require-docker --hostname testhost \ --rm -it \ -v "/tmp/testhost:/bin/yadm:ro" \ - yadm/testbed:latest \ + yadm/testbed:2019-09-25 \ bash -l .PHONY: scripthost @@ -124,7 +124,7 @@ scripthost: require-docker --rm -it \ -v "$$PWD/script.gz:/script.gz:rw" \ -v "/tmp/testhost:/bin/yadm:ro" \ - yadm/testbed:latest \ + yadm/testbed:2019-09-25 \ bash -c "script /tmp/script -q -c 'bash -l'; gzip < /tmp/script > /script.gz" @echo @echo "Script saved to $$PWD/script.gz" @@ -137,10 +137,10 @@ testenv: virtualenv --python=python3 testenv testenv/bin/pip3 install --upgrade pip setuptools testenv/bin/pip3 install --upgrade \ - flake8==3.5.0 \ - pylint==1.9.2 \ - pytest \ - yamllint==1.15.0 \ + flake8==3.7.8 \ + pylint==2.4.1 \ + pytest==5.1.3 \ + yamllint==1.17.0 \ ; @echo @echo 'To activate this test environment type:' diff --git a/docker-compose.yml b/docker-compose.yml index 4fe0b86..4936532 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -4,4 +4,4 @@ services: testbed: volumes: - .:/yadm:ro - image: yadm/testbed:latest + image: yadm/testbed:2019-09-25 diff --git a/test/conftest.py b/test/conftest.py index 0b1f904..e263a69 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -19,19 +19,19 @@ def shellcheck_version(): @pytest.fixture(scope='session') def pylint_version(): """Version of pylint supported""" - return '1.9.2' + return '2.4.1' @pytest.fixture(scope='session') def flake8_version(): """Version of flake8 supported""" - return '3.5.0' + return '3.7.8' @pytest.fixture(scope='session') def yamllint_version(): """Version of yamllint supported""" - return '1.15.0' + return '1.17.0' @pytest.fixture(scope='session') From d2afab68469f629aae2d412abfc8dca3413d6c4f Mon Sep 17 00:00:00 2001 From: Tim Byrne Date: Thu, 26 Sep 2019 08:46:05 -0500 Subject: [PATCH 048/137] Fix newly discovered linting errors --- test/conftest.py | 16 +++++----------- test/test_clone.py | 1 - test/test_list.py | 9 +++++---- 3 files changed, 10 insertions(+), 16 deletions(-) diff --git a/test/conftest.py b/test/conftest.py index e263a69..68ae77f 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -1,6 +1,7 @@ """Global tests configuration and fixtures""" import collections +import contextlib import copy import distutils.dir_util # pylint: disable=no-name-in-module,import-error import os @@ -50,11 +51,9 @@ def tst_host(): def tst_distro(runner): """Test session's distro""" distro = '' - try: + with contextlib.suppress(Exception): run = runner(command=['lsb_release', '-si'], report=False) distro = run.out.strip() - except BaseException: - pass return distro @@ -141,7 +140,7 @@ def supported_local_configs(supported_configs): return [c for c in supported_configs if c.startswith('local.')] -class Runner(object): +class Runner(): """Class for running commands Within yadm tests, this object should be used when running commands that @@ -238,7 +237,6 @@ def config_git(): os.system( 'git config user.email || ' 'git config --global user.email "test@test.test"') - return None @pytest.fixture() @@ -316,7 +314,7 @@ def yadm_y(paths): return command_list -class DataFile(object): +class DataFile(): """Datafile object""" def __init__(self, path, tracked=True, private=False): @@ -350,10 +348,9 @@ class DataFile(object): def relative_to(self, parent): """Update all relative paths to this py.path""" self.__parent = parent - return -class DataSet(object): +class DataSet(): """Dataset object""" def __init__(self): @@ -444,7 +441,6 @@ class DataSet(object): self.__relpath = relpath for datafile in self.files: datafile.relative_to(self.__relpath) - return @pytest.fixture(scope='session') @@ -526,7 +522,6 @@ def ds1_work_copy(ds1_data, paths): """Function scoped copy of ds1_data.work""" distutils.dir_util.copy_tree( # pylint: disable=no-member str(ds1_data.work), str(paths.work)) - return None @pytest.fixture() @@ -540,7 +535,6 @@ def ds1_repo_copy(runner, ds1_data, paths): command=['git', 'config', 'core.worktree', str(paths.work)], env=env, report=False) - return None @pytest.fixture() diff --git a/test/test_clone.py b/test/test_clone.py index 90febe1..231d967 100644 --- a/test/test_clone.py +++ b/test/test_clone.py @@ -271,4 +271,3 @@ def remote(paths, ds1_repo_copy): # cannot be applied to another fixture. paths.remote.remove() paths.repo.move(paths.remote) - return None diff --git a/test/test_list.py b/test/test_list.py index 44a5573..c2d8631 100644 --- a/test/test_list.py +++ b/test/test_list.py @@ -27,7 +27,7 @@ def test_list(runner, yadm_y, paths, ds1, location): assert run.success assert run.err == '' returned_files = set(run.out.splitlines()) - expected_files = set([e.path for e in ds1 if e.tracked]) + expected_files = {e.path for e in ds1 if e.tracked} assert returned_files == expected_files # test without '-a' # should get all tracked files, relative to the work path unless in a @@ -41,7 +41,8 @@ def test_list(runner, yadm_y, paths, ds1, location): basepath = os.path.basename(os.getcwd()) # only expect files within the subdir # names should be relative to subdir - expected_files = set( - [e.path[len(basepath)+1:] for e in ds1 - if e.tracked and e.path.startswith(basepath)]) + expected_files = { + e.path[len(basepath)+1:] + for e in ds1 if e.tracked and e.path.startswith(basepath) + } assert returned_files == expected_files From eeba216cfe1b39dd33337d7774ddf8ca765ab4e1 Mon Sep 17 00:00:00 2001 From: Tim Byrne Date: Fri, 27 Sep 2019 08:39:50 -0500 Subject: [PATCH 049/137] Mark deprecated tests --- pytest.ini | 2 ++ test/test_compat_alt.py | 3 +++ test/test_compat_jinja.py | 3 +++ 3 files changed, 8 insertions(+) diff --git a/pytest.ini b/pytest.ini index c7fc1db..d032ea5 100644 --- a/pytest.ini +++ b/pytest.ini @@ -1,3 +1,5 @@ [pytest] cache_dir = /tmp addopts = -ra +markers = + deprecated: marks tests for deprecated features (deselect with '-m "not deprecated"') diff --git a/test/test_compat_alt.py b/test/test_compat_alt.py index 6321252..e652a0c 100644 --- a/test/test_compat_alt.py +++ b/test/test_compat_alt.py @@ -6,6 +6,9 @@ import py import pytest import utils +# These tests are for the alternate processing in YADM_COMPATIBILITY=1 mode +pytestmark = pytest.mark.deprecated + # These test IDs are broken. During the writing of these tests, problems have # been discovered in the way yadm orders matching files. BROKEN_TEST_IDS = [ diff --git a/test/test_compat_jinja.py b/test/test_compat_jinja.py index 7d3c1aa..7e2b766 100644 --- a/test/test_compat_jinja.py +++ b/test/test_compat_jinja.py @@ -4,6 +4,9 @@ import os import pytest import utils +# These tests are for the template processing in YADM_COMPATIBILITY=1 mode +pytestmark = pytest.mark.deprecated + @pytest.fixture(scope='module') def envtpl_present(runner): From 36212cb752e1ab7bf177461983ee3499911e9c5d Mon Sep 17 00:00:00 2001 From: Tim Byrne Date: Sun, 29 Sep 2019 17:48:20 -0500 Subject: [PATCH 050/137] Add new alternates processing the cygwin copy testing --- test/conftest.py | 2 ++ test/test_cygwin_copy.py | 13 +++++++++---- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/test/conftest.py b/test/conftest.py index 68ae77f..b1cf16f 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -459,6 +459,8 @@ def ds1_dset(tst_sys, cygwin_sys): dset.add_file(f'test alt/test alt##C.S.H.U') dset.add_file(f'test_cygwin_copy##{tst_sys}') dset.add_file(f'test_cygwin_copy##{cygwin_sys}') + dset.add_file(f'test_cygwin_copy##os.{tst_sys}') + dset.add_file(f'test_cygwin_copy##os.{cygwin_sys}') dset.add_file('u1', tracked=False) dset.add_file('d2/u2', tracked=False) dset.add_file('.ssh/p1', tracked=False, private=True) diff --git a/test/test_cygwin_copy.py b/test/test_cygwin_copy.py index 92141e3..47dbb61 100644 --- a/test/test_cygwin_copy.py +++ b/test/test_cygwin_copy.py @@ -37,7 +37,11 @@ def test_cygwin_copy( if setting is not None: os.system(' '.join(yadm_y('config', 'yadm.cygwin-copy', str(setting)))) - expected_content = f'test_cygwin_copy##{tst_sys}' + if compatibility: + expected_content = f'test_cygwin_copy##{tst_sys}' + else: + expected_content = f'test_cygwin_copy##os.{tst_sys}' + alt_path = paths.work.join('test_cygwin_copy') if pre_existing == 'symlink': alt_path.mklinkto(expected_content) @@ -49,13 +53,14 @@ def test_cygwin_copy( uname = uname_path.join('uname') uname.write(f'#!/bin/sh\necho "{cygwin_sys}"\n') uname.chmod(0o777) - expected_content = f'test_cygwin_copy##{cygwin_sys}' + if compatibility: + expected_content = f'test_cygwin_copy##{cygwin_sys}' + else: + expected_content = f'test_cygwin_copy##os.{cygwin_sys}' env = os.environ.copy() env['PATH'] = ':'.join([str(uname_path), env['PATH']]) if compatibility: env['YADM_COMPATIBILITY'] = '1' - else: - pytest.xfail('Alternates 2.0.0 has not been implemented.') run = runner(yadm_y('alt'), env=env) assert run.success From 3ba17f41fd8d6088532fe7a3e94c5bd293ed719a Mon Sep 17 00:00:00 2001 From: Tim Byrne Date: Sun, 29 Sep 2019 18:33:41 -0500 Subject: [PATCH 051/137] Fix a known bug with hidden private files --- yadm | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/yadm b/yadm index f7cab19..fda4453 100755 --- a/yadm +++ b/yadm @@ -985,12 +985,12 @@ function perms() { # include all .ssh files (unless disabled) if [[ $(config --bool yadm.ssh-perms) != "false" ]] ; then - GLOBS+=(".ssh" ".ssh/*") + GLOBS+=(".ssh" ".ssh/*" ".ssh/.*") fi # include all gpg files (unless disabled) if [[ $(config --bool yadm.gpg-perms) != "false" ]] ; then - GLOBS+=(".gnupg" ".gnupg/*") + GLOBS+=(".gnupg" ".gnupg/*" ".gnupg/.*") fi # include any files we encrypt From c8a9165293e5885fcb75d17ee612fe916faff339 Mon Sep 17 00:00:00 2001 From: Tim Byrne Date: Sun, 29 Sep 2019 18:34:36 -0500 Subject: [PATCH 052/137] Remove warnings from tests These warnings are related to bugs which are fully fixed. --- test/test_enter.py | 16 ++++------------ test/test_perms.py | 11 ----------- 2 files changed, 4 insertions(+), 23 deletions(-) diff --git a/test/test_enter.py b/test/test_enter.py index 202df32..469e3ee 100644 --- a/test/test_enter.py +++ b/test/test_enter.py @@ -1,13 +1,12 @@ """Test enter""" import os -import warnings import pytest @pytest.mark.parametrize( 'shell, success', [ - ('delete', True), + ('delete', True), # if there is no shell variable, bash creates it ('', False), ('/usr/bin/env', True), ('noexec', False), @@ -33,6 +32,7 @@ def test_enter(runner, yadm_y, paths, shell, success): env['SHELL'] = str(noexec) else: env['SHELL'] = shell + run = runner(command=yadm_y('enter'), env=env) assert run.success == success assert run.err == '' @@ -40,20 +40,12 @@ def test_enter(runner, yadm_y, paths, shell, success): if success: assert run.out.startswith('Entering yadm repo') assert run.out.rstrip().endswith('Leaving yadm repo') - if shell == 'delete': - # When SHELL is empty (unlikely), it is attempted to be run anyway. - # This is a but which must be fixed. - warnings.warn('Unhandled bug: SHELL executed when empty', Warning) - else: - assert f'PROMPT={prompt}' in run.out - assert f'PS1={prompt}' in run.out - assert f'GIT_DIR={paths.repo}' in run.out if not success: assert 'does not refer to an executable' in run.out if 'env' in shell: assert f'GIT_DIR={paths.repo}' in run.out - assert 'PROMPT=yadm shell' in run.out - assert 'PS1=yadm shell' in run.out + assert f'PROMPT={prompt}' in run.out + assert f'PS1={prompt}' in run.out @pytest.mark.parametrize( diff --git a/test/test_perms.py b/test/test_perms.py index eb7ad8f..ddf5260 100644 --- a/test/test_perms.py +++ b/test/test_perms.py @@ -1,7 +1,6 @@ """Test perms""" import os -import warnings import pytest @@ -47,11 +46,6 @@ def test_perms(runner, yadm_y, paths, ds1, autoperms): # these paths should be secured if processing perms for private in privatepaths: - if '.p2' in private.basename or '.p4' in private.basename: - # Dot files within .ssh/.gnupg are not protected. - # This is a but which must be fixed - warnings.warn('Unhandled bug: private dot files', Warning) - continue if autoperms == 'false': assert not oct(private.stat().mode).endswith('00'), ( 'Path should not be secured') @@ -94,11 +88,6 @@ def test_perms_control(runner, yadm_y, paths, ds1, sshperms, gpgperms): # these paths should be secured if processing perms for private in privatepaths: - if '.p2' in private.basename or '.p4' in private.basename: - # Dot files within .ssh/.gnupg are not protected. - # This is a but which must be fixed - warnings.warn('Unhandled bug: private dot files', Warning) - continue if ( (sshperms == 'false' and 'ssh' in str(private)) or From 0438e383e5e6dce5d409cbb554a15fc0b9ab0d8f Mon Sep 17 00:00:00 2001 From: Tim Byrne Date: Mon, 30 Sep 2019 08:43:35 -0500 Subject: [PATCH 053/137] Unify the way alternate file strings are matched --- yadm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/yadm b/yadm index fda4453..ea4ee63 100755 --- a/yadm +++ b/yadm @@ -390,7 +390,7 @@ function alt() { local IFS=$'\n' for possible_alt in "${tracked_files[@]}" "${ENCRYPT_INCLUDE_FILES[@]}"; do if [[ $possible_alt =~ .\#\#. ]]; then - possible_alts+=("$YADM_WORK/${possible_alt%##*}") + possible_alts+=("$YADM_WORK/${possible_alt%%##*}") fi done local alt_linked From b411f9d74fa886dba6a569dba5a86f977346aab1 Mon Sep 17 00:00:00 2001 From: Tim Byrne Date: Mon, 30 Sep 2019 08:44:41 -0500 Subject: [PATCH 054/137] Split out processing of stale links --- test/test_unit_remove_stale_links.py | 37 ++++++++++++++++++++++++++++ yadm | 6 ++++- 2 files changed, 42 insertions(+), 1 deletion(-) create mode 100644 test/test_unit_remove_stale_links.py diff --git a/test/test_unit_remove_stale_links.py b/test/test_unit_remove_stale_links.py new file mode 100644 index 0000000..0bd960b --- /dev/null +++ b/test/test_unit_remove_stale_links.py @@ -0,0 +1,37 @@ +"""Unit tests: remove_stale_links""" +import os +import pytest + + +@pytest.mark.parametrize('linked', [True, False]) +@pytest.mark.parametrize('kind', ['file', 'symlink']) +def test_remove_stale_links(runner, yadm, tmpdir, kind, linked): + """Test remove_stale_links()""" + + source_file = tmpdir.join('source_file') + source_file.write('source file', ensure=True) + link = tmpdir.join('link') + + if kind == 'file': + link.write('link file', ensure=True) + else: + os.system(f'ln -s {source_file} {link}') + + alt_linked = '' + if linked: + alt_linked = source_file + + script = f""" + YADM_TEST=1 source {yadm} + possible_alts=({link}) + alt_linked=({alt_linked}) + function rm() {{ echo rm "$@"; }} + remove_stale_links + """ + + run = runner(command=['bash'], inp=script) + assert run.err == '' + if kind == 'symlink' and not linked: + assert f'rm -f {link}' in run.out + else: + assert run.out == '' diff --git a/yadm b/yadm index ea4ee63..a3f9893 100755 --- a/yadm +++ b/yadm @@ -402,6 +402,11 @@ function alt() { alt_future_linking fi + remove_stale_links + +} + +function remove_stale_links() { # review alternate candidates for stale links # if a possible alt IS linked, but it's target is not part of alt_linked, # remove it. @@ -419,7 +424,6 @@ function alt() { fi done fi - } function set_local_alt_values() { From 234055190b90f4a316103b10296418dbab6b58f2 Mon Sep 17 00:00:00 2001 From: Tim Byrne Date: Tue, 1 Oct 2019 08:03:09 -0500 Subject: [PATCH 055/137] Move min-similarity-lines to new section This is the correct place for a newer version of pylint. --- pylintrc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pylintrc b/pylintrc index 1ced5a6..13f768e 100644 --- a/pylintrc +++ b/pylintrc @@ -6,6 +6,8 @@ max-args=14 max-locals=28 max-attributes=8 max-statements=65 + +[SIMILARITIES] min-similarity-lines=6 [MESSAGES CONTROL] From e999929818e5afea4109d4e5878a0cfeab5dd6be Mon Sep 17 00:00:00 2001 From: Tim Byrne Date: Tue, 1 Oct 2019 08:17:28 -0500 Subject: [PATCH 056/137] Change builtin templates to resemble jinja --- test/test_unit_template_builtin.py | 84 +++++++++++++++--------------- yadm | 25 +++++---- 2 files changed, 56 insertions(+), 53 deletions(-) diff --git a/test/test_unit_template_builtin.py b/test/test_unit_template_builtin.py index ac141e4..4d87a12 100644 --- a/test/test_unit_template_builtin.py +++ b/test/test_unit_template_builtin.py @@ -8,56 +8,56 @@ LOCAL_USER = "builtin_Test+@-!^User" LOCAL_DISTRO = "builtin_Test+@-!^Distro" TEMPLATE = f''' start of template -builtin class = >YADM_CLASS< -builtin os = >YADM_OS< -builtin host = >YADM_HOSTNAME< -builtin user = >YADM_USER< -builtin distro = >YADM_DISTRO< -YADM_IF CLASS="wrongclass1" +builtin class = >{{{{yadm.class}}}}< +builtin os = >{{{{yadm.os}}}}< +builtin host = >{{{{yadm.hostname}}}}< +builtin user = >{{{{yadm.user}}}}< +builtin distro = >{{{{yadm.distro}}}}< +{{% if yadm.class == "wrongclass1" %}} wrong class 1 -YADM_END -YADM_IF CLASS="{LOCAL_CLASS}" -Included section for class = YADM_CLASS (YADM_CLASS repeated) -YADM_END -YADM_IF CLASS="wrongclass2" +{{% endif %}} +{{% if yadm.class == "{LOCAL_CLASS}" %}} +Included section for class = {{{{yadm.class}}}} ({{{{yadm.class}}}} repeated) +{{% endif %}} +{{% if yadm.class == "wrongclass2" %}} wrong class 2 -YADM_END -YADM_IF OS="wrongos1" +{{% endif %}} +{{% if yadm.os == "wrongos1" %}} wrong os 1 -YADM_END -YADM_IF OS="{LOCAL_SYSTEM}" -Included section for os = YADM_OS (YADM_OS repeated) -YADM_END -YADM_IF OS="wrongos2" +{{% endif %}} +{{% if yadm.os == "{LOCAL_SYSTEM}" %}} +Included section for os = {{{{yadm.os}}}} ({{{{yadm.os}}}} repeated) +{{% endif %}} +{{% if yadm.os == "wrongos2" %}} wrong os 2 -YADM_END -YADM_IF HOSTNAME="wronghost1" +{{% endif %}} +{{% if yadm.hostname == "wronghost1" %}} wrong host 1 -YADM_END -YADM_IF HOSTNAME="{LOCAL_HOST}" -Included section for host = YADM_HOSTNAME (YADM_HOSTNAME repeated) -YADM_END -YADM_IF HOSTNAME="wronghost2" +{{% endif %}} +{{% if yadm.hostname == "{LOCAL_HOST}" %}} +Included section for host = {{{{yadm.hostname}}}} ({{{{yadm.hostname}}}} again) +{{% endif %}} +{{% if yadm.hostname == "wronghost2" %}} wrong host 2 -YADM_END -YADM_IF USER="wronguser1" +{{% endif %}} +{{% if yadm.user == "wronguser1" %}} wrong user 1 -YADM_END -YADM_IF USER="{LOCAL_USER}" -Included section for user = YADM_USER (YADM_USER repeated) -YADM_END -YADM_IF USER="wronguser2" +{{% endif %}} +{{% if yadm.user == "{LOCAL_USER}" %}} +Included section for user = {{{{yadm.user}}}} ({{{{yadm.user}}}} repeated) +{{% endif %}} +{{% if yadm.user == "wronguser2" %}} wrong user 2 -YADM_END -YADM_IF DISTRO="wrongdistro1" +{{% endif %}} +{{% if yadm.distro == "wrongdistro1" %}} wrong distro 1 -YADM_END -YADM_IF DISTRO="{LOCAL_DISTRO}" -Included section for distro = YADM_DISTRO (YADM_DISTRO repeated) -YADM_END -YADM_IF DISTRO="wrongdistro2" +{{% endif %}} +{{% if yadm.distro == "{LOCAL_DISTRO}" %}} +Included section for distro = {{{{yadm.distro}}}} ({{{{yadm.distro}}}} again) +{{% endif %}} +{{% if yadm.distro == "wrongdistro2" %}} wrong distro 2 -YADM_END +{{% endif %}} end of template ''' EXPECTED = f''' @@ -69,9 +69,9 @@ builtin user = >{LOCAL_USER}< builtin distro = >{LOCAL_DISTRO}< Included section for class = {LOCAL_CLASS} ({LOCAL_CLASS} repeated) Included section for os = {LOCAL_SYSTEM} ({LOCAL_SYSTEM} repeated) -Included section for host = {LOCAL_HOST} ({LOCAL_HOST} repeated) +Included section for host = {LOCAL_HOST} ({LOCAL_HOST} again) Included section for user = {LOCAL_USER} ({LOCAL_USER} repeated) -Included section for distro = {LOCAL_DISTRO} ({LOCAL_DISTRO} repeated) +Included section for distro = {LOCAL_DISTRO} ({LOCAL_DISTRO} again) end of template ''' diff --git a/yadm b/yadm index a3f9893..2a9e989 100755 --- a/yadm +++ b/yadm @@ -279,35 +279,38 @@ function template_builtin() { input="$1" output="$2" + # the explicit "space + tab" character class used below is used because not + # all versions of awk seem to support the POSIX character classes [[:blank:]] awk_pgm=$(cat << "EOF" # built-in template processor BEGIN { - c["CLASS"] = class - c["OS"] = os - c["HOSTNAME"] = host - c["USER"] = user - c["DISTRO"] = distro + blank = "[ ]" + c["class"] = class + c["os"] = os + c["hostname"] = host + c["user"] = user + c["distro"] = distro valid = conditions() - end = "^YADM_END$" - skip = "^YADM_(IF|END$)" + end = "^{%" blank "*endif" blank "*%}$" + skip = "^{%" blank "*(if|endif)" } { replace_vars() } # variable replacements $0 ~ valid, $0 ~ end { if ($0 ~ skip) next } # valid conditional blocks -/^YADM_IF/, $0 ~ end { next } # invalid conditional blocks +$0 ~ ("^{%" blank "*if"), $0 ~ end { next } # invalid conditional blocks { print } function replace_vars() { for (label in c) { - gsub(("YADM_" label), c[label]) + gsub(("{{" blank "*yadm\\." label blank "*}}"), c[label]) } } function conditions() { - pattern = "^(" + pattern = "^{%" blank "*if" blank "*(" for (label in c) { value = c[label] gsub(/[\\.^$(){}\[\]|*+?]/, "\\\\&", value) - pattern = sprintf("%sYADM_IF +%s *= *\"%s\"|", pattern, label, value) + pattern = sprintf("%syadm\\.%s" blank "*==" blank "*\"%s\"|", pattern, label, value) } sub(/\|$/,")",pattern) return pattern From 81134c8edbf59df08f8d59d06058961aa112250f Mon Sep 17 00:00:00 2001 From: Tim Byrne Date: Wed, 2 Oct 2019 18:59:57 -0500 Subject: [PATCH 057/137] Update documentation * XDG Base Directory Specification * New alternates processing --- yadm.1 | 303 +++++++++++++++++++++++++++++++++++++++------------------ 1 file changed, 211 insertions(+), 92 deletions(-) diff --git a/yadm.1 b/yadm.1 index b50bf4d..3ad0318 100644 --- a/yadm.1 +++ b/yadm.1 @@ -1,8 +1,12 @@ ." vim: set spell so=8: .TH yadm 1 "25 October 2017" "1.12.0" + .SH NAME + yadm \- Yet Another Dotfiles Manager + .SH SYNOPSIS + .B yadm .I command .RI [ options ] @@ -52,19 +56,23 @@ list .BR yadm " introspect .I category + .SH DESCRIPTION + .B yadm is a tool for managing a collection of files across multiple computers, using a shared Git repository. In addition, .B yadm -provides a feature to select alternate versions of files -based on the operating system or host name. +provides a feature to select alternate versions of files for particular +systems. Lastly, .B yadm supplies the ability to manage a subset of secure files, which are encrypted before they are included in the repository. + .SH COMMANDS + .TP .IR git-command " or " git-alias Any command not internally handled by @@ -94,9 +102,9 @@ Instead use the command (see below). .TP .B alt -Create symbolic links and process Jinja templates for any managed files -matching the naming rules described in the ALTERNATES and JINJA sections. It is -usually unnecessary to run this command, as +Create symbolic links and process templates for any managed files matching the +naming rules described in the ALTERNATES and TEMPLATES sections. It is usually +unnecessary to run this command, as .B yadm automatically processes alternates by default. This automatic behavior can be disabled by setting the configuration @@ -268,6 +276,30 @@ to "false". .B version Print the version of .BR yadm . + +.SH COMPATIBILITY + +Beginning with version 2.0.0, +.B yadm +introduced a couple major changes which may require you to adjust your configurations. + +First, +.B yadm +now uses the "XDG Base Directory Specification" to find its configurations. +You can read https://yadm.io/docs/xdg_config_home for more information. + +Second, the naming conventions for alternate files have been changed. +You can read https://yadm.io/docs/alternates for more information. + +If you want to retain the old functionality, you can set an environment variable, +.IR YADM_COMPATIBILITY=1 . +Doing so will automatically use the old +.B yadm +directory, and process alternates the same as the pre-2.0.0 version. +This compatibility mode is deprecated, and will be removed in future versions. +This mode exists solely for transitioning to the new paths and naming of +alternates. + .SH OPTIONS .B yadm @@ -317,7 +349,9 @@ encrypted files archive. Override the location of the .B yadm bootstrap program. + .SH CONFIGURATION + .B yadm uses a configuration file named .IR $HOME/.config/yadm/config . @@ -386,43 +420,83 @@ symbolic links. This might be desirable, because non-Cygwin software may not properly interpret Cygwin symlinks. .RE -These last four "local" configurations are not stored in the +The following four "local" configurations are not stored in the .IR $HOME/.config/yadm/config, they are stored in the local repository. .TP .B local.class -Specify a CLASS for the purpose of symlinking alternate files. -By default, no CLASS will be matched. +Specify a class for the purpose of symlinking alternate files. +By default, no class will be matched. .TP .B local.os Override the OS for the purpose of symlinking alternate files. .TP .B local.hostname -Override the HOSTNAME for the purpose of symlinking alternate files. +Override the hostname for the purpose of symlinking alternate files. .TP .B local.user -Override the USER for the purpose of symlinking alternate files. +Override the user for the purpose of symlinking alternate files. + .SH ALTERNATES + When managing a set of files across different systems, it can be useful to have an automated way of choosing an alternate version of a file for a different -operating system, host, or user. -.B yadm -implements a feature which will automatically create a symbolic link to -the appropriate version of a file, as long as you follow a specific naming -convention. -.B yadm -can detect files with names ending in any of the following: +operating system, host, user, etc. - ## - ##CLASS - ##CLASS.OS - ##CLASS.OS.HOSTNAME - ##CLASS.OS.HOSTNAME.USER - ##OS - ##OS.HOSTNAME - ##OS.HOSTNAME.USER +yadm will automatically create a symbolic link to the appropriate version of a +file, when a valid suffix is appended to the filename. The suffix contains +the conditions that must be met for that file to be used. +The suffix begins with "##", followed by any number of conditions separated by +commas. + + ##[,,...] + +Each condition is an attribute/value pair, separated by a period. Some +conditions do not require a "value", and in that case, the period and value can +be omitted. + + [.] + +These are the supported attributes, in the order of the weighted precedence: + +.TP +.BR template , " t +Valid when the value matches a supported template processor. +See the TEMPLATES section for more details. +.TP +.BR user , " u +Valid if the value matches the current user. +Current user is calculated by running +.BR "id -u -n" . +.TP +.BR os , " o +Valid if the value matches the OS. +OS is calculated by running +.BR "uname -s" . +.TP +.BR class , " c +Valid if the value matches the +.B local.class +configuration. +Class must be manually set using +.BR "yadm config local.class " . +See the CONFIGURATION section for more details about setting +.BR local.class . +.TP +.BR hostname , " h +Valid if the value matches the short hostname. +Hostname is calculated by running +.BR "hostname" , +and trimming off any domain. +.TP +.B default +Valid when no other alternate is valid. +.LP + +You may use any number of conditions, in any order. +An alternate will only be used if ALL conditions are valid. If there are any files managed by .BR yadm \'s repository, @@ -430,125 +504,137 @@ or listed in .IR $HOME/.config/yadm/encrypt , which match this naming convention, symbolic links will be created for the most appropriate version. -This may best be demonstrated by example. Assume the following files are managed by + +The "most appropriate" version is determined by calculating a score for each +version of a file. A template is always scored higher than any symlink +condition. The number of conditions is the next largest factor in scoring. +Files with more conditions will always be favored. Any invalid condition will +disqualify that file completely. + +Alternate linking may best be demonstrated by example. Assume the following +files are managed by .BR yadm \'s repository: - - $HOME/path/example.txt## - - $HOME/path/example.txt##Work - - $HOME/path/example.txt##Darwin - - $HOME/path/example.txt##Darwin.host1 - - $HOME/path/example.txt##Darwin.host2 - - $HOME/path/example.txt##Linux - - $HOME/path/example.txt##Linux.host1 - - $HOME/path/example.txt##Linux.host2 + - $HOME/path/example.txt##default + - $HOME/path/example.txt##class.Work + - $HOME/path/example.txt##os.Darwin + - $HOME/path/example.txt##os.Darwin,hostname.host1 + - $HOME/path/example.txt##os.Darwin,hostname.host2 + - $HOME/path/example.txt##os.Linux + - $HOME/path/example.txt##os.Linux,hostname.host1 + - $HOME/path/example.txt##os.Linux,hostname.host2 If running on a Macbook named "host2", .B yadm will create a symbolic link which looks like this: -.IR $HOME/path/example.txt " -> " $HOME/path/example.txt##Darwin.host2 +.IR $HOME/path/example.txt " -> " $HOME/path/example.txt##os.Darwin,hostname.host2 However, on another Mackbook named "host3", .B yadm will create a symbolic link which looks like this: -.IR $HOME/path/example.txt " -> " $HOME/path/example.txt##Darwin +.IR $HOME/path/example.txt " -> " $HOME/path/example.txt##os.Darwin Since the hostname doesn't match any of the managed files, the more generic version is chosen. If running on a Linux server named "host4", the link will be: -.IR $HOME/path/example.txt " -> " $HOME/path/example.txt##Linux +.IR $HOME/path/example.txt " -> " $HOME/path/example.txt##os.Linux -If running on a Solaris server, the link use the default "##" version: +If running on a Solaris server, the link will use the default version: -.IR $HOME/path/example.txt " -> " $HOME/path/example.txt## +.IR $HOME/path/example.txt " -> " $HOME/path/example.txt##default -If running on a system, with CLASS set to "Work", the link will be: +If running on a system, with class set to "Work", the link will be: -.IR $HOME/path/example.txt " -> " $HOME/path/example.txt##WORK +.IR $HOME/path/example.txt " -> " $HOME/path/example.txt##class.Work -If no "##" version exists and no files match the current CLASS/OS/HOSTNAME/USER, then no link will be created. +If no "##default" version exists and no files have valid conditions, then no +link will be created. Links are also created for directories named this way, as long as they have at least one .B yadm managed file within them. -CLASS must be manually set using -.BR yadm\ config\ local.class\ . -OS is determined by running -.BR uname\ -s , -HOSTNAME by running -.BR hostname , -and USER by running -.BR id\ -u\ -n . .B yadm will automatically create these links by default. This can be disabled using the .I yadm.auto-alt configuration. Even if disabled, links can be manually created by running -.BR yadm\ alt . +.BR "yadm alt" . -It is possible to use "%" as a "wildcard" in place of CLASS, OS, HOSTNAME, or -USER. For example, The following file could be linked for any host when the -user is "harvey". - -.IR $HOME/path/example.txt##%.%.harvey - -CLASS is a special value which is stored locally on each host (inside the local -repository). To use alternate symlinks using CLASS, you must set the value of +Class is a special value which is stored locally on each host (inside the local +repository). To use alternate symlinks using class, you must set the value of class using the configuration .BR local.class . This is set like any other .B yadm configuration with the .B yadm config -command. The following sets the CLASS to be "Work". +command. The following sets the class to be "Work". yadm config local.class Work -Similarly, the values of OS, HOSTNAME, and USER can be manually overridden +Similarly, the values of os, hostname, and user can be manually overridden using the configuration options .BR local.os , .BR local.hostname , and .BR local.user . -.SH JINJA -If the +.SH TEMPLATES + +If a template condition is defined in an alternate file's "##" suffix, and the +necessary dependencies for the template are available, then the file will be +processed to create or overwrite files. + +Supported template processors: +.TP +.B builtin +This is +.BR yadm \'s +built-in template processor. This processor is very basic, with a Jinja-like +syntax. The advantage of this processor is that it only depends upon \fBawk\fR, +which is available on most *nix systems. To use this processor, specify the +value of "builtin" or just leave the value off (e.g. "##template"). +.TP +.B j2cli +To use the j2cli Jinja template processor, specify the value of "j2" or +"j2cli". +.TP .B envtpl -command is available, -.B Jinja -templates will also be processed to create or overwrite real files. -.B yadm -will treat files ending in +To use the envtpl Jinja template processor, specify the value of "j2" or "envtpl". +.LP - ##yadm.j2 +.BR NOTE : +Specifying "j2" as the processor will attempt to use j2cli or envtpl, whichever +is available. -as Jinja templates. During processing, the following variables are set -according to the rules explained in the ALTERNATES section: +If the template processor specified is available, templates will be processed +to create or overwrite files. - YADM_CLASS - YADM_OS - YADM_HOSTNAME - YADM_USER +During processing, the following variables are available in the template: -In addition YADM_DISTRO is exposed as the value of -.I lsb_release -si -if -.B lsb_release -is locally available. + Builtin Jinja Description + ------------- ------------- -------------------------- + yadm.class YADM_CLASS Locally defined yadm class + yadm.distro YADM_DISTRO lsb_release -si + yadm.hostname YADM_HOSTNAME hostname (without domain) + yadm.os YADM_OS uname -s + yadm.user YADM_USER id -u -n -For example, a file named -.I whatever##yadm.j2 +Examples: + +.I whatever##template with the following content - {% if YADM_USER == 'harvey' -%} - config={{YADM_CLASS}}-{{ YADM_OS }} - {% else -%} + {% if yadm.user == 'harvey' %} + config={{yadm.class}}-{{yadm.os}} + {% else %} config=dev-whatever - {% endif -%} + {% endif %} would output a file named .I whatever @@ -560,16 +646,24 @@ and the following otherwise: config=dev-whatever -See http://jinja.pocoo.org/ for an overview of -.BR Jinja . +An equivalent Jinja template named +.I whatever##template.j2 +would look like: + + {% if YADM_USER == 'harvey' -%} + config={{YADM_CLASS}}-{{YADM_OS}} + {% else -%} + config=dev-whatever + {% endif -%} .SH ENCRYPTION + It can be useful to manage confidential files, like SSH or GPG keys, across multiple systems. However, doing so would put plain text data into a Git repository, which often resides on a public system. .B yadm -implements a feature which can make it easy to encrypt and decrypt a set of -files so the encrypted version can be maintained in the Git repository. +can make it easy to encrypt and decrypt a set of files so the encrypted version +can be maintained in the Git repository. This feature will only work if the .BR gpg (1) command is available. @@ -601,7 +695,7 @@ The patterns and files.gpg should be added to the repository so they are available across multiple systems. To decrypt these files later, or on another system run -.BR yadm\ decrypt +.B yadm decrypt and provide the correct password. After files are decrypted, permissions are automatically updated as described in the PERMISSIONS section. @@ -614,7 +708,9 @@ configuration. .BR NOTE : It is recommended that you use a private repository when keeping confidential files, even though they are encrypted. + .SH PERMISSIONS + When files are checked out of a Git repository, their initial permissions are dependent upon the user's umask. Because of this, .B yadm @@ -636,7 +732,7 @@ will automatically update the permissions of some file paths. The "group" and will automatically update permissions by default. This can be disabled using the .I yadm.auto-perms configuration. Even if disabled, permissions can be manually updated by running -.BR yadm\ perms . +.BR "yadm perms" . The .I .ssh directory processing can be disabled using the @@ -662,7 +758,9 @@ will create those directories with mask 0700 prior to running the Git command. This can be disabled using the .I yadm.auto-private-dirs configuration. + .SH HOOKS + For every command .B yadm supports, a program can be provided to run before or after that command. These @@ -718,7 +816,21 @@ repository .TP .B YADM_HOOK_WORK The path to the work-tree + .SH FILES + +All of +.BR yadm \'s +configurations are relative to the "yadm directory". +.B yadm +uses the "XDG Base Directory Specification" to determine this directory. If the +environment variable +.B $XDG_CONFIG_HOME +is defined as a fully qualified path, this directory will be +.IR "$XDG_CONFIG_HOME/yadm" . +Otherwise it will be +.IR "$HOME/.config/yadm" . + The following are the default paths .B yadm uses for its own data. @@ -747,7 +859,9 @@ List of globs used for encrypt/decrypt All files encrypted with .B yadm encrypt are stored in this file. + .SH EXAMPLES + .TP .B yadm init Create an empty repo for managing files @@ -768,12 +882,17 @@ Add a new pattern to the list of encrypted files .TP .B yadm encrypt ; yadm add ~/.config/yadm/files.gpg ; yadm commit Commit a new set of encrypted files + .SH REPORTING BUGS + Report issues or create pull requests at GitHub: https://github.com/TheLocehiliosan/yadm/issues + .SH AUTHOR + Tim Byrne + .SH SEE ALSO .BR git (1), From 444622a658794a9090d9c8b792894a03b987c1b6 Mon Sep 17 00:00:00 2001 From: Tim Byrne Date: Sat, 5 Oct 2019 11:01:13 -0500 Subject: [PATCH 058/137] Support `else` statements in builtin templates --- test/test_unit_template_builtin.py | 10 ++++++++++ yadm | 23 ++++++++++++++++------- 2 files changed, 26 insertions(+), 7 deletions(-) diff --git a/test/test_unit_template_builtin.py b/test/test_unit_template_builtin.py index 4d87a12..a699f6a 100644 --- a/test/test_unit_template_builtin.py +++ b/test/test_unit_template_builtin.py @@ -13,11 +13,19 @@ builtin os = >{{{{yadm.os}}}}< builtin host = >{{{{yadm.hostname}}}}< builtin user = >{{{{yadm.user}}}}< builtin distro = >{{{{yadm.distro}}}}< +{{% if yadm.class == "else1" %}} +wrong else 1 +{{% else %}} +Included section from else +{{% endif %}} {{% if yadm.class == "wrongclass1" %}} wrong class 1 {{% endif %}} {{% if yadm.class == "{LOCAL_CLASS}" %}} Included section for class = {{{{yadm.class}}}} ({{{{yadm.class}}}} repeated) +Multiple lines +{{% else %}} +Should not be included... {{% endif %}} {{% if yadm.class == "wrongclass2" %}} wrong class 2 @@ -67,7 +75,9 @@ builtin os = >{LOCAL_SYSTEM}< builtin host = >{LOCAL_HOST}< builtin user = >{LOCAL_USER}< builtin distro = >{LOCAL_DISTRO}< +Included section from else Included section for class = {LOCAL_CLASS} ({LOCAL_CLASS} repeated) +Multiple lines Included section for os = {LOCAL_SYSTEM} ({LOCAL_SYSTEM} repeated) Included section for host = {LOCAL_HOST} ({LOCAL_HOST} again) Included section for user = {LOCAL_USER} ({LOCAL_USER} repeated) diff --git a/yadm b/yadm index 2a9e989..e4df305 100755 --- a/yadm +++ b/yadm @@ -290,21 +290,30 @@ BEGIN { c["hostname"] = host c["user"] = user c["distro"] = distro - valid = conditions() + vld = conditions() + ifs = "^{%" blank "*if" + els = "^{%" blank "*else" blank "*%}$" end = "^{%" blank "*endif" blank "*%}$" - skip = "^{%" blank "*(if|endif)" + skp = "^{%" blank "*(if|else|endif)" + prt = 1 } { replace_vars() } # variable replacements -$0 ~ valid, $0 ~ end { if ($0 ~ skip) next } # valid conditional blocks -$0 ~ ("^{%" blank "*if"), $0 ~ end { next } # invalid conditional blocks -{ print } - +$0 ~ vld, $0 ~ end { + if ($0 ~ vld || $0 ~ end) prt=1; + if ($0 ~ els) prt=0; + if ($0 ~ skp) next; +} +($0 ~ ifs && $0 !~ vld), $0 ~ end { + if ($0 ~ ifs && $0 !~ vld) prt=0; + if ($0 ~ els || $0 ~ end) prt=1; + if ($0 ~ skp) next; +} +{ if (prt) print } function replace_vars() { for (label in c) { gsub(("{{" blank "*yadm\\." label blank "*}}"), c[label]) } } - function conditions() { pattern = "^{%" blank "*if" blank "*(" for (label in c) { From 4f6b0f09cd6affd4579187f883d90e9f32872151 Mon Sep 17 00:00:00 2001 From: Tim Byrne Date: Sat, 5 Oct 2019 12:01:48 -0500 Subject: [PATCH 059/137] Remove unnecessary formatting --- yadm.1 | 294 ++++++++++++++++++++------------------------------------- 1 file changed, 104 insertions(+), 190 deletions(-) diff --git a/yadm.1 b/yadm.1 index 3ad0318..1ad3e11 100644 --- a/yadm.1 +++ b/yadm.1 @@ -59,29 +59,20 @@ list .SH DESCRIPTION -.B yadm -is a tool for managing a collection of files across multiple computers, +yadm is a tool for managing a collection of files across multiple computers, using a shared Git repository. -In addition, -.B yadm -provides a feature to select alternate versions of files for particular -systems. -Lastly, -.B yadm -supplies the ability to manage a subset of secure files, which are +In addition, yadm provides a feature to select alternate versions of files for +particular systems. +Lastly, yadm supplies the ability to manage a subset of secure files, which are encrypted before they are included in the repository. .SH COMMANDS .TP .IR git-command " or " git-alias -Any command not internally handled by -.B yadm -is passed through to +Any command not internally handled by yadm is passed through to .BR git (1). -Git commands or aliases are invoked with the -.B yadm -managed repository. +Git commands or aliases are invoked with the yadm managed repository. The working directory for Git commands will be the configured .IR work-tree " (usually .IR $HOME ). @@ -104,10 +95,8 @@ command (see below). .B alt Create symbolic links and process templates for any managed files matching the naming rules described in the ALTERNATES and TEMPLATES sections. It is usually -unnecessary to run this command, as -.B yadm -automatically processes alternates by default. -This automatic behavior can be disabled by setting the configuration +unnecessary to run this command, as yadm automatically processes alternates by +default. This automatic behavior can be disabled by setting the configuration .I yadm.auto-alt to "false". .TP @@ -155,21 +144,17 @@ will be used as the .IR work-tree , but this can be overridden with the .BR -w " option. -.B yadm -can be forced to overwrite an existing repository by providing the +yadm can be forced to overwrite an existing repository by providing the .BR -f " option. -By default -.B yadm -will ask the user if the bootstrap program should be run (if it exists). The -options +By default yadm will ask the user if the bootstrap program should be run (if it +exists). The options .BR --bootstrap " or " --no-bootstrap will either force the bootstrap to be run, or prevent it from being run, without prompting the user. .RE .TP .B config -This command manages configurations for -.BR yadm . +This command manages configurations for yadm. This command works exactly they way .BR git-config (1) does. @@ -193,11 +178,9 @@ See the ENCRYPTION section for more details. .B enter Run a sub-shell with all Git variables set. Exit the sub-shell the same way you leave your normal shell (usually with the "exit" command). This sub-shell can -be used to easily interact with your -.B yadm -repository using "git" commands. This could be useful if you are using a tool -which uses Git directly. For example, Emacs Tramp and Magit can manage files by -using this configuration: +be used to easily interact with your yadm repository using "git" commands. This +could be useful if you are using a tool which uses Git directly. For example, +Emacs Tramp and Magit can manage files by using this configuration: .RS (add-to-list 'tramp-methods '("yadm" @@ -210,17 +193,16 @@ using this configuration: .B gitconfig Pass options to the .B git config -command. Since -.B yadm -already uses the +command. Since yadm already uses the .I config command to manage its own configurations, -this command is provided as a way to change configurations of the repository managed by -.BR yadm . -One useful case might be to configure the repository so untracked files are shown in status commands. -.B yadm -initially configures its repository so that untracked files are not shown. -If you wish use the default Git behavior (to show untracked files and directories), you can remove this configuration. +this command is provided as a way to change configurations of the repository +managed by yadm. +One useful case might be to configure the repository so untracked files are +shown in status commands. yadm initially configures its repository so that +untracked files are not shown. +If you wish use the default Git behavior (to show untracked files and +directories), you can remove this configuration. .RS .RS @@ -229,8 +211,7 @@ yadm gitconfig --unset status.showUntrackedFiles .RE .TP .B help -Print a summary of -.BR yadm " commands. +Print a summary of yadm commands. .TP .B init Initialize a new, empty repository for tracking dotfiles. @@ -242,21 +223,17 @@ will be used as the .IR work-tree , but this can be overridden with the .BR -w " option. -.B yadm -can be forced to overwrite an existing repository by providing the +yadm can be forced to overwrite an existing repository by providing the .BR -f " option. .TP .B list -Print a list of files managed by -.BR yadm . +Print a list of files managed by yadm. .RB The " -a option will cause all managed files to be listed. Otherwise, the list will only include files from the current directory or below. .TP .BI introspect " category -Report internal -.B yadm -data. Supported categories are +Report internal yadm data. Supported categories are .IR commands , .IR configs , .IR repo, @@ -266,50 +243,42 @@ The purpose of introspection is to support command line completion. .TP .B perms Update permissions as described in the PERMISSIONS section. -It is usually unnecessary to run this command, as -.B yadm -automatically processes permissions by default. -This automatic behavior can be disabled by setting the configuration +It is usually unnecessary to run this command, as yadm automatically processes +permissions by default. This automatic behavior can be disabled by setting the +configuration .I yadm.auto-perms to "false". .TP .B version -Print the version of -.BR yadm . +Print the version of yadm. .SH COMPATIBILITY -Beginning with version 2.0.0, -.B yadm -introduced a couple major changes which may require you to adjust your configurations. +Beginning with version 2.0.0, yadm introduced a couple major changes which may +require you to adjust your configurations. -First, -.B yadm -now uses the "XDG Base Directory Specification" to find its configurations. -You can read https://yadm.io/docs/xdg_config_home for more information. +First, yadm now uses the "XDG Base Directory Specification" to find its +configurations. You can read https://yadm.io/docs/xdg_config_home for more +information. Second, the naming conventions for alternate files have been changed. You can read https://yadm.io/docs/alternates for more information. If you want to retain the old functionality, you can set an environment variable, .IR YADM_COMPATIBILITY=1 . -Doing so will automatically use the old -.B yadm -directory, and process alternates the same as the pre-2.0.0 version. -This compatibility mode is deprecated, and will be removed in future versions. -This mode exists solely for transitioning to the new paths and naming of -alternates. +Doing so will automatically use the old yadm directory, and process alternates +the same as the pre-2.0.0 version. This compatibility mode is deprecated, and +will be removed in future versions. This mode exists solely for transitioning +to the new paths and naming of alternates. .SH OPTIONS -.B yadm -supports a set of universal options that alter the paths it uses. -The default paths are documented in the FILES section. -Any path specified by these options must be fully qualified. -If you always want to override one or more of these paths, it may be useful to create an alias for the -.B yadm -command. -For example, the following alias could be used to override the repository directory. +yadm supports a set of universal options that alter the paths it uses. The +default paths are documented in the FILES section. Any path specified by these +options must be fully qualified. If you always want to override one or more of +these paths, it may be useful to create an alias for the yadm command. +For example, the following alias could be used to override the repository +directory. .RS alias yadm='yadm --yadm-repo /alternate/path/to/repo' @@ -319,41 +288,27 @@ The following is the full list of universal options. Each option should be followed by a fully qualified path. .TP .B -Y,--yadm-dir -Override the -.B yadm -directory. -.B yadm -stores its data relative to this directory. +Override the yadm directory. +yadm stores its data relative to this directory. .TP .B --yadm-repo -Override the location of the -.B yadm -repository. +Override the location of the yadm repository. .TP .B --yadm-config -Override the location of the -.B yadm -configuration file. +Override the location of the yadm configuration file. .TP .B --yadm-encrypt -Override the location of the -.B yadm -encryption configuration. +Override the location of the yadm encryption configuration. .TP .B --yadm-archive -Override the location of the -.B yadm -encrypted files archive. +Override the location of the yadm encrypted files archive. .TP .B --yadm-bootstrap -Override the location of the -.B yadm -bootstrap program. +Override the location of the yadm bootstrap program. .SH CONFIGURATION -.B yadm -uses a configuration file named +yadm uses a configuration file named .IR $HOME/.config/yadm/config . This file uses the same format as .BR git-config (1). @@ -371,11 +326,9 @@ yadm config yadm.auto-alt false The following is the full list of supported configurations: .TP .B yadm.auto-alt -Disable the automatic linking described in the section ALTERNATES. -If disabled, you may still run -.B yadm alt -manually to create the alternate links. -This feature is enabled by default. +Disable the automatic linking described in the section ALTERNATES. If disabled, +you may still run "yadm alt" manually to create the alternate links. This +feature is enabled by default. .TP .B yadm.auto-perms Disable the automatic permission changes described in the section PERMISSIONS. @@ -497,10 +450,7 @@ Valid when no other alternate is valid. You may use any number of conditions, in any order. An alternate will only be used if ALL conditions are valid. -If there are any files managed by -.BR yadm \'s -repository, -or listed in +If there are any files managed by yadm's repository, or listed in .IR $HOME/.config/yadm/encrypt , which match this naming convention, symbolic links will be created for the most appropriate version. @@ -512,9 +462,7 @@ Files with more conditions will always be favored. Any invalid condition will disqualify that file completely. Alternate linking may best be demonstrated by example. Assume the following -files are managed by -.BR yadm \'s -repository: +files are managed by yadm's repository: - $HOME/path/example.txt##default - $HOME/path/example.txt##class.Work @@ -526,14 +474,12 @@ repository: - $HOME/path/example.txt##os.Linux,hostname.host2 If running on a Macbook named "host2", -.B yadm -will create a symbolic link which looks like this: +yadm will create a symbolic link which looks like this: .IR $HOME/path/example.txt " -> " $HOME/path/example.txt##os.Darwin,hostname.host2 -However, on another Mackbook named "host3", -.B yadm -will create a symbolic link which looks like this: +However, on another Mackbook named "host3", yadm will create a symbolic link +which looks like this: .IR $HOME/path/example.txt " -> " $HOME/path/example.txt##os.Darwin @@ -554,12 +500,11 @@ If running on a system, with class set to "Work", the link will be: If no "##default" version exists and no files have valid conditions, then no link will be created. -Links are also created for directories named this way, as long as they have at least one -.B yadm -managed file within them. +Links are also created for directories named this way, as long as they have at +least one yadm managed file within them. -.B yadm -will automatically create these links by default. This can be disabled using the +yadm will automatically create these links by default. This can be disabled +using the .I yadm.auto-alt configuration. Even if disabled, links can be manually created by running @@ -569,9 +514,7 @@ Class is a special value which is stored locally on each host (inside the local repository). To use alternate symlinks using class, you must set the value of class using the configuration .BR local.class . -This is set like any other -.B yadm -configuration with the +This is set like any other yadm configuration with the .B yadm config command. The following sets the class to be "Work". @@ -593,12 +536,12 @@ processed to create or overwrite files. Supported template processors: .TP .B builtin -This is -.BR yadm \'s -built-in template processor. This processor is very basic, with a Jinja-like -syntax. The advantage of this processor is that it only depends upon \fBawk\fR, -which is available on most *nix systems. To use this processor, specify the -value of "builtin" or just leave the value off (e.g. "##template"). +This is yadm's built-in template processor. This processor is very basic, with +a Jinja-like syntax. The advantage of this processor is that it only depends +upon +.BR awk , +which is available on most *nix systems. To use this processor, +specify the value of "builtin" or just leave the value off (e.g. "##template"). .TP .B j2cli To use the j2cli Jinja template processor, specify the value of "j2" or @@ -660,10 +603,9 @@ would look like: It can be useful to manage confidential files, like SSH or GPG keys, across multiple systems. However, doing so would put plain text data into a Git -repository, which often resides on a public system. -.B yadm -can make it easy to encrypt and decrypt a set of files so the encrypted version -can be maintained in the Git repository. +repository, which often resides on a public system. yadm can make it easy to +encrypt and decrypt a set of files so the encrypted version can be maintained +in the Git repository. This feature will only work if the .BR gpg (1) command is available. @@ -690,9 +632,8 @@ The command will find all files matching the patterns, and prompt for a password. Once a password has confirmed, the matching files will be encrypted and saved as .IR $HOME/.config/yadm/files.gpg . -The patterns and files.gpg should be added to the -.B yadm -repository so they are available across multiple systems. +The patterns and files.gpg should be added to the yadm repository so they are +available across multiple systems. To decrypt these files later, or on another system run .B yadm decrypt @@ -712,10 +653,9 @@ files, even though they are encrypted. .SH PERMISSIONS When files are checked out of a Git repository, their initial permissions are -dependent upon the user's umask. Because of this, -.B yadm -will automatically update the permissions of some file paths. The "group" and -"others" permissions will be removed from the following files: +dependent upon the user's umask. Because of this, yadm will automatically +update the permissions of some file paths. The "group" and "others" permissions +will be removed from the following files: .RI - " $HOME/.config/yadm/files.gpg @@ -728,8 +668,8 @@ will automatically update the permissions of some file paths. The "group" and - The GPG directory and files, .I .gnupg/* -.B yadm -will automatically update permissions by default. This can be disabled using the +yadm will automatically update permissions by default. This can be disabled +using the .I yadm.auto-perms configuration. Even if disabled, permissions can be manually updated by running .BR "yadm perms" . @@ -745,29 +685,22 @@ configuration. When cloning a repo which includes data in a .IR .ssh " or " .gnupg -directory, if those directories do not exist at the time of cloning, -.B yadm -will create the directories with mask 0700 prior to merging the fetched data -into the work-tree. +directory, if those directories do not exist at the time of cloning, yadm will +create the directories with mask 0700 prior to merging the fetched data into +the work-tree. When running a Git command and .IR .ssh " or " .gnupg -directories do not exist, -.B yadm -will create those directories with mask 0700 prior to running the Git command. -This can be disabled using the +directories do not exist, yadm will create those directories with mask 0700 +prior to running the Git command. This can be disabled using the .I yadm.auto-private-dirs configuration. .SH HOOKS -For every command -.B yadm -supports, a program can be provided to run before or after that command. These -are referred to as "hooks". -.B yadm -looks for -hooks in the directory +For every command yadm supports, a program can be provided to run before or +after that command. These are referred to as "hooks". yadm looks for hooks in +the directory .IR $HOME/.config/yadm/hooks . Each hook is named using a prefix of .I pre_ @@ -782,11 +715,8 @@ Hooks must have the executable file permission set. If a .I pre_ -hook is defined, and the hook terminates with a non-zero exit status, -.B yadm -will refuse to run the -.B yadm -command. For example, if a +hook is defined, and the hook terminates with a non-zero exit status, yadm will +refuse to run the yadm command. For example, if a .I pre_commit hook is defined, but that command ends with a non-zero exit status, the .I yadm commit @@ -800,57 +730,41 @@ Hooks have the following environment variables available to them at runtime: The command which triggered the hook .TP .B YADM_HOOK_EXIT -The exit status of the -.B yadm -command +The exit status of the yadm command .TP .B YADM_HOOK_FULL_COMMAND -The -.B yadm -command with all command line arguments +The yadm command with all command line arguments .TP .B YADM_HOOK_REPO -The path to the -.B yadm -repository +The path to the yadm repository .TP .B YADM_HOOK_WORK The path to the work-tree .SH FILES -All of -.BR yadm \'s -configurations are relative to the "yadm directory". -.B yadm -uses the "XDG Base Directory Specification" to determine this directory. If the -environment variable +All of yadm's configurations are relative to the "yadm directory". +yadm uses the "XDG Base Directory Specification" to determine this directory. +If the environment variable .B $XDG_CONFIG_HOME is defined as a fully qualified path, this directory will be .IR "$XDG_CONFIG_HOME/yadm" . Otherwise it will be .IR "$HOME/.config/yadm" . -The following are the default paths -.B yadm -uses for its own data. +The following are the default paths yadm uses for its own data. These paths can be altered using universal options. See the OPTIONS section for details. .TP .I $HOME/.config/yadm -The -.B yadm -directory. By default, all data -.B yadm -stores is relative to this directory. +The yadm directory. By default, all data yadm stores is relative to this +directory. .TP .I $YADM_DIR/config -Configuration file for -.BR yadm . +Configuration file for yadm. .TP .I $YADM_DIR/repo.git -Git repository used by -.BR yadm . +Git repository used by yadm. .TP .I $YADM_DIR/encrypt List of globs used for encrypt/decrypt From 6a3199ceea5bda3130173b44bc789f657409970d Mon Sep 17 00:00:00 2001 From: Tim Byrne Date: Sun, 6 Oct 2019 11:04:21 -0500 Subject: [PATCH 060/137] Support DISTRO in alternates (#72) --- test/test_alt.py | 4 ++- test/test_unit_score_file.py | 51 +++++++++++++++++++++++++++++------- yadm | 13 ++++++--- yadm.1 | 5 ++++ 4 files changed, 60 insertions(+), 13 deletions(-) diff --git a/test/test_alt.py b/test/test_alt.py index e8a0001..b446c2c 100644 --- a/test/test_alt.py +++ b/test/test_alt.py @@ -49,13 +49,14 @@ def test_alt_source( @pytest.mark.parametrize('suffix', [ '##default', '##o.$tst_sys', '##os.$tst_sys', + '##d.$tst_distro', '##distro.$tst_distro', '##c.$tst_class', '##class.$tst_class', '##h.$tst_host', '##hostname.$tst_host', '##u.$tst_user', '##user.$tst_user', ]) def test_alt_conditions( runner, yadm_y, paths, - tst_sys, tst_host, tst_user, suffix): + tst_sys, tst_distro, tst_host, tst_user, suffix): """Test conditions supported by yadm alt""" # set the class @@ -64,6 +65,7 @@ def test_alt_conditions( suffix = string.Template(suffix).substitute( tst_sys=tst_sys, + tst_distro=tst_distro, tst_class=tst_class, tst_host=tst_host, tst_user=tst_user, diff --git a/test/test_unit_score_file.py b/test/test_unit_score_file.py index 122d580..5719fe4 100644 --- a/test/test_unit_score_file.py +++ b/test/test_unit_score_file.py @@ -10,17 +10,21 @@ CONDITION = { 'labels': ['o', 'os'], 'modifier': 1, }, + 'distro': { + 'labels': ['d', 'distro'], + 'modifier': 2, + }, 'class': { 'labels': ['c', 'class'], - 'modifier': 2, + 'modifier': 4, }, 'hostname': { 'labels': ['h', 'hostname'], - 'modifier': 4, + 'modifier': 8, }, 'user': { 'labels': ['u', 'user'], - 'modifier': 8, + 'modifier': 16, }, } TEMPLATE_LABELS = ['t', 'template', 'yadm'] @@ -44,24 +48,35 @@ def calculate_score(filename): if value == 'testsystem': score += 1000 + CONDITION['system']['modifier'] else: - return 0 + score = 0 + break + elif label in CONDITION['distro']['labels']: + if value == 'testdistro': + score += 1000 + CONDITION['distro']['modifier'] + else: + score = 0 + break elif label in CONDITION['class']['labels']: if value == 'testclass': score += 1000 + CONDITION['class']['modifier'] else: - return 0 + score = 0 + break elif label in CONDITION['hostname']['labels']: if value == 'testhost': score += 1000 + CONDITION['hostname']['modifier'] else: - return 0 + score = 0 + break elif label in CONDITION['user']['labels']: if value == 'testuser': score += 1000 + CONDITION['user']['modifier'] else: - return 0 + score = 0 + break elif label in TEMPLATE_LABELS: - return 0 + score = 0 + break return score @@ -69,6 +84,8 @@ def calculate_score(filename): 'default', ['default', None], ids=['default', 'no-default']) @pytest.mark.parametrize( 'system', ['system', None], ids=['system', 'no-system']) +@pytest.mark.parametrize( + 'distro', ['distro', None], ids=['distro', 'no-distro']) @pytest.mark.parametrize( 'cla', ['class', None], ids=['class', 'no-class']) @pytest.mark.parametrize( @@ -76,11 +93,12 @@ def calculate_score(filename): @pytest.mark.parametrize( 'user', ['user', None], ids=['user', 'no-user']) def test_score_values( - runner, yadm, default, system, cla, host, user): + runner, yadm, default, system, distro, cla, host, user): """Test score results""" # pylint: disable=too-many-branches local_class = 'testclass' local_system = 'testsystem' + local_distro = 'testdistro' local_host = 'testhost' local_user = 'testuser' filenames = {'filename##': 0} @@ -105,6 +123,18 @@ def test_score_values( local_system if match else 'badsys' ]) filenames[newfile] = calculate_score(newfile) + if distro: + for filename in list(filenames): + for match in [True, False]: + for label in CONDITION[distro]['labels']: + newfile = filename + if not newfile.endswith('##'): + newfile += ',' + newfile += '.'.join([ + label, + local_distro if match else 'baddistro' + ]) + filenames[newfile] = calculate_score(newfile) if cla: for filename in list(filenames): for match in [True, False]: @@ -147,6 +177,7 @@ def test_score_values( score=0 local_class={local_class} local_system={local_system} + local_distro={local_distro} local_host={local_host} local_user={local_user} """ @@ -169,6 +200,7 @@ def test_score_values_templates(runner, yadm): """Test score results""" local_class = 'testclass' local_system = 'testsystem' + local_distro = 'testdistro' local_host = 'testhost' local_user = 'testuser' filenames = {'filename##': 0} @@ -186,6 +218,7 @@ def test_score_values_templates(runner, yadm): score=0 local_class={local_class} local_system={local_system} + local_distro={local_distro} local_host={local_host} local_user={local_user} """ diff --git a/yadm b/yadm index e4df305..656e508 100755 --- a/yadm +++ b/yadm @@ -154,23 +154,30 @@ function score_file() { score=0 return fi + elif [[ "$label" =~ ^(d|distro)$ ]]; then + if [ "$value" = "$local_distro" ]; then + score=$((score + 2)) + else + score=0 + return + fi elif [[ "$label" =~ ^(c|class)$ ]]; then if [ "$value" = "$local_class" ]; then - score=$((score + 2)) + score=$((score + 4)) else score=0 return fi elif [[ "$label" =~ ^(h|hostname)$ ]]; then if [ "$value" = "$local_host" ]; then - score=$((score + 4)) + score=$((score + 8)) else score=0 return fi elif [[ "$label" =~ ^(u|user)$ ]]; then if [ "$value" = "$local_user" ]; then - score=$((score + 8)) + score=$((score + 16)) else score=0 return diff --git a/yadm.1 b/yadm.1 index 1ad3e11..526b6f9 100644 --- a/yadm.1 +++ b/yadm.1 @@ -424,6 +424,11 @@ Valid if the value matches the current user. Current user is calculated by running .BR "id -u -n" . .TP +.BR distro , " d +Valid if the value matches the distro. +Distro is calculated by running +.BR "lsb_release -si" . +.TP .BR os , " o Valid if the value matches the OS. OS is calculated by running From f3bde37f785fa9ea5782e6a155e66fdbc3e4064e Mon Sep 17 00:00:00 2001 From: Tim Byrne Date: Mon, 7 Oct 2019 08:36:32 -0500 Subject: [PATCH 061/137] Support `-b ` when cloning (#133) --- completion/yadm.bash_completion | 2 +- test/test_clone.py | 45 +++++++++++++++++++++ test/test_unit_is_valid_branch_name.py | 40 +++++++++++++++++++ yadm | 54 +++++++++++++++++++------- yadm.1 | 10 ++++- 5 files changed, 134 insertions(+), 17 deletions(-) create mode 100644 test/test_unit_is_valid_branch_name.py diff --git a/completion/yadm.bash_completion b/completion/yadm.bash_completion index 1091e55..0f21da8 100644 --- a/completion/yadm.bash_completion +++ b/completion/yadm.bash_completion @@ -55,7 +55,7 @@ if declare -F _git > /dev/null; then case "$antepenultimate" in clone) - COMPREPLY=( $(compgen -W "-f -w --bootstrap --no-bootstrap" -- "$current") ) + COMPREPLY=( $(compgen -W "-f -w -b --bootstrap --no-bootstrap" -- "$current") ) return 0 ;; esac diff --git a/test/test_clone.py b/test/test_clone.py index 231d967..1650e92 100644 --- a/test/test_clone.py +++ b/test/test_clone.py @@ -251,6 +251,51 @@ def test_clone_perms( f'.{private_type} has not been secured by auto.perms') +@pytest.mark.usefixtures('remote') +@pytest.mark.parametrize('branch', ['master', 'valid', 'invalid']) +def test_alternate_branch(runner, paths, yadm_y, repo_config, branch): + """Test cloning a branch other than master""" + + # add a "valid" branch to the remote + os.system(f'GIT_DIR="{paths.remote}" git checkout -b valid') + os.system( + f'GIT_DIR="{paths.remote}" git commit ' + f'--allow-empty -m "This branch is valid"') + + # clear out the work path + paths.work.remove() + paths.work.mkdir() + + remote_url = f'file://{paths.remote}' + + # run the clone command + args = ['clone', '-w', paths.work] + if branch != 'master': + args += ['-b', branch] + args += [remote_url] + run = runner(command=yadm_y(*args)) + + if branch == 'invalid': + assert run.failure + assert 'ERROR: Clone failed' in run.out + assert f"'origin/{branch}' does not exist in {remote_url}" in run.out + else: + assert successful_clone(run, paths, repo_config) + + # confirm correct Git origin + run = runner( + command=('git', 'remote', '-v', 'show'), + env={'GIT_DIR': paths.repo}) + assert run.success + assert run.err == '' + assert f'origin\t{remote_url}' in run.out + run = runner(command=yadm_y('show')) + if branch == 'valid': + assert 'This branch is valid' in run.out + else: + assert 'Initial commit' in run.out + + def successful_clone(run, paths, repo_config, expected_code=0): """Assert clone is successful""" assert run.code == expected_code diff --git a/test/test_unit_is_valid_branch_name.py b/test/test_unit_is_valid_branch_name.py new file mode 100644 index 0000000..9e5b6d1 --- /dev/null +++ b/test/test_unit_is_valid_branch_name.py @@ -0,0 +1,40 @@ +"""Unit tests: is_valid_branch_name""" +import pytest + +# Git branches do not allow: +# * path component that begins with "." +# * double dot +# * "~", "^", ":", "\", space +# * end with a "/" +# * end with ".lock" + + +@pytest.mark.parametrize( + 'branch, expected', [ + ('master', 'valid'), + ('path/branch', 'valid'), + ('path/.branch', 'invalid'), + ('path..branch', 'invalid'), + ('path~branch', 'invalid'), + ('path^branch', 'invalid'), + ('path:branch', 'invalid'), + ('path\\branch', 'invalid'), + ('path branch', 'invalid'), + ('path/branch/', 'invalid'), + ('branch.lock', 'invalid'), + ]) +def test_is_valid_branch_name(runner, yadm, branch, expected): + """Test function is_valid_branch_name()""" + + script = f""" + YADM_TEST=1 source {yadm} + if is_valid_branch_name "{branch}"; then + echo valid + else + echo invalid + fi + """ + run = runner(command=['bash'], inp=script) + assert run.success + assert run.err == '' + assert run.out.strip() == expected diff --git a/yadm b/yadm index 656e508..fc5fa29 100755 --- a/yadm +++ b/yadm @@ -610,11 +610,20 @@ function clean() { function clone() { DO_BOOTSTRAP=1 + local branch + branch="master" clone_args=() while [[ $# -gt 0 ]] ; do key="$1" case $key in + -b) + if ! is_valid_branch_name "$2"; then + error_out "You must provide a branch name when using '-b'" + fi + branch="$2" + shift + ;; --bootstrap) # force bootstrap, without prompt DO_BOOTSTRAP=2 ;; @@ -634,12 +643,12 @@ function clone() { local empty= init $empty - # add the specified remote, and configure the repo to track origin/master + # add the specified remote, and configure the repo to track origin/$branch debug "Adding remote to new repo" "$GIT_PROGRAM" remote add origin "${clone_args[@]}" - debug "Configuring new repo to track origin/master" - "$GIT_PROGRAM" config branch.master.remote origin - "$GIT_PROGRAM" config branch.master.merge refs/heads/master + debug "Configuring new repo to track origin/${branch}" + "$GIT_PROGRAM" config "branch.${branch}.remote" origin + "$GIT_PROGRAM" config "branch.${branch}.merge" "refs/heads/${branch}" # fetch / merge (and possibly fallback to reset) debug "Doing an initial fetch of the origin" @@ -648,30 +657,36 @@ function clone() { rm -rf "$YADM_REPO" error_out "Unable to fetch origin ${clone_args[0]}" } + debug "Verifying '${branch}' is a valid branch to merge" + [ -f "${YADM_REPO}/refs/remotes/origin/${branch}" ] || { + debug "Removing repo after failed clone" + rm -rf "$YADM_REPO" + error_out "Clone failed, 'origin/${branch}' does not exist in ${clone_args[0]}" + } debug "Determining if repo tracks private directories" for private_dir in .ssh/ .gnupg/; do - found_log=$("$GIT_PROGRAM" log -n 1 origin/master -- "$private_dir" 2>/dev/null) + found_log=$("$GIT_PROGRAM" log -n 1 "origin/${branch}" -- "$private_dir" 2>/dev/null) if [ -n "$found_log" ]; then debug "Private directory $private_dir is tracked by repo" assert_private_dirs "$private_dir" fi done [ -n "$DEBUG" ] && display_private_perms "pre-merge" - debug "Doing an initial merge of origin/master" - "$GIT_PROGRAM" merge origin/master || { + debug "Doing an initial merge of origin/${branch}" + "$GIT_PROGRAM" merge "origin/${branch}" || { debug "Merge failed, doing a reset and stashing conflicts." - "$GIT_PROGRAM" reset origin/master + "$GIT_PROGRAM" reset "origin/${branch}" if cd "$YADM_WORK"; then # necessary because of a bug in Git "$GIT_PROGRAM" -c user.name='yadm clone' -c user.email='yadm' stash save Conflicts preserved from yadm clone command 2>&1 cat </dev/null 2>&1; then diff --git a/yadm.1 b/yadm.1 index 526b6f9..a1b5be5 100644 --- a/yadm.1 +++ b/yadm.1 @@ -19,13 +19,15 @@ yadm \- Yet Another Dotfiles Manager init .RB [ -f ] .RB [ -w -.IR directory ] +.IR dir ] .B yadm .RI clone " url .RB [ -f ] .RB [ -w -.IR directory ] +.IR dir ] +.RB [ -b +.IR branch ] .RB [ --bootstrap ] .RB [ --no-bootstrap ] @@ -146,6 +148,10 @@ but this can be overridden with the .BR -w " option. yadm can be forced to overwrite an existing repository by providing the .BR -f " option. +If you want to use a branch other than +.IR origin/master , +you can specify it using the +.BR -b " option. By default yadm will ask the user if the bootstrap program should be run (if it exists). The options .BR --bootstrap " or " --no-bootstrap From e51166b7e815f118a62e73b95c1c330f9ed92064 Mon Sep 17 00:00:00 2001 From: Tim Byrne Date: Tue, 8 Oct 2019 08:40:33 -0500 Subject: [PATCH 062/137] Improve clone testing --- test/test_clone.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/test/test_clone.py b/test/test_clone.py index 1650e92..7cbbb5f 100644 --- a/test/test_clone.py +++ b/test/test_clone.py @@ -28,6 +28,10 @@ def test_clone( good_remote, repo_exists, force, conflicts): """Test basic clone operation""" + # clear out the work path + paths.work.remove() + paths.work.mkdir() + # determine remote url remote_url = f'file://{paths.remote}' if not good_remote: From 574945f0101ce39ca7fe1da25e534fc3ca9e027b Mon Sep 17 00:00:00 2001 From: Tim Byrne Date: Wed, 9 Oct 2019 08:25:02 -0500 Subject: [PATCH 063/137] Change yadm.cygwin-copy to yadm.alt-copy This removes the constraint of only allowing the copy option on Cygwin systems. Now any system can configure this option. --- test/conftest.py | 16 +++------ test/test_alt_copy.py | 61 ++++++++++++++++++++++++++++++++++ test/test_cygwin_copy.py | 71 ---------------------------------------- yadm | 9 ++--- yadm.1 | 16 ++++++--- 5 files changed, 81 insertions(+), 92 deletions(-) create mode 100644 test/test_alt_copy.py delete mode 100644 test/test_cygwin_copy.py diff --git a/test/conftest.py b/test/conftest.py index b1cf16f..805ee92 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -63,12 +63,6 @@ def tst_sys(): return platform.system() -@pytest.fixture(scope='session') -def cygwin_sys(): - """CYGWIN uname id""" - return 'CYGWIN_NT-6.1-WOW64' - - @pytest.fixture(scope='session') def supported_commands(): """List of supported commands @@ -105,10 +99,10 @@ def supported_configs(): 'local.hostname', 'local.os', 'local.user', + 'yadm.alt-copy', 'yadm.auto-alt', 'yadm.auto-perms', 'yadm.auto-private-dirs', - 'yadm.cygwin-copy', 'yadm.git-program', 'yadm.gpg-perms', 'yadm.gpg-program', @@ -444,7 +438,7 @@ class DataSet(): @pytest.fixture(scope='session') -def ds1_dset(tst_sys, cygwin_sys): +def ds1_dset(tst_sys): """Meta-data for dataset one files""" dset = DataSet() dset.add_file('t1') @@ -457,10 +451,8 @@ def ds1_dset(tst_sys, cygwin_sys): dset.add_file(f'test alt/test alt##S.H') dset.add_file(f'test alt/test alt##S.H.U') dset.add_file(f'test alt/test alt##C.S.H.U') - dset.add_file(f'test_cygwin_copy##{tst_sys}') - dset.add_file(f'test_cygwin_copy##{cygwin_sys}') - dset.add_file(f'test_cygwin_copy##os.{tst_sys}') - dset.add_file(f'test_cygwin_copy##os.{cygwin_sys}') + dset.add_file(f'test_alt_copy##{tst_sys}') + dset.add_file(f'test_alt_copy##os.{tst_sys}') dset.add_file('u1', tracked=False) dset.add_file('d2/u2', tracked=False) dset.add_file('.ssh/p1', tracked=False, private=True) diff --git a/test/test_alt_copy.py b/test/test_alt_copy.py new file mode 100644 index 0000000..b26848f --- /dev/null +++ b/test/test_alt_copy.py @@ -0,0 +1,61 @@ +"""Test yadm.alt-copy""" + +import os +import pytest + + +@pytest.mark.parametrize( + 'cygwin', + [pytest.param(True, marks=pytest.mark.deprecated), False], + ids=['cygwin', 'no-cygwin']) +@pytest.mark.parametrize( + 'compatibility', [True, False], ids=['compat', 'no-compat']) +@pytest.mark.parametrize( + 'setting, expect_link, pre_existing', [ + (None, True, None), + (True, False, None), + (False, True, None), + (True, False, 'link'), + (True, False, 'file'), + ], + ids=[ + 'unset', + 'true', + 'false', + 'pre-existing symlink', + 'pre-existing file', + ]) +@pytest.mark.usefixtures('ds1_copy') +def test_alt_copy( + runner, yadm_y, paths, tst_sys, + setting, expect_link, pre_existing, + compatibility, cygwin): + """Test yadm.alt-copy""" + + option = 'yadm.cygwin-copy' if cygwin else 'yadm.alt-copy' + + if setting is not None: + os.system(' '.join(yadm_y('config', option, str(setting)))) + + if compatibility: + expected_content = f'test_alt_copy##{tst_sys}' + else: + expected_content = f'test_alt_copy##os.{tst_sys}' + + alt_path = paths.work.join('test_alt_copy') + if pre_existing == 'symlink': + alt_path.mklinkto(expected_content) + elif pre_existing == 'file': + alt_path.write('wrong content') + + env = os.environ.copy() + if compatibility: + env['YADM_COMPATIBILITY'] = '1' + + run = runner(yadm_y('alt'), env=env) + assert run.success + assert run.err == '' + assert 'Linking' in run.out + + assert alt_path.read() == expected_content + assert alt_path.islink() == expect_link diff --git a/test/test_cygwin_copy.py b/test/test_cygwin_copy.py deleted file mode 100644 index 47dbb61..0000000 --- a/test/test_cygwin_copy.py +++ /dev/null @@ -1,71 +0,0 @@ -"""Test yadm.cygwin_copy""" - -import os -import pytest - - -@pytest.mark.parametrize( - 'compatibility', [True, False], ids=['compat', 'no-compat']) -@pytest.mark.parametrize( - 'setting, is_cygwin, expect_link, pre_existing', [ - (None, False, True, None), - (True, False, True, None), - (False, False, True, None), - (None, True, True, None), - (True, True, False, None), - (False, True, True, None), - (True, True, False, 'link'), - (True, True, False, 'file'), - ], - ids=[ - 'unset, non-cygwin', - 'true, non-cygwin', - 'false, non-cygwin', - 'unset, cygwin', - 'true, cygwin', - 'false, cygwin', - 'pre-existing symlink', - 'pre-existing file', - ]) -@pytest.mark.usefixtures('ds1_copy') -def test_cygwin_copy( - runner, yadm_y, paths, cygwin_sys, tst_sys, - setting, is_cygwin, expect_link, pre_existing, - compatibility): - """Test yadm.cygwin_copy""" - - if setting is not None: - os.system(' '.join(yadm_y('config', 'yadm.cygwin-copy', str(setting)))) - - if compatibility: - expected_content = f'test_cygwin_copy##{tst_sys}' - else: - expected_content = f'test_cygwin_copy##os.{tst_sys}' - - alt_path = paths.work.join('test_cygwin_copy') - if pre_existing == 'symlink': - alt_path.mklinkto(expected_content) - elif pre_existing == 'file': - alt_path.write('wrong content') - - uname_path = paths.root.join('tmp').mkdir() - if is_cygwin: - uname = uname_path.join('uname') - uname.write(f'#!/bin/sh\necho "{cygwin_sys}"\n') - uname.chmod(0o777) - if compatibility: - expected_content = f'test_cygwin_copy##{cygwin_sys}' - else: - expected_content = f'test_cygwin_copy##os.{cygwin_sys}' - env = os.environ.copy() - env['PATH'] = ':'.join([str(uname_path), env['PATH']]) - if compatibility: - env['YADM_COMPATIBILITY'] = '1' - - run = runner(yadm_y('alt'), env=env) - assert run.success - assert run.err == '' - assert 'Linking' in run.out - - assert alt_path.read() == expected_content - assert alt_path.islink() == expect_link diff --git a/yadm b/yadm index c3ec5c5..887da84 100755 --- a/yadm +++ b/yadm @@ -389,9 +389,10 @@ function alt() { # decide if a copy should be done instead of a symbolic link local do_copy=0 - [[ $OPERATING_SYSTEM =~ ^(CYGWIN|MSYS) ]] && - [ "$(config --bool yadm.cygwin-copy)" == "true" ] && - do_copy=1 + [ "$(config --bool yadm.alt-copy)" == "true" ] && do_copy=1 + + # deprecated yadm.cygwin-copy option (to be removed) + [ "$(config --bool yadm.cygwin-copy)" == "true" ] && do_copy=1 cd_work "Alternates" || return @@ -966,10 +967,10 @@ local.class local.hostname local.os local.user +yadm.alt-copy yadm.auto-alt yadm.auto-perms yadm.auto-private-dirs -yadm.cygwin-copy yadm.git-program yadm.gpg-perms yadm.gpg-program diff --git a/yadm.1 b/yadm.1 index a1b5be5..35a2294 100644 --- a/yadm.1 +++ b/yadm.1 @@ -331,6 +331,17 @@ yadm config yadm.auto-alt false The following is the full list of supported configurations: .TP +.B yadm.alt-copy +If set to "true", alternate files will be copies instead of symbolic links. +This might be desirable, because some systems may not properly support +symlinks. + +NOTE: The deprecated +.I yadm.cygwin-copy +option used by older versions of yadm has been replaced by +.IR yadm.alt-copy . +The old option will be removed in the next version of yadm. +.TP .B yadm.auto-alt Disable the automatic linking described in the section ALTERNATES. If disabled, you may still run "yadm alt" manually to create the alternate links. This @@ -372,11 +383,6 @@ By default, the first "gpg" found in $PATH is used. .B yadm.git-program Specify an alternate program to use instead of "git". By default, the first "git" found in $PATH is used. -.TP -.B yadm.cygwin-copy -If set to "true", for Cygwin hosts, alternate files will be copies instead of -symbolic links. This might be desirable, because non-Cygwin software may not -properly interpret Cygwin symlinks. .RE The following four "local" configurations are not stored in the From 117541fd217b3993eb41f801461ee33d8a834c0d Mon Sep 17 00:00:00 2001 From: Tim Byrne Date: Wed, 9 Oct 2019 18:40:25 -0500 Subject: [PATCH 064/137] Reorder descriptions of options --- yadm.1 | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/yadm.1 b/yadm.1 index 35a2294..be4d6b3 100644 --- a/yadm.1 +++ b/yadm.1 @@ -357,16 +357,19 @@ This feature is enabled by default. .B yadm.auto-private-dirs Disable the automatic creating of private directories described in the section PERMISSIONS. .TP -.B yadm.ssh-perms -Disable the permission changes to -.IR $HOME/.ssh/* . -This feature is enabled by default. +.B yadm.git-program +Specify an alternate program to use instead of "git". +By default, the first "git" found in $PATH is used. .TP .B yadm.gpg-perms Disable the permission changes to .IR $HOME/.gnupg/* . This feature is enabled by default. .TP +.B yadm.gpg-program +Specify an alternate program to use instead of "gpg". +By default, the first "gpg" found in $PATH is used. +.TP .B yadm.gpg-recipient Asymmetrically encrypt files with a gpg public/private key pair. Provide a "key ID" to specify which public key to encrypt with. @@ -376,13 +379,10 @@ If set to "ASK", gpg will interactively ask for recipients. See the ENCRYPTION section for more details. This feature is disabled by default. .TP -.B yadm.gpg-program -Specify an alternate program to use instead of "gpg". -By default, the first "gpg" found in $PATH is used. -.TP -.B yadm.git-program -Specify an alternate program to use instead of "git". -By default, the first "git" found in $PATH is used. +.B yadm.ssh-perms +Disable the permission changes to +.IR $HOME/.ssh/* . +This feature is enabled by default. .RE The following four "local" configurations are not stored in the @@ -394,12 +394,12 @@ they are stored in the local repository. Specify a class for the purpose of symlinking alternate files. By default, no class will be matched. .TP -.B local.os -Override the OS for the purpose of symlinking alternate files. -.TP .B local.hostname Override the hostname for the purpose of symlinking alternate files. .TP +.B local.os +Override the OS for the purpose of symlinking alternate files. +.TP .B local.user Override the user for the purpose of symlinking alternate files. From 3a192db4201218cb72c1b688b776e3d19da3e3c4 Mon Sep 17 00:00:00 2001 From: Tim Byrne Date: Fri, 11 Oct 2019 07:20:03 -0500 Subject: [PATCH 065/137] Remove symlinks before processing a template --- test/test_alt.py | 27 +++++++++++++++++++++++++++ yadm | 7 ++++--- 2 files changed, 31 insertions(+), 3 deletions(-) diff --git a/test/test_alt.py b/test/test_alt.py index b446c2c..def2037 100644 --- a/test/test_alt.py +++ b/test/test_alt.py @@ -193,3 +193,30 @@ def test_stale_link_removal(runner, yadm_y, paths): source_file = stale_path + '##class.' + tst_class assert not paths.work.join(stale_path).exists() assert str(paths.work.join(source_file)) not in linked + + +@pytest.mark.usefixtures('ds1_repo_copy') +def test_template_overwrite_symlink(runner, yadm_y, paths, tst_sys): + """Remove symlinks before processing a template + + If a symlink is in the way of the output of a template, the target of the + symlink will get the template content. To prevent this, the symlink should + be removed just before processing a template. + """ + + target = paths.work.join(f'test_link##os.{tst_sys}') + target.write('target') + + link = paths.work.join('test_link') + link.mksymlinkto(target, absolute=1) + + template = paths.work.join('test_link##template.builtin') + template.write('test-data') + + run = runner(yadm_y('add', target, template)) + assert run.success + assert run.err == '' + assert run.out == '' + assert not link.islink() + assert target.read().strip() == 'target' + assert link.read().strip() == 'test-data' diff --git a/yadm b/yadm index 887da84..6491b87 100755 --- a/yadm +++ b/yadm @@ -498,15 +498,16 @@ function alt_future_linking() { # a template is defined, process the template debug "Creating $filename from template $target" [ -n "$loud" ] && echo "Creating $filename from template $target" + # remove any existing symlink before processing template + [ -L "$filename" ] && rm -f "$filename" "$template_cmd" "$target" "$filename" elif [ -n "$target" ]; then # a link target is defined, create symlink debug "Linking $target to $filename" [ -n "$loud" ] && echo "Linking $target to $filename" if [ "$do_copy" -eq 1 ]; then - if [ -L "$filename" ]; then - rm -f "$filename" - fi + # remove any existing symlink before copying + [ -L "$filename" ] && rm -f "$filename" cp -f "$target" "$filename" else ln -nfs "$target" "$filename" From 0c7aec6dd70dca6f7f012d13d7ee37e97f2bcb88 Mon Sep 17 00:00:00 2001 From: Tim Byrne Date: Fri, 11 Oct 2019 07:22:38 -0500 Subject: [PATCH 066/137] Clarify xfails are for deprecated features --- test/test_compat_alt.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test_compat_alt.py b/test/test_compat_alt.py index e652a0c..a2c9ad1 100644 --- a/test/test_compat_alt.py +++ b/test/test_compat_alt.py @@ -153,7 +153,7 @@ def test_wild(request, runner, yadm_y, paths, if request.node.name in BROKEN_TEST_IDS: pytest.xfail( 'This test is known to be broken. ' - 'This bug needs to be fixed.') + 'This bug only affects deprecated features.') tst_class = 'testclass' From aeb6a54ad7e417d4732a3dc1e36a5663b3b860de Mon Sep 17 00:00:00 2001 From: Tim Byrne Date: Thu, 10 Oct 2019 08:09:31 -0500 Subject: [PATCH 067/137] Add `source` to templates (#163) A new variable is exposed to templates, which holds the filename of the template source. The primary use case is to be able to include a warning message within the template. For example: # Do not edit. This file auto-generated from {{ yadm.source }}. --- test/test_unit_template_builtin.py | 17 +++++++++++++++++ test/test_unit_template_j2.py | 18 ++++++++++++++++++ yadm | 4 ++++ yadm.1 | 1 + 4 files changed, 40 insertions(+) diff --git a/test/test_unit_template_builtin.py b/test/test_unit_template_builtin.py index a699f6a..bfc7747 100644 --- a/test/test_unit_template_builtin.py +++ b/test/test_unit_template_builtin.py @@ -106,3 +106,20 @@ def test_template_builtin(runner, yadm, tmpdir): assert run.success assert run.err == '' assert output_file.read() == EXPECTED + + +def test_source(runner, yadm, tmpdir): + """Test yadm.source""" + + input_file = tmpdir.join('input') + input_file.write('{{yadm.source}}', ensure=True) + output_file = tmpdir.join('output') + + script = f""" + YADM_TEST=1 source {yadm} + template_builtin "{input_file}" "{output_file}" + """ + run = runner(command=['bash'], inp=script) + assert run.success + assert run.err == '' + assert output_file.read().strip() == str(input_file) diff --git a/test/test_unit_template_j2.py b/test/test_unit_template_j2.py index e511576..85c6822 100644 --- a/test/test_unit_template_j2.py +++ b/test/test_unit_template_j2.py @@ -97,3 +97,21 @@ def test_template_j2(runner, yadm, tmpdir, processor): assert run.success assert run.err == '' assert output_file.read() == EXPECTED + + +@pytest.mark.parametrize('processor', ('j2cli', 'envtpl')) +def test_source(runner, yadm, tmpdir, processor): + """Test YADM_SOURCE""" + + input_file = tmpdir.join('input') + input_file.write('{{YADM_SOURCE}}', ensure=True) + output_file = tmpdir.join('output') + + script = f""" + YADM_TEST=1 source {yadm} + template_{processor} "{input_file}" "{output_file}" + """ + run = runner(command=['bash'], inp=script) + assert run.success + assert run.err == '' + assert output_file.read().strip() == str(input_file) diff --git a/yadm b/yadm index 6491b87..1529785 100755 --- a/yadm +++ b/yadm @@ -297,6 +297,7 @@ BEGIN { c["hostname"] = host c["user"] = user c["distro"] = distro + c["source"] = source vld = conditions() ifs = "^{%" blank "*if" els = "^{%" blank "*else" blank "*%}$" @@ -340,6 +341,7 @@ EOF -v host="$local_host" \ -v user="$local_user" \ -v distro="$local_distro" \ + -v source="$input" \ "$awk_pgm" \ "$input" > "$output" } @@ -353,6 +355,7 @@ function template_j2cli() { YADM_HOSTNAME="$local_host" \ YADM_USER="$local_user" \ YADM_DISTRO="$local_distro" \ + YADM_SOURCE="$input" \ "$J2CLI_PROGRAM" "$input" -o "$output" } @@ -365,6 +368,7 @@ function template_envtpl() { YADM_HOSTNAME="$local_host" \ YADM_USER="$local_user" \ YADM_DISTRO="$local_distro" \ + YADM_SOURCE="$input" \ "$ENVTPL_PROGRAM" --keep-template "$input" -o "$output" } diff --git a/yadm.1 b/yadm.1 index be4d6b3..925650f 100644 --- a/yadm.1 +++ b/yadm.1 @@ -584,6 +584,7 @@ During processing, the following variables are available in the template: yadm.hostname YADM_HOSTNAME hostname (without domain) yadm.os YADM_OS uname -s yadm.user YADM_USER id -u -n + yadm.source YADM_SOURCE Template filename Examples: From 4ea3ed9e2af11421d8fc029a8438ce74c1bbdd1c Mon Sep 17 00:00:00 2001 From: Tim Byrne Date: Thu, 10 Oct 2019 08:23:36 -0500 Subject: [PATCH 068/137] Allow storing alternates elsewhere (#90) This change allows alternates to be stored in "$YADM_DIR/alt". The correct path within the work tree will be symlinked. Storing alternates within the work tree is still allowed. Both locations will be considered when choosing an appropriate alternate file. --- test/test_alt.py | 54 ++++++++++++++++++++++++++++++++---------------- test/utils.py | 35 ++++++++++++++++++------------- yadm | 7 +++++++ yadm.1 | 13 +++++++++++- 4 files changed, 76 insertions(+), 33 deletions(-) diff --git a/test/test_alt.py b/test/test_alt.py index def2037..cbac403 100644 --- a/test/test_alt.py +++ b/test/test_alt.py @@ -9,6 +9,7 @@ TEST_PATHS = [utils.ALT_FILE1, utils.ALT_FILE2, utils.ALT_DIR] @pytest.mark.usefixtures('ds1_copy') +@pytest.mark.parametrize('yadm_alt', [True, False], ids=['alt', 'worktree']) @pytest.mark.parametrize( 'tracked,encrypt,exclude', [ (False, False, False), @@ -17,32 +18,39 @@ TEST_PATHS = [utils.ALT_FILE1, utils.ALT_FILE2, utils.ALT_DIR] (False, True, True), ], ids=['untracked', 'tracked', 'encrypted', 'excluded']) def test_alt_source( - runner, yadm_y, paths, - tracked, encrypt, exclude): + runner, paths, + tracked, encrypt, exclude, + yadm_alt): """Test yadm alt operates on all expected sources of alternates""" + yadm_dir = setup_standard_yadm_dir(paths) utils.create_alt_files( - paths, '##default', tracked=tracked, encrypt=encrypt, exclude=exclude) - run = runner(yadm_y('alt')) + paths, '##default', tracked=tracked, encrypt=encrypt, exclude=exclude, + yadm_alt=yadm_alt, yadm_dir=yadm_dir) + run = runner([paths.pgm, '-Y', yadm_dir, 'alt']) assert run.success assert run.err == '' linked = utils.parse_alt_output(run.out) + basepath = yadm_dir.join('alt') if yadm_alt else paths.work + for link_path in TEST_PATHS: - source_file = link_path + '##default' + source_file_content = link_path + '##default' + source_file = basepath.join(source_file_content) + link_file = paths.work.join(link_path) if tracked or (encrypt and not exclude): - assert paths.work.join(link_path).islink() - target = py.path.local(paths.work.join(link_path).readlink()) + assert link_file.islink() + target = py.path.local(link_file.readlink()) if target.isfile(): - assert paths.work.join(link_path).read() == source_file - assert str(paths.work.join(source_file)) in linked + assert link_file.read() == source_file_content + assert str(source_file) in linked else: - assert paths.work.join(link_path).join( - utils.CONTAINED).read() == source_file - assert str(paths.work.join(source_file)) in linked + assert link_file.join( + utils.CONTAINED).read() == source_file_content + assert str(source_file) in linked else: - assert not paths.work.join(link_path).exists() - assert str(paths.work.join(source_file)) not in linked + assert not link_file.exists() + assert str(source_file) not in linked @pytest.mark.usefixtures('ds1_copy') @@ -55,9 +63,10 @@ def test_alt_source( '##u.$tst_user', '##user.$tst_user', ]) def test_alt_conditions( - runner, yadm_y, paths, + runner, paths, tst_sys, tst_distro, tst_host, tst_user, suffix): """Test conditions supported by yadm alt""" + yadm_dir = setup_standard_yadm_dir(paths) # set the class tst_class = 'testclass' @@ -72,7 +81,7 @@ def test_alt_conditions( ) utils.create_alt_files(paths, suffix) - run = runner(yadm_y('alt')) + run = runner([paths.pgm, '-Y', yadm_dir, 'alt']) assert run.success assert run.err == '' linked = utils.parse_alt_output(run.out) @@ -94,12 +103,13 @@ def test_alt_conditions( @pytest.mark.parametrize('kind', ['builtin', '', 'envtpl', 'j2cli', 'j2']) @pytest.mark.parametrize('label', ['t', 'template', 'yadm', ]) def test_alt_templates( - runner, yadm_y, paths, kind, label): + runner, paths, kind, label): """Test templates supported by yadm alt""" + yadm_dir = setup_standard_yadm_dir(paths) suffix = f'##{label}.{kind}' utils.create_alt_files(paths, suffix) - run = runner(yadm_y('alt')) + run = runner([paths.pgm, '-Y', yadm_dir, 'alt']) assert run.success assert run.err == '' created = utils.parse_alt_output(run.out, linked=False) @@ -220,3 +230,11 @@ def test_template_overwrite_symlink(runner, yadm_y, paths, tst_sys): assert not link.islink() assert target.read().strip() == 'target' assert link.read().strip() == 'test-data' + + +def setup_standard_yadm_dir(paths): + """Configure a yadm home within the work tree""" + std_yadm_dir = paths.work.mkdir('.config').mkdir('yadm') + std_yadm_dir.join('repo.git').mksymlinkto(paths.repo, absolute=1) + std_yadm_dir.join('encrypt').mksymlinkto(paths.encrypt, absolute=1) + return std_yadm_dir diff --git a/test/utils.py b/test/utils.py index 00a0fdd..32db94a 100644 --- a/test/utils.py +++ b/test/utils.py @@ -32,7 +32,8 @@ def set_local(paths, variable, value): def create_alt_files(paths, suffix, preserve=False, tracked=True, encrypt=False, exclude=False, - content=None, includefile=False): + content=None, includefile=False, + yadm_alt=False, yadm_dir=None): """Create new files, and add to the repo This is used for testing alternate files. In each case, a suffix is @@ -40,17 +41,19 @@ def create_alt_files(paths, suffix, repo handling are dependent upon the function arguments. """ + basepath = yadm_dir.join('alt') if yadm_alt else paths.work + if not preserve: for remove_path in (ALT_FILE1, ALT_FILE2, ALT_DIR): - if paths.work.join(remove_path).exists(): - paths.work.join(remove_path).remove(rec=1, ignore_errors=True) - assert not paths.work.join(remove_path).exists() + if basepath.join(remove_path).exists(): + basepath.join(remove_path).remove(rec=1, ignore_errors=True) + assert not basepath.join(remove_path).exists() - new_file1 = paths.work.join(ALT_FILE1 + suffix) + new_file1 = basepath.join(ALT_FILE1 + suffix) new_file1.write(ALT_FILE1 + suffix, ensure=True) - new_file2 = paths.work.join(ALT_FILE2 + suffix) + new_file2 = basepath.join(ALT_FILE2 + suffix) new_file2.write(ALT_FILE2 + suffix, ensure=True) - new_dir = paths.work.join(ALT_DIR + suffix).join(CONTAINED) + new_dir = basepath.join(ALT_DIR + suffix).join(CONTAINED) new_dir.write(ALT_DIR + suffix, ensure=True) # Do not test directory support for jinja alternates @@ -65,9 +68,11 @@ def create_alt_files(paths, suffix, test_path.write('\n' + content, mode='a', ensure=True) assert test_path.exists() - _create_includefiles(includefile, paths, test_paths) + _create_includefiles(includefile, test_paths, basepath) _create_tracked(tracked, test_paths, paths) - _create_encrypt(encrypt, test_names, suffix, paths, exclude) + + prefix = '.config/yadm/alt/' if yadm_alt else '' + _create_encrypt(encrypt, test_names, suffix, paths, exclude, prefix) def parse_alt_output(output, linked=True): @@ -86,10 +91,10 @@ def parse_alt_output(output, linked=True): return parsed_list.values() -def _create_includefiles(includefile, paths, test_paths): +def _create_includefiles(includefile, test_paths, basepath): if includefile: for dpath in INCLUDE_DIRS: - incfile = paths.work.join(dpath + '/' + INCLUDE_FILE) + incfile = basepath.join(dpath + '/' + INCLUDE_FILE) incfile.write(INCLUDE_CONTENT, ensure=True) test_paths += [incfile] @@ -101,9 +106,11 @@ def _create_tracked(tracked, test_paths, paths): os.system(f'GIT_DIR={str(paths.repo)} git commit -m "Add test files"') -def _create_encrypt(encrypt, test_names, suffix, paths, exclude): +def _create_encrypt(encrypt, test_names, suffix, paths, exclude, prefix): if encrypt: for encrypt_name in test_names: - paths.encrypt.write(f'{encrypt_name + suffix}\n', mode='a') + paths.encrypt.write( + f'{prefix + encrypt_name + suffix}\n', mode='a') if exclude: - paths.encrypt.write(f'!{encrypt_name + suffix}\n', mode='a') + paths.encrypt.write( + f'!{prefix + encrypt_name + suffix}\n', mode='a') diff --git a/yadm b/yadm index 1529785..687264a 100755 --- a/yadm +++ b/yadm @@ -33,6 +33,7 @@ YADM_ENCRYPT="encrypt" YADM_ARCHIVE="files.gpg" YADM_BOOTSTRAP="bootstrap" YADM_HOOKS="hooks" +YADM_ALT="alt" HOOK_COMMAND="" FULL_COMMAND="" @@ -137,6 +138,11 @@ function score_file() { target="$1" filename="${target%%##*}" conditions="${target#*##}" + + if [ "${filename#$YADM_ALT/}" != "${filename}" ]; then + filename="${YADM_WORK}/${filename#$YADM_ALT/}" + fi + score=0 IFS=',' read -ra fields <<< "$conditions" for field in "${fields[@]}"; do @@ -1223,6 +1229,7 @@ function configure_paths() { YADM_ARCHIVE="$YADM_DIR/$YADM_ARCHIVE" YADM_BOOTSTRAP="$YADM_DIR/$YADM_BOOTSTRAP" YADM_HOOKS="$YADM_DIR/$YADM_HOOKS" + YADM_ALT="$YADM_DIR/$YADM_ALT" # independent overrides for paths if [ -n "$YADM_OVERRIDE_REPO" ]; then diff --git a/yadm.1 b/yadm.1 index 925650f..aff25f3 100644 --- a/yadm.1 +++ b/yadm.1 @@ -478,6 +478,12 @@ condition. The number of conditions is the next largest factor in scoring. Files with more conditions will always be favored. Any invalid condition will disqualify that file completely. +If you don't care to have all versions of alternates stored in the same +directory as the generated symlink, you can place them in the +.I $HOME/.config/yadm/alt +directory. The generated symlink or processed template will be created using +same relative path. + Alternate linking may best be demonstrated by example. Assume the following files are managed by yadm's repository: @@ -771,7 +777,7 @@ Otherwise it will be .IR "$HOME/.config/yadm" . The following are the default paths yadm uses for its own data. -These paths can be altered using universal options. +Most of these paths can be altered using universal options. See the OPTIONS section for details. .TP .I $HOME/.config/yadm @@ -781,6 +787,11 @@ directory. .I $YADM_DIR/config Configuration file for yadm. .TP +.I $YADM_DIR/alt +This is a directory to keep "alternate files" without having them side-by-side +with the resulting symlink or processed template. Alternate files placed in +this directory will be created relative to $HOME instead. +.TP .I $YADM_DIR/repo.git Git repository used by yadm. .TP From 6d5467951a5a4bc9a98add128365bb5d315442f5 Mon Sep 17 00:00:00 2001 From: Tim Byrne Date: Sat, 12 Oct 2019 09:14:45 -0500 Subject: [PATCH 069/137] Properly handle missing "." in alternate conditions --- test/test_alt.py | 5 ++++- test/utils.py | 2 +- yadm | 1 + 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/test/test_alt.py b/test/test_alt.py index cbac403..68954a0 100644 --- a/test/test_alt.py +++ b/test/test_alt.py @@ -100,7 +100,8 @@ def test_alt_conditions( @pytest.mark.usefixtures('ds1_copy') -@pytest.mark.parametrize('kind', ['builtin', '', 'envtpl', 'j2cli', 'j2']) +@pytest.mark.parametrize( + 'kind', ['builtin', '', None, 'envtpl', 'j2cli', 'j2']) @pytest.mark.parametrize('label', ['t', 'template', 'yadm', ]) def test_alt_templates( runner, paths, kind, label): @@ -108,6 +109,8 @@ def test_alt_templates( yadm_dir = setup_standard_yadm_dir(paths) suffix = f'##{label}.{kind}' + if kind is None: + suffix = f'##{label}' utils.create_alt_files(paths, suffix) run = runner([paths.pgm, '-Y', yadm_dir, 'alt']) assert run.success diff --git a/test/utils.py b/test/utils.py index 32db94a..2291fc5 100644 --- a/test/utils.py +++ b/test/utils.py @@ -59,7 +59,7 @@ def create_alt_files(paths, suffix, # Do not test directory support for jinja alternates test_paths = [new_file1, new_file2] test_names = [ALT_FILE1, ALT_FILE2] - if not re.match(r'##(t|template|yadm)\.', suffix): + if not re.match(r'##(t$|t\.|template|yadm)', suffix): test_paths += [new_dir] test_names += [ALT_DIR] diff --git a/yadm b/yadm index 687264a..c85ae4f 100755 --- a/yadm +++ b/yadm @@ -148,6 +148,7 @@ function score_file() { for field in "${fields[@]}"; do label=${field%%.*} value=${field#*.} + [ "$field" = "$label" ] && value="" # when .value is omitted score=$((score + 1000)) # default condition if [[ "$label" =~ ^(default)$ ]]; then From f3249e00b5f001d0c3e382b2aad2764031e0b4be Mon Sep 17 00:00:00 2001 From: Tim Byrne Date: Sat, 12 Oct 2019 18:22:02 -0500 Subject: [PATCH 070/137] Ensure base directories for alternates before creation --- test/test_alt.py | 16 ++++++++++++++++ yadm | 5 +++++ 2 files changed, 21 insertions(+) diff --git a/test/test_alt.py b/test/test_alt.py index 68954a0..dd62f07 100644 --- a/test/test_alt.py +++ b/test/test_alt.py @@ -235,6 +235,22 @@ def test_template_overwrite_symlink(runner, yadm_y, paths, tst_sys): assert link.read().strip() == 'test-data' +@pytest.mark.usefixtures('ds1_copy') +@pytest.mark.parametrize('style', ['symlink', 'template']) +def test_ensure_alt_path(runner, paths, style): + """Test that directories are created before making alternates""" + yadm_dir = setup_standard_yadm_dir(paths) + suffix = 'default' if style == 'symlink' else 'template' + filename = 'a/b/c/file' + source = yadm_dir.join(f'alt/{filename}##{suffix}') + source.write('test-data', ensure=True) + run = runner([paths.pgm, '-Y', yadm_dir, 'add', source]) + assert run.success + assert run.err == '' + assert run.out == '' + assert paths.work.join(filename).read().strip() == 'test-data' + + def setup_standard_yadm_dir(paths): """Configure a yadm home within the work tree""" std_yadm_dir = paths.work.mkdir('.config').mkdir('yadm') diff --git a/yadm b/yadm index c85ae4f..eb55852 100755 --- a/yadm +++ b/yadm @@ -505,10 +505,13 @@ function alt_future_linking() { filename="${alt_filenames[$index]}" target="${alt_targets[$index]}" template_cmd="${alt_template_cmds[$index]}" + basedir=${filename%/*} if [ -n "$template_cmd" ]; then # a template is defined, process the template debug "Creating $filename from template $target" [ -n "$loud" ] && echo "Creating $filename from template $target" + # ensure the destination path exists + [ -e "$basedir" ] || mkdir -p "$basedir" # remove any existing symlink before processing template [ -L "$filename" ] && rm -f "$filename" "$template_cmd" "$target" "$filename" @@ -516,6 +519,8 @@ function alt_future_linking() { # a link target is defined, create symlink debug "Linking $target to $filename" [ -n "$loud" ] && echo "Linking $target to $filename" + # ensure the destination path exists + [ -e "$basedir" ] || mkdir -p "$basedir" if [ "$do_copy" -eq 1 ]; then # remove any existing symlink before copying [ -L "$filename" ] && rm -f "$filename" From 0c9468c9b5a79abb0e8f2b28be7201641815bdf1 Mon Sep 17 00:00:00 2001 From: Tim Byrne Date: Tue, 15 Oct 2019 07:17:38 -0500 Subject: [PATCH 071/137] Ignore encrypted files (#69) Append the contents of .config/yadm/encrypt to the repo's git ignore list. This is to help prevent accidentally committing unencrypted sensitive data. --- test/conftest.py | 1 + test/test_encryption.py | 23 ++++++++++ test/test_introspect.py | 2 +- test/test_unit_exclude_encrypted.py | 66 +++++++++++++++++++++++++++++ yadm | 51 ++++++++++++++++++++++ yadm.1 | 17 ++++++++ 6 files changed, 159 insertions(+), 1 deletion(-) create mode 100644 test/test_unit_exclude_encrypted.py diff --git a/test/conftest.py b/test/conftest.py index 805ee92..a037223 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -101,6 +101,7 @@ def supported_configs(): 'local.user', 'yadm.alt-copy', 'yadm.auto-alt', + 'yadm.auto-exclude', 'yadm.auto-perms', 'yadm.auto-private-dirs', 'yadm.git-program', diff --git a/test/test_encryption.py b/test/test_encryption.py index f107ad5..3d10786 100644 --- a/test/test_encryption.py +++ b/test/test_encryption.py @@ -372,6 +372,29 @@ def test_offer_to_add(runner, yadm_y, paths, encrypt_targets, untracked): assert f'AM {worktree_archive.basename}' in run.out +def test_encrypt_added_to_exclude(runner, yadm_y, paths): + """Confirm that .config/yadm/encrypt is added to exclude""" + + expect = [ + ('passphrase:', PASSPHRASE), + ('passphrase:', PASSPHRASE), + ] + + exclude_file = paths.repo.join('info/exclude') + paths.encrypt.write('test-encrypt-data\n') + exclude_file.write('original-data', ensure=True) + + run = runner( + yadm_y('encrypt'), + expect=expect, + ) + + assert 'test-encrypt-data' in paths.repo.join('info/exclude').read() + assert 'original-data' in paths.repo.join('info/exclude').read() + assert run.success + assert run.err == '' + + def encrypted_data_valid(runner, encrypted, expected): """Verify encrypted data matches expectations""" run = runner([ diff --git a/test/test_introspect.py b/test/test_introspect.py index 9026e98..fcadf14 100644 --- a/test/test_introspect.py +++ b/test/test_introspect.py @@ -27,7 +27,7 @@ def test_introspect_category( expected = [] if name == 'commands': expected = supported_commands - elif name == 'config': + elif name == 'configs': expected = supported_configs elif name == 'switches': expected = supported_switches diff --git a/test/test_unit_exclude_encrypted.py b/test/test_unit_exclude_encrypted.py new file mode 100644 index 0000000..9d9a074 --- /dev/null +++ b/test/test_unit_exclude_encrypted.py @@ -0,0 +1,66 @@ +"""Unit tests: exclude_encrypted""" +import pytest + + +@pytest.mark.parametrize( + 'exclude', ['missing', 'outdated', 'up-to-date']) +@pytest.mark.parametrize( + 'encrypt_exists', [True, False], ids=['encrypt', 'no-encrypt']) +@pytest.mark.parametrize( + 'auto_exclude', [True, False], ids=['enabled', 'disabled']) +def test_exclude_encrypted( + runner, tmpdir, yadm, encrypt_exists, auto_exclude, exclude): + """Test exclude_encrypted()""" + + header = ( + "# yadm-auto-excludes\n" + "# This section is managed by yadm.\n" + "# Any edits below will be lost.\n" + ) + + config_function = 'function config() { echo "false";}' + if auto_exclude: + config_function = 'function config() { return; }' + + encrypt_file = tmpdir.join('encrypt_file') + repo_dir = tmpdir.join('repodir') + exclude_file = repo_dir.join('info/exclude') + + if encrypt_exists: + encrypt_file.write('test-encrypt-data\n', ensure=True) + if exclude == 'outdated': + exclude_file.write( + f'original-exclude\n{header}outdated\n', ensure=True) + elif exclude == 'up-to-date': + exclude_file.write( + f'original-exclude\n{header}test-encrypt-data\n', ensure=True) + + script = f""" + YADM_TEST=1 source {yadm} + {config_function} + DEBUG=1 + YADM_ENCRYPT="{encrypt_file}" + YADM_REPO="{repo_dir}" + exclude_encrypted + """ + run = runner(command=['bash'], inp=script) + assert run.success + assert run.err == '' + + if auto_exclude: + if encrypt_exists: + assert exclude_file.exists() + if exclude == 'missing': + assert exclude_file.read() == f'{header}test-encrypt-data\n' + else: + assert exclude_file.read() == ( + 'original-exclude\n' + f'{header}test-encrypt-data\n') + if exclude != 'up-to-date': + assert f'Updating {exclude_file}' in run.out + else: + assert run.out == '' + else: + assert run.out == '' + else: + assert run.out == '' diff --git a/yadm b/yadm index eb55852..7f70b77 100755 --- a/yadm +++ b/yadm @@ -804,6 +804,7 @@ function encrypt() { require_gpg require_encrypt + exclude_encrypted parse_encrypt cd_work "Encryption" || return @@ -986,6 +987,7 @@ local.os local.user yadm.alt-copy yadm.auto-alt +yadm.auto-exclude yadm.auto-perms yadm.auto-private-dirs yadm.git-program @@ -1069,6 +1071,55 @@ function version() { # ****** Utility Functions ****** +function exclude_encrypted() { + + auto_exclude=$(config --bool yadm.auto-exclude) + [ "$auto_exclude" == "false" ] && return 0 + + exclude_path="${YADM_REPO}/info/exclude" + newline=$'\n' + exclude_flag="# yadm-auto-excludes" + exclude_header="${exclude_flag}${newline}" + exclude_header="${exclude_header}# This section is managed by yadm." + exclude_header="${exclude_header}${newline}" + exclude_header="${exclude_header}# Any edits below will be lost." + exclude_header="${exclude_header}${newline}" + + # do nothing if there is no YADM_ENCRYPT + [ -e "$YADM_ENCRYPT" ] || return 0 + + # read encrypt + encrypt_data="" + while IFS='' read -r line || [ -n "$line" ]; do + encrypt_data="${encrypt_data}${line}${newline}" + done < "$YADM_ENCRYPT" + + # read info/exclude + unmanaged="" + managed="" + if [ -e "$exclude_path" ]; then + flag_seen=0 + while IFS='' read -r line || [ -n "$line" ]; do + [ "$line" = "$exclude_flag" ] && flag_seen=1 + if [ "$flag_seen" -eq 0 ]; then + unmanaged="${unmanaged}${line}${newline}" + else + managed="${managed}${line}${newline}" + fi + done < "$exclude_path" + fi + + if [ "${exclude_header}${encrypt_data}" != "$managed" ]; then + basedir=${exclude_path%/*} + [ -e "$basedir" ] || mkdir -p "$basedir" # assert path + debug "Updating ${exclude_path}" + printf "%s" "${unmanaged}${exclude_header}${encrypt_data}" > "$exclude_path" + fi + + return 0 + +} + function is_valid_branch_name() { # Git branches do not allow: # * path component that begins with "." diff --git a/yadm.1 b/yadm.1 index aff25f3..72032bc 100644 --- a/yadm.1 +++ b/yadm.1 @@ -347,6 +347,11 @@ Disable the automatic linking described in the section ALTERNATES. If disabled, you may still run "yadm alt" manually to create the alternate links. This feature is enabled by default. .TP +.B yadm.auto-exclude +Disable the automatic exclusion of patterns defined in +.IR $HOME/.config/yadm/encrypt . +This feature is enabled by default. +.TP .B yadm.auto-perms Disable the automatic permission changes described in the section PERMISSIONS. If disabled, you may still run @@ -674,6 +679,18 @@ configuration. It is recommended that you use a private repository when keeping confidential files, even though they are encrypted. +Patterns found in +.I $HOME/.config/yadm/encrypt +are automatically added to the repository's +.I info/exclude +file every time +.B yadm encrypt +is run. +This is to prevent accidentally committing sensitive data to the repository. +This can be disabled using the +.I yadm.auto-exclude +configuration. + .SH PERMISSIONS When files are checked out of a Git repository, their initial permissions are From b62a4c77a6988bf2073f267bf45a767f0d3503a3 Mon Sep 17 00:00:00 2001 From: Tim Byrne Date: Tue, 22 Oct 2019 17:47:41 -0500 Subject: [PATCH 072/137] Create an upgrade command This command will assist users with migration from 1.x.x to 2.0.0. --- test/conftest.py | 1 + test/test_unit_issue_legacy_path_warning.py | 13 ++- test/test_unit_upgrade.py | 101 ++++++++++++++++++++ yadm | 100 +++++++++++++++---- yadm.1 | 26 ++++- 5 files changed, 219 insertions(+), 22 deletions(-) create mode 100644 test/test_unit_upgrade.py diff --git a/test/conftest.py b/test/conftest.py index a037223..3e2d475 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -84,6 +84,7 @@ def supported_commands(): 'introspect', 'list', 'perms', + 'upgrade', 'version', ] diff --git a/test/test_unit_issue_legacy_path_warning.py b/test/test_unit_issue_legacy_path_warning.py index e4bed27..c14857b 100644 --- a/test/test_unit_issue_legacy_path_warning.py +++ b/test/test_unit_issue_legacy_path_warning.py @@ -10,25 +10,30 @@ import pytest 'encrypt', 'files.gpg', 'bootstrap', - 'hooks', + 'hooks/pre_command', + 'hooks/post_command', ], ) -def test_legacy_warning(tmpdir, runner, yadm, legacy_path): +@pytest.mark.parametrize( + 'upgrade', [True, False], ids=['upgrade', 'no-upgrade']) +def test_legacy_warning(tmpdir, runner, yadm, upgrade, legacy_path): """Use issue_legacy_path_warning""" home = tmpdir.mkdir('home') if legacy_path: - home.mkdir(f'.yadm').mkdir(legacy_path) + home.mkdir(f'.yadm').ensure(legacy_path) + main_args = 'MAIN_ARGS=("upgrade")' if upgrade else '' script = f""" HOME={home} YADM_TEST=1 source {yadm} + {main_args} issue_legacy_path_warning """ run = runner(command=['bash'], inp=script) assert run.success assert run.err == '' - if legacy_path: + if legacy_path and not upgrade: assert 'Legacy configuration paths have been detected' in run.out else: assert run.out.rstrip() == '' diff --git a/test/test_unit_upgrade.py b/test/test_unit_upgrade.py new file mode 100644 index 0000000..9b57828 --- /dev/null +++ b/test/test_unit_upgrade.py @@ -0,0 +1,101 @@ +"""Unit tests: upgrade""" +import pytest + +LEGACY_PATHS = [ + 'config', + 'encrypt', + 'files.gpg', + 'bootstrap', + 'hooks/pre_command', + 'hooks/post_command', +] + +# used: +# YADM_COMPATIBILITY +# YADM_DIR +# YADM_LEGACY_DIR +# GIT_PROGRAM +@pytest.mark.parametrize('condition', ['compat', 'equal', 'existing_repo']) +def test_upgrade_errors(tmpdir, runner, yadm, condition): + """Test upgrade() error conditions""" + + compatibility = 'YADM_COMPATIBILITY=1' if condition == 'compat' else '' + + home = tmpdir.mkdir('home') + yadm_dir = home.join('.config/yadm') + legacy_dir = home.join('.yadm') + if condition == 'equal': + legacy_dir = yadm_dir + if condition == 'existing_repo': + yadm_dir.ensure_dir('repo.git') + legacy_dir.ensure_dir('repo.git') + + script = f""" + YADM_TEST=1 source {yadm} + {compatibility} + YADM_DIR="{yadm_dir}" + YADM_REPO="{yadm_dir}/repo.git" + YADM_LEGACY_DIR="{legacy_dir}" + upgrade + """ + run = runner(command=['bash'], inp=script) + assert run.failure + assert run.err == '' + assert 'Unable to upgrade' in run.out + if condition == 'compat': + assert 'YADM_COMPATIBILITY' in run.out + if condition == 'equal': + assert 'has been resolved as' in run.out + if condition == 'existing_repo': + assert 'already exists' in run.out + + +@pytest.mark.parametrize('condition', ['no-paths', 'untracked', 'tracked']) +def test_upgrade(tmpdir, runner, yadm, condition): + """Test upgrade() + + When testing the condition of git-tracked data, "echo" will be used as a + mock for git. echo will return true, simulating a positive result from "git + ls-files". Also echo will report the parameters for "git mv". + """ + home = tmpdir.mkdir('home') + yadm_dir = home.join('.config/yadm') + legacy_dir = home.join('.yadm') + + if condition != 'no-paths': + legacy_dir.join('repo.git/config').write('test-repo', ensure=True) + for lpath in LEGACY_PATHS: + legacy_dir.join(lpath).write(lpath, ensure=True) + + git = 'echo' if condition == 'tracked' else 'git' + + script = f""" + YADM_TEST=1 source {yadm} + YADM_DIR="{yadm_dir}" + YADM_REPO="{yadm_dir}/repo.git" + YADM_LEGACY_DIR="{legacy_dir}" + GIT_PROGRAM="{git}" + upgrade + """ + run = runner(command=['bash'], inp=script) + assert run.success + assert run.err == '' + if condition == 'no-paths': + assert 'Upgrade is not necessary' in run.out + else: + for lpath in LEGACY_PATHS + ['repo.git']: + expected = ( + f'Moving {legacy_dir.join(lpath)} ' + f'to {yadm_dir.join(lpath)}') + assert expected in run.out + if condition == 'untracked': + assert 'test-repo' in yadm_dir.join('repo.git/config').read() + for lpath in LEGACY_PATHS: + assert lpath in yadm_dir.join(lpath).read() + elif condition == 'tracked': + for lpath in LEGACY_PATHS: + expected = ( + f'mv {legacy_dir.join(lpath)} ' + f'{yadm_dir.join(lpath)}') + assert expected in run.out + assert 'files tracked by yadm have been renamed' in run.out diff --git a/yadm b/yadm index 7f70b77..6c87c45 100755 --- a/yadm +++ b/yadm @@ -72,7 +72,7 @@ function main() { # parse command line arguments local retval=0 - internal_commands="^(alt|bootstrap|clean|clone|config|decrypt|encrypt|enter|help|init|introspect|list|perms|version)$" + internal_commands="^(alt|bootstrap|clean|clone|config|decrypt|encrypt|enter|help|init|introspect|list|perms|upgrade|version)$" if [ -z "$*" ] ; then # no argumnts will result in help() help @@ -505,13 +505,12 @@ function alt_future_linking() { filename="${alt_filenames[$index]}" target="${alt_targets[$index]}" template_cmd="${alt_template_cmds[$index]}" - basedir=${filename%/*} if [ -n "$template_cmd" ]; then # a template is defined, process the template debug "Creating $filename from template $target" [ -n "$loud" ] && echo "Creating $filename from template $target" # ensure the destination path exists - [ -e "$basedir" ] || mkdir -p "$basedir" + assert_parent "$filename" # remove any existing symlink before processing template [ -L "$filename" ] && rm -f "$filename" "$template_cmd" "$target" "$filename" @@ -520,7 +519,7 @@ function alt_future_linking() { debug "Linking $target to $filename" [ -n "$loud" ] && echo "Linking $target to $filename" # ensure the destination path exists - [ -e "$basedir" ] || mkdir -p "$basedir" + assert_parent "$filename" if [ "$do_copy" -eq 1 ]; then # remove any existing symlink before copying [ -L "$filename" ] && rm -f "$filename" @@ -975,6 +974,7 @@ init introspect list perms +upgrade version EOF } @@ -1062,6 +1062,66 @@ function perms() { } +function upgrade() { + + local actions_performed + actions_performed=0 + local repo_updates + repo_updates=0 + + [ "$YADM_COMPATIBILITY" = "1" ] && \ + error_out "Unable to upgrade. YADM_COMPATIBILITY is set to '1'." + + [ "$YADM_DIR" = "$YADM_LEGACY_DIR" ] && \ + error_out "Unable to upgrade. yadm dir has been resolved as '$YADM_LEGACY_DIR'." + + # handle legacy repo + if [ -d "$YADM_LEGACY_DIR/repo.git" ]; then + # legacy repo detected, it must be moved to YADM_REPO + if [ -e "$YADM_REPO" ]; then + error_out "Unable to upgrade. '$YADM_REPO' already exists. Refusing to overwrite it." + else + actions_performed=1 + echo "Moving $YADM_LEGACY_DIR/repo.git to $YADM_REPO" + assert_parent "$YADM_REPO" + mv "$YADM_LEGACY_DIR/repo.git" "$YADM_REPO" + fi + fi + + # handle other legacy paths + for legacy_path in \ + "$YADM_LEGACY_DIR/config" \ + "$YADM_LEGACY_DIR/encrypt" \ + "$YADM_LEGACY_DIR/files.gpg" \ + "$YADM_LEGACY_DIR/bootstrap" \ + "$YADM_LEGACY_DIR"/hooks/{pre,post}_* \ + ; \ + do + if [ -e "$legacy_path" ]; then + new_filename=${legacy_path#$YADM_LEGACY_DIR/} + new_filename="$YADM_DIR/$new_filename" + actions_performed=1 + echo "Moving $legacy_path to $new_filename" + assert_parent "$new_filename" + # test to see if path is "tracked" in repo, if so 'git mv' must be used + if GIT_DIR="$YADM_REPO" "$GIT_PROGRAM" ls-files --error-unmatch "$legacy_path" >/dev/null 2>&1; then + GIT_DIR="$YADM_REPO" "$GIT_PROGRAM" mv "$legacy_path" "$new_filename" && repo_updates=1 + else + mv -i "$legacy_path" "$new_filename" + fi + fi + done + + [ "$actions_performed" -eq 0 ] && \ + echo "No legacy paths found. Upgrade is not necessary" + + [ "$repo_updates" -eq 1 ] && \ + echo "Some files tracked by yadm have been renamed. This changes should probably be commited now." + + exit 0 + +} + function version() { echo "yadm $VERSION" @@ -1110,9 +1170,8 @@ function exclude_encrypted() { fi if [ "${exclude_header}${encrypt_data}" != "$managed" ]; then - basedir=${exclude_path%/*} - [ -e "$basedir" ] || mkdir -p "$basedir" # assert path debug "Updating ${exclude_path}" + assert_parent "$exclude_path" printf "%s" "${unmanaged}${exclude_header}${encrypt_data}" > "$exclude_path" fi @@ -1221,6 +1280,9 @@ function set_yadm_dir() { function issue_legacy_path_warning() { + # no warnings during upgrade + [[ "${MAIN_ARGS[*]}" =~ upgrade ]] && return + # no warnings if YADM_DIR is resolved as the leacy path [ "$YADM_DIR" = "$YADM_LEGACY_DIR" ] && return @@ -1231,14 +1293,14 @@ function issue_legacy_path_warning() { local legacy_found legacy_found=() # this is ordered by importance - for legacy_path in \ - "$YADM_LEGACY_DIR/$YADM_REPO" \ - "$YADM_LEGACY_DIR/$YADM_CONFIG" \ - "$YADM_LEGACY_DIR/$YADM_ENCRYPT" \ - "$YADM_LEGACY_DIR/$YADM_ARCHIVE" \ - "$YADM_LEGACY_DIR/$YADM_BOOTSTRAP" \ - "$YADM_LEGACY_DIR/$YADM_HOOKS" \ - ; \ + for legacy_path in \ + "$YADM_LEGACY_DIR/$YADM_REPO" \ + "$YADM_LEGACY_DIR/$YADM_CONFIG" \ + "$YADM_LEGACY_DIR/$YADM_ENCRYPT" \ + "$YADM_LEGACY_DIR/$YADM_ARCHIVE" \ + "$YADM_LEGACY_DIR/$YADM_BOOTSTRAP" \ + "$YADM_LEGACY_DIR/$YADM_HOOKS"/{pre,post}_* \ + ; \ do [ -e "$legacy_path" ] && legacy_found+=("$legacy_path") done @@ -1258,14 +1320,15 @@ function issue_legacy_path_warning() { Beginning with version 2.0.0, yadm uses the XDG Base Directory Specification to find its configurations. Read more about this change here: - https://yadm.io/docs/xdg_config_home + https://yadm.io/docs/upgrade_from_1.x.x In your environment, the configuration directory has been resolved to: $YADM_DIR To remove this warning do one of the following: - * Move yadm configurations to the directory listed above. (RECOMMENDED) + * Run "yadm upgrade" to move the yadm data to the new directory. (RECOMMENDED) + * Manually move yadm configurations to the directory listed above. * Specify your preferred yadm directory with -Y each execution. * Define an environment variable "YADM_COMPATIBILITY=1" to run in version 1 compatibility mode. (DEPRECATED) @@ -1421,6 +1484,11 @@ function assert_private_dirs() { done } +function assert_parent() { + basedir=${1%/*} + [ -e "$basedir" ] || mkdir -p "$basedir" +} + function display_private_perms() { when="$1" for private_dir in .ssh .gnupg; do diff --git a/yadm.1 b/yadm.1 index 72032bc..3f8368f 100644 --- a/yadm.1 +++ b/yadm.1 @@ -56,6 +56,8 @@ list .BR yadm " perms +.BR yadm " upgrade + .BR yadm " introspect .I category @@ -255,6 +257,22 @@ configuration .I yadm.auto-perms to "false". .TP +.B upgrade +Version 2 of yadm uses a different directory for storing your configurations. +When you start to use version 2 for the first time, you may see warnings about +moving your data to this new directory. +The easiest way to accomplish this is by running "yadm upgrade". +This command will start by moving your yadm repo to the new path. +Next it will move any configuration data to the new path. +If the configurations are tracked within your yadm repo, this command will +"stage" the renaming of those files in the repo's index. +After running "yadm upgrade", you should run "yadm status" to review changes +which have been staged, and commit them to your repository. + +You can read +https://yadm.io/docs/upgrade_from_1.x.x +for more information. +.TP .B version Print the version of yadm. @@ -262,10 +280,14 @@ Print the version of yadm. Beginning with version 2.0.0, yadm introduced a couple major changes which may require you to adjust your configurations. +See the +.B upgrade +command for help making those adjustments. First, yadm now uses the "XDG Base Directory Specification" to find its -configurations. You can read https://yadm.io/docs/xdg_config_home for more -information. +configurations. You can read +https://yadm.io/docs/upgrade_from_1.x.x +for more information. Second, the naming conventions for alternate files have been changed. You can read https://yadm.io/docs/alternates for more information. From 616baaeac683eb51e69d769086041c915bd25db3 Mon Sep 17 00:00:00 2001 From: Tim Byrne Date: Wed, 30 Oct 2019 17:29:17 -0500 Subject: [PATCH 073/137] Rename default template processor --- test/test_alt.py | 4 +- test/test_unit_choose_template_cmd.py | 8 ++-- ...iltin.py => test_unit_template_default.py} | 40 +++++++++---------- yadm | 8 ++-- 4 files changed, 30 insertions(+), 30 deletions(-) rename test/{test_unit_template_builtin.py => test_unit_template_default.py} (79%) diff --git a/test/test_alt.py b/test/test_alt.py index dd62f07..d616869 100644 --- a/test/test_alt.py +++ b/test/test_alt.py @@ -101,7 +101,7 @@ def test_alt_conditions( @pytest.mark.usefixtures('ds1_copy') @pytest.mark.parametrize( - 'kind', ['builtin', '', None, 'envtpl', 'j2cli', 'j2']) + 'kind', ['default', '', None, 'envtpl', 'j2cli', 'j2']) @pytest.mark.parametrize('label', ['t', 'template', 'yadm', ]) def test_alt_templates( runner, paths, kind, label): @@ -223,7 +223,7 @@ def test_template_overwrite_symlink(runner, yadm_y, paths, tst_sys): link = paths.work.join('test_link') link.mksymlinkto(target, absolute=1) - template = paths.work.join('test_link##template.builtin') + template = paths.work.join('test_link##template.default') template.write('test-data') run = runner(yadm_y('add', target, template)) diff --git a/test/test_unit_choose_template_cmd.py b/test/test_unit_choose_template_cmd.py index b592331..536735d 100644 --- a/test/test_unit_choose_template_cmd.py +++ b/test/test_unit_choose_template_cmd.py @@ -2,12 +2,12 @@ import pytest -@pytest.mark.parametrize('label', ['', 'builtin', 'other']) +@pytest.mark.parametrize('label', ['', 'default', 'other']) @pytest.mark.parametrize('awk', [True, False], ids=['awk', 'no-awk']) -def test_kind_builtin(runner, yadm, awk, label): - """Test kind: builtin""" +def test_kind_default(runner, yadm, awk, label): + """Test kind: default""" - expected = 'template_builtin' + expected = 'template_default' awk_avail = 'true' if not awk: diff --git a/test/test_unit_template_builtin.py b/test/test_unit_template_default.py similarity index 79% rename from test/test_unit_template_builtin.py rename to test/test_unit_template_default.py index bfc7747..b8e8faf 100644 --- a/test/test_unit_template_builtin.py +++ b/test/test_unit_template_default.py @@ -1,18 +1,18 @@ -"""Unit tests: template_builtin""" +"""Unit tests: template_default""" # these values are also testing the handling of bizarre characters -LOCAL_CLASS = "builtin_Test+@-!^Class" -LOCAL_SYSTEM = "builtin_Test+@-!^System" -LOCAL_HOST = "builtin_Test+@-!^Host" -LOCAL_USER = "builtin_Test+@-!^User" -LOCAL_DISTRO = "builtin_Test+@-!^Distro" +LOCAL_CLASS = "default_Test+@-!^Class" +LOCAL_SYSTEM = "default_Test+@-!^System" +LOCAL_HOST = "default_Test+@-!^Host" +LOCAL_USER = "default_Test+@-!^User" +LOCAL_DISTRO = "default_Test+@-!^Distro" TEMPLATE = f''' start of template -builtin class = >{{{{yadm.class}}}}< -builtin os = >{{{{yadm.os}}}}< -builtin host = >{{{{yadm.hostname}}}}< -builtin user = >{{{{yadm.user}}}}< -builtin distro = >{{{{yadm.distro}}}}< +default class = >{{{{yadm.class}}}}< +default os = >{{{{yadm.os}}}}< +default host = >{{{{yadm.hostname}}}}< +default user = >{{{{yadm.user}}}}< +default distro = >{{{{yadm.distro}}}}< {{% if yadm.class == "else1" %}} wrong else 1 {{% else %}} @@ -70,11 +70,11 @@ end of template ''' EXPECTED = f''' start of template -builtin class = >{LOCAL_CLASS}< -builtin os = >{LOCAL_SYSTEM}< -builtin host = >{LOCAL_HOST}< -builtin user = >{LOCAL_USER}< -builtin distro = >{LOCAL_DISTRO}< +default class = >{LOCAL_CLASS}< +default os = >{LOCAL_SYSTEM}< +default host = >{LOCAL_HOST}< +default user = >{LOCAL_USER}< +default distro = >{LOCAL_DISTRO}< Included section from else Included section for class = {LOCAL_CLASS} ({LOCAL_CLASS} repeated) Multiple lines @@ -86,8 +86,8 @@ end of template ''' -def test_template_builtin(runner, yadm, tmpdir): - """Test template_builtin""" +def test_template_default(runner, yadm, tmpdir): + """Test template_default""" input_file = tmpdir.join('input') input_file.write(TEMPLATE, ensure=True) @@ -100,7 +100,7 @@ def test_template_builtin(runner, yadm, tmpdir): local_host="{LOCAL_HOST}" local_user="{LOCAL_USER}" local_distro="{LOCAL_DISTRO}" - template_builtin "{input_file}" "{output_file}" + template_default "{input_file}" "{output_file}" """ run = runner(command=['bash'], inp=script) assert run.success @@ -117,7 +117,7 @@ def test_source(runner, yadm, tmpdir): script = f""" YADM_TEST=1 source {yadm} - template_builtin "{input_file}" "{output_file}" + template_default "{input_file}" "{output_file}" """ run = runner(command=['bash'], inp=script) assert run.success diff --git a/yadm b/yadm index 6c87c45..96f5754 100755 --- a/yadm +++ b/yadm @@ -275,8 +275,8 @@ function record_template() { function choose_template_cmd() { kind="$1" - if [ "$kind" = "builtin" ] || [ "$kind" = "" ] && awk_available; then - echo "template_builtin" + if [ "$kind" = "default" ] || [ "$kind" = "" ] && awk_available; then + echo "template_default" elif [ "$kind" = "j2cli" ] || [ "$kind" = "j2" ] && j2cli_available; then echo "template_j2cli" elif [ "$kind" = "envtpl" ] || [ "$kind" = "j2" ] && envtpl_available; then @@ -289,14 +289,14 @@ function choose_template_cmd() { # ****** Template Processors ****** -function template_builtin() { +function template_default() { input="$1" output="$2" # the explicit "space + tab" character class used below is used because not # all versions of awk seem to support the POSIX character classes [[:blank:]] awk_pgm=$(cat << "EOF" -# built-in template processor +# built-in default template processor BEGIN { blank = "[ ]" c["class"] = class From 6442313abf7b4b3bef520a8199a6b633fd8bd435 Mon Sep 17 00:00:00 2001 From: Tim Byrne Date: Wed, 30 Oct 2019 17:34:34 -0500 Subject: [PATCH 074/137] Improve manpage --- yadm.1 | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/yadm.1 b/yadm.1 index 3f8368f..80c665d 100644 --- a/yadm.1 +++ b/yadm.1 @@ -270,7 +270,7 @@ After running "yadm upgrade", you should run "yadm status" to review changes which have been staged, and commit them to your repository. You can read -https://yadm.io/docs/upgrade_from_1.x.x +https://yadm.io/docs/upgrade_from_1 for more information. .TP .B version @@ -286,7 +286,7 @@ command for help making those adjustments. First, yadm now uses the "XDG Base Directory Specification" to find its configurations. You can read -https://yadm.io/docs/upgrade_from_1.x.x +https://yadm.io/docs/upgrade_from_1 for more information. Second, the naming conventions for alternate files have been changed. @@ -447,7 +447,7 @@ commas. Each condition is an attribute/value pair, separated by a period. Some conditions do not require a "value", and in that case, the period and value can -be omitted. +be omitted. Most attributes can be abbreviated as a single letter. [.] @@ -494,9 +494,9 @@ Valid when no other alternate is valid. You may use any number of conditions, in any order. An alternate will only be used if ALL conditions are valid. -If there are any files managed by yadm's repository, or listed in +For all files managed by yadm’s repository or listed in .IR $HOME/.config/yadm/encrypt , -which match this naming convention, +if they match this naming convention, symbolic links will be created for the most appropriate version. The "most appropriate" version is determined by calculating a score for each @@ -509,7 +509,7 @@ If you don't care to have all versions of alternates stored in the same directory as the generated symlink, you can place them in the .I $HOME/.config/yadm/alt directory. The generated symlink or processed template will be created using -same relative path. +the same relative path. Alternate linking may best be demonstrated by example. Assume the following files are managed by yadm's repository: @@ -585,13 +585,13 @@ processed to create or overwrite files. Supported template processors: .TP -.B builtin +.B default This is yadm's built-in template processor. This processor is very basic, with a Jinja-like syntax. The advantage of this processor is that it only depends upon .BR awk , which is available on most *nix systems. To use this processor, -specify the value of "builtin" or just leave the value off (e.g. "##template"). +specify the value of "default" or just leave the value off (e.g. "##template"). .TP .B j2cli To use the j2cli Jinja template processor, specify the value of "j2" or From fc53cfd1f8c8109fb54c8d6ff703e8cc47da3469 Mon Sep 17 00:00:00 2001 From: Tim Byrne Date: Sun, 3 Nov 2019 13:35:44 -0600 Subject: [PATCH 075/137] Remove `--local` `--local` isn't supported by versions of Git older than 1.8. But it should be the default if the `--local` is omitted. --- yadm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/yadm b/yadm index 96f5754..e564b75 100755 --- a/yadm +++ b/yadm @@ -763,7 +763,7 @@ EOF # operate on the yadm repo's configuration file # this is always local to the machine - "$GIT_PROGRAM" config --local "$@" + "$GIT_PROGRAM" config "$@" CHANGES_POSSIBLE=1 From f5287f1588f4a6c88556e7137cb84c41b9a19df0 Mon Sep 17 00:00:00 2001 From: Tim Byrne Date: Sun, 3 Nov 2019 14:13:46 -0600 Subject: [PATCH 076/137] Properly format supported configs --- yadm | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/yadm b/yadm index e564b75..9d81443 100755 --- a/yadm +++ b/yadm @@ -748,6 +748,7 @@ function config() { # with no parameters, provide some helpful documentation echo "yadm supports the following configurations:" echo + local IFS=$'\n' for supported_config in $(introspect_configs); do echo " ${supported_config}" done @@ -980,7 +981,7 @@ EOF } function introspect_configs() { - cat << EOF + cat <<-EOF local.class local.hostname local.os From 5a802c8afd08c06a1905f771dcae0851cff4edf4 Mon Sep 17 00:00:00 2001 From: Tim Byrne Date: Sun, 3 Nov 2019 14:21:59 -0600 Subject: [PATCH 077/137] Allow `-l` to pass thru to the `yadm config` command --- yadm | 1 + 1 file changed, 1 insertion(+) diff --git a/yadm b/yadm index 9d81443..cce1b41 100755 --- a/yadm +++ b/yadm @@ -96,6 +96,7 @@ function main() { ;; -l) # used by decrypt() DO_LIST="YES" + [ "$YADM_COMMAND" = "config" ] && YADM_ARGS+=("$1") ;; -w) # used by init() and clone() if [[ ! "$2" =~ ^/ ]] ; then From 9362b93820e3374b98b79d128a2011c35b835a5a Mon Sep 17 00:00:00 2001 From: Tim Byrne Date: Wed, 17 Apr 2019 08:26:19 -0500 Subject: [PATCH 078/137] Update specfile for OBS --- yadm.spec | 48 ++++++++++++++++++++++++++++++++---------------- 1 file changed, 32 insertions(+), 16 deletions(-) diff --git a/yadm.spec b/yadm.spec index bc5fbb2..bfa7d63 100644 --- a/yadm.spec +++ b/yadm.spec @@ -1,12 +1,21 @@ -Summary: Yet Another Dotfiles Manager +%{!?_pkgdocdir: %global _pkgdocdir %{_docdir}/%{name}-%{version}} Name: yadm +Summary: Yet Another Dotfiles Manager Version: 1.12.0 +Group: Development/Tools Release: 1%{?dist} -URL: https://github.com/TheLocehiliosan/yadm -License: GPLv3 -BuildRequires: hostname git gnupg bats expect -Requires: bash hostname git -Source: https://github.com/TheLocehiliosan/%{name}/archive/%{version}.tar.gz#/%{name}-%{version}.tar.gz +URL: https://yadm.io +License: GPL-3.0-only +Requires: bash +Requires: git +%if 0%{?fedora} || 0%{?rhel_version} || 0%{?centos_version} >= 700 +Requires: /usr/bin/hostname +%else +Requires: /bin/hostname +%endif + +Source: %{name}.tar.gz +BuildRoot: %{_tmppath}/%{name}-%{version}-build BuildArch: noarch %description @@ -17,24 +26,31 @@ yadm supplies the ability to manage a subset of secure files, which are encrypted before they are included in the repository. %prep -%setup -q +%setup -c %build -%check -bats test - %install -mkdir -p ${RPM_BUILD_ROOT}%{_bindir} -mkdir -p ${RPM_BUILD_ROOT}%{_mandir}/man1 -install -m 755 yadm ${RPM_BUILD_ROOT}%{_bindir} -install -m 644 yadm.1 ${RPM_BUILD_ROOT}%{_mandir}/man1 + +# this is done to allow paths other than yadm-x.x.x (for example, when building +# from branches instead of release tags) +cd *yadm-* + +%{__mkdir} -p %{buildroot}%{_bindir} +%{__cp} yadm %{buildroot}%{_bindir} + +%{__mkdir} -p %{buildroot}%{_mandir}/man1 +%{__cp} yadm.1 %{buildroot}%{_mandir}/man1 + +%{__mkdir} -p %{buildroot}%{_pkgdocdir} +%{__cp} README.md %{buildroot}%{_pkgdocdir}/README +%{__cp} CHANGES CONTRIBUTORS LICENSE %{buildroot}%{_pkgdocdir} +%{__cp} -r completion contrib %{buildroot}%{_pkgdocdir} %files %attr(755,root,root) %{_bindir}/yadm %attr(644,root,root) %{_mandir}/man1/* -%license LICENSE -%doc CHANGES CONTRIBUTORS README.md completion/* +%doc %{_pkgdocdir} %changelog * Wed Oct 25 2017 Tim Byrne - 1.12.0-1 From a217537b26b5e1042d1e6a7d0b9867c90e9cc3a0 Mon Sep 17 00:00:00 2001 From: Tim Byrne Date: Mon, 4 Nov 2019 17:31:55 -0600 Subject: [PATCH 079/137] Fix URL for upgrade help --- yadm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/yadm b/yadm index cce1b41..f4f07ee 100755 --- a/yadm +++ b/yadm @@ -1322,7 +1322,7 @@ function issue_legacy_path_warning() { Beginning with version 2.0.0, yadm uses the XDG Base Directory Specification to find its configurations. Read more about this change here: - https://yadm.io/docs/upgrade_from_1.x.x + https://yadm.io/docs/upgrade_from_1 In your environment, the configuration directory has been resolved to: From f2b2d505a230e17c847714791aaafe576ad45753 Mon Sep 17 00:00:00 2001 From: Tim Byrne Date: Tue, 5 Nov 2019 16:36:05 -0600 Subject: [PATCH 080/137] Reinitialize submodules during upgrade --- test/test_unit_upgrade.py | 3 +++ yadm | 15 +++++++++++++-- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/test/test_unit_upgrade.py b/test/test_unit_upgrade.py index 9b57828..affd641 100644 --- a/test/test_unit_upgrade.py +++ b/test/test_unit_upgrade.py @@ -75,6 +75,7 @@ def test_upgrade(tmpdir, runner, yadm, condition): YADM_REPO="{yadm_dir}/repo.git" YADM_LEGACY_DIR="{legacy_dir}" GIT_PROGRAM="{git}" + function cd {{ echo "$@";}} upgrade """ run = runner(command=['bash'], inp=script) @@ -99,3 +100,5 @@ def test_upgrade(tmpdir, runner, yadm, condition): f'{yadm_dir.join(lpath)}') assert expected in run.out assert 'files tracked by yadm have been renamed' in run.out + assert 'submodule deinit -f .' in run.out + assert 'submodule update --init --recursive' in run.out diff --git a/yadm b/yadm index f4f07ee..57898f7 100755 --- a/yadm +++ b/yadm @@ -1091,6 +1091,8 @@ function upgrade() { fi # handle other legacy paths + GIT_DIR="$YADM_REPO" + export GIT_DIR for legacy_path in \ "$YADM_LEGACY_DIR/config" \ "$YADM_LEGACY_DIR/encrypt" \ @@ -1106,14 +1108,23 @@ function upgrade() { echo "Moving $legacy_path to $new_filename" assert_parent "$new_filename" # test to see if path is "tracked" in repo, if so 'git mv' must be used - if GIT_DIR="$YADM_REPO" "$GIT_PROGRAM" ls-files --error-unmatch "$legacy_path" >/dev/null 2>&1; then - GIT_DIR="$YADM_REPO" "$GIT_PROGRAM" mv "$legacy_path" "$new_filename" && repo_updates=1 + if "$GIT_PROGRAM" ls-files --error-unmatch "$legacy_path" >/dev/null 2>&1; then + "$GIT_PROGRAM" mv "$legacy_path" "$new_filename" && repo_updates=1 else mv -i "$legacy_path" "$new_filename" fi fi done + # handle submodules, which need to be reinitialized + if [ "$actions_performed" -ne 0 ]; then + cd_work "Upgrade submodules" + if "$GIT_PROGRAM" ls-files --error-unmatch .gitmodules >/dev/null 2>&1; then + "$GIT_PROGRAM" submodule deinit -f . + "$GIT_PROGRAM" submodule update --init --recursive + fi + fi + [ "$actions_performed" -eq 0 ] && \ echo "No legacy paths found. Upgrade is not necessary" From 375a34b97aba51c9e120a4beecc6c9e1986c771c Mon Sep 17 00:00:00 2001 From: Tim Byrne Date: Thu, 7 Nov 2019 07:48:42 -0600 Subject: [PATCH 081/137] Test conditional submodule upgrade processing --- test/test_unit_upgrade.py | 28 ++++++++++++++++++++++------ 1 file changed, 22 insertions(+), 6 deletions(-) diff --git a/test/test_unit_upgrade.py b/test/test_unit_upgrade.py index affd641..73f4cac 100644 --- a/test/test_unit_upgrade.py +++ b/test/test_unit_upgrade.py @@ -50,7 +50,8 @@ def test_upgrade_errors(tmpdir, runner, yadm, condition): assert 'already exists' in run.out -@pytest.mark.parametrize('condition', ['no-paths', 'untracked', 'tracked']) +@pytest.mark.parametrize( + 'condition', ['no-paths', 'untracked', 'tracked', 'submodules']) def test_upgrade(tmpdir, runner, yadm, condition): """Test upgrade() @@ -67,14 +68,25 @@ def test_upgrade(tmpdir, runner, yadm, condition): for lpath in LEGACY_PATHS: legacy_dir.join(lpath).write(lpath, ensure=True) - git = 'echo' if condition == 'tracked' else 'git' + mock_git = "" + if condition in ['tracked', 'submodules']: + mock_git = f''' + function git() {{ + echo "$@" + if [[ "$*" == *.gitmodules* ]]; then + return { '0' if condition == 'submodules' else '1' } + fi + return 0 + }} + ''' script = f""" YADM_TEST=1 source {yadm} YADM_DIR="{yadm_dir}" YADM_REPO="{yadm_dir}/repo.git" YADM_LEGACY_DIR="{legacy_dir}" - GIT_PROGRAM="{git}" + GIT_PROGRAM="git" + {mock_git} function cd {{ echo "$@";}} upgrade """ @@ -93,12 +105,16 @@ def test_upgrade(tmpdir, runner, yadm, condition): assert 'test-repo' in yadm_dir.join('repo.git/config').read() for lpath in LEGACY_PATHS: assert lpath in yadm_dir.join(lpath).read() - elif condition == 'tracked': + elif condition in ['tracked', 'submodules']: for lpath in LEGACY_PATHS: expected = ( f'mv {legacy_dir.join(lpath)} ' f'{yadm_dir.join(lpath)}') assert expected in run.out assert 'files tracked by yadm have been renamed' in run.out - assert 'submodule deinit -f .' in run.out - assert 'submodule update --init --recursive' in run.out + if condition == 'submodules': + assert 'submodule deinit -f .' in run.out + assert 'submodule update --init --recursive' in run.out + else: + assert 'submodule deinit -f .' not in run.out + assert 'submodule update --init --recursive' not in run.out From dc699e0b4e3a8c0acae897f96f00be1c7abc2ee3 Mon Sep 17 00:00:00 2001 From: Tim Byrne Date: Thu, 7 Nov 2019 20:36:53 -0600 Subject: [PATCH 082/137] Improve portability of hosted bootstrap curl-pipe --- bootstrap | 33 ++++++++++++++++++++++----------- 1 file changed, 22 insertions(+), 11 deletions(-) diff --git a/bootstrap b/bootstrap index c53d04d..1371bb4 100755 --- a/bootstrap +++ b/bootstrap @@ -4,6 +4,8 @@ # This script can be "curl-piped" into bash to bootstrap a dotfiles repo when # yadm is not locally installed. Read below for instructions. # +# This script is hosted at bootstrap.yadm.io to make it easy to remember/type. +# # DISCLAIMER: In general, I would advise against piping someone's code directly # from the Internet into an interpreter (like Bash). You should # probably review any code like this prior to executing it. I leave @@ -13,29 +15,38 @@ # (allowing the yadm project to be a submodule of my dotfiles # repo). # -# Invoke with: +# Invoke bootstrap with: # -# curl -fsSL 'https://tinyurl.com/yadm-bootstrap' | bash +# curl -L bootstrap.yadm.io | bash # -# OR +# OR +# +# curl -L bootstrap.yadm.io | bash [-s -- REPO_URL [YADM_RELEASE]] +# +# Alternatively, source in this file to export a yadm() function which uses +# yadm remotely until it is locally installed. +# +# source <(curl -L bootstrap.yadm.io) # -# curl -fsSL 'https://github.com/TheLocehiliosan/yadm/raw/master/bootstrap' | bash [-s -- REPO_URL [YADM_RELEASE]] YADM_REPO="https://github.com/TheLocehiliosan/yadm" -YADM_RELEASE="master" +YADM_RELEASE=${release:-master} REPO_URL="" -function yadm() { - if command -v which >/dev/null 2>&1 && which yadm >/dev/null 2>&1; then +function _private_yadm() { + unset -f yadm + if command -v yadm >/dev/null 2>&1; then echo "Found yadm installed locally, removing remote yadm() function" - unset -f yadm + unset -f _private_yadm command yadm "$@" else + function yadm() { _private_yadm "$@"; }; export -f yadm echo WARNING: Using yadm remotely. You should install yadm locally. curl -fsSL "$YADM_REPO/raw/$YADM_RELEASE/yadm" | bash -s -- "$@" fi } -export -f yadm +export -f _private_yadm +function yadm() { _private_yadm "$@"; }; export -f yadm # if being sourced, return here, otherwise continue processing return 2>/dev/null @@ -51,13 +62,13 @@ function ask_about_source() { echo "***************************************************" echo "yadm is NOT currently installed." echo "You should install it locally, this link may help:" - echo "https://thelocehiliosan.github.io/yadm/docs/install" + echo "https://yadm.io/docs/install" echo "***************************************************" echo echo "If installation is not possible right now, you can temporarily \"source\"" echo "in a yadm() function which fetches yadm remotely each time it is called." echo - echo " source <(curl -fsSL '$YADM_REPO/raw/$YADM_RELEASE/bootstrap')" + echo " source <(curl -L bootstrap.yadm.io)" echo fi } From 84ef8709e4518eb05b39f7964871714d0a52c0f8 Mon Sep 17 00:00:00 2001 From: Tim Byrne Date: Thu, 7 Nov 2019 21:09:37 -0600 Subject: [PATCH 083/137] Fix typo --- yadm.1 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/yadm.1 b/yadm.1 index 80c665d..539cc29 100644 --- a/yadm.1 +++ b/yadm.1 @@ -1,5 +1,5 @@ ." vim: set spell so=8: -.TH yadm 1 "25 October 2017" "1.12.0" +.TH yadm 1 "8 November 2019" "2.0.0" .SH NAME @@ -494,7 +494,7 @@ Valid when no other alternate is valid. You may use any number of conditions, in any order. An alternate will only be used if ALL conditions are valid. -For all files managed by yadm’s repository or listed in +For all files managed by yadm's repository or listed in .IR $HOME/.config/yadm/encrypt , if they match this naming convention, symbolic links will be created for the most appropriate version. From 1fc52536ac0f5bba4474e5a95cf356d6594ee2d5 Mon Sep 17 00:00:00 2001 From: Tim Byrne Date: Thu, 7 Nov 2019 21:23:49 -0600 Subject: [PATCH 084/137] Fix contrib make target (dev has been changed to develop) --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index c6dbca2..00d2ce5 100644 --- a/Makefile +++ b/Makefile @@ -164,7 +164,7 @@ yadm.md: yadm.1 .PHONY: contrib contrib: @echo "CONTRIBUTORS\n" > CONTRIBUTORS - @git shortlog -ns master gh-pages dev dev-pages | cut -f2 >> CONTRIBUTORS + @git shortlog -ns master gh-pages develop dev-pages | cut -f2 >> CONTRIBUTORS .PHONY: sync-clock sync-clock: From 5b105e0687e43f8cf08692112a8149fd29f2cd92 Mon Sep 17 00:00:00 2001 From: Tim Byrne Date: Thu, 7 Nov 2019 21:36:53 -0600 Subject: [PATCH 085/137] Add missing details to manpage --- yadm.1 | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/yadm.1 b/yadm.1 index 539cc29..9915e3a 100644 --- a/yadm.1 +++ b/yadm.1 @@ -266,6 +266,8 @@ This command will start by moving your yadm repo to the new path. Next it will move any configuration data to the new path. If the configurations are tracked within your yadm repo, this command will "stage" the renaming of those files in the repo's index. +Upgrading will also re-initialize all submodules you have added (otherwise they +will be broken when the repo moves). After running "yadm upgrade", you should run "yadm status" to review changes which have been staged, and commit them to your repository. @@ -610,7 +612,7 @@ to create or overwrite files. During processing, the following variables are available in the template: - Builtin Jinja Description + Default Jinja Description ------------- ------------- -------------------------- yadm.class YADM_CLASS Locally defined yadm class yadm.distro YADM_DISTRO lsb_release -si From de73c9f4b4c13d03d1e384fed28cd9b75e42217f Mon Sep 17 00:00:00 2001 From: Tim Byrne Date: Thu, 7 Nov 2019 07:58:14 -0600 Subject: [PATCH 086/137] Release 2.0.0 Update version number and update documentation * Support XDG base directory specification * Redesign alternate processing * Add built-in default template processor * Allow storing alternates in yadm dir (#90) * Add support for j2cli template processor * Ignore encrypted files (#69) * Support DISTRO in alternates (#72) * Support `source` in templates (#163) * Change yadm.cygwin-copy to yadm.alt-copy * Support `-b ` when cloning (#133) * Support includes for j2-based templates (#114) * Remove stale/invalid linked alternates (#65) * Add support for Mingw/Msys (#102) * Allow `-l` to pass thru to the `yadm config` command * Improve processing of `yadm/encrypt` * Fix bugs in legacy alternate processing * Fix bug with hidden private files * Improve support for older versions of Git * Add upgrade command --- .notes.md | 55 ------ CHANGES | 21 +++ CONTRIBUTORS | 6 +- yadm | 2 +- yadm.md | 497 +++++++++++++++++++++++++++++++++------------------ yadm.spec | 47 +---- 6 files changed, 352 insertions(+), 276 deletions(-) delete mode 100644 .notes.md diff --git a/.notes.md b/.notes.md deleted file mode 100644 index a471950..0000000 --- a/.notes.md +++ /dev/null @@ -1,55 +0,0 @@ -## New release checklist - ○ Version bump EVERYTHING - ○ Copyright year update? - ○ Rebuild contribs - ○ Rebuild man - ○ Update specfile - ○ Update CHANGES - - ○ Tag X.XX - ○ Merge dev → master - ○ Update Homebrew - ○ Update Copr - - ○ Tweet - -## Homebrew update - brew update - cd $(brew --repo homebrew/core) - git fetch --unshallow origin # only if still a shallow clone, - #this might just fail if this was already done - git remote add TheLocehiliosan git@github.com:TheLocehiliosan/homebrew-core.git - git push -f TheLocehiliosan master:master - vim Formula/yadm.rb - - brew install --build-from-source yadm - brew reinstall --verbose --debug yadm (✗₂) - brew audit --strict yadm - brew test yadm - - git add Formula/yadm.rb - git commit -S -m 'yadm X.XX' - - git push TheLocehiliosan master:yadm-X.XX - - Open pull request - -## Copr update - checkout yadm-rpm - bring in yadm.spec from yadm repo - update version in Makefile - - make tarball - make buildhost - - cd yadm-rpm - - because centos 6,7... - add 'Group: Development/Tools' - disable BuildRequires - disable %check - - fedpkg --dist f25 local - that should leave a src RPM in the yadm-rpm dir - - create a new build by uploading the src rpm to copr diff --git a/CHANGES b/CHANGES index 7488361..c5ef9fe 100644 --- a/CHANGES +++ b/CHANGES @@ -1,3 +1,24 @@ +2.0.0 + * Support XDG base directory specification + * Redesign alternate processing + * Add built-in default template processor + * Allow storing alternates in yadm dir (#90) + * Add support for j2cli template processor + * Ignore encrypted files (#69) + * Support DISTRO in alternates (#72) + * Support `source` in templates (#163) + * Change yadm.cygwin-copy to yadm.alt-copy + * Support `-b ` when cloning (#133) + * Support includes for j2-based templates (#114) + * Remove stale/invalid linked alternates (#65) + * Add support for Mingw/Msys (#102) + * Allow `-l` to pass thru to the `yadm config` command + * Improve processing of `yadm/encrypt` + * Fix bugs in legacy alternate processing + * Fix bug with hidden private files + * Improve support for older versions of Git + * Add upgrade command + 1.12.0 * Add basic Zsh completion (#71, #79) * Support directories in `.yadm/encrypt` (#81, #82) diff --git a/CONTRIBUTORS b/CONTRIBUTORS index 55cc6cd..adbc226 100644 --- a/CONTRIBUTORS +++ b/CONTRIBUTORS @@ -3,15 +3,19 @@ CONTRIBUTORS Tim Byrne Espen Henriksen Cameron Eagans +Ross Smith II Klas Mellbourn Jan Schulz +Satoshi Ohki Siôn Le Roux Sébastien Gross Thomas Luzat Tomas Cernaj Uroš Golja +Brayden Banks japm48 +Daniel Wagenknecht Franciszek Madej +Mateusz Piotrowski Paraplegic Racehorse Patrick Hof -Satoshi Ohki diff --git a/yadm b/yadm index 57898f7..307c400 100755 --- a/yadm +++ b/yadm @@ -20,7 +20,7 @@ if [ -z "$BASH_VERSION" ]; then [ "$YADM_TEST" != 1 ] && exec bash "$0" "$@" fi -VERSION=1.12.0 +VERSION=2.0.0 YADM_WORK="$HOME" YADM_DIR= diff --git a/yadm.md b/yadm.md index ee75c77..f5855bb 100644 --- a/yadm.md +++ b/yadm.md @@ -4,14 +4,15 @@ ## NAME yadm - Yet Another Dotfiles Manager + ## SYNOPSIS yadm command [options] yadm git-command-or-alias [options] - yadm init [-f] [-w directory] + yadm init [-f] [-w dir] - yadm clone url [-f] [-w directory] [--bootstrap] [--no-bootstrap] + yadm clone url [-f] [-w dir] [-b branch] [--bootstrap] [--no-bootstrap] yadm config name [value] @@ -31,15 +32,18 @@ yadm perms + yadm upgrade + yadm introspect category + ## DESCRIPTION - yadm is a tool for managing a collection of files across multiple com- - puters, using a shared Git repository. In addition, yadm provides a - feature to select alternate versions of files based on the operating - system or host name. Lastly, yadm supplies the ability to manage a - subset of secure files, which are encrypted before they are included in - the repository. + yadm is a tool for managing a collection of files across multiple com- + puters, using a shared Git repository. In addition, yadm provides a + feature to select alternate versions of files for particular systems. + Lastly, yadm supplies the ability to manage a subset of secure files, + which are encrypted before they are included in the repository. + ## COMMANDS git-command or git-alias @@ -54,15 +58,15 @@ The config command is not passed directly through. Instead use the gitconfig command (see below). - alt Create symbolic links and process Jinja templates for any man- - aged files matching the naming rules described in the ALTERNATES - and JINJA sections. It is usually unnecessary to run this com- + alt Create symbolic links and process templates for any managed + files matching the naming rules described in the ALTERNATES and + TEMPLATES sections. It is usually unnecessary to run this com- mand, as yadm automatically processes alternates by default. This automatic behavior can be disabled by setting the configu- ration yadm.auto-alt to "false". bootstrap - Execute $HOME/.yadm/bootstrap if it exists. + Execute $HOME/.config/yadm/bootstrap if it exists. clone url Clone a remote repository for tracking dotfiles. After the con- @@ -83,29 +87,30 @@ or yadm stash pop - The repository is stored in $HOME/.yadm/repo.git. By default, - $HOME will be used as the work-tree, but this can be overridden - with the -w option. yadm can be forced to overwrite an existing - repository by providing the -f option. By default yadm will ask - the user if the bootstrap program should be run (if it exists). - The options --bootstrap or --no-bootstrap will either force the - bootstrap to be run, or prevent it from being run, without - prompting the user. + The repository is stored in $HOME/.config/yadm/repo.git. By + default, $HOME will be used as the work-tree, but this can be + overridden with the -w option. yadm can be forced to overwrite + an existing repository by providing the -f option. If you want + to use a branch other than origin/master, you can specify it + using the -b option. By default yadm will ask the user if the + bootstrap program should be run (if it exists). The options + --bootstrap or --no-bootstrap will either force the bootstrap to + be run, or prevent it from being run, without prompting the + user. config This command manages configurations for yadm. This command works exactly they way git-config(1) does. See the CONFIGURA- TION section for more details. decrypt - Decrypt all files stored in $HOME/.yadm/files.gpg. Files + Decrypt all files stored in $HOME/.config/yadm/files.gpg. Files decrypted will be relative to the configured work-tree (usually $HOME). Using the -l option will list the files stored without extracting them. encrypt - Encrypt all files matching the patterns found in - $HOME/.yadm/encrypt. See the ENCRYPTION section for more - details. + Encrypt all files matching the patterns found in $HOME/.con- + fig/yadm/encrypt. See the ENCRYPTION section for more details. enter Run a sub-shell with all Git variables set. Exit the sub-shell the same way you leave your normal shell (usually with the @@ -137,10 +142,10 @@ help Print a summary of yadm commands. init Initialize a new, empty repository for tracking dotfiles. The - repository is stored in $HOME/.yadm/repo.git. By default, $HOME - will be used as the work-tree, but this can be overridden with - the -w option. yadm can be forced to overwrite an existing - repository by providing the -f option. + repository is stored in $HOME/.config/yadm/repo.git. By + default, $HOME will be used as the work-tree, but this can be + overridden with the -w option. yadm can be forced to overwrite + an existing repository by providing the -f option. list Print a list of files managed by yadm. The -a option will cause all managed files to be listed. Otherwise, the list will only @@ -153,28 +158,67 @@ perms Update permissions as described in the PERMISSIONS section. It is usually unnecessary to run this command, as yadm automati- - cally processes permissions by default. This automatic behavior + cally processes permissions by default. This automatic behavior can be disabled by setting the configuration yadm.auto-perms to "false". + upgrade + Version 2 of yadm uses a different directory for storing your + configurations. When you start to use version 2 for the first + time, you may see warnings about moving your data to this new + directory. The easiest way to accomplish this is by running + "yadm upgrade". This command will start by moving your yadm + repo to the new path. Next it will move any configuration data + to the new path. If the configurations are tracked within your + yadm repo, this command will "stage" the renaming of those files + in the repo's index. Upgrading will also re-initialize all sub- + modules you have added (otherwise they will be broken when the + repo moves). After running "yadm upgrade", you should run "yadm + status" to review changes which have been staged, and commit + them to your repository. + + You can read https://yadm.io/docs/upgrade_from_1 for more infor- + mation. + version Print the version of yadm. + +## COMPATIBILITY + Beginning with version 2.0.0, yadm introduced a couple major changes + which may require you to adjust your configurations. See the upgrade + command for help making those adjustments. + + First, yadm now uses the "XDG Base Directory Specification" to find its + configurations. You can read https://yadm.io/docs/upgrade_from_1 for + more information. + + Second, the naming conventions for alternate files have been changed. + You can read https://yadm.io/docs/alternates for more information. + + If you want to retain the old functionality, you can set an environment + variable, YADM_COMPATIBILITY=1. Doing so will automatically use the + old yadm directory, and process alternates the same as the pre-2.0.0 + version. This compatibility mode is deprecated, and will be removed in + future versions. This mode exists solely for transitioning to the new + paths and naming of alternates. + + ## OPTIONS - yadm supports a set of universal options that alter the paths it uses. - The default paths are documented in the FILES section. Any path speci- - fied by these options must be fully qualified. If you always want to - override one or more of these paths, it may be useful to create an - alias for the yadm command. For example, the following alias could be + yadm supports a set of universal options that alter the paths it uses. + The default paths are documented in the FILES section. Any path speci- + fied by these options must be fully qualified. If you always want to + override one or more of these paths, it may be useful to create an + alias for the yadm command. For example, the following alias could be used to override the repository directory. alias yadm='yadm --yadm-repo /alternate/path/to/repo' - The following is the full list of universal options. Each option + The following is the full list of universal options. Each option should be followed by a fully qualified path. -Y,--yadm-dir - Override the yadm directory. yadm stores its data relative to + Override the yadm directory. yadm stores its data relative to this directory. --yadm-repo @@ -192,10 +236,11 @@ --yadm-bootstrap Override the location of the yadm bootstrap program. + ## CONFIGURATION - yadm uses a configuration file named $HOME/.yadm/config. This file - uses the same format as git-config(1). Also, you can control the con- - tents of the configuration file via the yadm config command (which + yadm uses a configuration file named $HOME/.config/yadm/config. This + file uses the same format as git-config(1). Also, you can control the + contents of the configuration file via the yadm config command (which works exactly like git-config). For example, to disable alternates you can run the command: @@ -203,10 +248,23 @@ The following is the full list of supported configurations: + yadm.alt-copy + If set to "true", alternate files will be copies instead of sym- + bolic links. This might be desirable, because some systems may + not properly support symlinks. + + NOTE: The deprecated yadm.cygwin-copy option used by older ver- + sions of yadm has been replaced by yadm.alt-copy. The old + option will be removed in the next version of yadm. + yadm.auto-alt - Disable the automatic linking described in the section ALTER- - NATES. If disabled, you may still run yadm alt manually to cre- - ate the alternate links. This feature is enabled by default. + Disable the automatic linking described in the section ALTER- + NATES. If disabled, you may still run "yadm alt" manually to + create the alternate links. This feature is enabled by default. + + yadm.auto-exclude + Disable the automatic exclusion of patterns defined in + $HOME/.config/yadm/encrypt. This feature is enabled by default. yadm.auto-perms Disable the automatic permission changes described in the sec- @@ -218,168 +276,231 @@ Disable the automatic creating of private directories described in the section PERMISSIONS. - yadm.ssh-perms - Disable the permission changes to $HOME/.ssh/*. This feature is - enabled by default. + yadm.git-program + Specify an alternate program to use instead of "git". By + default, the first "git" found in $PATH is used. yadm.gpg-perms Disable the permission changes to $HOME/.gnupg/*. This feature is enabled by default. - yadm.gpg-recipient - Asymmetrically encrypt files with a gpg public/private key pair. - Provide a "key ID" to specify which public key to encrypt with. - The key must exist in your public keyrings. If left blank or - not provided, symmetric encryption is used instead. If set to - "ASK", gpg will interactively ask for recipients. See the - ENCRYPTION section for more details. This feature is disabled - by default. - yadm.gpg-program Specify an alternate program to use instead of "gpg". By default, the first "gpg" found in $PATH is used. - yadm.git-program - Specify an alternate program to use instead of "git". By - default, the first "git" found in $PATH is used. + yadm.gpg-recipient + Asymmetrically encrypt files with a gpg public/private key pair. + Provide a "key ID" to specify which public key to encrypt with. + The key must exist in your public keyrings. If left blank or + not provided, symmetric encryption is used instead. If set to + "ASK", gpg will interactively ask for recipients. See the + ENCRYPTION section for more details. This feature is disabled + by default. - yadm.cygwin-copy - If set to "true", for Cygwin hosts, alternate files will be - copies instead of symbolic links. This might be desirable, - because non-Cygwin software may not properly interpret Cygwin - symlinks. + yadm.ssh-perms + Disable the permission changes to $HOME/.ssh/*. This feature is + enabled by default. - These last four "local" configurations are not stored in the - $HOME/.yadm/config, they are stored in the local repository. + The following four "local" configurations are not stored in the + $HOME/.config/yadm/config, they are stored in the local repository. local.class - Specify a CLASS for the purpose of symlinking alternate files. - By default, no CLASS will be matched. + Specify a class for the purpose of symlinking alternate files. + By default, no class will be matched. + + local.hostname + Override the hostname for the purpose of symlinking alternate + files. local.os Override the OS for the purpose of symlinking alternate files. - local.hostname - Override the HOSTNAME for the purpose of symlinking alternate - files. - local.user - Override the USER for the purpose of symlinking alternate files. + Override the user for the purpose of symlinking alternate files. + ## ALTERNATES When managing a set of files across different systems, it can be useful to have an automated way of choosing an alternate version of a file for - a different operating system, host, or user. yadm implements a feature - which will automatically create a symbolic link to the appropriate ver- - sion of a file, as long as you follow a specific naming convention. - yadm can detect files with names ending in any of the following: + a different operating system, host, user, etc. - ## - ##CLASS - ##CLASS.OS - ##CLASS.OS.HOSTNAME - ##CLASS.OS.HOSTNAME.USER - ##OS - ##OS.HOSTNAME - ##OS.HOSTNAME.USER + yadm will automatically create a symbolic link to the appropriate ver- + sion of a file, when a valid suffix is appended to the filename. The + suffix contains the conditions that must be met for that file to be + used. - If there are any files managed by yadm's repository, or listed in - $HOME/.yadm/encrypt, which match this naming convention, symbolic links - will be created for the most appropriate version. This may best be - demonstrated by example. Assume the following files are managed by - yadm's repository: + The suffix begins with "##", followed by any number of conditions sepa- + rated by commas. - - $HOME/path/example.txt## - - $HOME/path/example.txt##Work - - $HOME/path/example.txt##Darwin - - $HOME/path/example.txt##Darwin.host1 - - $HOME/path/example.txt##Darwin.host2 - - $HOME/path/example.txt##Linux - - $HOME/path/example.txt##Linux.host1 - - $HOME/path/example.txt##Linux.host2 + ##[,,...] + + Each condition is an attribute/value pair, separated by a period. Some + conditions do not require a "value", and in that case, the period and + value can be omitted. Most attributes can be abbreviated as a single + letter. + + [.] + + These are the supported attributes, in the order of the weighted prece- + dence: + + + template, t + Valid when the value matches a supported template processor. + See the TEMPLATES section for more details. + + user, u + Valid if the value matches the current user. Current user is + calculated by running id -u -n. + + distro, d + Valid if the value matches the distro. Distro is calculated by + running lsb_release -si. + + os, o Valid if the value matches the OS. OS is calculated by running + uname -s. + + class, c + Valid if the value matches the local.class configuration. Class + must be manually set using yadm config local.class . See + the CONFIGURATION section for more details about setting + local.class. + + hostname, h + Valid if the value matches the short hostname. Hostname is cal- + culated by running hostname, and trimming off any domain. + + default + Valid when no other alternate is valid. + + + You may use any number of conditions, in any order. An alternate will + only be used if ALL conditions are valid. For all files managed by + yadm's repository or listed in $HOME/.config/yadm/encrypt, if they + match this naming convention, symbolic links will be created for the + most appropriate version. + + The "most appropriate" version is determined by calculating a score for + each version of a file. A template is always scored higher than any + symlink condition. The number of conditions is the next largest factor + in scoring. Files with more conditions will always be favored. Any + invalid condition will disqualify that file completely. + + If you don't care to have all versions of alternates stored in the same + directory as the generated symlink, you can place them in the + $HOME/.config/yadm/alt directory. The generated symlink or processed + template will be created using the same relative path. + + Alternate linking may best be demonstrated by example. Assume the fol- + lowing files are managed by yadm's repository: + + - $HOME/path/example.txt##default + - $HOME/path/example.txt##class.Work + - $HOME/path/example.txt##os.Darwin + - $HOME/path/example.txt##os.Darwin,hostname.host1 + - $HOME/path/example.txt##os.Darwin,hostname.host2 + - $HOME/path/example.txt##os.Linux + - $HOME/path/example.txt##os.Linux,hostname.host1 + - $HOME/path/example.txt##os.Linux,hostname.host2 If running on a Macbook named "host2", yadm will create a symbolic link which looks like this: - $HOME/path/example.txt -> $HOME/path/example.txt##Darwin.host2 + $HOME/path/example.txt -> $HOME/path/example.txt##os.Darwin,host- + name.host2 However, on another Mackbook named "host3", yadm will create a symbolic link which looks like this: - $HOME/path/example.txt -> $HOME/path/example.txt##Darwin + $HOME/path/example.txt -> $HOME/path/example.txt##os.Darwin - Since the hostname doesn't match any of the managed files, the more + Since the hostname doesn't match any of the managed files, the more generic version is chosen. If running on a Linux server named "host4", the link will be: - $HOME/path/example.txt -> $HOME/path/example.txt##Linux + $HOME/path/example.txt -> $HOME/path/example.txt##os.Linux - If running on a Solaris server, the link use the default "##" version: + If running on a Solaris server, the link will use the default version: - $HOME/path/example.txt -> $HOME/path/example.txt## + $HOME/path/example.txt -> $HOME/path/example.txt##default - If running on a system, with CLASS set to "Work", the link will be: + If running on a system, with class set to "Work", the link will be: - $HOME/path/example.txt -> $HOME/path/example.txt##WORK + $HOME/path/example.txt -> $HOME/path/example.txt##class.Work - If no "##" version exists and no files match the current CLASS/OS/HOST- - NAME/USER, then no link will be created. + If no "##default" version exists and no files have valid conditions, + then no link will be created. - Links are also created for directories named this way, as long as they + Links are also created for directories named this way, as long as they have at least one yadm managed file within them. - CLASS must be manually set using yadm config local.class . OS - is determined by running uname -s, HOSTNAME by running hostname, and - USER by running id -u -n. yadm will automatically create these links - by default. This can be disabled using the yadm.auto-alt configuration. - Even if disabled, links can be manually created by running yadm alt. + yadm will automatically create these links by default. This can be dis- + abled using the yadm.auto-alt configuration. Even if disabled, links + can be manually created by running yadm alt. - It is possible to use "%" as a "wildcard" in place of CLASS, OS, HOST- - NAME, or USER. For example, The following file could be linked for any - host when the user is "harvey". - - $HOME/path/example.txt##%.%.harvey - - CLASS is a special value which is stored locally on each host (inside - the local repository). To use alternate symlinks using CLASS, you must - set the value of class using the configuration local.class. This is + Class is a special value which is stored locally on each host (inside + the local repository). To use alternate symlinks using class, you must + set the value of class using the configuration local.class. This is set like any other yadm configuration with the yadm config command. The - following sets the CLASS to be "Work". + following sets the class to be "Work". yadm config local.class Work - Similarly, the values of OS, HOSTNAME, and USER can be manually over- - ridden using the configuration options local.os, local.hostname, and + Similarly, the values of os, hostname, and user can be manually over- + ridden using the configuration options local.os, local.hostname, and local.user. -## JINJA - If the envtpl command is available, Jinja templates will also be pro- - cessed to create or overwrite real files. yadm will treat files ending - in +## TEMPLATES + If a template condition is defined in an alternate file's "##" suffix, + and the necessary dependencies for the template are available, then the + file will be processed to create or overwrite files. - ##yadm.j2 + Supported template processors: - as Jinja templates. During processing, the following variables are set - according to the rules explained in the ALTERNATES section: + default + This is yadm's built-in template processor. This processor is + very basic, with a Jinja-like syntax. The advantage of this pro- + cessor is that it only depends upon awk, which is available on + most *nix systems. To use this processor, specify the value of + "default" or just leave the value off (e.g. "##template"). - YADM_CLASS - YADM_OS - YADM_HOSTNAME - YADM_USER + j2cli To use the j2cli Jinja template processor, specify the value of + "j2" or "j2cli". - In addition YADM_DISTRO is exposed as the value of lsb_release -si if - lsb_release is locally available. + envtpl To use the envtpl Jinja template processor, specify the value of + "j2" or "envtpl". - For example, a file named whatever##yadm.j2 with the following content - {% if YADM_USER == 'harvey' -%} - config={{YADM_CLASS}}-{{ YADM_OS }} - {% else -%} + NOTE: Specifying "j2" as the processor will attempt to use j2cli or + envtpl, whichever is available. + + If the template processor specified is available, templates will be + processed to create or overwrite files. + + During processing, the following variables are available in the tem- + plate: + + Default Jinja Description + ------------- ------------- -------------------------- + yadm.class YADM_CLASS Locally defined yadm class + yadm.distro YADM_DISTRO lsb_release -si + yadm.hostname YADM_HOSTNAME hostname (without domain) + yadm.os YADM_OS uname -s + yadm.user YADM_USER id -u -n + yadm.source YADM_SOURCE Template filename + + Examples: + + whatever##template with the following content + + {% if yadm.user == 'harvey' %} + config={{yadm.class}}-{{yadm.os}} + {% else %} config=dev-whatever - {% endif -%} + {% endif %} would output a file named whatever with the following content if the user is "harvey": @@ -390,21 +511,27 @@ config=dev-whatever - See http://jinja.pocoo.org/ for an overview of Jinja. + An equivalent Jinja template named whatever##template.j2 would look + like: + + {% if YADM_USER == 'harvey' -%} + config={{YADM_CLASS}}-{{YADM_OS}} + {% else -%} + config=dev-whatever + {% endif -%} ## ENCRYPTION - It can be useful to manage confidential files, like SSH or GPG keys, - across multiple systems. However, doing so would put plain text data - into a Git repository, which often resides on a public system. yadm - implements a feature which can make it easy to encrypt and decrypt a - set of files so the encrypted version can be maintained in the Git - repository. This feature will only work if the gpg(1) command is - available. + It can be useful to manage confidential files, like SSH or GPG keys, + across multiple systems. However, doing so would put plain text data + into a Git repository, which often resides on a public system. yadm can + make it easy to encrypt and decrypt a set of files so the encrypted + version can be maintained in the Git repository. This feature will + only work if the gpg(1) command is available. To use this feature, a list of patterns must be created and saved as - $HOME/.yadm/encrypt. This list of patterns should be relative to the - configured work-tree (usually $HOME). For example: + $HOME/.config/yadm/encrypt. This list of patterns should be relative + to the configured work-tree (usually $HOME). For example: .ssh/*.key .gnupg/*.gpg @@ -417,9 +544,9 @@ The yadm encrypt command will find all files matching the patterns, and prompt for a password. Once a password has confirmed, the matching - files will be encrypted and saved as $HOME/.yadm/files.gpg. The pat- - terns and files.gpg should be added to the yadm repository so they are - available across multiple systems. + files will be encrypted and saved as $HOME/.config/yadm/files.gpg. The + patterns and files.gpg should be added to the yadm repository so they + are available across multiple systems. To decrypt these files later, or on another system run yadm decrypt and provide the correct password. After files are decrypted, permissions @@ -431,15 +558,21 @@ NOTE: It is recommended that you use a private repository when keeping confidential files, even though they are encrypted. + Patterns found in $HOME/.config/yadm/encrypt are automatically added to + the repository's info/exclude file every time yadm encrypt is run. + This is to prevent accidentally committing sensitive data to the repos- + itory. This can be disabled using the yadm.auto-exclude configuration. + + ## PERMISSIONS When files are checked out of a Git repository, their initial permis- sions are dependent upon the user's umask. Because of this, yadm will automatically update the permissions of some file paths. The "group" and "others" permissions will be removed from the following files: - - $HOME/.yadm/files.gpg + - $HOME/.config/yadm/files.gpg - - All files matching patterns in $HOME/.yadm/encrypt + - All files matching patterns in $HOME/.config/yadm/encrypt - The SSH directory and files, .ssh/* @@ -459,17 +592,18 @@ When running a Git command and .ssh or .gnupg directories do not exist, yadm will create those directories with mask 0700 prior to running the - Git command. This can be disabled using the yadm.auto-private-dirs - configuration. + Git command. This can be disabled using the yadm.auto-private-dirs con- + figuration. + ## HOOKS For every command yadm supports, a program can be provided to run - before or after that command. These are referred to as "hooks". yadm - looks for hooks in the directory $HOME/.yadm/hooks. Each hook is named - using a prefix of pre_ or post_, followed by the command which should - trigger the hook. For example, to create a hook which is run after - every yadm pull command, create a hook named post_pull. Hooks must - have the executable file permission set. + before or after that command. These are referred to as "hooks". yadm + looks for hooks in the directory $HOME/.config/yadm/hooks. Each hook + is named using a prefix of pre_ or post_, followed by the command which + should trigger the hook. For example, to create a hook which is run + after every yadm pull command, create a hook named post_pull. Hooks + must have the executable file permission set. If a pre_ hook is defined, and the hook terminates with a non-zero exit status, yadm will refuse to run the yadm command. For example, if a @@ -495,18 +629,31 @@ YADM_HOOK_WORK The path to the work-tree -## FILES - The following are the default paths yadm uses for its own data. These - paths can be altered using universal options. See the OPTIONS section - for details. - $HOME/.yadm +## FILES + All of yadm's configurations are relative to the "yadm directory". + yadm uses the "XDG Base Directory Specification" to determine this + directory. If the environment variable $XDG_CONFIG_HOME is defined as + a fully qualified path, this directory will be $XDG_CONFIG_HOME/yadm. + Otherwise it will be $HOME/.config/yadm. + + The following are the default paths yadm uses for its own data. Most + of these paths can be altered using universal options. See the OPTIONS + section for details. + + $HOME/.config/yadm The yadm directory. By default, all data yadm stores is relative to this directory. $YADM_DIR/config Configuration file for yadm. + $YADM_DIR/alt + This is a directory to keep "alternate files" without having + them side-by-side with the resulting symlink or processed tem- + plate. Alternate files placed in this directory will be created + relative to $HOME instead. + $YADM_DIR/repo.git Git repository used by yadm. @@ -516,6 +663,7 @@ $YADM_DIR/files.gpg All files encrypted with yadm encrypt are stored in this file. + ## EXAMPLES yadm init Create an empty repo for managing files @@ -529,24 +677,27 @@ yadm push -u origin master Initial push of master to origin - echo .ssh/*.key >> $HOME/.yadm/encrypt + echo .ssh/*.key >> $HOME/.config/yadm/encrypt Add a new pattern to the list of encrypted files - yadm encrypt ; yadm add ~/.yadm/files.gpg ; yadm commit + yadm encrypt ; yadm add ~/.config/yadm/files.gpg ; yadm commit Commit a new set of encrypted files + ## REPORTING BUGS Report issues or create pull requests at GitHub: https://github.com/TheLocehiliosan/yadm/issues + ## AUTHOR Tim Byrne + ## SEE ALSO git(1), gpg(1) - https://thelocehiliosan.github.io/yadm/ + https://yadm.io/ diff --git a/yadm.spec b/yadm.spec index bfa7d63..6f25ba2 100644 --- a/yadm.spec +++ b/yadm.spec @@ -1,7 +1,7 @@ %{!?_pkgdocdir: %global _pkgdocdir %{_docdir}/%{name}-%{version}} Name: yadm Summary: Yet Another Dotfiles Manager -Version: 1.12.0 +Version: 2.0.0 Group: Development/Tools Release: 1%{?dist} URL: https://yadm.io @@ -51,48 +51,3 @@ cd *yadm-* %attr(755,root,root) %{_bindir}/yadm %attr(644,root,root) %{_mandir}/man1/* %doc %{_pkgdocdir} - -%changelog -* Wed Oct 25 2017 Tim Byrne - 1.12.0-1 -- Bump version to 1.12.0 -- Include zsh completion - -* Wed Aug 23 2017 Tim Byrne - 1.11.1-1 -- Bump version to 1.11.1 - -* Mon Jul 10 2017 Tim Byrne - 1.11.0-1 -- Bump version to 1.11.0 - -* Wed May 10 2017 Tim Byrne - 1.10.0-1 -- Bump version to 1.10.0 -- Transition to semantic versioning - -* Thu May 4 2017 Tim Byrne - 1.09-1 -- Bump version to 1.09 -- Add yadm.bash_completion - -* Mon Apr 3 2017 Tim Byrne - 1.08-1 -- Bump version to 1.08 - -* Fri Feb 10 2017 Tim Byrne - 1.07-1 -- Bump version to 1.07 - -* Fri Jan 13 2017 Tim Byrne - 1.06-1 -- Bump version to 1.06 - -* Tue May 17 2016 Tim Byrne - 1.04-3 -- Add missing docs -- Fix changelog format -- Remove file attribute for docs and license - -* Mon May 16 2016 Tim Byrne - 1.04-2 -- Add %%check -- Add %%{?dist} -- Add build dependencies -- Add license and docs -- Remove %%defattr -- Remove group tag -- Sync RPM description with man page - -* Fri Apr 22 2016 Tim Byrne - 1.04-1 -- Initial RPM release From ab578c950249c5cbf4f1ca0e1c309536913f45c1 Mon Sep 17 00:00:00 2001 From: Tim Byrne Date: Fri, 8 Nov 2019 07:18:02 -0600 Subject: [PATCH 087/137] Add OBS badge --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index a0d681a..bed9d39 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ [![Latest Version][releases-badge]][releases-link] [![Homebrew Version][homebrew-badge]][homebrew-link] -[![Copr Version][copr-badge]][copr-link] +[![OBS Version][obs-badge]][obs-link] [![Arch Version][aur-badge]][aur-link] [![License][license-badge]][license-link]
[![Master Update][master-date]][master-commits] @@ -29,8 +29,6 @@ Features, usage, examples and installation instructions can be found on the [GnuPG]: https://gnupg.org/ [aur-badge]: https://img.shields.io/aur/version/yadm-git.svg [aur-link]: https://aur.archlinux.org/packages/yadm-git -[copr-badge]: https://img.shields.io/badge/dynamic/json.svg?label=copr&prefix=v&query=%24..version&url=https%3A%2F%2Fcopr.fedorainfracloud.org%2Fapi_2%2Fbuilds%3Fproject_id%3D7041%26limit%3D1 -[copr-link]: https://copr.fedorainfracloud.org/coprs/thelocehiliosan/yadm/ [dev-pages-badge]: https://img.shields.io/travis/TheLocehiliosan/yadm/dev-pages.svg?label=dev-pages [develop-badge]: https://img.shields.io/travis/TheLocehiliosan/yadm/develop.svg?label=develop [develop-commits]: https://github.com/TheLocehiliosan/yadm/commits/develop @@ -44,6 +42,8 @@ Features, usage, examples and installation instructions can be found on the [master-badge]: https://img.shields.io/travis/TheLocehiliosan/yadm/master.svg?label=master [master-commits]: https://github.com/TheLocehiliosan/yadm/commits/master [master-date]: https://img.shields.io/github/last-commit/TheLocehiliosan/yadm/master.svg?label=master +[obs-badge]: https://img.shields.io/badge/OBS-v2.0.0-blue +[obs-link]: https://software.opensuse.org//download.html?project=home%3ATheLocehiliosan%3Ayadm&package=yadm [releases-badge]: https://img.shields.io/github/tag/TheLocehiliosan/yadm.svg?label=latest+release [releases-link]: https://github.com/TheLocehiliosan/yadm/releases [travis-ci]: https://travis-ci.org/TheLocehiliosan/yadm/branches From 2eb8a9e362d44ae29ef6248364c879df98e5c87d Mon Sep 17 00:00:00 2001 From: Tim Byrne Date: Sat, 9 Nov 2019 15:48:14 -0600 Subject: [PATCH 088/137] Add make install --- Makefile | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/Makefile b/Makefile index 00d2ce5..b9e4777 100644 --- a/Makefile +++ b/Makefile @@ -65,6 +65,11 @@ usage: @echo ' make contrib' @echo ' - Generate the CONTRIBUTORS file, from the repo history.' @echo + @echo 'INSTALLATION' + @echo + @echo ' make install PREFIX=' + @echo ' - Install yadm, manpage, etc. to ' + @echo @echo 'UTILITIES' @echo @echo ' make sync-clock' @@ -166,6 +171,21 @@ contrib: @echo "CONTRIBUTORS\n" > CONTRIBUTORS @git shortlog -ns master gh-pages develop dev-pages | cut -f2 >> CONTRIBUTORS +.PHONY: install +install: + @[ -n "$(PREFIX)" ] || { echo "PREFIX is not set"; exit 1; } + @{\ + set -e ;\ + bin="$(PREFIX)/bin" ;\ + doc="$(PREFIX)/share/doc/yadm" ;\ + man="$(PREFIX)/share/man/man1" ;\ + install -d "$$bin" "$$doc" "$$man" ;\ + install -m 0755 yadm "$$bin" ;\ + install -m 0644 yadm.1 "$$man" ;\ + install -m 0644 CHANGES CONTRIBUTORS LICENSE "$$doc" ;\ + cp -r contrib "$$doc" ;\ + } + .PHONY: sync-clock sync-clock: docker run --rm --privileged alpine hwclock -s From 86330837166028e5b812595079a80a8b0537c09e Mon Sep 17 00:00:00 2001 From: Ross Smith II Date: Mon, 11 Nov 2019 20:30:50 -0800 Subject: [PATCH 089/137] Don't glob into parent dirs --- yadm | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/yadm b/yadm index 307c400..7df89cf 100755 --- a/yadm +++ b/yadm @@ -1045,12 +1045,12 @@ function perms() { # include all .ssh files (unless disabled) if [[ $(config --bool yadm.ssh-perms) != "false" ]] ; then - GLOBS+=(".ssh" ".ssh/*" ".ssh/.*") + GLOBS+=(".ssh" ".ssh/*" ".ssh/.[!.]*") fi # include all gpg files (unless disabled) if [[ $(config --bool yadm.gpg-perms) != "false" ]] ; then - GLOBS+=(".gnupg" ".gnupg/*" ".gnupg/.*") + GLOBS+=(".gnupg" ".gnupg/*" ".gnupg/.[!.]*") fi # include any files we encrypt From c29834ed86e0a760879fd74c1ffc59926a323513 Mon Sep 17 00:00:00 2001 From: Tim Byrne Date: Mon, 11 Nov 2019 23:28:16 -0600 Subject: [PATCH 090/137] Add test for permission bug (#174) --- test/test_perms.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/test/test_perms.py b/test/test_perms.py index ddf5260..d19b53b 100644 --- a/test/test_perms.py +++ b/test/test_perms.py @@ -98,3 +98,6 @@ def test_perms_control(runner, yadm_y, paths, ds1, sshperms, gpgperms): else: assert oct(private.stat().mode).endswith('00'), ( 'Path has not been secured') + + # verify permissions aren't changed for the worktree + assert oct(paths.work.stat().mode).endswith('0755') From 5aa1a7be755a99392a169f8795da17fa60aba96f Mon Sep 17 00:00:00 2001 From: Tim Byrne Date: Tue, 12 Nov 2019 07:02:27 -0600 Subject: [PATCH 091/137] Release 2.0.1 Update version number and update documentation * Fix bug with worktree permissions (#174) --- CHANGES | 3 +++ README.md | 2 +- yadm | 2 +- yadm.1 | 2 +- yadm.spec | 2 +- 5 files changed, 7 insertions(+), 4 deletions(-) diff --git a/CHANGES b/CHANGES index c5ef9fe..258f548 100644 --- a/CHANGES +++ b/CHANGES @@ -1,3 +1,6 @@ +2.0.1 + * Fix bug with worktree permissions (#174) + 2.0.0 * Support XDG base directory specification * Redesign alternate processing diff --git a/README.md b/README.md index bed9d39..2d3163a 100644 --- a/README.md +++ b/README.md @@ -42,7 +42,7 @@ Features, usage, examples and installation instructions can be found on the [master-badge]: https://img.shields.io/travis/TheLocehiliosan/yadm/master.svg?label=master [master-commits]: https://github.com/TheLocehiliosan/yadm/commits/master [master-date]: https://img.shields.io/github/last-commit/TheLocehiliosan/yadm/master.svg?label=master -[obs-badge]: https://img.shields.io/badge/OBS-v2.0.0-blue +[obs-badge]: https://img.shields.io/badge/OBS-v2.0.1-blue [obs-link]: https://software.opensuse.org//download.html?project=home%3ATheLocehiliosan%3Ayadm&package=yadm [releases-badge]: https://img.shields.io/github/tag/TheLocehiliosan/yadm.svg?label=latest+release [releases-link]: https://github.com/TheLocehiliosan/yadm/releases diff --git a/yadm b/yadm index 7df89cf..c9d8c4d 100755 --- a/yadm +++ b/yadm @@ -20,7 +20,7 @@ if [ -z "$BASH_VERSION" ]; then [ "$YADM_TEST" != 1 ] && exec bash "$0" "$@" fi -VERSION=2.0.0 +VERSION=2.0.1 YADM_WORK="$HOME" YADM_DIR= diff --git a/yadm.1 b/yadm.1 index 9915e3a..07616d5 100644 --- a/yadm.1 +++ b/yadm.1 @@ -1,5 +1,5 @@ ." vim: set spell so=8: -.TH yadm 1 "8 November 2019" "2.0.0" +.TH yadm 1 "12 November 2019" "2.0.1" .SH NAME diff --git a/yadm.spec b/yadm.spec index 6f25ba2..2aff646 100644 --- a/yadm.spec +++ b/yadm.spec @@ -1,7 +1,7 @@ %{!?_pkgdocdir: %global _pkgdocdir %{_docdir}/%{name}-%{version}} Name: yadm Summary: Yet Another Dotfiles Manager -Version: 2.0.0 +Version: 2.0.1 Group: Development/Tools Release: 1%{?dist} URL: https://yadm.io From 539ffd3ffc047fa99b60d4b40b9d893f8afb10f8 Mon Sep 17 00:00:00 2001 From: Ross Smith II Date: Wed, 13 Nov 2019 08:17:06 -0800 Subject: [PATCH 092/137] Make symlinks relative --- yadm | 26 ++++++++++++++++++++------ 1 file changed, 20 insertions(+), 6 deletions(-) diff --git a/yadm b/yadm index c9d8c4d..e653f9d 100755 --- a/yadm +++ b/yadm @@ -20,7 +20,7 @@ if [ -z "$BASH_VERSION" ]; then [ "$YADM_TEST" != 1 ] && exec bash "$0" "$@" fi -VERSION=2.0.1 +VERSION=2.0.0 YADM_WORK="$HOME" YADM_DIR= @@ -447,11 +447,11 @@ function remove_stale_links() { if [ -L "$stale_candidate" ]; then link_target=$(readlink "$stale_candidate" 2>/dev/null) if [ -n "$link_target" ]; then - removal=yes for review_link in "${alt_linked[@]}"; do - [ "$link_target" = "$review_link" ] && removal=no + [ "$link_target" = "$review_link" ] && continue 2 + [ "$link_target" = "$(basename "$review_link")" ] && continue 2 done - [ "$removal" = "yes" ] && rm -f "$stale_candidate" + rm -f "$stale_candidate" fi fi done @@ -526,7 +526,7 @@ function alt_future_linking() { [ -L "$filename" ] && rm -f "$filename" cp -f "$target" "$filename" else - ln -nfs "$target" "$filename" + alt_ln "$target" "$filename" alt_linked+=("$target") fi fi @@ -570,7 +570,7 @@ function alt_past_linking() { fi cp -f "$alt_path" "$new_link" else - ln -nfs "$alt_path" "$new_link" + alt_ln "$alt_path" "$new_link" alt_linked+=("$alt_path") fi last_linked="$alt_path" @@ -607,6 +607,20 @@ function alt_past_linking() { } +function alt_ln() { + local alt_dir alt_base new_base + + alt_dir="$(dirname "$1")" + alt_base="$(basename "$1")" + new_base="$(basename "$2")" + if pushd "$alt_dir" >/dev/null ; then + ln -nfs "$alt_base" "$new_base" + popd &>/dev/null + else + ln -nfs "$1" "$2" + fi +} + function bootstrap() { bootstrap_available || error_out "Cannot execute bootstrap\n'$YADM_BOOTSTRAP' is not an executable program." From eed59388cbcd77bf8fe208f768404f15297baf49 Mon Sep 17 00:00:00 2001 From: Tim Byrne Date: Thu, 14 Nov 2019 08:23:41 -0600 Subject: [PATCH 093/137] Note exception for WSL (#113) --- yadm.1 | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/yadm.1 b/yadm.1 index 07616d5..c913722 100644 --- a/yadm.1 +++ b/yadm.1 @@ -494,6 +494,10 @@ and trimming off any domain. Valid when no other alternate is valid. .LP +.BR NOTE : +The OS for "Windows Subsystem for Linux" is reported as "WSL", even +though uname identifies as "Linux". + You may use any number of conditions, in any order. An alternate will only be used if ALL conditions are valid. For all files managed by yadm's repository or listed in @@ -621,6 +625,10 @@ During processing, the following variables are available in the template: yadm.user YADM_USER id -u -n yadm.source YADM_SOURCE Template filename +.BR NOTE : +The OS for "Windows Subsystem for Linux" is reported as "WSL", even +though uname identifies as "Linux". + Examples: .I whatever##template From daa55b1af0a947127a7f1ce425db9a7fabea5b0b Mon Sep 17 00:00:00 2001 From: Sheng Yang Date: Fri, 15 Nov 2019 20:52:31 -0600 Subject: [PATCH 094/137] Update documentation about using yadm with magit --- yadm.1 | 3 +++ 1 file changed, 3 insertions(+) diff --git a/yadm.1 b/yadm.1 index c913722..a028b29 100644 --- a/yadm.1 +++ b/yadm.1 @@ -194,9 +194,12 @@ Emacs Tramp and Magit can manage files by using this configuration: '("yadm" (tramp-login-program "yadm") (tramp-login-args (("enter"))) + (tramp-login-env (("SHELL") ("/bin/sh"))) (tramp-remote-shell "/bin/sh") (tramp-remote-shell-args ("-c")))) .RE +and use (magit-status "/yadm::"). If you find issue with Emacs 27 and zsh, +trying running (setenv "SHELL" "/bin/bash"). .TP .B gitconfig Pass options to the From abf21873f6ef248184769bddbc89cbc51a225410 Mon Sep 17 00:00:00 2001 From: Tim Byrne Date: Sat, 16 Nov 2019 15:26:50 -0600 Subject: [PATCH 095/137] Adjust formatting --- yadm.1 | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/yadm.1 b/yadm.1 index a028b29..d8ae1dc 100644 --- a/yadm.1 +++ b/yadm.1 @@ -187,8 +187,11 @@ See the ENCRYPTION section for more details. Run a sub-shell with all Git variables set. Exit the sub-shell the same way you leave your normal shell (usually with the "exit" command). This sub-shell can be used to easily interact with your yadm repository using "git" commands. This -could be useful if you are using a tool which uses Git directly. For example, -Emacs Tramp and Magit can manage files by using this configuration: +could be useful if you are using a tool which uses Git directly. + +For example, Emacs Tramp and Magit can manage files by using this +configuration: + .RS (add-to-list 'tramp-methods '("yadm" @@ -198,8 +201,11 @@ Emacs Tramp and Magit can manage files by using this configuration: (tramp-remote-shell "/bin/sh") (tramp-remote-shell-args ("-c")))) .RE -and use (magit-status "/yadm::"). If you find issue with Emacs 27 and zsh, + +.RS +With this config, use (magit-status "/yadm::"). If you find issue with Emacs 27 and zsh, trying running (setenv "SHELL" "/bin/bash"). +.RE .TP .B gitconfig Pass options to the From 98915151a31af06678d89137cf7c243fea9520e3 Mon Sep 17 00:00:00 2001 From: Tim Byrne Date: Sun, 17 Nov 2019 12:39:05 -0600 Subject: [PATCH 096/137] Revert version change --- yadm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/yadm b/yadm index e653f9d..800b097 100755 --- a/yadm +++ b/yadm @@ -20,7 +20,7 @@ if [ -z "$BASH_VERSION" ]; then [ "$YADM_TEST" != 1 ] && exec bash "$0" "$@" fi -VERSION=2.0.0 +VERSION=2.0.1 YADM_WORK="$HOME" YADM_DIR= From 2bf98a5ade6d077bb74cdceb5f7dc23f9232ae16 Mon Sep 17 00:00:00 2001 From: Ross Smith II Date: Sat, 23 Nov 2019 19:39:57 -0800 Subject: [PATCH 097/137] Use gawk on OpenWrt, as awk is BusyBox version The BusyBox version of awk fails with this error: awk: bad regex '[\\.^$(){}\[\]|*+?]': Repetition not preceded by valid expression --- yadm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/yadm b/yadm index c9d8c4d..a333b14 100755 --- a/yadm +++ b/yadm @@ -40,7 +40,7 @@ FULL_COMMAND="" GPG_PROGRAM="gpg" GIT_PROGRAM="git" -AWK_PROGRAM="awk" +AWK_PROGRAM="${AWK_PROGRAM:-awk}" J2CLI_PROGRAM="j2" ENVTPL_PROGRAM="envtpl" LSB_RELEASE_PROGRAM="lsb_release" From f8d6d2b0e426a88e4e851f46fbe21f60f3a10863 Mon Sep 17 00:00:00 2001 From: Tim Byrne Date: Sun, 17 Nov 2019 13:07:04 -0600 Subject: [PATCH 098/137] Change tests to expect relative links --- test/test_alt.py | 34 ++++++++++++++++++++++++++++++---- test/test_compat_alt.py | 19 +++++++++++-------- 2 files changed, 41 insertions(+), 12 deletions(-) diff --git a/test/test_alt.py b/test/test_alt.py index d616869..359f32d 100644 --- a/test/test_alt.py +++ b/test/test_alt.py @@ -40,7 +40,7 @@ def test_alt_source( link_file = paths.work.join(link_path) if tracked or (encrypt and not exclude): assert link_file.islink() - target = py.path.local(link_file.readlink()) + target = py.path.local(os.path.realpath(link_file)) if target.isfile(): assert link_file.read() == source_file_content assert str(source_file) in linked @@ -53,6 +53,31 @@ def test_alt_source( assert str(source_file) not in linked +@pytest.mark.usefixtures('ds1_copy') +@pytest.mark.parametrize('yadm_alt', [True, False], ids=['alt', 'worktree']) +def test_relative_link(runner, paths, yadm_alt): + """Confirm links created are relative""" + yadm_dir = setup_standard_yadm_dir(paths) + + utils.create_alt_files( + paths, '##default', tracked=True, encrypt=False, exclude=False, + yadm_alt=yadm_alt, yadm_dir=yadm_dir) + run = runner([paths.pgm, '-Y', yadm_dir, 'alt']) + assert run.success + assert run.err == '' + + basepath = yadm_dir.join('alt') if yadm_alt else paths.work + + for link_path in TEST_PATHS: + source_file_content = link_path + '##default' + source_file = basepath.join(source_file_content) + link_file = paths.work.join(link_path) + link = link_file.readlink() + relpath = os.path.relpath( + source_file, start=os.path.dirname(link_file)) + assert link == relpath + + @pytest.mark.usefixtures('ds1_copy') @pytest.mark.parametrize('suffix', [ '##default', @@ -89,7 +114,7 @@ def test_alt_conditions( for link_path in TEST_PATHS: source_file = link_path + suffix assert paths.work.join(link_path).islink() - target = py.path.local(paths.work.join(link_path).readlink()) + target = py.path.local(os.path.realpath(paths.work.join(link_path))) if target.isfile(): assert paths.work.join(link_path).read() == source_file assert str(paths.work.join(source_file)) in linked @@ -146,7 +171,8 @@ def test_auto_alt(runner, yadm_y, paths, autoalt): assert not paths.work.join(link_path).exists() else: assert paths.work.join(link_path).islink() - target = py.path.local(paths.work.join(link_path).readlink()) + target = py.path.local( + os.path.realpath(paths.work.join(link_path))) if target.isfile(): assert paths.work.join(link_path).read() == source_file # no linking output when run via auto-alt @@ -183,7 +209,7 @@ def test_stale_link_removal(runner, yadm_y, paths): for stale_path in TEST_PATHS: source_file = stale_path + '##class.' + tst_class assert paths.work.join(stale_path).islink() - target = py.path.local(paths.work.join(stale_path).readlink()) + target = py.path.local(os.path.realpath(paths.work.join(stale_path))) if target.isfile(): assert paths.work.join(stale_path).read() == source_file assert str(paths.work.join(source_file)) in linked diff --git a/test/test_compat_alt.py b/test/test_compat_alt.py index a2c9ad1..da7a8cf 100644 --- a/test/test_compat_alt.py +++ b/test/test_compat_alt.py @@ -101,7 +101,8 @@ def test_alt(runner, yadm_y, paths, source_file = file_path + precedence[precedence_index] if tracked or (encrypt and not exclude): assert paths.work.join(file_path).islink() - target = py.path.local(paths.work.join(file_path).readlink()) + target = py.path.local( + os.path.realpath(paths.work.join(file_path))) if target.isfile(): assert paths.work.join(file_path).read() == source_file assert str(paths.work.join(source_file)) in linked @@ -203,7 +204,7 @@ def test_wild(request, runner, yadm_y, paths, for file_path in TEST_PATHS: source_file = file_path + wild_suffix assert paths.work.join(file_path).islink() - target = py.path.local(paths.work.join(file_path).readlink()) + target = py.path.local(os.path.realpath(paths.work.join(file_path))) if target.isfile(): assert paths.work.join(file_path).read() == source_file assert str(paths.work.join(source_file)) in linked @@ -228,7 +229,7 @@ def test_wild(request, runner, yadm_y, paths, for file_path in TEST_PATHS: source_file = file_path + std_suffix assert paths.work.join(file_path).islink() - target = py.path.local(paths.work.join(file_path).readlink()) + target = py.path.local(os.path.realpath(paths.work.join(file_path))) if target.isfile(): assert paths.work.join(file_path).read() == source_file assert str(paths.work.join(source_file)) in linked @@ -268,7 +269,7 @@ def test_local_override(runner, yadm_y, paths, for file_path in TEST_PATHS: source_file = file_path + '##or-class.or-os.or-hostname.or-user' assert paths.work.join(file_path).islink() - target = py.path.local(paths.work.join(file_path).readlink()) + target = py.path.local(os.path.realpath(paths.work.join(file_path))) if target.isfile(): assert paths.work.join(file_path).read() == source_file assert str(paths.work.join(source_file)) in linked @@ -308,7 +309,7 @@ def test_class_case(runner, yadm_y, paths, tst_sys, suffix): for file_path in TEST_PATHS: source_file = file_path + f'##{suffix}' assert paths.work.join(file_path).islink() - target = py.path.local(paths.work.join(file_path).readlink()) + target = py.path.local(os.path.realpath(paths.work.join(file_path))) if target.isfile(): assert paths.work.join(file_path).read() == source_file assert str(paths.work.join(source_file)) in linked @@ -346,7 +347,8 @@ def test_auto_alt(runner, yadm_y, paths, autoalt): assert not paths.work.join(file_path).exists() else: assert paths.work.join(file_path).islink() - target = py.path.local(paths.work.join(file_path).readlink()) + target = py.path.local( + os.path.realpath(paths.work.join(file_path))) if target.isfile(): assert paths.work.join(file_path).read() == source_file # no linking output when run via auto-alt @@ -383,7 +385,8 @@ def test_delimiter(runner, yadm_y, paths, source_file = file_path + suffix if delimiter == '.': assert paths.work.join(file_path).islink() - target = py.path.local(paths.work.join(file_path).readlink()) + target = py.path.local( + os.path.realpath(paths.work.join(file_path))) if target.isfile(): assert paths.work.join(file_path).read() == source_file assert str(paths.work.join(source_file)) in linked @@ -423,7 +426,7 @@ def test_invalid_links_removed(runner, yadm_y, paths): for file_path in TEST_PATHS: source_file = file_path + '##' + tst_class assert paths.work.join(file_path).islink() - target = py.path.local(paths.work.join(file_path).readlink()) + target = py.path.local(os.path.realpath(paths.work.join(file_path))) if target.isfile(): assert paths.work.join(file_path).read() == source_file assert str(paths.work.join(source_file)) in linked From 98392b9a9c340fb5c0ad86310e0a29a2b15e493c Mon Sep 17 00:00:00 2001 From: Tim Byrne Date: Sun, 17 Nov 2019 12:58:49 -0600 Subject: [PATCH 099/137] Add function relative_path This function will create a path relative to another, without the use of an external program like dirname. --- test/test_unit_relative_path.py | 31 +++++++++++++++++ yadm | 60 +++++++++++++++++++++++++++++++++ 2 files changed, 91 insertions(+) create mode 100644 test/test_unit_relative_path.py diff --git a/test/test_unit_relative_path.py b/test/test_unit_relative_path.py new file mode 100644 index 0000000..f723c84 --- /dev/null +++ b/test/test_unit_relative_path.py @@ -0,0 +1,31 @@ +"""Unit tests: relative_path""" +import pytest + + +@pytest.mark.parametrize( + 'base,full_path,expected', + [ + ("/A/B/C", "/A", "../.."), + ("/A/B/C", "/A/B", ".."), + ("/A/B/C", "/A/B/C", ""), + ("/A/B/C", "/A/B/C/D", "D"), + ("/A/B/C", "/A/B/C/D/E", "D/E"), + ("/A/B/C", "/A/B/D", "../D"), + ("/A/B/C", "/A/B/D/E", "../D/E"), + ("/A/B/C", "/A/D", "../../D"), + ("/A/B/C", "/A/D/E", "../../D/E"), + ("/A/B/C", "/D/E/F", "../../../D/E/F"), + ], +) +def test_relative_path(runner, paths, base, full_path, expected): + """Test translate_to_relative""" + + script = f""" + YADM_TEST=1 source {paths.pgm} + relative_path "{base}" "{full_path}" + """ + + run = runner(command=['bash'], inp=script) + assert run.success + assert run.err == '' + assert run.out.strip() == expected diff --git a/yadm b/yadm index 800b097..dc79359 100755 --- a/yadm +++ b/yadm @@ -1592,6 +1592,66 @@ function parse_encrypt() { } +function builtin_dirname() { + # dirname is not builtin, and universally available, this is a built-in + # replacement using parameter expansion + path="$1" + dname="${path%/*}" + if ! [[ "$path" =~ / ]]; then + echo "." + elif [ "$dname" = "" ]; then + echo "/" + else + echo "$dname" + fi +} + +function relative_path() { + # Output a path to $2/full, relative to $1/base + # + # This fucntion created with ideas from + # https://stackoverflow.com/questions/2564634 + base="$1" + full="$2" + + common_part="$base" + result="" + + count=0 + while [ "${full#$common_part}" == "${full}" ]; do + [ "$count" = "500" ] && return # this is a failsafe + # no match, means that candidate common part is not correct + # go up one level (reduce common part) + common_part="$(builtin_dirname "$common_part")" + # and record that we went back, with correct / handling + if [[ -z $result ]]; then + result=".." + else + result="../$result" + fi + count=$((count+1)) + done + + if [[ $common_part == "/" ]]; then + # special case for root (no common path) + result="$result/" + fi + + # since we now have identified the common part, + # compute the non-common part + forward_part="${full#$common_part}" + + # and now stick all parts together + if [[ -n $result ]] && [[ -n $forward_part ]]; then + result="$result$forward_part" + elif [[ -n $forward_part ]]; then + # extra slash removal + result="${forward_part:1}" + fi + + echo "$result" +} + # ****** Auto Functions ****** function auto_alt() { From 61576a6ae1c4a1e42df464c2213118bdcaf51074 Mon Sep 17 00:00:00 2001 From: Tim Byrne Date: Sun, 17 Nov 2019 13:07:29 -0600 Subject: [PATCH 100/137] Use relative symlinks for alt (#100) * Fix broken support for .config/yadm/alt * Removes dependence on external basename / dirname --- yadm | 27 ++++++++++++--------------- 1 file changed, 12 insertions(+), 15 deletions(-) diff --git a/yadm b/yadm index dc79359..1182320 100755 --- a/yadm +++ b/yadm @@ -422,7 +422,12 @@ function alt() { local IFS=$'\n' for possible_alt in "${tracked_files[@]}" "${ENCRYPT_INCLUDE_FILES[@]}"; do if [[ $possible_alt =~ .\#\#. ]]; then - possible_alts+=("$YADM_WORK/${possible_alt%%##*}") + base_alt="${possible_alt%%##*}" + yadm_alt="${YADM_WORK}/${base_alt}" + if [ "${yadm_alt#$YADM_ALT/}" != "${yadm_alt}" ]; then + base_alt="${yadm_alt#$YADM_ALT/}" + fi + possible_alts+=("$YADM_WORK/${base_alt}") fi done local alt_linked @@ -449,7 +454,6 @@ function remove_stale_links() { if [ -n "$link_target" ]; then for review_link in "${alt_linked[@]}"; do [ "$link_target" = "$review_link" ] && continue 2 - [ "$link_target" = "$(basename "$review_link")" ] && continue 2 done rm -f "$stale_candidate" fi @@ -527,7 +531,6 @@ function alt_future_linking() { cp -f "$target" "$filename" else alt_ln "$target" "$filename" - alt_linked+=("$target") fi fi done @@ -571,7 +574,6 @@ function alt_past_linking() { cp -f "$alt_path" "$new_link" else alt_ln "$alt_path" "$new_link" - alt_linked+=("$alt_path") fi last_linked="$alt_path" fi @@ -608,17 +610,12 @@ function alt_past_linking() { } function alt_ln() { - local alt_dir alt_base new_base - - alt_dir="$(dirname "$1")" - alt_base="$(basename "$1")" - new_base="$(basename "$2")" - if pushd "$alt_dir" >/dev/null ; then - ln -nfs "$alt_base" "$new_base" - popd &>/dev/null - else - ln -nfs "$1" "$2" - fi + local full_link base + full_link="$2" + base="${full_link%/*}" + rel_source=$(relative_path "$base" "$1") + ln -nfs "$rel_source" "$full_link" + alt_linked+=("$rel_source") } function bootstrap() { From 5634c09a8a1f8e9c88a5d3d18d0432dbce3989bd Mon Sep 17 00:00:00 2001 From: Tim Byrne Date: Mon, 25 Nov 2019 07:44:44 -0600 Subject: [PATCH 101/137] Refactor symlink code Update variable names, favoring the terminology used by `ln`. * source (original file containing data) * target (the symlink file, pointing to source) --- test/test_unit_record_score.py | 44 ++++++------ test/test_unit_record_template.py | 30 ++++---- yadm | 111 +++++++++++++++--------------- 3 files changed, 93 insertions(+), 92 deletions(-) diff --git a/test/test_unit_record_score.py b/test/test_unit_record_score.py index 3bb1b25..525c967 100644 --- a/test/test_unit_record_score.py +++ b/test/test_unit_record_score.py @@ -8,16 +8,16 @@ INIT_VARS = """ local_host=testhost local_user=testuser alt_scores=() - alt_filenames=() alt_targets=() + alt_sources=() alt_template_cmds=() """ REPORT_RESULTS = """ echo "SIZE:${#alt_scores[@]}" echo "SCORES:${alt_scores[@]}" - echo "FILENAMES:${alt_filenames[@]}" echo "TARGETS:${alt_targets[@]}" + echo "SOURCES:${alt_sources[@]}" """ @@ -27,7 +27,7 @@ def test_dont_record_zeros(runner, yadm): script = f""" YADM_TEST=1 source {yadm} {INIT_VARS} - record_score "0" "testfile" "testtarget" + record_score "0" "testtgt" "testsrc" {REPORT_RESULTS} """ run = runner(command=['bash'], inp=script) @@ -35,8 +35,8 @@ def test_dont_record_zeros(runner, yadm): assert run.err == '' assert 'SIZE:0\n' in run.out assert 'SCORES:\n' in run.out - assert 'FILENAMES:\n' in run.out assert 'TARGETS:\n' in run.out + assert 'SOURCES:\n' in run.out def test_new_scores(runner, yadm): @@ -45,9 +45,9 @@ def test_new_scores(runner, yadm): script = f""" YADM_TEST=1 source {yadm} {INIT_VARS} - record_score "1" "file_one" "targ_one" - record_score "2" "file_two" "targ_two" - record_score "4" "file_three" "targ_three" + record_score "1" "tgt_one" "src_one" + record_score "2" "tgt_two" "src_two" + record_score "4" "tgt_three" "src_three" {REPORT_RESULTS} """ run = runner(command=['bash'], inp=script) @@ -55,8 +55,8 @@ def test_new_scores(runner, yadm): assert run.err == '' assert 'SIZE:3\n' in run.out assert 'SCORES:1 2 4\n' in run.out - assert 'FILENAMES:file_one file_two file_three\n' in run.out - assert 'TARGETS:targ_one targ_two targ_three\n' in run.out + assert 'TARGETS:tgt_one tgt_two tgt_three\n' in run.out + assert 'SOURCES:src_one src_two src_three\n' in run.out @pytest.mark.parametrize('difference', ['lower', 'equal', 'higher']) @@ -64,7 +64,7 @@ def test_existing_scores(runner, yadm, difference): """Test existing scores""" expected_score = '2' - expected_target = 'existing_target' + expected_src = 'existing_src' if difference == 'lower': score = '1' elif difference == 'equal': @@ -72,15 +72,15 @@ def test_existing_scores(runner, yadm, difference): else: score = '4' expected_score = '4' - expected_target = 'new_target' + expected_src = 'new_src' script = f""" YADM_TEST=1 source {yadm} {INIT_VARS} alt_scores=(2) - alt_filenames=("testfile") - alt_targets=("existing_target") - record_score "{score}" "testfile" "new_target" + alt_targets=("testtgt") + alt_sources=("existing_src") + record_score "{score}" "testtgt" "new_src" {REPORT_RESULTS} """ run = runner(command=['bash'], inp=script) @@ -88,21 +88,21 @@ def test_existing_scores(runner, yadm, difference): assert run.err == '' assert 'SIZE:1\n' in run.out assert f'SCORES:{expected_score}\n' in run.out - assert 'FILENAMES:testfile\n' in run.out - assert f'TARGETS:{expected_target}\n' in run.out + assert 'TARGETS:testtgt\n' in run.out + assert f'SOURCES:{expected_src}\n' in run.out def test_existing_template(runner, yadm): - """Record nothing if a template command is registered for this file""" + """Record nothing if a template command is registered for this target""" script = f""" YADM_TEST=1 source {yadm} {INIT_VARS} alt_scores=(1) - alt_filenames=("testfile") - alt_targets=() + alt_targets=("testtgt") + alt_sources=() alt_template_cmds=("existing_template") - record_score "2" "testfile" "new_target" + record_score "2" "testtgt" "new_src" {REPORT_RESULTS} """ run = runner(command=['bash'], inp=script) @@ -110,5 +110,5 @@ def test_existing_template(runner, yadm): assert run.err == '' assert 'SIZE:1\n' in run.out assert 'SCORES:1\n' in run.out - assert 'FILENAMES:testfile\n' in run.out - assert 'TARGETS:\n' in run.out + assert 'TARGETS:testtgt\n' in run.out + assert 'SOURCES:\n' in run.out diff --git a/test/test_unit_record_template.py b/test/test_unit_record_template.py index f2e0a83..6bfd012 100644 --- a/test/test_unit_record_template.py +++ b/test/test_unit_record_template.py @@ -1,16 +1,16 @@ """Unit tests: record_template""" INIT_VARS = """ - alt_filenames=() - alt_template_cmds=() alt_targets=() + alt_template_cmds=() + alt_sources=() """ REPORT_RESULTS = """ - echo "SIZE:${#alt_filenames[@]}" - echo "FILENAMES:${alt_filenames[@]}" + echo "SIZE:${#alt_targets[@]}" + echo "TARGETS:${alt_targets[@]}" echo "CMDS:${alt_template_cmds[@]}" - echo "TARGS:${alt_targets[@]}" + echo "SOURCES:${alt_sources[@]}" """ @@ -20,18 +20,18 @@ def test_new_template(runner, yadm): script = f""" YADM_TEST=1 source {yadm} {INIT_VARS} - record_template "file_one" "cmd_one" "targ_one" - record_template "file_two" "cmd_two" "targ_two" - record_template "file_three" "cmd_three" "targ_three" + record_template "tgt_one" "cmd_one" "src_one" + record_template "tgt_two" "cmd_two" "src_two" + record_template "tgt_three" "cmd_three" "src_three" {REPORT_RESULTS} """ run = runner(command=['bash'], inp=script) assert run.success assert run.err == '' assert 'SIZE:3\n' in run.out - assert 'FILENAMES:file_one file_two file_three\n' in run.out + assert 'TARGETS:tgt_one tgt_two tgt_three\n' in run.out assert 'CMDS:cmd_one cmd_two cmd_three\n' in run.out - assert 'TARGS:targ_one targ_two targ_three\n' in run.out + assert 'SOURCES:src_one src_two src_three\n' in run.out def test_existing_template(runner, yadm): @@ -40,16 +40,16 @@ def test_existing_template(runner, yadm): script = f""" YADM_TEST=1 source {yadm} {INIT_VARS} - alt_filenames=("testfile") + alt_targets=("testtgt") alt_template_cmds=("existing_cmd") - alt_targets=("existing_targ") - record_template "testfile" "new_cmd" "new_targ" + alt_sources=("existing_src") + record_template "testtgt" "new_cmd" "new_src" {REPORT_RESULTS} """ run = runner(command=['bash'], inp=script) assert run.success assert run.err == '' assert 'SIZE:1\n' in run.out - assert 'FILENAMES:testfile\n' in run.out + assert 'TARGETS:testtgt\n' in run.out assert 'CMDS:new_cmd\n' in run.out - assert 'TARGS:new_targ\n' in run.out + assert 'SOURCES:new_src\n' in run.out diff --git a/yadm b/yadm index 1182320..dd63f78 100755 --- a/yadm +++ b/yadm @@ -136,12 +136,12 @@ function main() { # ****** Alternate Processing ****** function score_file() { - target="$1" - filename="${target%%##*}" - conditions="${target#*##}" + src="$1" + tgt="${src%%##*}" + conditions="${src#*##}" - if [ "${filename#$YADM_ALT/}" != "${filename}" ]; then - filename="${YADM_WORK}/${filename#$YADM_ALT/}" + if [ "${tgt#$YADM_ALT/}" != "${tgt}" ]; then + tgt="${YADM_WORK}/${tgt#$YADM_ALT/}" fi score=0 @@ -195,10 +195,10 @@ function score_file() { score=0 cmd=$(choose_template_cmd "$value") if [ -n "$cmd" ]; then - record_template "$filename" "$cmd" "$target" + record_template "$tgt" "$cmd" "$src" else - debug "No supported template processor for template $target" - [ -n "$loud" ] && echo "No supported template processor for template $target" + debug "No supported template processor for template $src" + [ -n "$loud" ] && echo "No supported template processor for template $src" fi return 0 # unsupported values @@ -208,30 +208,30 @@ function score_file() { fi done - record_score "$score" "$filename" "$target" + record_score "$score" "$tgt" "$src" } function record_score() { score="$1" - filename="$2" - target="$3" + tgt="$2" + src="$3" # record nothing if the score is zero [ "$score" -eq 0 ] && return - # search for the index of this filename, to see if we already are tracking it + # search for the index of this target, to see if we already are tracking it index=-1 - for search_index in "${!alt_filenames[@]}"; do - if [ "${alt_filenames[$search_index]}" = "$filename" ]; then + for search_index in "${!alt_targets[@]}"; do + if [ "${alt_targets[$search_index]}" = "$tgt" ]; then index="$search_index" break fi done # if we don't find an existing index, create one by appending to the array if [ "$index" -eq -1 ]; then - alt_filenames+=("$filename") + alt_targets+=("$tgt") # set index to the last index (newly created one) - for index in "${!alt_filenames[@]}"; do :; done + for index in "${!alt_targets[@]}"; do :; done # and set its initial score to zero alt_scores[$index]=0 fi @@ -239,37 +239,37 @@ function record_score() { # record nothing if a template command is registered for this file [ "${alt_template_cmds[$index]+isset}" ] && return - # record higher scoring targets + # record higher scoring sources if [ "$score" -gt "${alt_scores[$index]}" ]; then alt_scores[$index]="$score" - alt_targets[$index]="$target" + alt_sources[$index]="$src" fi } function record_template() { - filename="$1" + tgt="$1" cmd="$2" - target="$3" + src="$3" - # search for the index of this filename, to see if we already are tracking it + # search for the index of this target, to see if we already are tracking it index=-1 - for search_index in "${!alt_filenames[@]}"; do - if [ "${alt_filenames[$search_index]}" = "$filename" ]; then + for search_index in "${!alt_targets[@]}"; do + if [ "${alt_targets[$search_index]}" = "$tgt" ]; then index="$search_index" break fi done # if we don't find an existing index, create one by appending to the array if [ "$index" -eq -1 ]; then - alt_filenames+=("$filename") + alt_targets+=("$tgt") # set index to the last index (newly created one) - for index in "${!alt_filenames[@]}"; do :; done + for index in "${!alt_targets[@]}"; do :; done fi # record the template command, last one wins alt_template_cmds[$index]="$cmd" - alt_targets[$index]="$target" + alt_sources[$index]="$src" } @@ -445,15 +445,15 @@ function alt() { function remove_stale_links() { # review alternate candidates for stale links - # if a possible alt IS linked, but it's target is not part of alt_linked, + # if a possible alt IS linked, but it's source is not part of alt_linked, # remove it. if readlink_available; then for stale_candidate in "${possible_alts[@]}"; do if [ -L "$stale_candidate" ]; then - link_target=$(readlink "$stale_candidate" 2>/dev/null) - if [ -n "$link_target" ]; then + src=$(readlink "$stale_candidate" 2>/dev/null) + if [ -n "$src" ]; then for review_link in "${alt_linked[@]}"; do - [ "$link_target" = "$review_link" ] && continue 2 + [ "$src" = "$review_link" ] && continue 2 done rm -f "$stale_candidate" fi @@ -489,12 +489,12 @@ function set_local_alt_values() { function alt_future_linking() { local alt_scores - local alt_filenames local alt_targets + local alt_sources local alt_template_cmds alt_scores=() - alt_filenames=() alt_targets=() + alt_sources=() alt_template_cmds=() for alt_path in $(for tracked in "${tracked_files[@]}"; do printf "%s\n" "$tracked" "${tracked%/*}"; done | LC_ALL=C sort -u) "${ENCRYPT_INCLUDE_FILES[@]}"; do @@ -506,31 +506,31 @@ function alt_future_linking() { fi done - for index in "${!alt_filenames[@]}"; do - filename="${alt_filenames[$index]}" - target="${alt_targets[$index]}" + for index in "${!alt_targets[@]}"; do + tgt="${alt_targets[$index]}" + src="${alt_sources[$index]}" template_cmd="${alt_template_cmds[$index]}" if [ -n "$template_cmd" ]; then # a template is defined, process the template - debug "Creating $filename from template $target" - [ -n "$loud" ] && echo "Creating $filename from template $target" + debug "Creating $tgt from template $src" + [ -n "$loud" ] && echo "Creating $tgt from template $src" # ensure the destination path exists - assert_parent "$filename" + assert_parent "$tgt" # remove any existing symlink before processing template - [ -L "$filename" ] && rm -f "$filename" - "$template_cmd" "$target" "$filename" - elif [ -n "$target" ]; then - # a link target is defined, create symlink - debug "Linking $target to $filename" - [ -n "$loud" ] && echo "Linking $target to $filename" + [ -L "$tgt" ] && rm -f "$tgt" + "$template_cmd" "$src" "$tgt" + elif [ -n "$src" ]; then + # a link source is defined, create symlink + debug "Linking $src to $tgt" + [ -n "$loud" ] && echo "Linking $src to $tgt" # ensure the destination path exists - assert_parent "$filename" + assert_parent "$tgt" if [ "$do_copy" -eq 1 ]; then # remove any existing symlink before copying - [ -L "$filename" ] && rm -f "$filename" - cp -f "$target" "$filename" + [ -L "$tgt" ] && rm -f "$tgt" + cp -f "$src" "$tgt" else - alt_ln "$target" "$filename" + ln_relative "$src" "$tgt" fi fi done @@ -573,7 +573,7 @@ function alt_past_linking() { fi cp -f "$alt_path" "$new_link" else - alt_ln "$alt_path" "$new_link" + ln_relative "$alt_path" "$new_link" fi last_linked="$alt_path" fi @@ -609,12 +609,13 @@ function alt_past_linking() { } -function alt_ln() { - local full_link base - full_link="$2" - base="${full_link%/*}" - rel_source=$(relative_path "$base" "$1") - ln -nfs "$rel_source" "$full_link" +function ln_relative() { + local full_source full_target target_dir + full_source="$1" + full_target="$2" + target_dir="${full_target%/*}" + rel_source=$(relative_path "$target_dir" "$full_source") + ln -nfs "$rel_source" "$full_target" alt_linked+=("$rel_source") } From 60e0fbbf427d78d102fadd83336b6af24e720dae Mon Sep 17 00:00:00 2001 From: David Mandelberg Date: Fri, 4 Jan 2019 21:17:34 -0500 Subject: [PATCH 102/137] Fix completion after a command-line flag. Before: yadm checkout -f # Completes filenames. yadm checkout --yadm-dir # Completes filenames. After: yadm checkout -f # Completes branch names. yadm checkout --yadm-dir # Completes filenames. --- completion/yadm.bash_completion | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/completion/yadm.bash_completion b/completion/yadm.bash_completion index 1091e55..34f199f 100644 --- a/completion/yadm.bash_completion +++ b/completion/yadm.bash_completion @@ -60,15 +60,17 @@ if declare -F _git > /dev/null; then ;; esac + local yadm_switches=( $(yadm introspect switches 2>/dev/null) ) + # this condition is so files are completed properly for --yadm-xxx options - if [[ ! "$penultimate" =~ ^- ]]; then + if [[ " ${yadm_switches[*]} " != *" $penultimate "* ]]; then # TODO: somehow solve the problem with [--yadm-xxx option] being # incompatible with what git expects, namely [--arg=option] _git fi if [[ "$current" =~ ^- ]]; then local matching - matching=$(compgen -W "$(yadm introspect switches 2>/dev/null)" -- "$current") + matching=$(compgen -W "${yadm_switches[*]}" -- "$current") __gitcompappend "$matching" fi From 5d9e0a7133647b6b7fadc2a5261fc0c4030dc4da Mon Sep 17 00:00:00 2001 From: David Mandelberg Date: Fri, 4 Jan 2019 21:26:47 -0500 Subject: [PATCH 103/137] Mark GIT_DIR for export. Before: yadm push # Completes filenames. After: yadm push # Completes names of git remotes. --- completion/yadm.bash_completion | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/completion/yadm.bash_completion b/completion/yadm.bash_completion index 34f199f..60a71ff 100644 --- a/completion/yadm.bash_completion +++ b/completion/yadm.bash_completion @@ -18,7 +18,7 @@ if declare -F _git > /dev/null; then antepenultimate=${COMP_WORDS[COMP_CWORD-2]} fi - local GIT_DIR + local -x GIT_DIR # shellcheck disable=SC2034 GIT_DIR="$(yadm introspect repo 2>/dev/null)" From bcf6531da68dcc14c6f5716bd1af4e454cb91157 Mon Sep 17 00:00:00 2001 From: David Mandelberg Date: Fri, 4 Jan 2019 21:44:49 -0500 Subject: [PATCH 104/137] Only add yadm commands to the completion list when applicable. Before: yadm # Completes git and yadm commands. yadm -Y . # Completes yadm commands. yadm p -u origin foo # Completes yadm+git commands like p*. yadm push -u origin # Completes branch names and yadm commands. After: yadm # Completes git and yadm commands. yadm -Y . # Completes yadm commands. yadm p -u origin foo # Completes yadm+git commands like p*. yadm push -u origin # Completes branch names. --- completion/yadm.bash_completion | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/completion/yadm.bash_completion b/completion/yadm.bash_completion index 60a71ff..228bc34 100644 --- a/completion/yadm.bash_completion +++ b/completion/yadm.bash_completion @@ -74,7 +74,19 @@ if declare -F _git > /dev/null; then __gitcompappend "$matching" fi - if [ "$COMP_CWORD" == 1 ] || [[ "$antepenultimate" =~ ^- ]] ; then + # Find the index of where the sub-command argument should go. + local command_idx + for (( command_idx=1 ; command_idx < ${#COMP_WORDS[@]} ; command_idx++ )); do + local command_idx_arg="${COMP_WORDS[$command_idx]}" + if [[ " ${yadm_switches[*]} " = *" $command_idx_arg "* ]]; then + let command_idx++ + elif [[ "$command_idx_arg" = -* ]]; then + : + else + break + fi + done + if [[ "$COMP_CWORD" = "$command_idx" ]]; then local matching matching=$(compgen -W "$(yadm introspect commands 2>/dev/null)" -- "$current") __gitcompappend "$matching" From 24e6e817130d1c4d67151280b1e031884910a645 Mon Sep 17 00:00:00 2001 From: Tim Byrne Date: Tue, 26 Nov 2019 16:24:36 -0600 Subject: [PATCH 105/137] Test support for double-star globs --- test/test_unit_parse_encrypt.py | 32 ++++++++++++++++++++++++++------ 1 file changed, 26 insertions(+), 6 deletions(-) diff --git a/test/test_unit_parse_encrypt.py b/test/test_unit_parse_encrypt.py index 7ab1d30..e6365b4 100644 --- a/test/test_unit_parse_encrypt.py +++ b/test/test_unit_parse_encrypt.py @@ -46,12 +46,8 @@ def test_empty(runner, paths, encrypt): assert 'EIF_COUNT:0' in run.out, 'EIF should be empty' -@pytest.mark.usefixtures('ds1_repo_copy') -def test_file_parse_encrypt(runner, paths): - """Test parse_encrypt - - Test an array of supported features of the encrypt configuration. - """ +def create_test_encrypt_data(paths): + """Generate test data for testing encrypt""" edata = '' expected = set() @@ -126,6 +122,30 @@ def test_file_parse_encrypt(runner, paths): expected.add('ex ex/file4') expected.add('ex ex/file6.text') + # double star + edata += 'doublestar/**/file*\n' + edata += '!**/file3\n' + paths.work.join('doublestar/a/b/file1').write('', ensure=True) + paths.work.join('doublestar/c/d/file2').write('', ensure=True) + paths.work.join('doublestar/e/f/file3').write('', ensure=True) + paths.work.join('doublestar/g/h/nomatch').write('', ensure=True) + expected.add('doublestar/a/b/file1') + expected.add('doublestar/c/d/file2') + # doublestar/e/f/file3 is excluded + + return edata, expected + + +@pytest.mark.usefixtures('ds1_repo_copy') +def test_file_parse_encrypt(runner, paths): + """Test parse_encrypt + + Test an array of supported features of the encrypt configuration. + """ + + # generate test data & expectations + edata, expected = create_test_encrypt_data(paths) + # write encrypt file print(f'ENCRYPT:\n---\n{edata}---\n') paths.encrypt.write(edata) From 510169eb7fc0e93998c55a91636390a7186d2666 Mon Sep 17 00:00:00 2001 From: Tim Byrne Date: Tue, 26 Nov 2019 16:24:51 -0600 Subject: [PATCH 106/137] Support double-star globs in encrypt (#109) This will only work for Bash >=4, there the shell option "globstar" is supported. --- yadm | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/yadm b/yadm index dd63f78..331fa64 100755 --- a/yadm +++ b/yadm @@ -1545,6 +1545,14 @@ function parse_encrypt() { cd_work "Parsing encrypt" || return + # setting globstar to allow ** in encrypt patterns + # (only supported on Bash >= 4) + local unset_globstar + if ! shopt globstar &>/dev/null; then + unset_globstar=1 + fi + shopt -s globstar &>/dev/null + exclude_pattern="^!(.+)" if [ -f "$YADM_ENCRYPT" ] ; then # parse both included/excluded @@ -1588,6 +1596,10 @@ function parse_encrypt() { unset IFS fi + if [ "$unset_globstar" = "1" ]; then + shopt -u globstar &>/dev/null + fi + } function builtin_dirname() { From ecbffdbb28edb144ad699a0f4adb7d97ff6cdc65 Mon Sep 17 00:00:00 2001 From: Tim Byrne Date: Tue, 26 Nov 2019 16:59:19 -0600 Subject: [PATCH 107/137] Update manpage for double-star support --- yadm.1 | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/yadm.1 b/yadm.1 index d8ae1dc..fdd598b 100644 --- a/yadm.1 +++ b/yadm.1 @@ -692,10 +692,12 @@ For example: .gnupg/*.gpg .RE -Standard filename expansions (*, ?, [) are supported. Other shell expansions -like brace and tilde are not supported. Spaces in paths are supported, and -should not be quoted. If a directory is specified, its contents will be -included, but not recursively. Paths beginning with a "!" will be excluded. +Standard filename expansions (*, ?, [) are supported. +If you have Bash version 4, you may use "**" to match all subdirectories. +Other shell expansions like brace and tilde are not supported. +Spaces in paths are supported, and should not be quoted. +If a directory is specified, its contents will be included, but not recursively. +Paths beginning with a "!" will be excluded. The .B yadm encrypt From 75c19c9cc00dfb73a8ab68406b1729c844d2dc42 Mon Sep 17 00:00:00 2001 From: Tim Byrne Date: Wed, 27 Nov 2019 07:21:44 -0600 Subject: [PATCH 108/137] Release 2.1.0 Update version number and update documentation * Use relative symlinks for alternates (#100, #177) * Support double-star globs in .config/yadm/encrypt (#109) * Improve bash completion (#136) * Update docs about using magit (#123) * Note exception for WSL (#113) --- CHANGES | 7 ++ CONTRIBUTORS | 5 +- README.md | 2 +- yadm | 2 +- yadm.1 | 2 +- yadm.md | 196 +++++++++++++++++++++++++++------------------------ yadm.spec | 2 +- 7 files changed, 120 insertions(+), 96 deletions(-) diff --git a/CHANGES b/CHANGES index 258f548..d9edf5b 100644 --- a/CHANGES +++ b/CHANGES @@ -1,3 +1,10 @@ +2.1.0 + * Use relative symlinks for alternates (#100, #177) + * Support double-star globs in .config/yadm/encrypt (#109) + * Improve bash completion (#136) + * Update docs about using magit (#123) + * Note exception for WSL (#113) + 2.0.1 * Fix bug with worktree permissions (#174) diff --git a/CONTRIBUTORS b/CONTRIBUTORS index adbc226..895a980 100644 --- a/CONTRIBUTORS +++ b/CONTRIBUTORS @@ -2,11 +2,13 @@ CONTRIBUTORS Tim Byrne Espen Henriksen -Cameron Eagans Ross Smith II +Cameron Eagans +David Mandelberg Klas Mellbourn Jan Schulz Satoshi Ohki +Sheng Yang Siôn Le Roux Sébastien Gross Thomas Luzat @@ -14,6 +16,7 @@ Tomas Cernaj Uroš Golja Brayden Banks japm48 +Daniel Gray Daniel Wagenknecht Franciszek Madej Mateusz Piotrowski diff --git a/README.md b/README.md index 2d3163a..3895466 100644 --- a/README.md +++ b/README.md @@ -42,7 +42,7 @@ Features, usage, examples and installation instructions can be found on the [master-badge]: https://img.shields.io/travis/TheLocehiliosan/yadm/master.svg?label=master [master-commits]: https://github.com/TheLocehiliosan/yadm/commits/master [master-date]: https://img.shields.io/github/last-commit/TheLocehiliosan/yadm/master.svg?label=master -[obs-badge]: https://img.shields.io/badge/OBS-v2.0.1-blue +[obs-badge]: https://img.shields.io/badge/OBS-v2.1.0-blue [obs-link]: https://software.opensuse.org//download.html?project=home%3ATheLocehiliosan%3Ayadm&package=yadm [releases-badge]: https://img.shields.io/github/tag/TheLocehiliosan/yadm.svg?label=latest+release [releases-link]: https://github.com/TheLocehiliosan/yadm/releases diff --git a/yadm b/yadm index 331fa64..df8d35c 100755 --- a/yadm +++ b/yadm @@ -20,7 +20,7 @@ if [ -z "$BASH_VERSION" ]; then [ "$YADM_TEST" != 1 ] && exec bash "$0" "$@" fi -VERSION=2.0.1 +VERSION=2.1.0 YADM_WORK="$HOME" YADM_DIR= diff --git a/yadm.1 b/yadm.1 index fdd598b..9008f6f 100644 --- a/yadm.1 +++ b/yadm.1 @@ -1,5 +1,5 @@ ." vim: set spell so=8: -.TH yadm 1 "12 November 2019" "2.0.1" +.TH yadm 1 "27 November 2019" "2.1.0" .SH NAME diff --git a/yadm.md b/yadm.md index f5855bb..685bd48 100644 --- a/yadm.md +++ b/yadm.md @@ -116,65 +116,72 @@ the same way you leave your normal shell (usually with the "exit" command). This sub-shell can be used to easily interact with your yadm repository using "git" commands. This could be - useful if you are using a tool which uses Git directly. For - example, Emacs Tramp and Magit can manage files by using this - configuration: + useful if you are using a tool which uses Git directly. + + For example, Emacs Tramp and Magit can manage files by using + this configuration: + (add-to-list 'tramp-methods '("yadm" (tramp-login-program "yadm") (tramp-login-args (("enter"))) + (tramp-login-env (("SHELL") ("/bin/sh"))) (tramp-remote-shell "/bin/sh") (tramp-remote-shell-args ("-c")))) + With this config, use (magit-status "/yadm::"). If you find + issue with Emacs 27 and zsh, trying running (setenv "SHELL" + "/bin/bash"). + gitconfig - Pass options to the git config command. Since yadm already uses - the config command to manage its own configurations, this com- + Pass options to the git config command. Since yadm already uses + the config command to manage its own configurations, this com- mand is provided as a way to change configurations of the repos- - itory managed by yadm. One useful case might be to configure - the repository so untracked files are shown in status commands. + itory managed by yadm. One useful case might be to configure + the repository so untracked files are shown in status commands. yadm initially configures its repository so that untracked files - are not shown. If you wish use the default Git behavior (to - show untracked files and directories), you can remove this con- + are not shown. If you wish use the default Git behavior (to + show untracked files and directories), you can remove this con- figuration. yadm gitconfig --unset status.showUntrackedFiles help Print a summary of yadm commands. - init Initialize a new, empty repository for tracking dotfiles. The - repository is stored in $HOME/.config/yadm/repo.git. By - default, $HOME will be used as the work-tree, but this can be - overridden with the -w option. yadm can be forced to overwrite + init Initialize a new, empty repository for tracking dotfiles. The + repository is stored in $HOME/.config/yadm/repo.git. By + default, $HOME will be used as the work-tree, but this can be + overridden with the -w option. yadm can be forced to overwrite an existing repository by providing the -f option. list Print a list of files managed by yadm. The -a option will cause - all managed files to be listed. Otherwise, the list will only + all managed files to be listed. Otherwise, the list will only include files from the current directory or below. introspect category - Report internal yadm data. Supported categories are commands, + Report internal yadm data. Supported categories are commands, configs, repo, and switches. The purpose of introspection is to support command line completion. - perms Update permissions as described in the PERMISSIONS section. It - is usually unnecessary to run this command, as yadm automati- - cally processes permissions by default. This automatic behavior - can be disabled by setting the configuration yadm.auto-perms to + perms Update permissions as described in the PERMISSIONS section. It + is usually unnecessary to run this command, as yadm automati- + cally processes permissions by default. This automatic behavior + can be disabled by setting the configuration yadm.auto-perms to "false". upgrade - Version 2 of yadm uses a different directory for storing your - configurations. When you start to use version 2 for the first - time, you may see warnings about moving your data to this new - directory. The easiest way to accomplish this is by running - "yadm upgrade". This command will start by moving your yadm - repo to the new path. Next it will move any configuration data - to the new path. If the configurations are tracked within your + Version 2 of yadm uses a different directory for storing your + configurations. When you start to use version 2 for the first + time, you may see warnings about moving your data to this new + directory. The easiest way to accomplish this is by running + "yadm upgrade". This command will start by moving your yadm + repo to the new path. Next it will move any configuration data + to the new path. If the configurations are tracked within your yadm repo, this command will "stage" the renaming of those files in the repo's index. Upgrading will also re-initialize all sub- - modules you have added (otherwise they will be broken when the + modules you have added (otherwise they will be broken when the repo moves). After running "yadm upgrade", you should run "yadm - status" to review changes which have been staged, and commit + status" to review changes which have been staged, and commit them to your repository. You can read https://yadm.io/docs/upgrade_from_1 for more infor- @@ -185,40 +192,40 @@ ## COMPATIBILITY - Beginning with version 2.0.0, yadm introduced a couple major changes - which may require you to adjust your configurations. See the upgrade + Beginning with version 2.0.0, yadm introduced a couple major changes + which may require you to adjust your configurations. See the upgrade command for help making those adjustments. First, yadm now uses the "XDG Base Directory Specification" to find its - configurations. You can read https://yadm.io/docs/upgrade_from_1 for + configurations. You can read https://yadm.io/docs/upgrade_from_1 for more information. - Second, the naming conventions for alternate files have been changed. + Second, the naming conventions for alternate files have been changed. You can read https://yadm.io/docs/alternates for more information. If you want to retain the old functionality, you can set an environment - variable, YADM_COMPATIBILITY=1. Doing so will automatically use the - old yadm directory, and process alternates the same as the pre-2.0.0 - version. This compatibility mode is deprecated, and will be removed in - future versions. This mode exists solely for transitioning to the new + variable, YADM_COMPATIBILITY=1. Doing so will automatically use the + old yadm directory, and process alternates the same as the pre-2.0.0 + version. This compatibility mode is deprecated, and will be removed in + future versions. This mode exists solely for transitioning to the new paths and naming of alternates. ## OPTIONS - yadm supports a set of universal options that alter the paths it uses. - The default paths are documented in the FILES section. Any path speci- - fied by these options must be fully qualified. If you always want to - override one or more of these paths, it may be useful to create an - alias for the yadm command. For example, the following alias could be + yadm supports a set of universal options that alter the paths it uses. + The default paths are documented in the FILES section. Any path speci- + fied by these options must be fully qualified. If you always want to + override one or more of these paths, it may be useful to create an + alias for the yadm command. For example, the following alias could be used to override the repository directory. alias yadm='yadm --yadm-repo /alternate/path/to/repo' - The following is the full list of universal options. Each option + The following is the full list of universal options. Each option should be followed by a fully qualified path. -Y,--yadm-dir - Override the yadm directory. yadm stores its data relative to + Override the yadm directory. yadm stores its data relative to this directory. --yadm-repo @@ -238,9 +245,9 @@ ## CONFIGURATION - yadm uses a configuration file named $HOME/.config/yadm/config. This - file uses the same format as git-config(1). Also, you can control the - contents of the configuration file via the yadm config command (which + yadm uses a configuration file named $HOME/.config/yadm/config. This + file uses the same format as git-config(1). Also, you can control the + contents of the configuration file via the yadm config command (which works exactly like git-config). For example, to disable alternates you can run the command: @@ -250,67 +257,67 @@ yadm.alt-copy If set to "true", alternate files will be copies instead of sym- - bolic links. This might be desirable, because some systems may + bolic links. This might be desirable, because some systems may not properly support symlinks. - NOTE: The deprecated yadm.cygwin-copy option used by older ver- - sions of yadm has been replaced by yadm.alt-copy. The old + NOTE: The deprecated yadm.cygwin-copy option used by older ver- + sions of yadm has been replaced by yadm.alt-copy. The old option will be removed in the next version of yadm. yadm.auto-alt - Disable the automatic linking described in the section ALTER- - NATES. If disabled, you may still run "yadm alt" manually to - create the alternate links. This feature is enabled by default. + Disable the automatic linking described in the section ALTER- + NATES. If disabled, you may still run "yadm alt" manually to + create the alternate links. This feature is enabled by default. yadm.auto-exclude - Disable the automatic exclusion of patterns defined in + Disable the automatic exclusion of patterns defined in $HOME/.config/yadm/encrypt. This feature is enabled by default. yadm.auto-perms - Disable the automatic permission changes described in the sec- - tion PERMISSIONS. If disabled, you may still run yadm perms - manually to update permissions. This feature is enabled by + Disable the automatic permission changes described in the sec- + tion PERMISSIONS. If disabled, you may still run yadm perms + manually to update permissions. This feature is enabled by default. yadm.auto-private-dirs - Disable the automatic creating of private directories described + Disable the automatic creating of private directories described in the section PERMISSIONS. yadm.git-program - Specify an alternate program to use instead of "git". By + Specify an alternate program to use instead of "git". By default, the first "git" found in $PATH is used. yadm.gpg-perms - Disable the permission changes to $HOME/.gnupg/*. This feature + Disable the permission changes to $HOME/.gnupg/*. This feature is enabled by default. yadm.gpg-program - Specify an alternate program to use instead of "gpg". By + Specify an alternate program to use instead of "gpg". By default, the first "gpg" found in $PATH is used. yadm.gpg-recipient Asymmetrically encrypt files with a gpg public/private key pair. - Provide a "key ID" to specify which public key to encrypt with. - The key must exist in your public keyrings. If left blank or - not provided, symmetric encryption is used instead. If set to - "ASK", gpg will interactively ask for recipients. See the - ENCRYPTION section for more details. This feature is disabled + Provide a "key ID" to specify which public key to encrypt with. + The key must exist in your public keyrings. If left blank or + not provided, symmetric encryption is used instead. If set to + "ASK", gpg will interactively ask for recipients. See the + ENCRYPTION section for more details. This feature is disabled by default. yadm.ssh-perms Disable the permission changes to $HOME/.ssh/*. This feature is enabled by default. - The following four "local" configurations are not stored in the + The following four "local" configurations are not stored in the $HOME/.config/yadm/config, they are stored in the local repository. local.class - Specify a class for the purpose of symlinking alternate files. + Specify a class for the purpose of symlinking alternate files. By default, no class will be matched. local.hostname - Override the hostname for the purpose of symlinking alternate + Override the hostname for the purpose of symlinking alternate files. local.os @@ -325,9 +332,9 @@ to have an automated way of choosing an alternate version of a file for a different operating system, host, user, etc. - yadm will automatically create a symbolic link to the appropriate ver- - sion of a file, when a valid suffix is appended to the filename. The - suffix contains the conditions that must be met for that file to be + yadm will automatically create a symbolic link to the appropriate ver- + sion of a file, when a valid suffix is appended to the filename. The + suffix contains the conditions that must be met for that file to be used. The suffix begins with "##", followed by any number of conditions sepa- @@ -335,9 +342,9 @@ ##[,,...] - Each condition is an attribute/value pair, separated by a period. Some - conditions do not require a "value", and in that case, the period and - value can be omitted. Most attributes can be abbreviated as a single + Each condition is an attribute/value pair, separated by a period. Some + conditions do not require a "value", and in that case, the period and + value can be omitted. Most attributes can be abbreviated as a single letter. [.] @@ -347,24 +354,24 @@ template, t - Valid when the value matches a supported template processor. + Valid when the value matches a supported template processor. See the TEMPLATES section for more details. user, u - Valid if the value matches the current user. Current user is + Valid if the value matches the current user. Current user is calculated by running id -u -n. distro, d - Valid if the value matches the distro. Distro is calculated by + Valid if the value matches the distro. Distro is calculated by running lsb_release -si. - os, o Valid if the value matches the OS. OS is calculated by running + os, o Valid if the value matches the OS. OS is calculated by running uname -s. class, c Valid if the value matches the local.class configuration. Class must be manually set using yadm config local.class . See - the CONFIGURATION section for more details about setting + the CONFIGURATION section for more details about setting local.class. hostname, h @@ -375,6 +382,9 @@ Valid when no other alternate is valid. + NOTE: The OS for "Windows Subsystem for Linux" is reported as "WSL", + even though uname identifies as "Linux". + You may use any number of conditions, in any order. An alternate will only be used if ALL conditions are valid. For all files managed by yadm's repository or listed in $HOME/.config/yadm/encrypt, if they @@ -492,6 +502,9 @@ yadm.user YADM_USER id -u -n yadm.source YADM_SOURCE Template filename + NOTE: The OS for "Windows Subsystem for Linux" is reported as "WSL", + even though uname identifies as "Linux". + Examples: whatever##template with the following content @@ -502,7 +515,7 @@ config=dev-whatever {% endif %} - would output a file named whatever with the following content if the + would output a file named whatever with the following content if the user is "harvey": config=work-Linux @@ -511,7 +524,7 @@ config=dev-whatever - An equivalent Jinja template named whatever##template.j2 would look + An equivalent Jinja template named whatever##template.j2 would look like: {% if YADM_USER == 'harvey' -%} @@ -522,24 +535,25 @@ ## ENCRYPTION - It can be useful to manage confidential files, like SSH or GPG keys, - across multiple systems. However, doing so would put plain text data + It can be useful to manage confidential files, like SSH or GPG keys, + across multiple systems. However, doing so would put plain text data into a Git repository, which often resides on a public system. yadm can - make it easy to encrypt and decrypt a set of files so the encrypted - version can be maintained in the Git repository. This feature will + make it easy to encrypt and decrypt a set of files so the encrypted + version can be maintained in the Git repository. This feature will only work if the gpg(1) command is available. - To use this feature, a list of patterns must be created and saved as - $HOME/.config/yadm/encrypt. This list of patterns should be relative + To use this feature, a list of patterns must be created and saved as + $HOME/.config/yadm/encrypt. This list of patterns should be relative to the configured work-tree (usually $HOME). For example: .ssh/*.key .gnupg/*.gpg - Standard filename expansions (*, ?, [) are supported. Other shell - expansions like brace and tilde are not supported. Spaces in paths are - supported, and should not be quoted. If a directory is specified, its - contents will be included, but not recursively. Paths beginning with a + Standard filename expansions (*, ?, [) are supported. If you have Bash + version 4, you may use "**" to match all subdirectories. Other shell + expansions like brace and tilde are not supported. Spaces in paths are + supported, and should not be quoted. If a directory is specified, its + contents will be included, but not recursively. Paths beginning with a "!" will be excluded. The yadm encrypt command will find all files matching the patterns, and diff --git a/yadm.spec b/yadm.spec index 2aff646..f7cd484 100644 --- a/yadm.spec +++ b/yadm.spec @@ -1,7 +1,7 @@ %{!?_pkgdocdir: %global _pkgdocdir %{_docdir}/%{name}-%{version}} Name: yadm Summary: Yet Another Dotfiles Manager -Version: 2.0.1 +Version: 2.1.0 Group: Development/Tools Release: 1%{?dist} URL: https://yadm.io From 96839a57432d9e80ee3da328c71b1d094e95fbde Mon Sep 17 00:00:00 2001 From: Tim Byrne Date: Sat, 30 Nov 2019 10:27:28 -0600 Subject: [PATCH 109/137] Remove dependency on `hostname` (#182) --- yadm | 2 +- yadm.1 | 4 ++-- yadm.spec | 5 ----- 3 files changed, 3 insertions(+), 8 deletions(-) diff --git a/yadm b/yadm index df8d35c..a4ff860 100755 --- a/yadm +++ b/yadm @@ -473,7 +473,7 @@ function set_local_alt_values() { local_host="$(config local.hostname)" if [ -z "$local_host" ] ; then - local_host=$(hostname) + local_host=$(uname -n) local_host=${local_host%%.*} # trim any domain from hostname fi diff --git a/yadm.1 b/yadm.1 index 9008f6f..0e9c49c 100644 --- a/yadm.1 +++ b/yadm.1 @@ -496,7 +496,7 @@ See the CONFIGURATION section for more details about setting .BR hostname , " h Valid if the value matches the short hostname. Hostname is calculated by running -.BR "hostname" , +.BR "uname -n" , and trimming off any domain. .TP .B default @@ -629,7 +629,7 @@ During processing, the following variables are available in the template: ------------- ------------- -------------------------- yadm.class YADM_CLASS Locally defined yadm class yadm.distro YADM_DISTRO lsb_release -si - yadm.hostname YADM_HOSTNAME hostname (without domain) + yadm.hostname YADM_HOSTNAME uname -n (without domain) yadm.os YADM_OS uname -s yadm.user YADM_USER id -u -n yadm.source YADM_SOURCE Template filename diff --git a/yadm.spec b/yadm.spec index f7cd484..bd93c07 100644 --- a/yadm.spec +++ b/yadm.spec @@ -8,11 +8,6 @@ URL: https://yadm.io License: GPL-3.0-only Requires: bash Requires: git -%if 0%{?fedora} || 0%{?rhel_version} || 0%{?centos_version} >= 700 -Requires: /usr/bin/hostname -%else -Requires: /bin/hostname -%endif Source: %{name}.tar.gz BuildRoot: %{_tmppath}/%{name}-%{version}-build From 6bf085260949097b80e37024763fdbceb5d915c5 Mon Sep 17 00:00:00 2001 From: Tim Byrne Date: Sat, 30 Nov 2019 10:52:18 -0600 Subject: [PATCH 110/137] Use /etc/os-release if lsb_release is missing (#175) --- test/test_unit_query_distro.py | 34 +++++++++++++++++++--------------- yadm | 8 ++++++++ yadm.1 | 8 +++++++- 3 files changed, 34 insertions(+), 16 deletions(-) diff --git a/test/test_unit_query_distro.py b/test/test_unit_query_distro.py index 3c53c54..e83ff4e 100644 --- a/test/test_unit_query_distro.py +++ b/test/test_unit_query_distro.py @@ -1,26 +1,30 @@ """Unit tests: query_distro""" +import pytest -def test_lsb_release_present(runner, yadm, tst_distro): +@pytest.mark.parametrize( + 'condition', ['lsb_release', 'os-release', 'missing']) +def test_query_distro(runner, yadm, tst_distro, tmp_path, condition): """Match lsb_release -si when present""" + test_release = 'testrelease' + lsb_release = '' + os_release = tmp_path.joinpath('os-release') + if condition == 'os-release': + os_release.write_text(f"testing\nID={test_release}\nrelease") + if condition != 'lsb_release': + lsb_release = 'LSB_RELEASE_PROGRAM="missing_lsb_release"' script = f""" YADM_TEST=1 source {yadm} + {lsb_release} + OS_RELEASE="{os_release}" query_distro """ run = runner(command=['bash'], inp=script) assert run.success assert run.err == '' - assert run.out.rstrip() == tst_distro - - -def test_lsb_release_missing(runner, yadm): - """Empty value when missing""" - script = f""" - YADM_TEST=1 source {yadm} - LSB_RELEASE_PROGRAM="missing_lsb_release" - query_distro - """ - run = runner(command=['bash'], inp=script) - assert run.success - assert run.err == '' - assert run.out.rstrip() == '' + if condition == 'lsb_release': + assert run.out.rstrip() == tst_distro + elif condition == 'os-release': + assert run.out.rstrip() == test_release + else: + assert run.out.rstrip() == '' diff --git a/yadm b/yadm index a4ff860..650816f 100755 --- a/yadm +++ b/yadm @@ -45,6 +45,7 @@ J2CLI_PROGRAM="j2" ENVTPL_PROGRAM="envtpl" LSB_RELEASE_PROGRAM="lsb_release" +OS_RELEASE="/etc/os-release" PROC_VERSION="/proc/version" OPERATING_SYSTEM="Unknown" @@ -1219,6 +1220,13 @@ function query_distro() { distro="" if command -v "$LSB_RELEASE_PROGRAM" >/dev/null 2>&1; then distro=$($LSB_RELEASE_PROGRAM -si 2>/dev/null) + elif [ -f "$OS_RELEASE" ]; then + while IFS='' read -r line || [ -n "$line" ]; do + if [[ "$line" = ID=* ]]; then + distro="${line#ID=}" + break + fi + done < "$OS_RELEASE" fi echo "$distro" } diff --git a/yadm.1 b/yadm.1 index 0e9c49c..5dabce8 100644 --- a/yadm.1 +++ b/yadm.1 @@ -477,7 +477,9 @@ Current user is calculated by running .BR distro , " d Valid if the value matches the distro. Distro is calculated by running -.BR "lsb_release -si" . +.B "lsb_release -si" +or by inspecting the ID from +.BR "/etc/os-release" . .TP .BR os , " o Valid if the value matches the OS. @@ -638,6 +640,10 @@ During processing, the following variables are available in the template: The OS for "Windows Subsystem for Linux" is reported as "WSL", even though uname identifies as "Linux". +.BR NOTE : +If lsb_release is not available, DISTRO will be the ID specified in +/etc/os-release. + Examples: .I whatever##template From 3d10309665136cddee85ae5dc8601782b5ab4677 Mon Sep 17 00:00:00 2001 From: Tim Byrne Date: Tue, 3 Dec 2019 08:30:51 -0600 Subject: [PATCH 111/137] Issue warning for any invalid alternates found (#183) --- yadm | 40 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/yadm b/yadm index 650816f..f42514c 100755 --- a/yadm +++ b/yadm @@ -51,6 +51,9 @@ OPERATING_SYSTEM="Unknown" ENCRYPT_INCLUDE_FILES="unparsed" +LEGACY_WARNING_ISSUED=0 +INVALID_ALT=() + # flag causing path translations with cygpath USE_CYGPATH=0 @@ -204,6 +207,7 @@ function score_file() { return 0 # unsupported values else + INVALID_ALT+=("$src") score=0 return fi @@ -442,6 +446,40 @@ function alt() { remove_stale_links + report_invalid_alts + +} + +function report_invalid_alts() { + [ "$YADM_COMPATIBILITY" = "1" ] && return + [ "$LEGACY_WARNING_ISSUED" = "1" ] && return + [ "${#INVALID_ALT[@]}" = "0" ] && return + local path_list + for invalid in "${INVALID_ALT[@]}"; do + path_list="$path_list * $invalid"$'\n' + done + cat < + + Invalid alternates detected: +${path_list} +*********** +EOF } function remove_stale_links() { @@ -1371,6 +1409,8 @@ ${path_list} *********** EOF +LEGACY_WARNING_ISSUED=1 + } function configure_paths() { From 59da359e632c650039043196e46054cb45b37b83 Mon Sep 17 00:00:00 2001 From: Tim Byrne Date: Wed, 4 Dec 2019 08:17:03 -0600 Subject: [PATCH 112/137] Remove old-style alts from test data --- test/conftest.py | 9 --------- test/test_alt_copy.py | 15 +++------------ 2 files changed, 3 insertions(+), 21 deletions(-) diff --git a/test/conftest.py b/test/conftest.py index 3e2d475..5cd3c19 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -445,15 +445,6 @@ def ds1_dset(tst_sys): dset = DataSet() dset.add_file('t1') dset.add_file('d1/t2') - dset.add_file(f'test_alt##S') - dset.add_file(f'test_alt##S.H') - dset.add_file(f'test_alt##S.H.U') - dset.add_file(f'test_alt##C.S.H.U') - dset.add_file(f'test alt/test alt##S') - dset.add_file(f'test alt/test alt##S.H') - dset.add_file(f'test alt/test alt##S.H.U') - dset.add_file(f'test alt/test alt##C.S.H.U') - dset.add_file(f'test_alt_copy##{tst_sys}') dset.add_file(f'test_alt_copy##os.{tst_sys}') dset.add_file('u1', tracked=False) dset.add_file('d2/u2', tracked=False) diff --git a/test/test_alt_copy.py b/test/test_alt_copy.py index b26848f..c808348 100644 --- a/test/test_alt_copy.py +++ b/test/test_alt_copy.py @@ -8,8 +8,6 @@ import pytest 'cygwin', [pytest.param(True, marks=pytest.mark.deprecated), False], ids=['cygwin', 'no-cygwin']) -@pytest.mark.parametrize( - 'compatibility', [True, False], ids=['compat', 'no-compat']) @pytest.mark.parametrize( 'setting, expect_link, pre_existing', [ (None, True, None), @@ -29,7 +27,7 @@ import pytest def test_alt_copy( runner, yadm_y, paths, tst_sys, setting, expect_link, pre_existing, - compatibility, cygwin): + cygwin): """Test yadm.alt-copy""" option = 'yadm.cygwin-copy' if cygwin else 'yadm.alt-copy' @@ -37,10 +35,7 @@ def test_alt_copy( if setting is not None: os.system(' '.join(yadm_y('config', option, str(setting)))) - if compatibility: - expected_content = f'test_alt_copy##{tst_sys}' - else: - expected_content = f'test_alt_copy##os.{tst_sys}' + expected_content = f'test_alt_copy##os.{tst_sys}' alt_path = paths.work.join('test_alt_copy') if pre_existing == 'symlink': @@ -48,11 +43,7 @@ def test_alt_copy( elif pre_existing == 'file': alt_path.write('wrong content') - env = os.environ.copy() - if compatibility: - env['YADM_COMPATIBILITY'] = '1' - - run = runner(yadm_y('alt'), env=env) + run = runner(yadm_y('alt')) assert run.success assert run.err == '' assert 'Linking' in run.out From 66a3969c8a5c6e9a1bdb9df34fc897222880045d Mon Sep 17 00:00:00 2001 From: Tim Byrne Date: Wed, 4 Dec 2019 08:40:05 -0600 Subject: [PATCH 113/137] Add unit tests for reporting invalid alternates (#183) --- test/test_unit_report_invalid_alts.py | 39 +++++++++++++++++++++++++++ test/test_unit_score_file.py | 16 +++++++++++ 2 files changed, 55 insertions(+) create mode 100644 test/test_unit_report_invalid_alts.py diff --git a/test/test_unit_report_invalid_alts.py b/test/test_unit_report_invalid_alts.py new file mode 100644 index 0000000..7aa93bb --- /dev/null +++ b/test/test_unit_report_invalid_alts.py @@ -0,0 +1,39 @@ +"""Unit tests: report_invalid_alts""" +import pytest + + +@pytest.mark.parametrize( + 'condition', [ + 'compat', + 'previous-message', + 'invalid-alts', + 'no-invalid-alts', + ]) +def test_report_invalid_alts(runner, yadm, condition): + """Use report_invalid_alts""" + + compat = '' + previous = '' + alts = 'INVALID_ALT=()' + if condition == 'compat': + compat = 'YADM_COMPATIBILITY=1' + if condition == 'previous-message': + previous = 'LEGACY_WARNING_ISSUED=1' + if condition == 'invalid-alts': + alts = 'INVALID_ALT=("file##invalid")' + + script = f""" + YADM_TEST=1 source {yadm} + {compat} + {previous} + {alts} + report_invalid_alts + """ + run = runner(command=['bash'], inp=script) + assert run.success + assert run.err == '' + if condition == 'invalid-alts': + assert 'WARNING' in run.out + assert 'file##invalid' in run.out + else: + assert run.out == '' diff --git a/test/test_unit_score_file.py b/test/test_unit_score_file.py index 5719fe4..679229f 100644 --- a/test/test_unit_score_file.py +++ b/test/test_unit_score_file.py @@ -260,3 +260,19 @@ def test_template_recording(runner, yadm, cmd_generated): assert run.success assert run.err == '' assert run.out.rstrip() == expected + + +def test_invalid(runner, yadm): + """Verify invalid alternates are noted in INVALID_ALT""" + + invalid_file = "file##invalid" + + script = f""" + YADM_TEST=1 source {yadm} + score_file "{invalid_file}" + echo "INVALID:${{INVALID_ALT[@]}}" + """ + run = runner(command=['bash'], inp=script) + assert run.success + assert run.err == '' + assert run.out.rstrip() == f'INVALID:{invalid_file}' From c1f779521c0f8ad8cf5bf8a830e04dff47ecd290 Mon Sep 17 00:00:00 2001 From: Tim Byrne Date: Wed, 4 Dec 2019 08:43:22 -0600 Subject: [PATCH 114/137] Confirm LEGACY_WARNING_ISSUED is set appropriately --- test/test_unit_issue_legacy_path_warning.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/test/test_unit_issue_legacy_path_warning.py b/test/test_unit_issue_legacy_path_warning.py index c14857b..3f5cd6f 100644 --- a/test/test_unit_issue_legacy_path_warning.py +++ b/test/test_unit_issue_legacy_path_warning.py @@ -29,11 +29,13 @@ def test_legacy_warning(tmpdir, runner, yadm, upgrade, legacy_path): YADM_TEST=1 source {yadm} {main_args} issue_legacy_path_warning + echo "LWI:$LEGACY_WARNING_ISSUED" """ run = runner(command=['bash'], inp=script) assert run.success assert run.err == '' if legacy_path and not upgrade: assert 'Legacy configuration paths have been detected' in run.out + assert 'LWI:1' in run.out else: - assert run.out.rstrip() == '' + assert run.out.rstrip() == 'LWI:0' From 758a2e0c269a5c69b6ed2e91cfbaa06907064050 Mon Sep 17 00:00:00 2001 From: Tim Byrne Date: Wed, 4 Dec 2019 22:18:22 -0600 Subject: [PATCH 115/137] Automatically prefer `gawk` over `awk` --- yadm | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/yadm b/yadm index 852e9f5..4467f51 100755 --- a/yadm +++ b/yadm @@ -40,7 +40,7 @@ FULL_COMMAND="" GPG_PROGRAM="gpg" GIT_PROGRAM="git" -AWK_PROGRAM="${AWK_PROGRAM:-awk}" +AWK_PROGRAM=("gawk" "awk") J2CLI_PROGRAM="j2" ENVTPL_PROGRAM="envtpl" LSB_RELEASE_PROGRAM="lsb_release" @@ -348,7 +348,7 @@ function conditions() { EOF ) - "$AWK_PROGRAM" \ + "${AWK_PROGRAM[0]}" \ -v class="$local_class" \ -v os="$local_system" \ -v host="$local_host" \ @@ -1490,6 +1490,13 @@ function set_operating_system() { } +function set_awk() { + local pgm + for pgm in "${AWK_PROGRAM[@]}"; do + command -v "$pgm" >/dev/null 2>&1 && AWK_PROGRAM=("$pgm") && return + done +} + function debug() { [ -n "$DEBUG" ] && echo_e "DEBUG: $*" @@ -1802,7 +1809,7 @@ function bootstrap_available() { return 1 } function awk_available() { - command -v "$AWK_PROGRAM" >/dev/null 2>&1 && return + command -v "${AWK_PROGRAM[0]}" >/dev/null 2>&1 && return return 1 } function j2cli_available() { @@ -1856,6 +1863,7 @@ function echo_e() { if [ "$YADM_TEST" != 1 ] ; then process_global_args "$@" set_operating_system + set_awk set_yadm_dir configure_paths main "${MAIN_ARGS[@]}" From f7485915ed9b0c56d932f25cf5229ef6695b083d Mon Sep 17 00:00:00 2001 From: Tim Byrne Date: Wed, 4 Dec 2019 22:18:55 -0600 Subject: [PATCH 116/137] Update tests for gawk support --- test/test_unit_template_default.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/test_unit_template_default.py b/test/test_unit_template_default.py index b8e8faf..42464b8 100644 --- a/test/test_unit_template_default.py +++ b/test/test_unit_template_default.py @@ -95,6 +95,7 @@ def test_template_default(runner, yadm, tmpdir): script = f""" YADM_TEST=1 source {yadm} + set_awk local_class="{LOCAL_CLASS}" local_system="{LOCAL_SYSTEM}" local_host="{LOCAL_HOST}" @@ -117,6 +118,7 @@ def test_source(runner, yadm, tmpdir): script = f""" YADM_TEST=1 source {yadm} + set_awk template_default "{input_file}" "{output_file}" """ run = runner(command=['bash'], inp=script) From 4d23bbcf111c1945b8a523da1577dd653e4a7c79 Mon Sep 17 00:00:00 2001 From: Tim Byrne Date: Wed, 4 Dec 2019 22:36:58 -0600 Subject: [PATCH 117/137] Standardize on &> when not appending output --- Makefile | 4 ++-- bootstrap | 4 ++-- yadm | 30 +++++++++++++++--------------- 3 files changed, 19 insertions(+), 19 deletions(-) diff --git a/Makefile b/Makefile index b9e4777..4aea086 100644 --- a/Makefile +++ b/Makefile @@ -92,7 +92,7 @@ test: cd /yadm && \ py.test -v $(testargs); \ else \ - if command -v "docker-compose" >/dev/null 2>&1; then \ + if command -v "docker-compose" &> /dev/null; then \ docker-compose run --rm testbed make test testargs="$(testargs)"; \ else \ echo "Sorry, this make test requires docker-compose to be installed."; \ @@ -192,7 +192,7 @@ sync-clock: .PHONY: require-docker require-docker: - @if ! command -v "docker" >/dev/null 2>&1; then \ + @if ! command -v "docker" &> /dev/null; then \ echo "Sorry, this make target requires docker to be installed."; \ false; \ fi diff --git a/bootstrap b/bootstrap index 1371bb4..ab62aa3 100755 --- a/bootstrap +++ b/bootstrap @@ -35,7 +35,7 @@ REPO_URL="" function _private_yadm() { unset -f yadm - if command -v yadm >/dev/null 2>&1; then + if command -v yadm &> /dev/null; then echo "Found yadm installed locally, removing remote yadm() function" unset -f _private_yadm command yadm "$@" @@ -57,7 +57,7 @@ function remote_yadm() { } function ask_about_source() { - if ! command -v yadm >/dev/null 2>&1; then + if ! command -v yadm &> /dev/null; then echo echo "***************************************************" echo "yadm is NOT currently installed." diff --git a/yadm b/yadm index 4467f51..09e852d 100755 --- a/yadm +++ b/yadm @@ -1110,7 +1110,7 @@ function perms() { # remove group/other permissions from collected globs #shellcheck disable=SC2068 #(SC2068 is disabled because in this case, we desire globbing) - chmod -f go-rwx ${GLOBS[@]} >/dev/null 2>&1 + chmod -f go-rwx ${GLOBS[@]} &> /dev/null # TODO: detect and report changing permissions in a portable way } @@ -1159,7 +1159,7 @@ function upgrade() { echo "Moving $legacy_path to $new_filename" assert_parent "$new_filename" # test to see if path is "tracked" in repo, if so 'git mv' must be used - if "$GIT_PROGRAM" ls-files --error-unmatch "$legacy_path" >/dev/null 2>&1; then + if "$GIT_PROGRAM" ls-files --error-unmatch "$legacy_path" &> /dev/null; then "$GIT_PROGRAM" mv "$legacy_path" "$new_filename" && repo_updates=1 else mv -i "$legacy_path" "$new_filename" @@ -1170,7 +1170,7 @@ function upgrade() { # handle submodules, which need to be reinitialized if [ "$actions_performed" -ne 0 ]; then cd_work "Upgrade submodules" - if "$GIT_PROGRAM" ls-files --error-unmatch .gitmodules >/dev/null 2>&1; then + if "$GIT_PROGRAM" ls-files --error-unmatch .gitmodules &> /dev/null; then "$GIT_PROGRAM" submodule deinit -f . "$GIT_PROGRAM" submodule update --init --recursive fi @@ -1256,7 +1256,7 @@ function is_valid_branch_name() { function query_distro() { distro="" - if command -v "$LSB_RELEASE_PROGRAM" >/dev/null 2>&1; then + if command -v "$LSB_RELEASE_PROGRAM" &> /dev/null; then distro=$($LSB_RELEASE_PROGRAM -si 2>/dev/null) elif [ -f "$OS_RELEASE" ]; then while IFS='' read -r line || [ -n "$line" ]; do @@ -1493,7 +1493,7 @@ function set_operating_system() { function set_awk() { local pgm for pgm in "${AWK_PROGRAM[@]}"; do - command -v "$pgm" >/dev/null 2>&1 && AWK_PROGRAM=("$pgm") && return + command -v "$pgm" &> /dev/null && AWK_PROGRAM=("$pgm") && return done } @@ -1559,7 +1559,7 @@ function assert_private_dirs() { if [ ! -d "$work/$private_dir" ]; then debug "Creating $work/$private_dir" #shellcheck disable=SC2174 - mkdir -m 0700 -p "$work/$private_dir" >/dev/null 2>&1 + mkdir -m 0700 -p "$work/$private_dir" &> /dev/null fi done } @@ -1603,10 +1603,10 @@ function parse_encrypt() { # setting globstar to allow ** in encrypt patterns # (only supported on Bash >= 4) local unset_globstar - if ! shopt globstar &>/dev/null; then + if ! shopt globstar &> /dev/null; then unset_globstar=1 fi - shopt -s globstar &>/dev/null + shopt -s globstar &> /dev/null exclude_pattern="^!(.+)" if [ -f "$YADM_ENCRYPT" ] ; then @@ -1652,7 +1652,7 @@ function parse_encrypt() { fi if [ "$unset_globstar" = "1" ]; then - shopt -u globstar &>/dev/null + shopt -u globstar &> /dev/null fi } @@ -1781,7 +1781,7 @@ function require_git() { GIT_PROGRAM="$alt_git" more_info="\nThis command has been set via the yadm.git-program configuration." fi - command -v "$GIT_PROGRAM" >/dev/null 2>&1 || + command -v "$GIT_PROGRAM" &> /dev/null || error_out "This functionality requires Git to be installed, but the command '$GIT_PROGRAM' cannot be located.$more_info" } function require_gpg() { @@ -1795,7 +1795,7 @@ function require_gpg() { GPG_PROGRAM="$alt_gpg" more_info="\nThis command has been set via the yadm.gpg-program configuration." fi - command -v "$GPG_PROGRAM" >/dev/null 2>&1 || + command -v "$GPG_PROGRAM" &> /dev/null || error_out "This functionality requires GPG to be installed, but the command '$GPG_PROGRAM' cannot be located.$more_info" } function require_repo() { @@ -1809,19 +1809,19 @@ function bootstrap_available() { return 1 } function awk_available() { - command -v "${AWK_PROGRAM[0]}" >/dev/null 2>&1 && return + command -v "${AWK_PROGRAM[0]}" &> /dev/null && return return 1 } function j2cli_available() { - command -v "$J2CLI_PROGRAM" >/dev/null 2>&1 && return + command -v "$J2CLI_PROGRAM" &> /dev/null && return return 1 } function envtpl_available() { - command -v "$ENVTPL_PROGRAM" >/dev/null 2>&1 && return + command -v "$ENVTPL_PROGRAM" &> /dev/null && return return 1 } function readlink_available() { - command -v "readlink" >/dev/null 2>&1 && return + command -v "readlink" &> /dev/null && return return 1 } From 3aefeeff0f82de97c2892cbb62f5100aaecf45e2 Mon Sep 17 00:00:00 2001 From: Tim Byrne Date: Sat, 30 Nov 2019 18:56:17 -0600 Subject: [PATCH 118/137] Create pinentry-mock This is a program which adheres to the pinentry protocol. It always provides the password located in /etc/mock-password. --- test/pinentry-mock | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100755 test/pinentry-mock diff --git a/test/pinentry-mock b/test/pinentry-mock new file mode 100755 index 0000000..d40033b --- /dev/null +++ b/test/pinentry-mock @@ -0,0 +1,12 @@ +#!/bin/bash +# This program is a custom mock pinentry program +# It always uses whatever password is found in the /tmp directory +password="$(cat /tmp/mock-password 2>/dev/null)" +echo "OK Pleased to meet you" +while read -r line; do + if [[ $line =~ GETPIN ]]; then + echo -n "D " + echo "$password" + fi + echo "OK"; +done From fe96cfce28a57c2190e478c9456b900a600cd368 Mon Sep 17 00:00:00 2001 From: Tim Byrne Date: Sat, 30 Nov 2019 20:15:46 -0600 Subject: [PATCH 119/137] Update testbed image to use GnuPG 2 --- .travis.yml | 4 ++-- Dockerfile | 5 +---- Makefile | 4 ++-- docker-compose.yml | 2 +- 4 files changed, 6 insertions(+), 9 deletions(-) diff --git a/.travis.yml b/.travis.yml index 3c2b8ea..617deba 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,6 +3,6 @@ language: minimal services: - docker before_install: - - docker pull yadm/testbed:2019-09-25 + - docker pull yadm/testbed:2019-12-02 script: - - docker run -t --rm -v "$PWD:/yadm:ro" yadm/testbed:2019-09-25 + - docker run -t --rm -v "$PWD:/yadm:ro" yadm/testbed:2019-12-02 diff --git a/Dockerfile b/Dockerfile index be8ae09..a8d689a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -19,7 +19,7 @@ RUN \ curl \ expect \ git \ - gnupg1 \ + gnupg \ lsb-release \ make \ python3-pip \ @@ -35,9 +35,6 @@ RUN pip3 install \ yamllint==1.17.0 \ ; -# Force GNUPG version 1 at path /usr/bin/gpg -RUN ln -fs /usr/bin/gpg1 /usr/bin/gpg - # Create a flag to identify when running inside the yadm testbed RUN touch /.yadmtestbed diff --git a/Makefile b/Makefile index 4aea086..5f7ebeb 100644 --- a/Makefile +++ b/Makefile @@ -112,7 +112,7 @@ testhost: require-docker --hostname testhost \ --rm -it \ -v "/tmp/testhost:/bin/yadm:ro" \ - yadm/testbed:2019-09-25 \ + yadm/testbed:2019-12-02 \ bash -l .PHONY: scripthost @@ -129,7 +129,7 @@ scripthost: require-docker --rm -it \ -v "$$PWD/script.gz:/script.gz:rw" \ -v "/tmp/testhost:/bin/yadm:ro" \ - yadm/testbed:2019-09-25 \ + yadm/testbed:2019-12-02 \ bash -c "script /tmp/script -q -c 'bash -l'; gzip < /tmp/script > /script.gz" @echo @echo "Script saved to $$PWD/script.gz" diff --git a/docker-compose.yml b/docker-compose.yml index 4936532..85afb09 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -4,4 +4,4 @@ services: testbed: volumes: - .:/yadm:ro - image: yadm/testbed:2019-09-25 + image: yadm/testbed:2019-12-02 From e5ff95d09c972d23725e0020799176ddfbd4a0fe Mon Sep 17 00:00:00 2001 From: Tim Byrne Date: Sat, 30 Nov 2019 20:16:44 -0600 Subject: [PATCH 120/137] Create gnupg fixture This fixture is a session scoped gnupg home directory, along with a method to set the mocked password which will be used by the pinentry-mock program. --- test/conftest.py | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/test/conftest.py b/test/conftest.py index 5cd3c19..8982a6c 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -8,6 +8,7 @@ import os import platform import pwd from subprocess import Popen, PIPE +import py import pytest @@ -544,3 +545,29 @@ def ds1(ds1_work_copy, paths, ds1_dset): dscopy = copy.deepcopy(ds1_dset) dscopy.relative_to(copy.deepcopy(paths.work)) return dscopy + + +@pytest.fixture(scope='session') +def gnupg(tmpdir_factory, runner): + """Location of GNUPGHOME""" + + def register_gpg_password(password): + """Publish a new GPG mock password""" + py.path.local('/tmp/mock-password').write(password) + + home = tmpdir_factory.mktemp('gnupghome') + home.chmod(0o700) + conf = home.join('gpg-agent.conf') + conf.write( + f'pinentry-program {os.path.abspath("test/pinentry-mock")}\n' + 'max-cache-ttl 0\n' + ) + conf.chmod(0o600) + data = collections.namedtuple('GNUPG', ['home', 'pw']) + env = os.environ.copy() + env['GNUPGHOME'] = home + + # this pre-populates std files in the GNUPGHOME + runner(['gpg', '-k'], env=env) + + return data(home, register_gpg_password) From 5d484ca825f3e953a9b88427d06cad7a01038e1d Mon Sep 17 00:00:00 2001 From: Tim Byrne Date: Mon, 2 Dec 2019 17:57:57 -0600 Subject: [PATCH 121/137] Test with GnuPG 2 (#179) Take advantage of pinentry-mock to obtain passphrases, instead of using "expect" (which requires GnuPG 1). --- test/test_encryption.py | 175 +++++++++++++++++++++++++--------------- 1 file changed, 111 insertions(+), 64 deletions(-) diff --git a/test/test_encryption.py b/test/test_encryption.py index 3d10786..ec3b330 100644 --- a/test/test_encryption.py +++ b/test/test_encryption.py @@ -2,6 +2,7 @@ import os import pipes +import time import pytest KEY_FILE = 'test/test_key' @@ -13,26 +14,50 @@ PASSPHRASE = 'ExamplePassword' pytestmark = pytest.mark.usefixtures('config_git') -def add_asymmetric_key(): +def add_asymmetric_key(runner, gnupg): """Add asymmetric key""" - os.system(f'gpg --import {pipes.quote(KEY_FILE)}') - os.system(f'gpg --import-ownertrust < {pipes.quote(KEY_TRUST)}') + env = os.environ.copy() + env['GNUPGHOME'] = gnupg.home + runner( + ['gpg', '--import', pipes.quote(KEY_FILE)], + env=env, + shell=True, + ) + runner( + ['gpg', '--import-ownertrust', '<', pipes.quote(KEY_TRUST)], + env=env, + shell=True, + ) -def remove_asymmetric_key(): +def remove_asymmetric_key(runner, gnupg): """Remove asymmetric key""" - os.system( - f'gpg --batch --yes ' - f'--delete-secret-keys {pipes.quote(KEY_FINGERPRINT)}') - os.system(f'gpg --batch --yes --delete-key {pipes.quote(KEY_FINGERPRINT)}') + env = os.environ.copy() + env['GNUPGHOME'] = gnupg.home + runner( + [ + 'gpg', '--batch', '--yes', + '--delete-secret-keys', pipes.quote(KEY_FINGERPRINT) + ], + env=env, + shell=True, + ) + runner( + [ + 'gpg', '--batch', '--yes', + '--delete-key', pipes.quote(KEY_FINGERPRINT) + ], + env=env, + shell=True, + ) @pytest.fixture -def asymmetric_key(): +def asymmetric_key(runner, gnupg): """Fixture for asymmetric key, removed in teardown""" - add_asymmetric_key() + add_asymmetric_key(runner, gnupg) yield KEY_NAME - remove_asymmetric_key() + remove_asymmetric_key(runner, gnupg) @pytest.fixture @@ -95,7 +120,7 @@ def encrypt_targets(yadm_y, paths): @pytest.fixture(scope='session') -def decrypt_targets(tmpdir_factory, runner): +def decrypt_targets(tmpdir_factory, runner, gnupg): """Fixture for setting data to decrypt This fixture: @@ -117,17 +142,21 @@ def decrypt_targets(tmpdir_factory, runner): tmpdir.join('subdir/decrypt3').write('subdir/decrypt3') expected.append('subdir/decrypt3') + gnupg.pw(PASSPHRASE) + env = os.environ.copy() + env['GNUPGHOME'] = gnupg.home run = runner( ['tar', 'cvf', '-'] + expected + ['|', 'gpg', '--batch', '--yes', '-c'] + - ['--passphrase', pipes.quote(PASSPHRASE)] + ['--output', pipes.quote(str(symmetric))], cwd=tmpdir, + env=env, shell=True) assert run.success - add_asymmetric_key() + gnupg.pw('') + add_asymmetric_key(runner, gnupg) run = runner( ['tar', 'cvf', '-'] + expected + @@ -135,9 +164,10 @@ def decrypt_targets(tmpdir_factory, runner): ['-r', pipes.quote(KEY_NAME)] + ['--output', pipes.quote(str(asymmetric))], cwd=tmpdir, + env=env, shell=True) assert run.success - remove_asymmetric_key() + remove_asymmetric_key(runner, gnupg) return { 'asymmetric': asymmetric, @@ -147,8 +177,8 @@ def decrypt_targets(tmpdir_factory, runner): @pytest.mark.parametrize( - 'mismatched_phrase', [False, True], - ids=['matching_phrase', 'mismatched_phrase']) + 'bad_phrase', [False, True], + ids=['good_phrase', 'bad_phrase']) @pytest.mark.parametrize( 'missing_encrypt', [False, True], ids=['encrypt_exists', 'encrypt_missing']) @@ -157,25 +187,25 @@ def decrypt_targets(tmpdir_factory, runner): ids=['clean', 'overwrite']) def test_symmetric_encrypt( runner, yadm_y, paths, encrypt_targets, - overwrite, missing_encrypt, mismatched_phrase): + gnupg, bad_phrase, overwrite, missing_encrypt): """Test symmetric encryption""" if missing_encrypt: paths.encrypt.remove() - matched_phrase = PASSPHRASE - if mismatched_phrase: - matched_phrase = 'mismatched' + if bad_phrase: + gnupg.pw('') + else: + gnupg.pw(PASSPHRASE) if overwrite: paths.archive.write('existing archive') - run = runner(yadm_y('encrypt'), expect=[ - ('passphrase:', PASSPHRASE), - ('passphrase:', matched_phrase), - ]) + env = os.environ.copy() + env['GNUPGHOME'] = gnupg.home + run = runner(yadm_y('encrypt'), env=env) - if missing_encrypt or mismatched_phrase: + if missing_encrypt or bad_phrase: assert run.failure else: assert run.success @@ -183,15 +213,16 @@ def test_symmetric_encrypt( if missing_encrypt: assert 'does not exist' in run.out - elif mismatched_phrase: - assert 'invalid passphrase' in run.out + elif bad_phrase: + assert 'Invalid passphrase' in run.err else: - assert encrypted_data_valid(runner, paths.archive, encrypt_targets) + assert encrypted_data_valid( + runner, gnupg, paths.archive, encrypt_targets) @pytest.mark.parametrize( - 'wrong_phrase', [False, True], - ids=['correct_phrase', 'wrong_phrase']) + 'bad_phrase', [False, True], + ids=['good_phrase', 'bad_phrase']) @pytest.mark.parametrize( 'archive_exists', [True, False], ids=['archive_exists', 'archive_missing']) @@ -199,16 +230,18 @@ def test_symmetric_encrypt( 'dolist', [False, True], ids=['decrypt', 'list']) def test_symmetric_decrypt( - runner, yadm_y, paths, decrypt_targets, - dolist, archive_exists, wrong_phrase): + runner, yadm_y, paths, decrypt_targets, gnupg, + dolist, archive_exists, bad_phrase): """Test decryption""" # init empty yadm repo os.system(' '.join(yadm_y('init', '-w', str(paths.work), '-f'))) - phrase = PASSPHRASE - if wrong_phrase: - phrase = 'wrong-phrase' + if bad_phrase: + gnupg.pw('') + time.sleep(1) # allow gpg-agent cache to expire + else: + gnupg.pw(PASSPHRASE) if archive_exists: decrypt_targets['symmetric'].copy(paths.archive) @@ -216,15 +249,18 @@ def test_symmetric_decrypt( # to test overwriting paths.work.join('decrypt1').write('pre-existing file') + env = os.environ.copy() + env['GNUPGHOME'] = gnupg.home + args = [] if dolist: args.append('-l') - run = runner(yadm_y('decrypt') + args, expect=[('passphrase:', phrase)]) + run = runner(yadm_y('decrypt') + args, env=env) - if archive_exists and not wrong_phrase: + if archive_exists and not bad_phrase: assert run.success - assert run.err == '' + assert 'encrypted with 1 passphrase' in run.err if dolist: for filename in decrypt_targets['expected']: if filename != 'decrypt1': # this one should exist @@ -248,7 +284,7 @@ def test_symmetric_decrypt( 'overwrite', [False, True], ids=['clean', 'overwrite']) def test_asymmetric_encrypt( - runner, yadm_y, paths, encrypt_targets, + runner, yadm_y, paths, encrypt_targets, gnupg, overwrite, key_exists, ask): """Test asymmetric encryption""" @@ -264,13 +300,17 @@ def test_asymmetric_encrypt( paths.archive.write('existing archive') if not key_exists: - remove_asymmetric_key() + remove_asymmetric_key(runner, gnupg) - run = runner(yadm_y('encrypt'), expect=expect) + env = os.environ.copy() + env['GNUPGHOME'] = gnupg.home + + run = runner(yadm_y('encrypt'), env=env, expect=expect) if key_exists: assert run.success - assert encrypted_data_valid(runner, paths.archive, encrypt_targets) + assert encrypted_data_valid( + runner, gnupg, paths.archive, encrypt_targets) else: assert run.failure assert 'Unable to write' in run.out @@ -287,7 +327,7 @@ def test_asymmetric_encrypt( 'dolist', [False, True], ids=['decrypt', 'list']) def test_asymmetric_decrypt( - runner, yadm_y, paths, decrypt_targets, + runner, yadm_y, paths, decrypt_targets, gnupg, dolist, key_exists): """Test decryption""" @@ -300,13 +340,15 @@ def test_asymmetric_decrypt( paths.work.join('decrypt1').write('pre-existing file') if not key_exists: - remove_asymmetric_key() + remove_asymmetric_key(runner, gnupg) args = [] if dolist: args.append('-l') - run = runner(yadm_y('decrypt') + args) + env = os.environ.copy() + env['GNUPGHOME'] = gnupg.home + run = runner(yadm_y('decrypt') + args, env=env) if key_exists: assert run.success @@ -327,7 +369,8 @@ def test_asymmetric_decrypt( 'untracked', [False, 'y', 'n'], ids=['tracked', 'untracked_answer_y', 'untracked_answer_n']) -def test_offer_to_add(runner, yadm_y, paths, encrypt_targets, untracked): +def test_offer_to_add( + runner, yadm_y, paths, encrypt_targets, gnupg, untracked): """Test offer to add encrypted archive All the other encryption tests use an archive outside of the work tree. @@ -336,10 +379,12 @@ def test_offer_to_add(runner, yadm_y, paths, encrypt_targets, untracked): """ worktree_archive = paths.work.join('worktree-archive.tar.gpg') - expect = [ - ('passphrase:', PASSPHRASE), - ('passphrase:', PASSPHRASE), - ] + + expect = [] + + gnupg.pw(PASSPHRASE) + env = os.environ.copy() + env['GNUPGHOME'] = gnupg.home if untracked: expect.append(('add it now', untracked)) @@ -349,12 +394,14 @@ def test_offer_to_add(runner, yadm_y, paths, encrypt_targets, untracked): run = runner( yadm_y('encrypt', '--yadm-archive', str(worktree_archive)), + env=env, expect=expect ) assert run.success assert run.err == '' - assert encrypted_data_valid(runner, worktree_archive, encrypt_targets) + assert encrypted_data_valid( + runner, gnupg, worktree_archive, encrypt_targets) run = runner( yadm_y('status', '--porcelain', '-uall', str(worktree_archive))) @@ -372,22 +419,20 @@ def test_offer_to_add(runner, yadm_y, paths, encrypt_targets, untracked): assert f'AM {worktree_archive.basename}' in run.out -def test_encrypt_added_to_exclude(runner, yadm_y, paths): +@pytest.mark.usefixtures('ds1_copy') +def test_encrypt_added_to_exclude(runner, yadm_y, paths, gnupg): """Confirm that .config/yadm/encrypt is added to exclude""" - expect = [ - ('passphrase:', PASSPHRASE), - ('passphrase:', PASSPHRASE), - ] + gnupg.pw(PASSPHRASE) + env = os.environ.copy() + env['GNUPGHOME'] = gnupg.home exclude_file = paths.repo.join('info/exclude') paths.encrypt.write('test-encrypt-data\n') + paths.work.join('test-encrypt-data').write('') exclude_file.write('original-data', ensure=True) - run = runner( - yadm_y('encrypt'), - expect=expect, - ) + run = runner(yadm_y('encrypt'), env=env) assert 'test-encrypt-data' in paths.repo.join('info/exclude').read() assert 'original-data' in paths.repo.join('info/exclude').read() @@ -395,14 +440,16 @@ def test_encrypt_added_to_exclude(runner, yadm_y, paths): assert run.err == '' -def encrypted_data_valid(runner, encrypted, expected): +def encrypted_data_valid(runner, gnupg, encrypted, expected): """Verify encrypted data matches expectations""" + gnupg.pw(PASSPHRASE) + env = os.environ.copy() + env['GNUPGHOME'] = gnupg.home run = runner([ 'gpg', - '--passphrase', pipes.quote(PASSPHRASE), '-d', pipes.quote(str(encrypted)), '2>/dev/null', - '|', 'tar', 't'], shell=True, report=False) + '|', 'tar', 't'], env=env, shell=True, report=False) file_count = 0 for filename in run.out.splitlines(): if filename.endswith('/'): From 437ae2b71942d224b347b02cee49737157a0f23a Mon Sep 17 00:00:00 2001 From: Tim Byrne Date: Tue, 3 Dec 2019 07:54:59 -0600 Subject: [PATCH 122/137] Add --force-linters option to pylint (#179) When this option is provided, linters will be run regardless of the version installed. Normally tests are skipped if the linters are not the supported version. --- test/conftest.py | 10 ++++++++++ test/test_syntax.py | 36 ++++++++++++++++++++---------------- 2 files changed, 30 insertions(+), 16 deletions(-) diff --git a/test/conftest.py b/test/conftest.py index 8982a6c..4b6208b 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -12,6 +12,16 @@ import py import pytest +def pytest_addoption(parser): + """Add options to pytest""" + parser.addoption( + "--force-linters", + action="store_true", + default=False, + help="Run linters regardless of installed versions", + ) + + @pytest.fixture(scope='session') def shellcheck_version(): """Version of shellcheck supported""" diff --git a/test/test_syntax.py b/test/test_syntax.py index 5e39b3a..f78fd51 100644 --- a/test/test_syntax.py +++ b/test/test_syntax.py @@ -10,20 +10,22 @@ def test_yadm_syntax(runner, yadm): assert run.success -def test_shellcheck(runner, yadm, shellcheck_version): +def test_shellcheck(pytestconfig, runner, yadm, shellcheck_version): """Passes shellcheck""" - run = runner(command=['shellcheck', '-V'], report=False) - if f'version: {shellcheck_version}' not in run.out: - pytest.skip('Unsupported shellcheck version') + if not pytestconfig.getoption("--force-linters"): + run = runner(command=['shellcheck', '-V'], report=False) + if f'version: {shellcheck_version}' not in run.out: + pytest.skip('Unsupported shellcheck version') run = runner(command=['shellcheck', '-s', 'bash', yadm]) assert run.success -def test_pylint(runner, pylint_version): +def test_pylint(pytestconfig, runner, pylint_version): """Passes pylint""" - run = runner(command=['pylint', '--version'], report=False) - if f'pylint {pylint_version}' not in run.out: - pytest.skip('Unsupported pylint version') + if not pytestconfig.getoption("--force-linters"): + run = runner(command=['pylint', '--version'], report=False) + if f'pylint {pylint_version}' not in run.out: + pytest.skip('Unsupported pylint version') pyfiles = list() for tfile in os.listdir('test'): if tfile.endswith('.py'): @@ -32,20 +34,22 @@ def test_pylint(runner, pylint_version): assert run.success -def test_flake8(runner, flake8_version): +def test_flake8(pytestconfig, runner, flake8_version): """Passes flake8""" - run = runner(command=['flake8', '--version'], report=False) - if not run.out.startswith(flake8_version): - pytest.skip('Unsupported flake8 version') + if not pytestconfig.getoption("--force-linters"): + run = runner(command=['flake8', '--version'], report=False) + if not run.out.startswith(flake8_version): + pytest.skip('Unsupported flake8 version') run = runner(command=['flake8', 'test']) assert run.success -def test_yamllint(runner, yamllint_version): +def test_yamllint(pytestconfig, runner, yamllint_version): """Passes yamllint""" - run = runner(command=['yamllint', '--version'], report=False) - if not run.out.strip().endswith(yamllint_version): - pytest.skip('Unsupported yamllint version') + if not pytestconfig.getoption("--force-linters"): + run = runner(command=['yamllint', '--version'], report=False) + if not run.out.strip().endswith(yamllint_version): + pytest.skip('Unsupported yamllint version') run = runner( command=['yamllint', '-s', '$(find . -name \\*.yml)'], shell=True) From 397d45ccd075f765ba27951bd8edf90016032db1 Mon Sep 17 00:00:00 2001 From: Tim Byrne Date: Tue, 3 Dec 2019 10:40:40 -0600 Subject: [PATCH 123/137] Suppress insecure memory warnings --- test/conftest.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/test/conftest.py b/test/conftest.py index 4b6208b..941979e 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -567,12 +567,15 @@ def gnupg(tmpdir_factory, runner): home = tmpdir_factory.mktemp('gnupghome') home.chmod(0o700) - conf = home.join('gpg-agent.conf') - conf.write( + conf = home.join('gpg.conf') + conf.write('no-secmem-warning\n') + conf.chmod(0o600) + agentconf = home.join('gpg-agent.conf') + agentconf.write( f'pinentry-program {os.path.abspath("test/pinentry-mock")}\n' 'max-cache-ttl 0\n' ) - conf.chmod(0o600) + agentconf.chmod(0o600) data = collections.namedtuple('GNUPG', ['home', 'pw']) env = os.environ.copy() env['GNUPGHOME'] = home From 96bce8dbac2bdc728be0cb63f9c22d4d565b9023 Mon Sep 17 00:00:00 2001 From: Tim Byrne Date: Thu, 5 Dec 2019 16:53:16 -0600 Subject: [PATCH 124/137] Release 2.2.0 Update version number and update documentation * Resolve hostname using `uname -n` (#182) * Use /etc/os-release if lsb_release is missing (#175) * Issue warning for any invalid alternates found (#183) * Add support for gawk (#180) --- CHANGES | 6 +++++ CONTRIBUTORS | 9 ++++--- README.md | 2 +- yadm | 2 +- yadm.1 | 2 +- yadm.md | 70 +++++++++++++++++++++++++++------------------------- yadm.spec | 2 +- 7 files changed, 52 insertions(+), 41 deletions(-) diff --git a/CHANGES b/CHANGES index d9edf5b..a8235dd 100644 --- a/CHANGES +++ b/CHANGES @@ -1,3 +1,9 @@ +2.2.0 + * Resolve hostname using `uname -n` (#182) + * Use /etc/os-release if lsb_release is missing (#175) + * Issue warning for any invalid alternates found (#183) + * Add support for gawk (#180) + 2.1.0 * Use relative symlinks for alternates (#100, #177) * Support double-star globs in .config/yadm/encrypt (#109) diff --git a/CONTRIBUTORS b/CONTRIBUTORS index 895a980..4d81c4b 100644 --- a/CONTRIBUTORS +++ b/CONTRIBUTORS @@ -4,21 +4,22 @@ Tim Byrne Espen Henriksen Ross Smith II Cameron Eagans -David Mandelberg Klas Mellbourn +David Mandelberg +Daniel Gray Jan Schulz -Satoshi Ohki -Sheng Yang Siôn Le Roux Sébastien Gross Thomas Luzat Tomas Cernaj Uroš Golja +con-f-use Brayden Banks japm48 -Daniel Gray Daniel Wagenknecht Franciszek Madej Mateusz Piotrowski Paraplegic Racehorse Patrick Hof +Satoshi Ohki +Sheng Yang diff --git a/README.md b/README.md index 3895466..2c1e922 100644 --- a/README.md +++ b/README.md @@ -42,7 +42,7 @@ Features, usage, examples and installation instructions can be found on the [master-badge]: https://img.shields.io/travis/TheLocehiliosan/yadm/master.svg?label=master [master-commits]: https://github.com/TheLocehiliosan/yadm/commits/master [master-date]: https://img.shields.io/github/last-commit/TheLocehiliosan/yadm/master.svg?label=master -[obs-badge]: https://img.shields.io/badge/OBS-v2.1.0-blue +[obs-badge]: https://img.shields.io/badge/OBS-v2.2.0-blue [obs-link]: https://software.opensuse.org//download.html?project=home%3ATheLocehiliosan%3Ayadm&package=yadm [releases-badge]: https://img.shields.io/github/tag/TheLocehiliosan/yadm.svg?label=latest+release [releases-link]: https://github.com/TheLocehiliosan/yadm/releases diff --git a/yadm b/yadm index 09e852d..ec3a3d8 100755 --- a/yadm +++ b/yadm @@ -20,7 +20,7 @@ if [ -z "$BASH_VERSION" ]; then [ "$YADM_TEST" != 1 ] && exec bash "$0" "$@" fi -VERSION=2.1.0 +VERSION=2.2.0 YADM_WORK="$HOME" YADM_DIR= diff --git a/yadm.1 b/yadm.1 index 5dabce8..ef24448 100644 --- a/yadm.1 +++ b/yadm.1 @@ -1,5 +1,5 @@ ." vim: set spell so=8: -.TH yadm 1 "27 November 2019" "2.1.0" +.TH yadm 1 "6 December 2019" "2.2.0" .SH NAME diff --git a/yadm.md b/yadm.md index 685bd48..9d465d7 100644 --- a/yadm.md +++ b/yadm.md @@ -363,46 +363,47 @@ distro, d Valid if the value matches the distro. Distro is calculated by - running lsb_release -si. + running lsb_release -si or by inspecting the ID from /etc/os- + release. - os, o Valid if the value matches the OS. OS is calculated by running + os, o Valid if the value matches the OS. OS is calculated by running uname -s. class, c Valid if the value matches the local.class configuration. Class must be manually set using yadm config local.class . See - the CONFIGURATION section for more details about setting + the CONFIGURATION section for more details about setting local.class. hostname, h Valid if the value matches the short hostname. Hostname is cal- - culated by running hostname, and trimming off any domain. + culated by running uname -n, and trimming off any domain. default Valid when no other alternate is valid. - NOTE: The OS for "Windows Subsystem for Linux" is reported as "WSL", + NOTE: The OS for "Windows Subsystem for Linux" is reported as "WSL", even though uname identifies as "Linux". - You may use any number of conditions, in any order. An alternate will - only be used if ALL conditions are valid. For all files managed by - yadm's repository or listed in $HOME/.config/yadm/encrypt, if they - match this naming convention, symbolic links will be created for the + You may use any number of conditions, in any order. An alternate will + only be used if ALL conditions are valid. For all files managed by + yadm's repository or listed in $HOME/.config/yadm/encrypt, if they + match this naming convention, symbolic links will be created for the most appropriate version. The "most appropriate" version is determined by calculating a score for - each version of a file. A template is always scored higher than any - symlink condition. The number of conditions is the next largest factor - in scoring. Files with more conditions will always be favored. Any + each version of a file. A template is always scored higher than any + symlink condition. The number of conditions is the next largest factor + in scoring. Files with more conditions will always be favored. Any invalid condition will disqualify that file completely. If you don't care to have all versions of alternates stored in the same directory as the generated symlink, you can place them in the - $HOME/.config/yadm/alt directory. The generated symlink or processed + $HOME/.config/yadm/alt directory. The generated symlink or processed template will be created using the same relative path. - Alternate linking may best be demonstrated by example. Assume the fol- + Alternate linking may best be demonstrated by example. Assume the fol- lowing files are managed by yadm's repository: - $HOME/path/example.txt##default @@ -425,7 +426,7 @@ $HOME/path/example.txt -> $HOME/path/example.txt##os.Darwin - Since the hostname doesn't match any of the managed files, the more + Since the hostname doesn't match any of the managed files, the more generic version is chosen. If running on a Linux server named "host4", the link will be: @@ -440,71 +441,74 @@ $HOME/path/example.txt -> $HOME/path/example.txt##class.Work - If no "##default" version exists and no files have valid conditions, + If no "##default" version exists and no files have valid conditions, then no link will be created. - Links are also created for directories named this way, as long as they + Links are also created for directories named this way, as long as they have at least one yadm managed file within them. yadm will automatically create these links by default. This can be dis- - abled using the yadm.auto-alt configuration. Even if disabled, links + abled using the yadm.auto-alt configuration. Even if disabled, links can be manually created by running yadm alt. - Class is a special value which is stored locally on each host (inside - the local repository). To use alternate symlinks using class, you must - set the value of class using the configuration local.class. This is + Class is a special value which is stored locally on each host (inside + the local repository). To use alternate symlinks using class, you must + set the value of class using the configuration local.class. This is set like any other yadm configuration with the yadm config command. The following sets the class to be "Work". yadm config local.class Work - Similarly, the values of os, hostname, and user can be manually over- - ridden using the configuration options local.os, local.hostname, and + Similarly, the values of os, hostname, and user can be manually over- + ridden using the configuration options local.os, local.hostname, and local.user. ## TEMPLATES - If a template condition is defined in an alternate file's "##" suffix, + If a template condition is defined in an alternate file's "##" suffix, and the necessary dependencies for the template are available, then the file will be processed to create or overwrite files. Supported template processors: default - This is yadm's built-in template processor. This processor is + This is yadm's built-in template processor. This processor is very basic, with a Jinja-like syntax. The advantage of this pro- - cessor is that it only depends upon awk, which is available on - most *nix systems. To use this processor, specify the value of + cessor is that it only depends upon awk, which is available on + most *nix systems. To use this processor, specify the value of "default" or just leave the value off (e.g. "##template"). - j2cli To use the j2cli Jinja template processor, specify the value of + j2cli To use the j2cli Jinja template processor, specify the value of "j2" or "j2cli". envtpl To use the envtpl Jinja template processor, specify the value of "j2" or "envtpl". - NOTE: Specifying "j2" as the processor will attempt to use j2cli or + NOTE: Specifying "j2" as the processor will attempt to use j2cli or envtpl, whichever is available. - If the template processor specified is available, templates will be + If the template processor specified is available, templates will be processed to create or overwrite files. - During processing, the following variables are available in the tem- + During processing, the following variables are available in the tem- plate: Default Jinja Description ------------- ------------- -------------------------- yadm.class YADM_CLASS Locally defined yadm class yadm.distro YADM_DISTRO lsb_release -si - yadm.hostname YADM_HOSTNAME hostname (without domain) + yadm.hostname YADM_HOSTNAME uname -n (without domain) yadm.os YADM_OS uname -s yadm.user YADM_USER id -u -n yadm.source YADM_SOURCE Template filename - NOTE: The OS for "Windows Subsystem for Linux" is reported as "WSL", + NOTE: The OS for "Windows Subsystem for Linux" is reported as "WSL", even though uname identifies as "Linux". + NOTE: If lsb_release is not available, DISTRO will be the ID specified + in /etc/os-release. + Examples: whatever##template with the following content diff --git a/yadm.spec b/yadm.spec index bd93c07..2900dc8 100644 --- a/yadm.spec +++ b/yadm.spec @@ -1,7 +1,7 @@ %{!?_pkgdocdir: %global _pkgdocdir %{_docdir}/%{name}-%{version}} Name: yadm Summary: Yet Another Dotfiles Manager -Version: 2.1.0 +Version: 2.2.0 Group: Development/Tools Release: 1%{?dist} URL: https://yadm.io From 32baf81b56191533ac3b8f708e0eaf0d7f2112d7 Mon Sep 17 00:00:00 2001 From: Tim Byrne Date: Sun, 8 Dec 2019 14:39:03 -0600 Subject: [PATCH 125/137] Support specifying a command after `yadm enter` --- test/test_enter.py | 18 ++++++++++++++++-- yadm | 12 +++++++++--- yadm.1 | 12 ++++++++---- 3 files changed, 33 insertions(+), 9 deletions(-) diff --git a/test/test_enter.py b/test/test_enter.py index 469e3ee..106eb5c 100644 --- a/test/test_enter.py +++ b/test/test_enter.py @@ -58,8 +58,9 @@ def test_enter(runner, yadm_y, paths, shell, success): 'csh', 'zsh', ]) +@pytest.mark.parametrize('cmd', [False, True], ids=['no-cmd', 'cmd']) @pytest.mark.usefixtures('ds1_copy') -def test_enter_shell_ops(runner, yadm_y, paths, shell, opts, path): +def test_enter_shell_ops(runner, yadm_y, paths, shell, opts, path, cmd): """Enter tests for specific shell options""" # Create custom shell to detect options passed @@ -67,11 +68,24 @@ def test_enter_shell_ops(runner, yadm_y, paths, shell, opts, path): custom_shell.write('#!/bin/sh\necho OPTS=$*\necho PROMPT=$PROMPT') custom_shell.chmod(0o775) + test_cmd = ['test1', 'test2', 'test3'] + + enter_cmd = ['enter'] + if cmd: + enter_cmd += test_cmd + env = os.environ.copy() env['SHELL'] = custom_shell - run = runner(command=yadm_y('enter'), env=env) + run = runner(command=yadm_y(*enter_cmd), env=env) assert run.success assert run.err == '' assert f'OPTS={opts}' in run.out assert f'PROMPT=yadm shell ({paths.repo}) {path} >' in run.out + if cmd: + assert '-c ' + ' '.join(test_cmd) in run.out + assert 'Entering yadm repo' not in run.out + assert 'Leaving yadm repo' not in run.out + else: + assert 'Entering yadm repo' in run.out + assert 'Leaving yadm repo' in run.out diff --git a/yadm b/yadm index ec3a3d8..b5bd5cb 100755 --- a/yadm +++ b/yadm @@ -900,6 +900,7 @@ function encrypt() { } function enter() { + command="$*" require_shell require_repo @@ -913,12 +914,17 @@ function enter() { shell_path="%~" fi - echo "Entering yadm repo" + shell_cmd=() + if [ -n "$command" ]; then + shell_cmd=('-c' "$*") + fi + + [ "${#shell_cmd[@]}" -eq 0 ] && echo "Entering yadm repo" yadm_prompt="yadm shell ($YADM_REPO) $shell_path > " - PROMPT="$yadm_prompt" PS1="$yadm_prompt" "$SHELL" $shell_opts + PROMPT="$yadm_prompt" PS1="$yadm_prompt" "$SHELL" $shell_opts "${shell_cmd[@]}" - echo "Leaving yadm repo" + [ "${#shell_cmd[@]}" -eq 0 ] && echo "Leaving yadm repo" } function git_command() { diff --git a/yadm.1 b/yadm.1 index ef24448..e2c3fc0 100644 --- a/yadm.1 +++ b/yadm.1 @@ -47,7 +47,7 @@ list .BR yadm " encrypt -.BR yadm " enter +.BR yadm " enter [ command ] .BR yadm " decrypt .RB [ -l ] @@ -187,10 +187,14 @@ See the ENCRYPTION section for more details. Run a sub-shell with all Git variables set. Exit the sub-shell the same way you leave your normal shell (usually with the "exit" command). This sub-shell can be used to easily interact with your yadm repository using "git" commands. This -could be useful if you are using a tool which uses Git directly. +could be useful if you are using a tool which uses Git directly, such as tig, +vim-fugitive, git-cola, etc. -For example, Emacs Tramp and Magit can manage files by using this -configuration: +Optionally, you can provide a command after "enter", and instead of invoking +your shell, that command will be run with all of the Git variables exposed to +the command's environment. + +Emacs Tramp and Magit can manage files by using this configuration: .RS (add-to-list 'tramp-methods From e7d2406af398e0a1d9c8cc46aa1b0f2e5ec445c0 Mon Sep 17 00:00:00 2001 From: Tim Byrne Date: Sun, 8 Dec 2019 14:42:04 -0600 Subject: [PATCH 126/137] Export GIT_WORK_TREE during `yadm enter` (#160) --- test/test_enter.py | 1 + yadm | 3 +++ 2 files changed, 4 insertions(+) diff --git a/test/test_enter.py b/test/test_enter.py index 106eb5c..d97ac2f 100644 --- a/test/test_enter.py +++ b/test/test_enter.py @@ -44,6 +44,7 @@ def test_enter(runner, yadm_y, paths, shell, success): assert 'does not refer to an executable' in run.out if 'env' in shell: assert f'GIT_DIR={paths.repo}' in run.out + assert f'GIT_WORK_TREE={paths.work}' in run.out assert f'PROMPT={prompt}' in run.out assert f'PS1={prompt}' in run.out diff --git a/yadm b/yadm index b5bd5cb..2bbecc8 100755 --- a/yadm +++ b/yadm @@ -919,6 +919,9 @@ function enter() { shell_cmd=('-c' "$*") fi + GIT_WORK_TREE=$(unix_path "$("$GIT_PROGRAM" config core.worktree)") + export GIT_WORK_TREE + [ "${#shell_cmd[@]}" -eq 0 ] && echo "Entering yadm repo" yadm_prompt="yadm shell ($YADM_REPO) $shell_path > " From 18e5fcfacc7f460473d8e3202c512273cad698bd Mon Sep 17 00:00:00 2001 From: Tim Byrne Date: Tue, 10 Dec 2019 08:16:42 -0600 Subject: [PATCH 127/137] Only assert private dirs, when worktree = $HOME --- test/test_assert_private_dirs.py | 25 ++++++++++++++++++------- test/test_clone.py | 5 ++++- yadm | 4 ++++ 3 files changed, 26 insertions(+), 8 deletions(-) diff --git a/test/test_assert_private_dirs.py b/test/test_assert_private_dirs.py index 65cb0b7..2d4d163 100644 --- a/test/test_assert_private_dirs.py +++ b/test/test_assert_private_dirs.py @@ -8,7 +8,8 @@ pytestmark = pytest.mark.usefixtures('ds1_copy') PRIVATE_DIRS = ['.gnupg', '.ssh'] -def test_pdirs_missing(runner, yadm_y, paths): +@pytest.mark.parametrize('home', [True, False], ids=['home', 'not-home']) +def test_pdirs_missing(runner, yadm_y, paths, home): """Private dirs (private dirs missing) When a git command is run @@ -23,8 +24,12 @@ def test_pdirs_missing(runner, yadm_y, paths): path.remove() assert not path.exists() + env = {'DEBUG': 'yes'} + if home: + env['HOME'] = paths.work + # run status - run = runner(command=yadm_y('status'), env={'DEBUG': 'yes'}) + run = runner(command=yadm_y('status'), env=env) assert run.success assert run.err == '' assert 'On branch master' in run.out @@ -33,13 +38,19 @@ def test_pdirs_missing(runner, yadm_y, paths): # and are protected for pdir in PRIVATE_DIRS: path = paths.work.join(pdir) - assert path.exists() - assert oct(path.stat().mode).endswith('00'), 'Directory is not secured' + if home: + assert path.exists() + assert oct(path.stat().mode).endswith('00'), ('Directory is ' + 'not secured') + else: + assert not path.exists() # confirm directories are created before command is run: - assert re.search( - r'Creating.+\.gnupg.+Creating.+\.ssh.+Running git command git status', - run.out, re.DOTALL), 'directories created before command is run' + if home: + assert re.search( + (r'Creating.+\.gnupg.+Creating.+\.ssh.+' + r'Running git command git status'), + run.out, re.DOTALL), 'directories created before command is run' def test_pdirs_missing_apd_false(runner, yadm_y, paths): diff --git a/test/test_clone.py b/test/test_clone.py index 7cbbb5f..a6df6d0 100644 --- a/test/test_clone.py +++ b/test/test_clone.py @@ -222,8 +222,11 @@ def test_clone_perms( paths.work.remove() paths.work.mkdir() + env = {'HOME': paths.work} run = runner( - yadm_y('clone', '-d', '-w', paths.work, f'file://{paths.remote}')) + yadm_y('clone', '-d', '-w', paths.work, f'file://{paths.remote}'), + env=env + ) assert successful_clone(run, paths, repo_config) if in_work: diff --git a/yadm b/yadm index 2bbecc8..d38376b 100755 --- a/yadm +++ b/yadm @@ -1564,6 +1564,10 @@ function invoke_hook() { function assert_private_dirs() { work=$(unix_path "$("$GIT_PROGRAM" config core.worktree)") + + # only assert private dirs if the worktree is the same as $HOME + [ "$work" != "$HOME" ] && return + for private_dir in "$@"; do if [ ! -d "$work/$private_dir" ]; then debug "Creating $work/$private_dir" From 46105aae47c1c895cf9178e3adebaa46315aec73 Mon Sep 17 00:00:00 2001 From: Tim Byrne Date: Thu, 12 Dec 2019 08:00:10 -0600 Subject: [PATCH 128/137] Set YADM_WORK within configure_paths --- test/test_unit_parse_encrypt.py | 2 ++ yadm | 25 +++++++++++++------------ 2 files changed, 15 insertions(+), 12 deletions(-) diff --git a/test/test_unit_parse_encrypt.py b/test/test_unit_parse_encrypt.py index e6365b4..ec3a6ee 100644 --- a/test/test_unit_parse_encrypt.py +++ b/test/test_unit_parse_encrypt.py @@ -186,6 +186,8 @@ def run_parse_encrypt( export YADM_ENCRYPT GIT_DIR={paths.repo} export GIT_DIR + YADM_WORK={paths.work} + export YADM_WORK {parse_cmd} export ENCRYPT_INCLUDE_FILES export PARSE_ENCRYPT_SHORT diff --git a/yadm b/yadm index d38376b..4db275f 100755 --- a/yadm +++ b/yadm @@ -833,8 +833,6 @@ function decrypt() { require_gpg require_archive - YADM_WORK=$(unix_path "$("$GIT_PROGRAM" config core.worktree)") - if [ "$DO_LIST" = "YES" ] ; then tar_option="t" else @@ -919,7 +917,7 @@ function enter() { shell_cmd=('-c' "$*") fi - GIT_WORK_TREE=$(unix_path "$("$GIT_PROGRAM" config core.worktree)") + GIT_WORK_TREE="$YADM_WORK" export GIT_WORK_TREE [ "${#shell_cmd[@]}" -eq 0 ] && echo "Entering yadm repo" @@ -1454,6 +1452,13 @@ function configure_paths() { GIT_DIR=$(mixed_path "$YADM_REPO") export GIT_DIR + # obtain YADM_WORK from repo if it exists + if [ -d "$GIT_DIR" ]; then + local work + work=$(unix_path "$("$GIT_PROGRAM" config core.worktree)") + [ -n "$work" ] && YADM_WORK="$work" + fi + } function configure_repo() { @@ -1536,12 +1541,11 @@ function invoke_hook() { debug "Invoking hook: $hook_command" # expose some internal data to all hooks - work=$(unix_path "$("$GIT_PROGRAM" config core.worktree)") YADM_HOOK_COMMAND=$HOOK_COMMAND YADM_HOOK_EXIT=$exit_status YADM_HOOK_FULL_COMMAND=$FULL_COMMAND YADM_HOOK_REPO=$YADM_REPO - YADM_HOOK_WORK=$work + YADM_HOOK_WORK=$YADM_WORK export YADM_HOOK_COMMAND export YADM_HOOK_EXIT export YADM_HOOK_FULL_COMMAND @@ -1563,16 +1567,14 @@ function invoke_hook() { } function assert_private_dirs() { - work=$(unix_path "$("$GIT_PROGRAM" config core.worktree)") - # only assert private dirs if the worktree is the same as $HOME - [ "$work" != "$HOME" ] && return + [ "$YADM_WORK" != "$HOME" ] && return for private_dir in "$@"; do - if [ ! -d "$work/$private_dir" ]; then - debug "Creating $work/$private_dir" + if [ ! -d "$YADM_WORK/$private_dir" ]; then + debug "Creating $YADM_WORK/$private_dir" #shellcheck disable=SC2174 - mkdir -m 0700 -p "$work/$private_dir" &> /dev/null + mkdir -m 0700 -p "$YADM_WORK/$private_dir" &> /dev/null fi done } @@ -1593,7 +1595,6 @@ function display_private_perms() { } function cd_work() { - YADM_WORK=$(unix_path "$("$GIT_PROGRAM" config core.worktree)") cd "$YADM_WORK" || { debug "$1 not processed, unable to cd to $YADM_WORK" return 1 From cc1993dc14a04b8522b5acde23b4ad2443d4e91e Mon Sep 17 00:00:00 2001 From: Tim Byrne Date: Thu, 12 Dec 2019 08:09:00 -0600 Subject: [PATCH 129/137] Move logic around assert_private_dirs to be more efficient --- yadm | 31 +++++++++++++++++-------------- 1 file changed, 17 insertions(+), 14 deletions(-) diff --git a/yadm b/yadm index 4db275f..e398822 100755 --- a/yadm +++ b/yadm @@ -732,14 +732,18 @@ function clone() { rm -rf "$YADM_REPO" error_out "Clone failed, 'origin/${branch}' does not exist in ${clone_args[0]}" } - debug "Determining if repo tracks private directories" - for private_dir in .ssh/ .gnupg/; do - found_log=$("$GIT_PROGRAM" log -n 1 "origin/${branch}" -- "$private_dir" 2>/dev/null) - if [ -n "$found_log" ]; then - debug "Private directory $private_dir is tracked by repo" - assert_private_dirs "$private_dir" - fi - done + + if [ "$YADM_WORK" = "$HOME" ]; then + debug "Determining if repo tracks private directories" + for private_dir in .ssh/ .gnupg/; do + found_log=$("$GIT_PROGRAM" log -n 1 "origin/${branch}" -- "$private_dir" 2>/dev/null) + if [ -n "$found_log" ]; then + debug "Private directory $private_dir is tracked by repo" + assert_private_dirs "$private_dir" + fi + done + fi + [ -n "$DEBUG" ] && display_private_perms "pre-merge" debug "Doing an initial merge of origin/${branch}" "$GIT_PROGRAM" merge "origin/${branch}" || { @@ -940,9 +944,11 @@ function git_command() { # ensure private .ssh and .gnupg directories exist first # TODO: consider restricting this to only commands which modify the work-tree - auto_private_dirs=$(config --bool yadm.auto-private-dirs) - if [ "$auto_private_dirs" != "false" ] ; then - assert_private_dirs .gnupg/ .ssh/ + if [ "$YADM_WORK" = "$HOME" ]; then + auto_private_dirs=$(config --bool yadm.auto-private-dirs) + if [ "$auto_private_dirs" != "false" ] ; then + assert_private_dirs .gnupg/ .ssh/ + fi fi CHANGES_POSSIBLE=1 @@ -1567,9 +1573,6 @@ function invoke_hook() { } function assert_private_dirs() { - # only assert private dirs if the worktree is the same as $HOME - [ "$YADM_WORK" != "$HOME" ] && return - for private_dir in "$@"; do if [ ! -d "$YADM_WORK/$private_dir" ]; then debug "Creating $YADM_WORK/$private_dir" From 84a173551ed96d06a52c012f52fbc916741cda0a Mon Sep 17 00:00:00 2001 From: Tim Byrne Date: Fri, 13 Dec 2019 07:34:06 -0600 Subject: [PATCH 130/137] Only assert private dirs, when worktree = $HOME (#171) --- test/test_perms.py | 4 ++-- yadm | 17 ++++++++++------- 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/test/test_perms.py b/test/test_perms.py index d19b53b..0eb8add 100644 --- a/test/test_perms.py +++ b/test/test_perms.py @@ -38,7 +38,7 @@ def test_perms(runner, yadm_y, paths, ds1, autoperms): cmd = 'perms' if autoperms != 'notest': cmd = 'status' - run = runner(yadm_y(cmd)) + run = runner(yadm_y(cmd), env={'HOME': paths.work}) assert run.success assert run.err == '' if cmd == 'perms': @@ -81,7 +81,7 @@ def test_perms_control(runner, yadm_y, paths, ds1, sshperms, gpgperms): assert not oct(private.stat().mode).endswith('00'), ( 'Path started secured') - run = runner(yadm_y('perms')) + run = runner(yadm_y('perms'), env={'HOME': paths.work}) assert run.success assert run.err == '' assert run.out == '' diff --git a/yadm b/yadm index e398822..42307e8 100755 --- a/yadm +++ b/yadm @@ -1107,14 +1107,17 @@ function perms() { # include the archive created by "encrypt" [ -f "$YADM_ARCHIVE" ] && GLOBS+=("$YADM_ARCHIVE") - # include all .ssh files (unless disabled) - if [[ $(config --bool yadm.ssh-perms) != "false" ]] ; then - GLOBS+=(".ssh" ".ssh/*" ".ssh/.[!.]*") - fi + # only include private globs if using HOME as worktree + if [ "$YADM_WORK" = "$HOME" ]; then + # include all .ssh files (unless disabled) + if [[ $(config --bool yadm.ssh-perms) != "false" ]] ; then + GLOBS+=(".ssh" ".ssh/*" ".ssh/.[!.]*") + fi - # include all gpg files (unless disabled) - if [[ $(config --bool yadm.gpg-perms) != "false" ]] ; then - GLOBS+=(".gnupg" ".gnupg/*" ".gnupg/.[!.]*") + # include all gpg files (unless disabled) + if [[ $(config --bool yadm.gpg-perms) != "false" ]] ; then + GLOBS+=(".gnupg" ".gnupg/*" ".gnupg/.[!.]*") + fi fi # include any files we encrypt From b9f5fdaafa046ef649c9582f10e5ba23e28b4778 Mon Sep 17 00:00:00 2001 From: Tim Byrne Date: Fri, 13 Dec 2019 08:37:34 -0600 Subject: [PATCH 131/137] Support GNUPGHOME environment variable (#134) --- test/test_assert_private_dirs.py | 2 +- test/test_unit_private_dirs.py | 34 ++++++++++++++++++++++++++++++++ yadm | 26 ++++++++++++++++++++---- 3 files changed, 57 insertions(+), 5 deletions(-) create mode 100644 test/test_unit_private_dirs.py diff --git a/test/test_assert_private_dirs.py b/test/test_assert_private_dirs.py index 2d4d163..606012f 100644 --- a/test/test_assert_private_dirs.py +++ b/test/test_assert_private_dirs.py @@ -48,7 +48,7 @@ def test_pdirs_missing(runner, yadm_y, paths, home): # confirm directories are created before command is run: if home: assert re.search( - (r'Creating.+\.gnupg.+Creating.+\.ssh.+' + (r'Creating.+\.(gnupg|ssh).+Creating.+\.(gnupg|ssh).+' r'Running git command git status'), run.out, re.DOTALL), 'directories created before command is run' diff --git a/test/test_unit_private_dirs.py b/test/test_unit_private_dirs.py new file mode 100644 index 0000000..4f182da --- /dev/null +++ b/test/test_unit_private_dirs.py @@ -0,0 +1,34 @@ +"""Unit tests: private_dirs""" +import pytest + + +@pytest.mark.parametrize( + 'gnupghome', + [True, False], + ids=['gnupghome-set', 'gnupghome-unset'], +) +@pytest.mark.parametrize('param', ['all', 'gnupg']) +def test_relative_path(runner, paths, gnupghome, param): + """Test translate_to_relative""" + + alt_gnupghome = 'alt/gnupghome' + env_gnupghome = paths.work.join(alt_gnupghome) + + script = f""" + YADM_TEST=1 source {paths.pgm} + YADM_WORK={paths.work} + private_dirs {param} + """ + + env = {} + if gnupghome: + env['GNUPGHOME'] = env_gnupghome + + expected = alt_gnupghome if gnupghome else '.gnupg' + if param == 'all': + expected = f'.ssh {expected}' + + run = runner(command=['bash'], inp=script, env=env) + assert run.success + assert run.err == '' + assert run.out.strip() == expected diff --git a/yadm b/yadm index 42307e8..9b328f4 100755 --- a/yadm +++ b/yadm @@ -735,7 +735,7 @@ function clone() { if [ "$YADM_WORK" = "$HOME" ]; then debug "Determining if repo tracks private directories" - for private_dir in .ssh/ .gnupg/; do + for private_dir in $(private_dirs all); do found_log=$("$GIT_PROGRAM" log -n 1 "origin/${branch}" -- "$private_dir" 2>/dev/null) if [ -n "$found_log" ]; then debug "Private directory $private_dir is tracked by repo" @@ -947,7 +947,9 @@ function git_command() { if [ "$YADM_WORK" = "$HOME" ]; then auto_private_dirs=$(config --bool yadm.auto-private-dirs) if [ "$auto_private_dirs" != "false" ] ; then - assert_private_dirs .gnupg/ .ssh/ + for pdir in $(private_dirs all); do + assert_private_dirs "$pdir" + done fi fi @@ -1115,8 +1117,9 @@ function perms() { fi # include all gpg files (unless disabled) + gnupghome="$(private_dirs gnupg)" if [[ $(config --bool yadm.gpg-perms) != "false" ]] ; then - GLOBS+=(".gnupg" ".gnupg/*" ".gnupg/.[!.]*") + GLOBS+=("${gnupghome}" "${gnupghome}/*" "${gnupghome}/.[!.]*") fi fi @@ -1575,6 +1578,21 @@ function invoke_hook() { } +function private_dirs() { + fetch="$1" + pdirs=(.ssh) + if [ -z "${GNUPGHOME:-}" ]; then + pdirs+=(.gnupg) + else + pdirs+=("$(relative_path "$YADM_WORK" "$GNUPGHOME")") + fi + if [ "$fetch" = "all" ]; then + echo "${pdirs[@]}" + else + echo "${pdirs[1]}" + fi +} + function assert_private_dirs() { for private_dir in "$@"; do if [ ! -d "$YADM_WORK/$private_dir" ]; then @@ -1592,7 +1610,7 @@ function assert_parent() { function display_private_perms() { when="$1" - for private_dir in .ssh .gnupg; do + for private_dir in $(private_dirs all); do if [ -d "$YADM_WORK/$private_dir" ]; then private_perms=$(ls -ld "$YADM_WORK/$private_dir") debug "$when" private dir perms "$private_perms" From d3a2a061844e48722bfc994f227f3e88c106d1a8 Mon Sep 17 00:00:00 2001 From: Tim Byrne Date: Sun, 15 Dec 2019 18:42:21 -0600 Subject: [PATCH 132/137] Support git-crypt (#168) Support is inherently provided by `enter`, which now supports a command. I've added a `git-crypt` command, which is really just an alias under-the-hood for "enter git-crypt". --- test/conftest.py | 1 + test/test_git_crypt.py | 42 ++++++++++++++++++++++ yadm | 80 ++++++++++++++++++++++++++---------------- 3 files changed, 92 insertions(+), 31 deletions(-) create mode 100644 test/test_git_crypt.py diff --git a/test/conftest.py b/test/conftest.py index 941979e..31d872b 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -89,6 +89,7 @@ def supported_commands(): 'decrypt', 'encrypt', 'enter', + 'git-crypt', 'gitconfig', 'help', 'init', diff --git a/test/test_git_crypt.py b/test/test_git_crypt.py new file mode 100644 index 0000000..6b92de9 --- /dev/null +++ b/test/test_git_crypt.py @@ -0,0 +1,42 @@ +"""Test git-crypt""" + +import pytest + + +@pytest.mark.parametrize( + 'crypt', + [False, 'installed', 'installed-but-failed'], + ids=['not-installed', 'installed', 'installed-but-failed'] +) +def test_git_crypt(runner, yadm, paths, tmpdir, crypt): + """git-crypt tests""" + + paths.repo.ensure(dir=True) + bindir = tmpdir.mkdir('bin') + pgm = bindir.join('test-git-crypt') + + if crypt: + pgm.write(f'#!/bin/sh\necho git-crypt ran\n') + pgm.chmod(0o775) + if crypt == 'installed-but-failed': + pgm.write('false\n', mode='a') + + script = f""" + YADM_TEST=1 source {yadm} + YADM_REPO={paths.repo} + GIT_CRYPT_PROGRAM="{pgm}" + git_crypt "param1" + """ + + run = runner(command=['bash'], inp=script) + + if crypt: + if crypt == 'installed-but-failed': + assert run.failure + else: + assert run.success + assert run.out.strip() == 'git-crypt ran' + else: + assert run.failure + assert f"command '{pgm}' cannot be located" in run.out + assert run.err == '' diff --git a/yadm b/yadm index 9b328f4..b5ce07e 100755 --- a/yadm +++ b/yadm @@ -41,6 +41,7 @@ FULL_COMMAND="" GPG_PROGRAM="gpg" GIT_PROGRAM="git" AWK_PROGRAM=("gawk" "awk") +GIT_CRYPT_PROGRAM="git-crypt" J2CLI_PROGRAM="j2" ENVTPL_PROGRAM="envtpl" LSB_RELEASE_PROGRAM="lsb_release" @@ -76,45 +77,50 @@ function main() { # parse command line arguments local retval=0 - internal_commands="^(alt|bootstrap|clean|clone|config|decrypt|encrypt|enter|help|init|introspect|list|perms|upgrade|version)$" + internal_commands="^(alt|bootstrap|clean|clone|config|decrypt|encrypt|enter|git-crypt|help|init|introspect|list|perms|upgrade|version)$" if [ -z "$*" ] ; then # no argumnts will result in help() help elif [[ "$1" =~ $internal_commands ]] ; then # for internal commands, process all of the arguments - YADM_COMMAND="$1" + YADM_COMMAND="${1/-/_}" YADM_ARGS=() shift - while [[ $# -gt 0 ]] ; do - key="$1" - case $key in - -a) # used by list() - LIST_ALL="YES" - ;; - -d) # used by all commands - DEBUG="YES" - ;; - -f) # used by init() and clone() - FORCE="YES" - ;; - -l) # used by decrypt() - DO_LIST="YES" - [ "$YADM_COMMAND" = "config" ] && YADM_ARGS+=("$1") - ;; - -w) # used by init() and clone() - if [[ ! "$2" =~ ^/ ]] ; then - error_out "You must specify a fully qualified work tree" - fi - YADM_WORK="$2" - shift - ;; - *) # any unhandled arguments - YADM_ARGS+=("$1") - ;; - esac - shift - done + # commands listed below do not process any of the parameters + if [[ "$YADM_COMMAND" =~ ^(enter|git_crypt)$ ]] ; then + YADM_ARGS=("$@") + else + while [[ $# -gt 0 ]] ; do + key="$1" + case $key in + -a) # used by list() + LIST_ALL="YES" + ;; + -d) # used by all commands + DEBUG="YES" + ;; + -f) # used by init() and clone() + FORCE="YES" + ;; + -l) # used by decrypt() + DO_LIST="YES" + [ "$YADM_COMMAND" = "config" ] && YADM_ARGS+=("$1") + ;; + -w) # used by init() and clone() + if [[ ! "$2" =~ ^/ ]] ; then + error_out "You must specify a fully qualified work tree" + fi + YADM_WORK="$2" + shift + ;; + *) # any unhandled arguments + YADM_ARGS+=("$1") + ;; + esac + shift + done + fi [ ! -d "$YADM_WORK" ] && error_out "Work tree does not exist: [$YADM_WORK]" HOOK_COMMAND="$YADM_COMMAND" invoke_hook "pre" @@ -901,6 +907,11 @@ function encrypt() { } +function git_crypt() { + require_git_crypt + enter "${GIT_CRYPT_PROGRAM} $*" +} + function enter() { command="$*" require_shell @@ -983,6 +994,8 @@ Commands: yadm encrypt - Encrypt files yadm decrypt [-l] - Decrypt files yadm perms - Fix perms for private files + yadm enter [COMMAND] - Run sub-shell with GIT variables set + yadm git-crypt [OPTIONS] - Run git-crypt commands for the yadm repo Files: \$HOME/.config/yadm/config - yadm's configuration file @@ -1037,6 +1050,7 @@ decrypt encrypt enter gitconfig +git-crypt help init introspect @@ -1842,6 +1856,10 @@ function require_repo() { function require_shell() { [ -x "$SHELL" ] || error_out "\$SHELL does not refer to an executable." } +function require_git_crypt() { + command -v "$GIT_CRYPT_PROGRAM" &> /dev/null || + error_out "This functionality requires git-crypt to be installed, but the command '$GIT_CRYPT_PROGRAM' cannot be located." +} function bootstrap_available() { [ -f "$YADM_BOOTSTRAP" ] && [ -x "$YADM_BOOTSTRAP" ] && return return 1 From 787de27b7c9bcffe5ee4594545264934908cd45c Mon Sep 17 00:00:00 2001 From: Tim Byrne Date: Mon, 16 Dec 2019 08:06:52 -0600 Subject: [PATCH 133/137] Reorder items in man page --- yadm.1 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/yadm.1 b/yadm.1 index e2c3fc0..11ce778 100644 --- a/yadm.1 +++ b/yadm.1 @@ -47,8 +47,6 @@ list .BR yadm " encrypt -.BR yadm " enter [ command ] - .BR yadm " decrypt .RB [ -l ] @@ -56,6 +54,8 @@ list .BR yadm " perms +.BR yadm " enter [ command ] + .BR yadm " upgrade .BR yadm " introspect From 2978c7dd8a8ba90e29376c7a17da3ff74ad5f3d8 Mon Sep 17 00:00:00 2001 From: Tim Byrne Date: Mon, 16 Dec 2019 08:25:11 -0600 Subject: [PATCH 134/137] Add git-crypt info to man page --- yadm.1 | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/yadm.1 b/yadm.1 index 11ce778..75d0cf7 100644 --- a/yadm.1 +++ b/yadm.1 @@ -56,6 +56,8 @@ list .BR yadm " enter [ command ] +.BR yadm " git-crypt [ options ] + .BR yadm " upgrade .BR yadm " introspect @@ -211,6 +213,15 @@ With this config, use (magit-status "/yadm::"). If you find issue with Emacs 27 trying running (setenv "SHELL" "/bin/bash"). .RE .TP +.BI git-crypt " options +If git-crypt is installed, this command allows you to pass options directly to +git-crypt, with the environment configured to use the yadm repository. + +git-crypt enables transparent encryption and decryption of files in a git repository. +You can read +https://github.com/AGWA/git-crypt +for details. +.TP .B gitconfig Pass options to the .B git config @@ -744,6 +755,17 @@ This can be disabled using the .I yadm.auto-exclude configuration. +.B Using git-crypt + +A completely separate option for encrypting data is to install and use git-crypt. +Once installed, you can run git-crypt commands for the yadm repo by running +.BR "yadm git-crypt" . +git-crypt enables transparent encryption and decryption of files in a git repository. +You can read +https://github.com/AGWA/git-crypt +for details. +.LP + .SH PERMISSIONS When files are checked out of a Git repository, their initial permissions are From 7ad28c3a970673a243e1ce20a3b3622875c575a0 Mon Sep 17 00:00:00 2001 From: Tim Byrne Date: Mon, 16 Dec 2019 08:36:48 -0600 Subject: [PATCH 135/137] Set exit status when running a command from `enter` --- yadm | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/yadm b/yadm index b5ce07e..3e4c964 100755 --- a/yadm +++ b/yadm @@ -939,8 +939,13 @@ function enter() { yadm_prompt="yadm shell ($YADM_REPO) $shell_path > " PROMPT="$yadm_prompt" PS1="$yadm_prompt" "$SHELL" $shell_opts "${shell_cmd[@]}" + return_code="$?" - [ "${#shell_cmd[@]}" -eq 0 ] && echo "Leaving yadm repo" + if [ "${#shell_cmd[@]}" -eq 0 ]; then + echo "Leaving yadm repo" + else + exit_with_hook "$return_code" + fi } function git_command() { From ba5829ad4849bc6cc233be3ded14d2cb2c12b85e Mon Sep 17 00:00:00 2001 From: Tim Byrne Date: Mon, 16 Dec 2019 08:47:35 -0600 Subject: [PATCH 136/137] Confirm exit status for enter w/cmd --- test/test_enter.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/test/test_enter.py b/test/test_enter.py index d97ac2f..d1f65d0 100644 --- a/test/test_enter.py +++ b/test/test_enter.py @@ -59,14 +59,21 @@ def test_enter(runner, yadm_y, paths, shell, success): 'csh', 'zsh', ]) -@pytest.mark.parametrize('cmd', [False, True], ids=['no-cmd', 'cmd']) +@pytest.mark.parametrize( + 'cmd', + [False, 'cmd', 'cmd-bad-exit'], + ids=['no-cmd', 'cmd', 'cmd-bad-exit']) @pytest.mark.usefixtures('ds1_copy') def test_enter_shell_ops(runner, yadm_y, paths, shell, opts, path, cmd): """Enter tests for specific shell options""" + change_exit = '\nfalse' if cmd == 'cmd-bad-exit' else '' + # Create custom shell to detect options passed custom_shell = paths.root.join(shell) - custom_shell.write('#!/bin/sh\necho OPTS=$*\necho PROMPT=$PROMPT') + custom_shell.write( + f'#!/bin/sh\necho OPTS=$*\necho PROMPT=$PROMPT{change_exit}' + ) custom_shell.chmod(0o775) test_cmd = ['test1', 'test2', 'test3'] @@ -79,7 +86,10 @@ def test_enter_shell_ops(runner, yadm_y, paths, shell, opts, path, cmd): env['SHELL'] = custom_shell run = runner(command=yadm_y(*enter_cmd), env=env) - assert run.success + if cmd == 'cmd-bad-exit': + assert run.failure + else: + assert run.success assert run.err == '' assert f'OPTS={opts}' in run.out assert f'PROMPT=yadm shell ({paths.repo}) {path} >' in run.out From b4fd9e19c2e96cb975e54892f0f439f571b5cfc5 Mon Sep 17 00:00:00 2001 From: Tim Byrne Date: Tue, 17 Dec 2019 07:15:58 -0600 Subject: [PATCH 137/137] Release 2.3.0 Update version number and update documentation * Support git-crypt (#168) * Support specifying a command after `yadm enter` * Expose GIT_WORK_TREE during `yadm enter` (#160) * Support GNUPGHOME environment variable (#134) * Assert private dirs, only when worktree = $HOME (#171) --- CHANGES | 7 ++ README.md | 2 +- yadm | 2 +- yadm.1 | 2 +- yadm.md | 337 +++++++++++++++++++++++++++++------------------------- yadm.spec | 2 +- 6 files changed, 192 insertions(+), 160 deletions(-) diff --git a/CHANGES b/CHANGES index a8235dd..f0b3fdb 100644 --- a/CHANGES +++ b/CHANGES @@ -1,3 +1,10 @@ +2.3.0 + * Support git-crypt (#168) + * Support specifying a command after `yadm enter` + * Expose GIT_WORK_TREE during `yadm enter` (#160) + * Support GNUPGHOME environment variable (#134) + * Assert private dirs, only when worktree = $HOME (#171) + 2.2.0 * Resolve hostname using `uname -n` (#182) * Use /etc/os-release if lsb_release is missing (#175) diff --git a/README.md b/README.md index 2c1e922..4526ebb 100644 --- a/README.md +++ b/README.md @@ -42,7 +42,7 @@ Features, usage, examples and installation instructions can be found on the [master-badge]: https://img.shields.io/travis/TheLocehiliosan/yadm/master.svg?label=master [master-commits]: https://github.com/TheLocehiliosan/yadm/commits/master [master-date]: https://img.shields.io/github/last-commit/TheLocehiliosan/yadm/master.svg?label=master -[obs-badge]: https://img.shields.io/badge/OBS-v2.2.0-blue +[obs-badge]: https://img.shields.io/badge/OBS-v2.3.0-blue [obs-link]: https://software.opensuse.org//download.html?project=home%3ATheLocehiliosan%3Ayadm&package=yadm [releases-badge]: https://img.shields.io/github/tag/TheLocehiliosan/yadm.svg?label=latest+release [releases-link]: https://github.com/TheLocehiliosan/yadm/releases diff --git a/yadm b/yadm index 3e4c964..b8f42fd 100755 --- a/yadm +++ b/yadm @@ -20,7 +20,7 @@ if [ -z "$BASH_VERSION" ]; then [ "$YADM_TEST" != 1 ] && exec bash "$0" "$@" fi -VERSION=2.2.0 +VERSION=2.3.0 YADM_WORK="$HOME" YADM_DIR= diff --git a/yadm.1 b/yadm.1 index 75d0cf7..fabf7ec 100644 --- a/yadm.1 +++ b/yadm.1 @@ -1,5 +1,5 @@ ." vim: set spell so=8: -.TH yadm 1 "6 December 2019" "2.2.0" +.TH yadm 1 "17 December 2019" "2.3.0" .SH NAME diff --git a/yadm.md b/yadm.md index 9d465d7..1ebba71 100644 --- a/yadm.md +++ b/yadm.md @@ -24,14 +24,16 @@ yadm encrypt - yadm enter - yadm decrypt [-l] yadm alt yadm perms + yadm enter [ command ] + + yadm git-crypt [ options ] + yadm upgrade yadm introspect category @@ -116,10 +118,15 @@ the same way you leave your normal shell (usually with the "exit" command). This sub-shell can be used to easily interact with your yadm repository using "git" commands. This could be - useful if you are using a tool which uses Git directly. + useful if you are using a tool which uses Git directly, such as + tig, vim-fugitive, git-cola, etc. - For example, Emacs Tramp and Magit can manage files by using - this configuration: + Optionally, you can provide a command after "enter", and instead + of invoking your shell, that command will be run with all of the + Git variables exposed to the command's environment. + + Emacs Tramp and Magit can manage files by using this configura- + tion: (add-to-list 'tramp-methods '("yadm" @@ -129,59 +136,68 @@ (tramp-remote-shell "/bin/sh") (tramp-remote-shell-args ("-c")))) - With this config, use (magit-status "/yadm::"). If you find - issue with Emacs 27 and zsh, trying running (setenv "SHELL" + With this config, use (magit-status "/yadm::"). If you find + issue with Emacs 27 and zsh, trying running (setenv "SHELL" "/bin/bash"). + git-crypt options + If git-crypt is installed, this command allows you to pass + options directly to git-crypt, with the environment configured + to use the yadm repository. + + git-crypt enables transparent encryption and decryption of files + in a git repository. You can read https://github.com/AGWA/git- + crypt for details. + gitconfig - Pass options to the git config command. Since yadm already uses - the config command to manage its own configurations, this com- + Pass options to the git config command. Since yadm already uses + the config command to manage its own configurations, this com- mand is provided as a way to change configurations of the repos- - itory managed by yadm. One useful case might be to configure - the repository so untracked files are shown in status commands. + itory managed by yadm. One useful case might be to configure + the repository so untracked files are shown in status commands. yadm initially configures its repository so that untracked files - are not shown. If you wish use the default Git behavior (to - show untracked files and directories), you can remove this con- + are not shown. If you wish use the default Git behavior (to + show untracked files and directories), you can remove this con- figuration. yadm gitconfig --unset status.showUntrackedFiles help Print a summary of yadm commands. - init Initialize a new, empty repository for tracking dotfiles. The - repository is stored in $HOME/.config/yadm/repo.git. By - default, $HOME will be used as the work-tree, but this can be - overridden with the -w option. yadm can be forced to overwrite + init Initialize a new, empty repository for tracking dotfiles. The + repository is stored in $HOME/.config/yadm/repo.git. By + default, $HOME will be used as the work-tree, but this can be + overridden with the -w option. yadm can be forced to overwrite an existing repository by providing the -f option. list Print a list of files managed by yadm. The -a option will cause - all managed files to be listed. Otherwise, the list will only + all managed files to be listed. Otherwise, the list will only include files from the current directory or below. introspect category - Report internal yadm data. Supported categories are commands, + Report internal yadm data. Supported categories are commands, configs, repo, and switches. The purpose of introspection is to support command line completion. - perms Update permissions as described in the PERMISSIONS section. It - is usually unnecessary to run this command, as yadm automati- - cally processes permissions by default. This automatic behavior - can be disabled by setting the configuration yadm.auto-perms to + perms Update permissions as described in the PERMISSIONS section. It + is usually unnecessary to run this command, as yadm automati- + cally processes permissions by default. This automatic behavior + can be disabled by setting the configuration yadm.auto-perms to "false". upgrade - Version 2 of yadm uses a different directory for storing your - configurations. When you start to use version 2 for the first - time, you may see warnings about moving your data to this new - directory. The easiest way to accomplish this is by running - "yadm upgrade". This command will start by moving your yadm - repo to the new path. Next it will move any configuration data - to the new path. If the configurations are tracked within your + Version 2 of yadm uses a different directory for storing your + configurations. When you start to use version 2 for the first + time, you may see warnings about moving your data to this new + directory. The easiest way to accomplish this is by running + "yadm upgrade". This command will start by moving your yadm + repo to the new path. Next it will move any configuration data + to the new path. If the configurations are tracked within your yadm repo, this command will "stage" the renaming of those files in the repo's index. Upgrading will also re-initialize all sub- - modules you have added (otherwise they will be broken when the + modules you have added (otherwise they will be broken when the repo moves). After running "yadm upgrade", you should run "yadm - status" to review changes which have been staged, and commit + status" to review changes which have been staged, and commit them to your repository. You can read https://yadm.io/docs/upgrade_from_1 for more infor- @@ -192,40 +208,40 @@ ## COMPATIBILITY - Beginning with version 2.0.0, yadm introduced a couple major changes - which may require you to adjust your configurations. See the upgrade + Beginning with version 2.0.0, yadm introduced a couple major changes + which may require you to adjust your configurations. See the upgrade command for help making those adjustments. First, yadm now uses the "XDG Base Directory Specification" to find its - configurations. You can read https://yadm.io/docs/upgrade_from_1 for + configurations. You can read https://yadm.io/docs/upgrade_from_1 for more information. - Second, the naming conventions for alternate files have been changed. + Second, the naming conventions for alternate files have been changed. You can read https://yadm.io/docs/alternates for more information. If you want to retain the old functionality, you can set an environment - variable, YADM_COMPATIBILITY=1. Doing so will automatically use the - old yadm directory, and process alternates the same as the pre-2.0.0 - version. This compatibility mode is deprecated, and will be removed in - future versions. This mode exists solely for transitioning to the new + variable, YADM_COMPATIBILITY=1. Doing so will automatically use the + old yadm directory, and process alternates the same as the pre-2.0.0 + version. This compatibility mode is deprecated, and will be removed in + future versions. This mode exists solely for transitioning to the new paths and naming of alternates. ## OPTIONS - yadm supports a set of universal options that alter the paths it uses. - The default paths are documented in the FILES section. Any path speci- - fied by these options must be fully qualified. If you always want to - override one or more of these paths, it may be useful to create an - alias for the yadm command. For example, the following alias could be + yadm supports a set of universal options that alter the paths it uses. + The default paths are documented in the FILES section. Any path speci- + fied by these options must be fully qualified. If you always want to + override one or more of these paths, it may be useful to create an + alias for the yadm command. For example, the following alias could be used to override the repository directory. alias yadm='yadm --yadm-repo /alternate/path/to/repo' - The following is the full list of universal options. Each option + The following is the full list of universal options. Each option should be followed by a fully qualified path. -Y,--yadm-dir - Override the yadm directory. yadm stores its data relative to + Override the yadm directory. yadm stores its data relative to this directory. --yadm-repo @@ -245,9 +261,9 @@ ## CONFIGURATION - yadm uses a configuration file named $HOME/.config/yadm/config. This - file uses the same format as git-config(1). Also, you can control the - contents of the configuration file via the yadm config command (which + yadm uses a configuration file named $HOME/.config/yadm/config. This + file uses the same format as git-config(1). Also, you can control the + contents of the configuration file via the yadm config command (which works exactly like git-config). For example, to disable alternates you can run the command: @@ -257,67 +273,67 @@ yadm.alt-copy If set to "true", alternate files will be copies instead of sym- - bolic links. This might be desirable, because some systems may + bolic links. This might be desirable, because some systems may not properly support symlinks. - NOTE: The deprecated yadm.cygwin-copy option used by older ver- - sions of yadm has been replaced by yadm.alt-copy. The old + NOTE: The deprecated yadm.cygwin-copy option used by older ver- + sions of yadm has been replaced by yadm.alt-copy. The old option will be removed in the next version of yadm. yadm.auto-alt - Disable the automatic linking described in the section ALTER- - NATES. If disabled, you may still run "yadm alt" manually to - create the alternate links. This feature is enabled by default. + Disable the automatic linking described in the section ALTER- + NATES. If disabled, you may still run "yadm alt" manually to + create the alternate links. This feature is enabled by default. yadm.auto-exclude - Disable the automatic exclusion of patterns defined in + Disable the automatic exclusion of patterns defined in $HOME/.config/yadm/encrypt. This feature is enabled by default. yadm.auto-perms - Disable the automatic permission changes described in the sec- - tion PERMISSIONS. If disabled, you may still run yadm perms - manually to update permissions. This feature is enabled by + Disable the automatic permission changes described in the sec- + tion PERMISSIONS. If disabled, you may still run yadm perms + manually to update permissions. This feature is enabled by default. yadm.auto-private-dirs - Disable the automatic creating of private directories described + Disable the automatic creating of private directories described in the section PERMISSIONS. yadm.git-program - Specify an alternate program to use instead of "git". By + Specify an alternate program to use instead of "git". By default, the first "git" found in $PATH is used. yadm.gpg-perms - Disable the permission changes to $HOME/.gnupg/*. This feature + Disable the permission changes to $HOME/.gnupg/*. This feature is enabled by default. yadm.gpg-program - Specify an alternate program to use instead of "gpg". By + Specify an alternate program to use instead of "gpg". By default, the first "gpg" found in $PATH is used. yadm.gpg-recipient Asymmetrically encrypt files with a gpg public/private key pair. - Provide a "key ID" to specify which public key to encrypt with. - The key must exist in your public keyrings. If left blank or - not provided, symmetric encryption is used instead. If set to - "ASK", gpg will interactively ask for recipients. See the - ENCRYPTION section for more details. This feature is disabled + Provide a "key ID" to specify which public key to encrypt with. + The key must exist in your public keyrings. If left blank or + not provided, symmetric encryption is used instead. If set to + "ASK", gpg will interactively ask for recipients. See the + ENCRYPTION section for more details. This feature is disabled by default. yadm.ssh-perms Disable the permission changes to $HOME/.ssh/*. This feature is enabled by default. - The following four "local" configurations are not stored in the + The following four "local" configurations are not stored in the $HOME/.config/yadm/config, they are stored in the local repository. local.class - Specify a class for the purpose of symlinking alternate files. + Specify a class for the purpose of symlinking alternate files. By default, no class will be matched. local.hostname - Override the hostname for the purpose of symlinking alternate + Override the hostname for the purpose of symlinking alternate files. local.os @@ -332,9 +348,9 @@ to have an automated way of choosing an alternate version of a file for a different operating system, host, user, etc. - yadm will automatically create a symbolic link to the appropriate ver- - sion of a file, when a valid suffix is appended to the filename. The - suffix contains the conditions that must be met for that file to be + yadm will automatically create a symbolic link to the appropriate ver- + sion of a file, when a valid suffix is appended to the filename. The + suffix contains the conditions that must be met for that file to be used. The suffix begins with "##", followed by any number of conditions sepa- @@ -342,9 +358,9 @@ ##[,,...] - Each condition is an attribute/value pair, separated by a period. Some - conditions do not require a "value", and in that case, the period and - value can be omitted. Most attributes can be abbreviated as a single + Each condition is an attribute/value pair, separated by a period. Some + conditions do not require a "value", and in that case, the period and + value can be omitted. Most attributes can be abbreviated as a single letter. [.] @@ -354,25 +370,25 @@ template, t - Valid when the value matches a supported template processor. + Valid when the value matches a supported template processor. See the TEMPLATES section for more details. user, u - Valid if the value matches the current user. Current user is + Valid if the value matches the current user. Current user is calculated by running id -u -n. distro, d - Valid if the value matches the distro. Distro is calculated by - running lsb_release -si or by inspecting the ID from /etc/os- + Valid if the value matches the distro. Distro is calculated by + running lsb_release -si or by inspecting the ID from /etc/os- release. - os, o Valid if the value matches the OS. OS is calculated by running + os, o Valid if the value matches the OS. OS is calculated by running uname -s. class, c Valid if the value matches the local.class configuration. Class must be manually set using yadm config local.class . See - the CONFIGURATION section for more details about setting + the CONFIGURATION section for more details about setting local.class. hostname, h @@ -383,27 +399,27 @@ Valid when no other alternate is valid. - NOTE: The OS for "Windows Subsystem for Linux" is reported as "WSL", + NOTE: The OS for "Windows Subsystem for Linux" is reported as "WSL", even though uname identifies as "Linux". - You may use any number of conditions, in any order. An alternate will - only be used if ALL conditions are valid. For all files managed by - yadm's repository or listed in $HOME/.config/yadm/encrypt, if they - match this naming convention, symbolic links will be created for the + You may use any number of conditions, in any order. An alternate will + only be used if ALL conditions are valid. For all files managed by + yadm's repository or listed in $HOME/.config/yadm/encrypt, if they + match this naming convention, symbolic links will be created for the most appropriate version. The "most appropriate" version is determined by calculating a score for - each version of a file. A template is always scored higher than any - symlink condition. The number of conditions is the next largest factor - in scoring. Files with more conditions will always be favored. Any + each version of a file. A template is always scored higher than any + symlink condition. The number of conditions is the next largest factor + in scoring. Files with more conditions will always be favored. Any invalid condition will disqualify that file completely. If you don't care to have all versions of alternates stored in the same directory as the generated symlink, you can place them in the - $HOME/.config/yadm/alt directory. The generated symlink or processed + $HOME/.config/yadm/alt directory. The generated symlink or processed template will be created using the same relative path. - Alternate linking may best be demonstrated by example. Assume the fol- + Alternate linking may best be demonstrated by example. Assume the fol- lowing files are managed by yadm's repository: - $HOME/path/example.txt##default @@ -426,7 +442,7 @@ $HOME/path/example.txt -> $HOME/path/example.txt##os.Darwin - Since the hostname doesn't match any of the managed files, the more + Since the hostname doesn't match any of the managed files, the more generic version is chosen. If running on a Linux server named "host4", the link will be: @@ -441,57 +457,57 @@ $HOME/path/example.txt -> $HOME/path/example.txt##class.Work - If no "##default" version exists and no files have valid conditions, + If no "##default" version exists and no files have valid conditions, then no link will be created. - Links are also created for directories named this way, as long as they + Links are also created for directories named this way, as long as they have at least one yadm managed file within them. yadm will automatically create these links by default. This can be dis- - abled using the yadm.auto-alt configuration. Even if disabled, links + abled using the yadm.auto-alt configuration. Even if disabled, links can be manually created by running yadm alt. - Class is a special value which is stored locally on each host (inside - the local repository). To use alternate symlinks using class, you must - set the value of class using the configuration local.class. This is + Class is a special value which is stored locally on each host (inside + the local repository). To use alternate symlinks using class, you must + set the value of class using the configuration local.class. This is set like any other yadm configuration with the yadm config command. The following sets the class to be "Work". yadm config local.class Work - Similarly, the values of os, hostname, and user can be manually over- - ridden using the configuration options local.os, local.hostname, and + Similarly, the values of os, hostname, and user can be manually over- + ridden using the configuration options local.os, local.hostname, and local.user. ## TEMPLATES - If a template condition is defined in an alternate file's "##" suffix, + If a template condition is defined in an alternate file's "##" suffix, and the necessary dependencies for the template are available, then the file will be processed to create or overwrite files. Supported template processors: default - This is yadm's built-in template processor. This processor is + This is yadm's built-in template processor. This processor is very basic, with a Jinja-like syntax. The advantage of this pro- - cessor is that it only depends upon awk, which is available on - most *nix systems. To use this processor, specify the value of + cessor is that it only depends upon awk, which is available on + most *nix systems. To use this processor, specify the value of "default" or just leave the value off (e.g. "##template"). - j2cli To use the j2cli Jinja template processor, specify the value of + j2cli To use the j2cli Jinja template processor, specify the value of "j2" or "j2cli". envtpl To use the envtpl Jinja template processor, specify the value of "j2" or "envtpl". - NOTE: Specifying "j2" as the processor will attempt to use j2cli or + NOTE: Specifying "j2" as the processor will attempt to use j2cli or envtpl, whichever is available. - If the template processor specified is available, templates will be + If the template processor specified is available, templates will be processed to create or overwrite files. - During processing, the following variables are available in the tem- + During processing, the following variables are available in the tem- plate: Default Jinja Description @@ -503,10 +519,10 @@ yadm.user YADM_USER id -u -n yadm.source YADM_SOURCE Template filename - NOTE: The OS for "Windows Subsystem for Linux" is reported as "WSL", + NOTE: The OS for "Windows Subsystem for Linux" is reported as "WSL", even though uname identifies as "Linux". - NOTE: If lsb_release is not available, DISTRO will be the ID specified + NOTE: If lsb_release is not available, DISTRO will be the ID specified in /etc/os-release. Examples: @@ -519,7 +535,7 @@ config=dev-whatever {% endif %} - would output a file named whatever with the following content if the + would output a file named whatever with the following content if the user is "harvey": config=work-Linux @@ -528,7 +544,7 @@ config=dev-whatever - An equivalent Jinja template named whatever##template.j2 would look + An equivalent Jinja template named whatever##template.j2 would look like: {% if YADM_USER == 'harvey' -%} @@ -539,53 +555,62 @@ ## ENCRYPTION - It can be useful to manage confidential files, like SSH or GPG keys, - across multiple systems. However, doing so would put plain text data + It can be useful to manage confidential files, like SSH or GPG keys, + across multiple systems. However, doing so would put plain text data into a Git repository, which often resides on a public system. yadm can - make it easy to encrypt and decrypt a set of files so the encrypted - version can be maintained in the Git repository. This feature will + make it easy to encrypt and decrypt a set of files so the encrypted + version can be maintained in the Git repository. This feature will only work if the gpg(1) command is available. - To use this feature, a list of patterns must be created and saved as - $HOME/.config/yadm/encrypt. This list of patterns should be relative + To use this feature, a list of patterns must be created and saved as + $HOME/.config/yadm/encrypt. This list of patterns should be relative to the configured work-tree (usually $HOME). For example: .ssh/*.key .gnupg/*.gpg Standard filename expansions (*, ?, [) are supported. If you have Bash - version 4, you may use "**" to match all subdirectories. Other shell + version 4, you may use "**" to match all subdirectories. Other shell expansions like brace and tilde are not supported. Spaces in paths are - supported, and should not be quoted. If a directory is specified, its + supported, and should not be quoted. If a directory is specified, its contents will be included, but not recursively. Paths beginning with a "!" will be excluded. The yadm encrypt command will find all files matching the patterns, and - prompt for a password. Once a password has confirmed, the matching + prompt for a password. Once a password has confirmed, the matching files will be encrypted and saved as $HOME/.config/yadm/files.gpg. The - patterns and files.gpg should be added to the yadm repository so they + patterns and files.gpg should be added to the yadm repository so they are available across multiple systems. To decrypt these files later, or on another system run yadm decrypt and - provide the correct password. After files are decrypted, permissions + provide the correct password. After files are decrypted, permissions are automatically updated as described in the PERMISSIONS section. - Symmetric encryption is used by default, but asymmetric encryption may + Symmetric encryption is used by default, but asymmetric encryption may be enabled using the yadm.gpg-recipient configuration. - NOTE: It is recommended that you use a private repository when keeping + NOTE: It is recommended that you use a private repository when keeping confidential files, even though they are encrypted. Patterns found in $HOME/.config/yadm/encrypt are automatically added to - the repository's info/exclude file every time yadm encrypt is run. + the repository's info/exclude file every time yadm encrypt is run. This is to prevent accidentally committing sensitive data to the repos- itory. This can be disabled using the yadm.auto-exclude configuration. + Using git-crypt + + A completely separate option for encrypting data is to install and use + git-crypt. Once installed, you can run git-crypt commands for the yadm + repo by running yadm git-crypt. git-crypt enables transparent encryp- + tion and decryption of files in a git repository. You can read + https://github.com/AGWA/git-crypt for details. + + ## PERMISSIONS - When files are checked out of a Git repository, their initial permis- - sions are dependent upon the user's umask. Because of this, yadm will - automatically update the permissions of some file paths. The "group" + When files are checked out of a Git repository, their initial permis- + sions are dependent upon the user's umask. Because of this, yadm will + automatically update the permissions of some file paths. The "group" and "others" permissions will be removed from the following files: - $HOME/.config/yadm/files.gpg @@ -597,39 +622,39 @@ - The GPG directory and files, .gnupg/* yadm will automatically update permissions by default. This can be dis- - abled using the yadm.auto-perms configuration. Even if disabled, per- - missions can be manually updated by running yadm perms. The .ssh - directory processing can be disabled using the yadm.ssh-perms configu- - ration. The .gnupg directory processing can be disabled using the + abled using the yadm.auto-perms configuration. Even if disabled, per- + missions can be manually updated by running yadm perms. The .ssh + directory processing can be disabled using the yadm.ssh-perms configu- + ration. The .gnupg directory processing can be disabled using the yadm.gpg-perms configuration. - When cloning a repo which includes data in a .ssh or .gnupg directory, - if those directories do not exist at the time of cloning, yadm will + When cloning a repo which includes data in a .ssh or .gnupg directory, + if those directories do not exist at the time of cloning, yadm will create the directories with mask 0700 prior to merging the fetched data into the work-tree. When running a Git command and .ssh or .gnupg directories do not exist, - yadm will create those directories with mask 0700 prior to running the + yadm will create those directories with mask 0700 prior to running the Git command. This can be disabled using the yadm.auto-private-dirs con- figuration. ## HOOKS - For every command yadm supports, a program can be provided to run - before or after that command. These are referred to as "hooks". yadm - looks for hooks in the directory $HOME/.config/yadm/hooks. Each hook + For every command yadm supports, a program can be provided to run + before or after that command. These are referred to as "hooks". yadm + looks for hooks in the directory $HOME/.config/yadm/hooks. Each hook is named using a prefix of pre_ or post_, followed by the command which - should trigger the hook. For example, to create a hook which is run - after every yadm pull command, create a hook named post_pull. Hooks + should trigger the hook. For example, to create a hook which is run + after every yadm pull command, create a hook named post_pull. Hooks must have the executable file permission set. If a pre_ hook is defined, and the hook terminates with a non-zero exit - status, yadm will refuse to run the yadm command. For example, if a - pre_commit hook is defined, but that command ends with a non-zero exit - status, the yadm commit will never be run. This allows one to "short- + status, yadm will refuse to run the yadm command. For example, if a + pre_commit hook is defined, but that command ends with a non-zero exit + status, the yadm commit will never be run. This allows one to "short- circuit" any operation using a pre_ hook. - Hooks have the following environment variables available to them at + Hooks have the following environment variables available to them at runtime: YADM_HOOK_COMMAND @@ -649,13 +674,13 @@ ## FILES - All of yadm's configurations are relative to the "yadm directory". - yadm uses the "XDG Base Directory Specification" to determine this - directory. If the environment variable $XDG_CONFIG_HOME is defined as - a fully qualified path, this directory will be $XDG_CONFIG_HOME/yadm. + All of yadm's configurations are relative to the "yadm directory". + yadm uses the "XDG Base Directory Specification" to determine this + directory. If the environment variable $XDG_CONFIG_HOME is defined as + a fully qualified path, this directory will be $XDG_CONFIG_HOME/yadm. Otherwise it will be $HOME/.config/yadm. - The following are the default paths yadm uses for its own data. Most + The following are the default paths yadm uses for its own data. Most of these paths can be altered using universal options. See the OPTIONS section for details. @@ -667,9 +692,9 @@ Configuration file for yadm. $YADM_DIR/alt - This is a directory to keep "alternate files" without having - them side-by-side with the resulting symlink or processed tem- - plate. Alternate files placed in this directory will be created + This is a directory to keep "alternate files" without having + them side-by-side with the resulting symlink or processed tem- + plate. Alternate files placed in this directory will be created relative to $HOME instead. $YADM_DIR/repo.git diff --git a/yadm.spec b/yadm.spec index 2900dc8..af378f0 100644 --- a/yadm.spec +++ b/yadm.spec @@ -1,7 +1,7 @@ %{!?_pkgdocdir: %global _pkgdocdir %{_docdir}/%{name}-%{version}} Name: yadm Summary: Yet Another Dotfiles Manager -Version: 2.2.0 +Version: 2.3.0 Group: Development/Tools Release: 1%{?dist} URL: https://yadm.io