From cfda485b34459979c35993b8aa0044a564b52218 Mon Sep 17 00:00:00 2001 From: Tim Byrne Date: Tue, 1 Oct 2019 08:12:18 -0500 Subject: [PATCH] Implement future alternate processing --- test/test_alt.py | 193 +++++++++++++++++ test/test_compat_alt.py | 29 +-- test/test_compat_jinja.py | 18 +- test/test_unit_choose_template_cmd.py | 61 ++++++ test/test_unit_record_score.py | 114 ++++++++++ test/test_unit_record_template.py | 55 +++++ test/test_unit_score_file.py | 229 ++++++++++++++++++++ test/test_unit_set_local_alt_values.py | 15 ++ test/test_unit_template_builtin.py | 98 +++++++++ test/test_unit_template_j2.py | 99 +++++++++ test/utils.py | 19 +- yadm | 276 ++++++++++++++++++++++++- 12 files changed, 1168 insertions(+), 38 deletions(-) create mode 100644 test/test_alt.py create mode 100644 test/test_unit_choose_template_cmd.py create mode 100644 test/test_unit_record_score.py create mode 100644 test/test_unit_record_template.py create mode 100644 test/test_unit_score_file.py create mode 100644 test/test_unit_template_builtin.py create mode 100644 test/test_unit_template_j2.py diff --git a/test/test_alt.py b/test/test_alt.py new file mode 100644 index 0000000..e8a0001 --- /dev/null +++ b/test/test_alt.py @@ -0,0 +1,193 @@ +"""Test alt""" +import os +import string +import py +import pytest +import utils + +TEST_PATHS = [utils.ALT_FILE1, utils.ALT_FILE2, utils.ALT_DIR] + + +@pytest.mark.usefixtures('ds1_copy') +@pytest.mark.parametrize( + 'tracked,encrypt,exclude', [ + (False, False, False), + (True, False, False), + (False, True, False), + (False, True, True), + ], ids=['untracked', 'tracked', 'encrypted', 'excluded']) +def test_alt_source( + runner, yadm_y, paths, + tracked, encrypt, exclude): + """Test yadm alt operates on all expected sources of alternates""" + + utils.create_alt_files( + paths, '##default', tracked=tracked, encrypt=encrypt, exclude=exclude) + run = runner(yadm_y('alt')) + assert run.success + assert run.err == '' + linked = utils.parse_alt_output(run.out) + + for link_path in TEST_PATHS: + source_file = link_path + '##default' + if tracked or (encrypt and not exclude): + assert paths.work.join(link_path).islink() + target = py.path.local(paths.work.join(link_path).readlink()) + if target.isfile(): + assert paths.work.join(link_path).read() == source_file + assert str(paths.work.join(source_file)) in linked + else: + assert paths.work.join(link_path).join( + utils.CONTAINED).read() == source_file + assert str(paths.work.join(source_file)) in linked + else: + assert not paths.work.join(link_path).exists() + assert str(paths.work.join(source_file)) not in linked + + +@pytest.mark.usefixtures('ds1_copy') +@pytest.mark.parametrize('suffix', [ + '##default', + '##o.$tst_sys', '##os.$tst_sys', + '##c.$tst_class', '##class.$tst_class', + '##h.$tst_host', '##hostname.$tst_host', + '##u.$tst_user', '##user.$tst_user', + ]) +def test_alt_conditions( + runner, yadm_y, paths, + tst_sys, tst_host, tst_user, suffix): + """Test conditions supported by yadm alt""" + + # set the class + tst_class = 'testclass' + utils.set_local(paths, 'class', tst_class) + + suffix = string.Template(suffix).substitute( + tst_sys=tst_sys, + tst_class=tst_class, + tst_host=tst_host, + tst_user=tst_user, + ) + + utils.create_alt_files(paths, suffix) + run = runner(yadm_y('alt')) + assert run.success + assert run.err == '' + linked = utils.parse_alt_output(run.out) + + for link_path in TEST_PATHS: + source_file = link_path + suffix + assert paths.work.join(link_path).islink() + target = py.path.local(paths.work.join(link_path).readlink()) + if target.isfile(): + assert paths.work.join(link_path).read() == source_file + assert str(paths.work.join(source_file)) in linked + else: + assert paths.work.join(link_path).join( + utils.CONTAINED).read() == source_file + assert str(paths.work.join(source_file)) in linked + + +@pytest.mark.usefixtures('ds1_copy') +@pytest.mark.parametrize('kind', ['builtin', '', 'envtpl', 'j2cli', 'j2']) +@pytest.mark.parametrize('label', ['t', 'template', 'yadm', ]) +def test_alt_templates( + runner, yadm_y, paths, kind, label): + """Test templates supported by yadm alt""" + + suffix = f'##{label}.{kind}' + utils.create_alt_files(paths, suffix) + run = runner(yadm_y('alt')) + assert run.success + assert run.err == '' + created = utils.parse_alt_output(run.out, linked=False) + + for created_path in TEST_PATHS: + if created_path != utils.ALT_DIR: + source_file = created_path + suffix + assert paths.work.join(created_path).isfile() + assert paths.work.join(created_path).read().strip() == source_file + assert str(paths.work.join(source_file)) in created + + +@pytest.mark.usefixtures('ds1_copy') +@pytest.mark.parametrize('autoalt', [None, 'true', 'false']) +def test_auto_alt(runner, yadm_y, paths, autoalt): + """Test auto alt""" + + # set the value of auto-alt + if autoalt: + os.system(' '.join(yadm_y('config', 'yadm.auto-alt', autoalt))) + + utils.create_alt_files(paths, '##default') + run = runner(yadm_y('status')) + assert run.success + assert run.err == '' + linked = utils.parse_alt_output(run.out) + + for link_path in TEST_PATHS: + source_file = link_path + '##default' + if autoalt == 'false': + assert not paths.work.join(link_path).exists() + else: + assert paths.work.join(link_path).islink() + target = py.path.local(paths.work.join(link_path).readlink()) + if target.isfile(): + assert paths.work.join(link_path).read() == source_file + # no linking output when run via auto-alt + assert str(paths.work.join(source_file)) not in linked + else: + assert paths.work.join(link_path).join( + utils.CONTAINED).read() == source_file + # no linking output when run via auto-alt + assert str(paths.work.join(source_file)) not in linked + + +@pytest.mark.usefixtures('ds1_copy') +def test_stale_link_removal(runner, yadm_y, paths): + """Stale links to alternative files are removed + + This test ensures that when an already linked alternative becomes invalid + due to a change in class, the alternate link is removed. + """ + + # set the class + tst_class = 'testclass' + utils.set_local(paths, 'class', tst_class) + + # create files which match the test class + utils.create_alt_files(paths, f'##class.{tst_class}') + + # run alt to trigger linking + run = runner(yadm_y('alt')) + assert run.success + assert run.err == '' + linked = utils.parse_alt_output(run.out) + + # assert the proper linking has occurred + for stale_path in TEST_PATHS: + source_file = stale_path + '##class.' + tst_class + assert paths.work.join(stale_path).islink() + target = py.path.local(paths.work.join(stale_path).readlink()) + if target.isfile(): + assert paths.work.join(stale_path).read() == source_file + assert str(paths.work.join(source_file)) in linked + else: + assert paths.work.join(stale_path).join( + utils.CONTAINED).read() == source_file + assert str(paths.work.join(source_file)) in linked + + # change the class so there are no valid alternates + utils.set_local(paths, 'class', 'changedclass') + + # run alt to trigger linking + run = runner(yadm_y('alt')) + assert run.success + assert run.err == '' + linked = utils.parse_alt_output(run.out) + + # assert the linking is removed + for stale_path in TEST_PATHS: + source_file = stale_path + '##class.' + tst_class + assert not paths.work.join(stale_path).exists() + assert str(paths.work.join(source_file)) not in linked diff --git a/test/test_compat_alt.py b/test/test_compat_alt.py index 466e7fc..6321252 100644 --- a/test/test_compat_alt.py +++ b/test/test_compat_alt.py @@ -1,7 +1,6 @@ """Test alt""" import os -import re import string import py import pytest @@ -92,7 +91,7 @@ def test_alt(runner, yadm_y, paths, run = runner(yadm_y('alt'), env=env) assert run.success assert run.err == '' - linked = linked_list(run.out) + linked = utils.parse_alt_output(run.out) # assert the proper linking has occurred for file_path in TEST_PATHS: @@ -195,7 +194,7 @@ def test_wild(request, runner, yadm_y, paths, run = runner(yadm_y('alt'), env=env) assert run.success assert run.err == '' - linked = linked_list(run.out) + linked = utils.parse_alt_output(run.out) # assert the proper linking has occurred for file_path in TEST_PATHS: @@ -220,7 +219,7 @@ def test_wild(request, runner, yadm_y, paths, run = runner(yadm_y('alt'), env=env) assert run.success assert run.err == '' - linked = linked_list(run.out) + linked = utils.parse_alt_output(run.out) # assert the proper linking has occurred for file_path in TEST_PATHS: @@ -260,7 +259,7 @@ def test_local_override(runner, yadm_y, paths, run = runner(yadm_y('alt'), env=env) assert run.success assert run.err == '' - linked = linked_list(run.out) + linked = utils.parse_alt_output(run.out) # assert the proper linking has occurred for file_path in TEST_PATHS: @@ -300,7 +299,7 @@ def test_class_case(runner, yadm_y, paths, tst_sys, suffix): run = runner(yadm_y('alt'), env=env) assert run.success assert run.err == '' - linked = linked_list(run.out) + linked = utils.parse_alt_output(run.out) # assert the proper linking has occurred for file_path in TEST_PATHS: @@ -335,7 +334,7 @@ def test_auto_alt(runner, yadm_y, paths, autoalt): run = runner(yadm_y('status'), env=env) assert run.success assert run.err == '' - linked = linked_list(run.out) + linked = utils.parse_alt_output(run.out) # assert the proper linking has occurred for file_path in TEST_PATHS: @@ -373,7 +372,7 @@ def test_delimiter(runner, yadm_y, paths, run = runner(yadm_y('alt'), env=env) assert run.success assert run.err == '' - linked = linked_list(run.out) + linked = utils.parse_alt_output(run.out) # assert the proper linking has occurred # only a delimiter of '.' is valid @@ -415,7 +414,7 @@ def test_invalid_links_removed(runner, yadm_y, paths): run = runner(yadm_y('alt'), env=env) assert run.success assert run.err == '' - linked = linked_list(run.out) + linked = utils.parse_alt_output(run.out) # assert the proper linking has occurred for file_path in TEST_PATHS: @@ -439,20 +438,10 @@ def test_invalid_links_removed(runner, yadm_y, paths): run = runner(yadm_y('alt'), env=env) assert run.success assert run.err == '' - linked = linked_list(run.out) + linked = utils.parse_alt_output(run.out) # assert the linking is removed for file_path in TEST_PATHS: source_file = file_path + '##' + tst_class assert not paths.work.join(file_path).exists() assert str(paths.work.join(source_file)) not in linked - - -def linked_list(output): - """Parse output, and return list of linked files""" - linked = dict() - for line in output.splitlines(): - match = re.match('Linking (.+) to (.+)$', line) - if match: - linked[match.group(2)] = match.group(1) - return linked.values() diff --git a/test/test_compat_jinja.py b/test/test_compat_jinja.py index 342d202..7d3c1aa 100644 --- a/test/test_compat_jinja.py +++ b/test/test_compat_jinja.py @@ -1,7 +1,6 @@ """Test jinja""" import os -import re import pytest import utils @@ -55,7 +54,7 @@ def test_local_override(runner, yadm_y, paths, run = runner(yadm_y('alt'), env=env) assert run.success assert run.err == '' - created = created_list(run.out) + created = utils.parse_alt_output(run.out, linked=False) # assert the proper creation has occurred for file_path in (utils.ALT_FILE1, utils.ALT_FILE2): @@ -90,7 +89,7 @@ def test_auto_alt(runner, yadm_y, paths, autoalt, tst_sys, run = runner(yadm_y('status'), env=env) assert run.success assert run.err == '' - created = created_list(run.out) + created = utils.parse_alt_output(run.out, linked=False) # assert the proper creation has occurred for file_path in (utils.ALT_FILE1, utils.ALT_FILE2): @@ -180,7 +179,7 @@ def test_jinja(runner, yadm_y, paths, run = runner(yadm_y('alt'), env=env) assert run.success assert run.err == '' - created = created_list(run.out) + created = utils.parse_alt_output(run.out, linked=False) # assert the proper creation has occurred for file_path in (utils.ALT_FILE1, utils.ALT_FILE2): @@ -194,14 +193,3 @@ def test_jinja(runner, yadm_y, paths, else: assert not paths.work.join(file_path).exists() assert str(paths.work.join(source_file)) not in created - - -def created_list(output): - """Parse output, and return list of created files""" - - created = dict() - for line in output.splitlines(): - match = re.match('Creating (.+) from template (.+)$', line) - if match: - created[match.group(1)] = match.group(2) - return created.values() diff --git a/test/test_unit_choose_template_cmd.py b/test/test_unit_choose_template_cmd.py new file mode 100644 index 0000000..b592331 --- /dev/null +++ b/test/test_unit_choose_template_cmd.py @@ -0,0 +1,61 @@ +"""Unit tests: choose_template_cmd""" +import pytest + + +@pytest.mark.parametrize('label', ['', 'builtin', 'other']) +@pytest.mark.parametrize('awk', [True, False], ids=['awk', 'no-awk']) +def test_kind_builtin(runner, yadm, awk, label): + """Test kind: builtin""" + + expected = 'template_builtin' + awk_avail = 'true' + + if not awk: + awk_avail = 'false' + expected = '' + + if label == 'other': + expected = '' + + script = f""" + YADM_TEST=1 source {yadm} + function awk_available {{ { awk_avail}; }} + template="$(choose_template_cmd "{label}")" + echo "TEMPLATE:$template" + """ + run = runner(command=['bash'], inp=script) + assert run.success + assert run.err == '' + assert f'TEMPLATE:{expected}\n' in run.out + + +@pytest.mark.parametrize('label', ['envtpl', 'j2cli', 'j2', 'other']) +@pytest.mark.parametrize('envtpl', [True, False], ids=['envtpl', 'no-envtpl']) +@pytest.mark.parametrize('j2cli', [True, False], ids=['j2cli', 'no-j2cli']) +def test_kind_j2cli_envtpl(runner, yadm, envtpl, j2cli, label): + """Test kind: j2 (both j2cli & envtpl) + + j2cli is preferred over envtpl if available. + """ + + envtpl_avail = 'true' if envtpl else 'false' + j2cli_avail = 'true' if j2cli else 'false' + + if label in ('j2cli', 'j2') and j2cli: + expected = 'template_j2cli' + elif label in ('envtpl', 'j2') and envtpl: + expected = 'template_envtpl' + else: + expected = '' + + script = f""" + YADM_TEST=1 source {yadm} + function envtpl_available {{ { envtpl_avail}; }} + function j2cli_available {{ { j2cli_avail}; }} + template="$(choose_template_cmd "{label}")" + echo "TEMPLATE:$template" + """ + run = runner(command=['bash'], inp=script) + assert run.success + assert run.err == '' + assert f'TEMPLATE:{expected}\n' in run.out diff --git a/test/test_unit_record_score.py b/test/test_unit_record_score.py new file mode 100644 index 0000000..3bb1b25 --- /dev/null +++ b/test/test_unit_record_score.py @@ -0,0 +1,114 @@ +"""Unit tests: record_score""" +import pytest + +INIT_VARS = """ + score=0 + local_class=testclass + local_system=testsystem + local_host=testhost + local_user=testuser + alt_scores=() + alt_filenames=() + alt_targets=() + alt_template_cmds=() +""" + +REPORT_RESULTS = """ + echo "SIZE:${#alt_scores[@]}" + echo "SCORES:${alt_scores[@]}" + echo "FILENAMES:${alt_filenames[@]}" + echo "TARGETS:${alt_targets[@]}" +""" + + +def test_dont_record_zeros(runner, yadm): + """Record nothing if the score is zero""" + + script = f""" + YADM_TEST=1 source {yadm} + {INIT_VARS} + record_score "0" "testfile" "testtarget" + {REPORT_RESULTS} + """ + run = runner(command=['bash'], inp=script) + assert run.success + assert run.err == '' + assert 'SIZE:0\n' in run.out + assert 'SCORES:\n' in run.out + assert 'FILENAMES:\n' in run.out + assert 'TARGETS:\n' in run.out + + +def test_new_scores(runner, yadm): + """Test new scores""" + + script = f""" + YADM_TEST=1 source {yadm} + {INIT_VARS} + record_score "1" "file_one" "targ_one" + record_score "2" "file_two" "targ_two" + record_score "4" "file_three" "targ_three" + {REPORT_RESULTS} + """ + run = runner(command=['bash'], inp=script) + assert run.success + assert run.err == '' + assert 'SIZE:3\n' in run.out + assert 'SCORES:1 2 4\n' in run.out + assert 'FILENAMES:file_one file_two file_three\n' in run.out + assert 'TARGETS:targ_one targ_two targ_three\n' in run.out + + +@pytest.mark.parametrize('difference', ['lower', 'equal', 'higher']) +def test_existing_scores(runner, yadm, difference): + """Test existing scores""" + + expected_score = '2' + expected_target = 'existing_target' + if difference == 'lower': + score = '1' + elif difference == 'equal': + score = '2' + else: + score = '4' + expected_score = '4' + expected_target = 'new_target' + + script = f""" + YADM_TEST=1 source {yadm} + {INIT_VARS} + alt_scores=(2) + alt_filenames=("testfile") + alt_targets=("existing_target") + record_score "{score}" "testfile" "new_target" + {REPORT_RESULTS} + """ + run = runner(command=['bash'], inp=script) + assert run.success + assert run.err == '' + assert 'SIZE:1\n' in run.out + assert f'SCORES:{expected_score}\n' in run.out + assert 'FILENAMES:testfile\n' in run.out + assert f'TARGETS:{expected_target}\n' in run.out + + +def test_existing_template(runner, yadm): + """Record nothing if a template command is registered for this file""" + + script = f""" + YADM_TEST=1 source {yadm} + {INIT_VARS} + alt_scores=(1) + alt_filenames=("testfile") + alt_targets=() + alt_template_cmds=("existing_template") + record_score "2" "testfile" "new_target" + {REPORT_RESULTS} + """ + run = runner(command=['bash'], inp=script) + assert run.success + assert run.err == '' + assert 'SIZE:1\n' in run.out + assert 'SCORES:1\n' in run.out + assert 'FILENAMES:testfile\n' in run.out + assert 'TARGETS:\n' in run.out diff --git a/test/test_unit_record_template.py b/test/test_unit_record_template.py new file mode 100644 index 0000000..f2e0a83 --- /dev/null +++ b/test/test_unit_record_template.py @@ -0,0 +1,55 @@ +"""Unit tests: record_template""" + +INIT_VARS = """ + alt_filenames=() + alt_template_cmds=() + alt_targets=() +""" + +REPORT_RESULTS = """ + echo "SIZE:${#alt_filenames[@]}" + echo "FILENAMES:${alt_filenames[@]}" + echo "CMDS:${alt_template_cmds[@]}" + echo "TARGS:${alt_targets[@]}" +""" + + +def test_new_template(runner, yadm): + """Test new template""" + + script = f""" + YADM_TEST=1 source {yadm} + {INIT_VARS} + record_template "file_one" "cmd_one" "targ_one" + record_template "file_two" "cmd_two" "targ_two" + record_template "file_three" "cmd_three" "targ_three" + {REPORT_RESULTS} + """ + run = runner(command=['bash'], inp=script) + assert run.success + assert run.err == '' + assert 'SIZE:3\n' in run.out + assert 'FILENAMES:file_one file_two file_three\n' in run.out + assert 'CMDS:cmd_one cmd_two cmd_three\n' in run.out + assert 'TARGS:targ_one targ_two targ_three\n' in run.out + + +def test_existing_template(runner, yadm): + """Overwrite existing templates""" + + script = f""" + YADM_TEST=1 source {yadm} + {INIT_VARS} + alt_filenames=("testfile") + alt_template_cmds=("existing_cmd") + alt_targets=("existing_targ") + record_template "testfile" "new_cmd" "new_targ" + {REPORT_RESULTS} + """ + run = runner(command=['bash'], inp=script) + assert run.success + assert run.err == '' + assert 'SIZE:1\n' in run.out + assert 'FILENAMES:testfile\n' in run.out + assert 'CMDS:new_cmd\n' in run.out + assert 'TARGS:new_targ\n' in run.out diff --git a/test/test_unit_score_file.py b/test/test_unit_score_file.py new file mode 100644 index 0000000..122d580 --- /dev/null +++ b/test/test_unit_score_file.py @@ -0,0 +1,229 @@ +"""Unit tests: score_file""" +import pytest + +CONDITION = { + 'default': { + 'labels': ['default'], + 'modifier': 0, + }, + 'system': { + 'labels': ['o', 'os'], + 'modifier': 1, + }, + 'class': { + 'labels': ['c', 'class'], + 'modifier': 2, + }, + 'hostname': { + 'labels': ['h', 'hostname'], + 'modifier': 4, + }, + 'user': { + 'labels': ['u', 'user'], + 'modifier': 8, + }, + } +TEMPLATE_LABELS = ['t', 'template', 'yadm'] + + +def calculate_score(filename): + """Calculate the expected score""" + # pylint: disable=too-many-branches + score = 0 + + _, conditions = filename.split('##', 1) + + for condition in conditions.split(','): + label = condition + value = None + if '.' in condition: + label, value = condition.split('.', 1) + if label in CONDITION['default']['labels']: + score += 1000 + elif label in CONDITION['system']['labels']: + if value == 'testsystem': + score += 1000 + CONDITION['system']['modifier'] + else: + return 0 + elif label in CONDITION['class']['labels']: + if value == 'testclass': + score += 1000 + CONDITION['class']['modifier'] + else: + return 0 + elif label in CONDITION['hostname']['labels']: + if value == 'testhost': + score += 1000 + CONDITION['hostname']['modifier'] + else: + return 0 + elif label in CONDITION['user']['labels']: + if value == 'testuser': + score += 1000 + CONDITION['user']['modifier'] + else: + return 0 + elif label in TEMPLATE_LABELS: + return 0 + return score + + +@pytest.mark.parametrize( + 'default', ['default', None], ids=['default', 'no-default']) +@pytest.mark.parametrize( + 'system', ['system', None], ids=['system', 'no-system']) +@pytest.mark.parametrize( + 'cla', ['class', None], ids=['class', 'no-class']) +@pytest.mark.parametrize( + 'host', ['hostname', None], ids=['hostname', 'no-host']) +@pytest.mark.parametrize( + 'user', ['user', None], ids=['user', 'no-user']) +def test_score_values( + runner, yadm, default, system, cla, host, user): + """Test score results""" + # pylint: disable=too-many-branches + local_class = 'testclass' + local_system = 'testsystem' + local_host = 'testhost' + local_user = 'testuser' + filenames = {'filename##': 0} + + if default: + for filename in list(filenames): + for label in CONDITION[default]['labels']: + newfile = filename + if not newfile.endswith('##'): + newfile += ',' + newfile += label + filenames[newfile] = calculate_score(newfile) + if system: + for filename in list(filenames): + for match in [True, False]: + for label in CONDITION[system]['labels']: + newfile = filename + if not newfile.endswith('##'): + newfile += ',' + newfile += '.'.join([ + label, + local_system if match else 'badsys' + ]) + filenames[newfile] = calculate_score(newfile) + if cla: + for filename in list(filenames): + for match in [True, False]: + for label in CONDITION[cla]['labels']: + newfile = filename + if not newfile.endswith('##'): + newfile += ',' + newfile += '.'.join([ + label, + local_class if match else 'badclass' + ]) + filenames[newfile] = calculate_score(newfile) + if host: + for filename in list(filenames): + for match in [True, False]: + for label in CONDITION[host]['labels']: + newfile = filename + if not newfile.endswith('##'): + newfile += ',' + newfile += '.'.join([ + label, + local_host if match else 'badhost' + ]) + filenames[newfile] = calculate_score(newfile) + if user: + for filename in list(filenames): + for match in [True, False]: + for label in CONDITION[user]['labels']: + newfile = filename + if not newfile.endswith('##'): + newfile += ',' + newfile += '.'.join([ + label, + local_user if match else 'baduser' + ]) + filenames[newfile] = calculate_score(newfile) + + script = f""" + YADM_TEST=1 source {yadm} + score=0 + local_class={local_class} + local_system={local_system} + local_host={local_host} + local_user={local_user} + """ + expected = '' + for filename in filenames: + script += f""" + score_file "{filename}" + echo "{filename}" + echo "$score" + """ + expected += filename + '\n' + expected += str(filenames[filename]) + '\n' + run = runner(command=['bash'], inp=script) + assert run.success + assert run.err == '' + assert run.out == expected + + +def test_score_values_templates(runner, yadm): + """Test score results""" + local_class = 'testclass' + local_system = 'testsystem' + local_host = 'testhost' + local_user = 'testuser' + filenames = {'filename##': 0} + + for filename in list(filenames): + for label in TEMPLATE_LABELS: + newfile = filename + if not newfile.endswith('##'): + newfile += ',' + newfile += '.'.join([label, 'testtemplate']) + filenames[newfile] = calculate_score(newfile) + + script = f""" + YADM_TEST=1 source {yadm} + score=0 + local_class={local_class} + local_system={local_system} + local_host={local_host} + local_user={local_user} + """ + expected = '' + for filename in filenames: + script += f""" + score_file "{filename}" + echo "{filename}" + echo "$score" + """ + expected += filename + '\n' + expected += str(filenames[filename]) + '\n' + run = runner(command=['bash'], inp=script) + assert run.success + assert run.err == '' + assert run.out == expected + + +@pytest.mark.parametrize( + 'cmd_generated', + [True, False], + ids=['supported-template', 'unsupported-template']) +def test_template_recording(runner, yadm, cmd_generated): + """Template should be recorded if choose_template_cmd outputs a command""" + + mock = 'function choose_template_cmd() { return; }' + expected = '' + if cmd_generated: + mock = 'function choose_template_cmd() { echo "test_cmd"; }' + expected = 'template recorded' + + script = f""" + YADM_TEST=1 source {yadm} + function record_template() {{ echo "template recorded"; }} + {mock} + score_file "testfile##template.kind" + """ + run = runner(command=['bash'], inp=script) + assert run.success + assert run.err == '' + assert run.out.rstrip() == expected diff --git a/test/test_unit_set_local_alt_values.py b/test/test_unit_set_local_alt_values.py index 9d34484..d3d2447 100644 --- a/test/test_unit_set_local_alt_values.py +++ b/test/test_unit_set_local_alt_values.py @@ -60,3 +60,18 @@ def test_set_local_alt_values( assert f"user='override'" in run.out else: assert f"user='{tst_user}'" in run.out + + +def test_distro(runner, yadm): + """Assert that local_distro is set""" + + script = f""" + YADM_TEST=1 source {yadm} + function query_distro() {{ echo "testdistro"; }} + set_local_alt_values + echo "distro='$local_distro'" + """ + run = runner(command=['bash'], inp=script) + assert run.success + assert run.err == '' + assert run.out.strip() == "distro='testdistro'" diff --git a/test/test_unit_template_builtin.py b/test/test_unit_template_builtin.py new file mode 100644 index 0000000..ac141e4 --- /dev/null +++ b/test/test_unit_template_builtin.py @@ -0,0 +1,98 @@ +"""Unit tests: template_builtin""" + +# these values are also testing the handling of bizarre characters +LOCAL_CLASS = "builtin_Test+@-!^Class" +LOCAL_SYSTEM = "builtin_Test+@-!^System" +LOCAL_HOST = "builtin_Test+@-!^Host" +LOCAL_USER = "builtin_Test+@-!^User" +LOCAL_DISTRO = "builtin_Test+@-!^Distro" +TEMPLATE = f''' +start of template +builtin class = >YADM_CLASS< +builtin os = >YADM_OS< +builtin host = >YADM_HOSTNAME< +builtin user = >YADM_USER< +builtin distro = >YADM_DISTRO< +YADM_IF CLASS="wrongclass1" +wrong class 1 +YADM_END +YADM_IF CLASS="{LOCAL_CLASS}" +Included section for class = YADM_CLASS (YADM_CLASS repeated) +YADM_END +YADM_IF CLASS="wrongclass2" +wrong class 2 +YADM_END +YADM_IF OS="wrongos1" +wrong os 1 +YADM_END +YADM_IF OS="{LOCAL_SYSTEM}" +Included section for os = YADM_OS (YADM_OS repeated) +YADM_END +YADM_IF OS="wrongos2" +wrong os 2 +YADM_END +YADM_IF HOSTNAME="wronghost1" +wrong host 1 +YADM_END +YADM_IF HOSTNAME="{LOCAL_HOST}" +Included section for host = YADM_HOSTNAME (YADM_HOSTNAME repeated) +YADM_END +YADM_IF HOSTNAME="wronghost2" +wrong host 2 +YADM_END +YADM_IF USER="wronguser1" +wrong user 1 +YADM_END +YADM_IF USER="{LOCAL_USER}" +Included section for user = YADM_USER (YADM_USER repeated) +YADM_END +YADM_IF USER="wronguser2" +wrong user 2 +YADM_END +YADM_IF DISTRO="wrongdistro1" +wrong distro 1 +YADM_END +YADM_IF DISTRO="{LOCAL_DISTRO}" +Included section for distro = YADM_DISTRO (YADM_DISTRO repeated) +YADM_END +YADM_IF DISTRO="wrongdistro2" +wrong distro 2 +YADM_END +end of template +''' +EXPECTED = f''' +start of template +builtin class = >{LOCAL_CLASS}< +builtin os = >{LOCAL_SYSTEM}< +builtin host = >{LOCAL_HOST}< +builtin user = >{LOCAL_USER}< +builtin distro = >{LOCAL_DISTRO}< +Included section for class = {LOCAL_CLASS} ({LOCAL_CLASS} repeated) +Included section for os = {LOCAL_SYSTEM} ({LOCAL_SYSTEM} repeated) +Included section for host = {LOCAL_HOST} ({LOCAL_HOST} repeated) +Included section for user = {LOCAL_USER} ({LOCAL_USER} repeated) +Included section for distro = {LOCAL_DISTRO} ({LOCAL_DISTRO} repeated) +end of template +''' + + +def test_template_builtin(runner, yadm, tmpdir): + """Test template_builtin""" + + input_file = tmpdir.join('input') + input_file.write(TEMPLATE, ensure=True) + output_file = tmpdir.join('output') + + script = f""" + YADM_TEST=1 source {yadm} + local_class="{LOCAL_CLASS}" + local_system="{LOCAL_SYSTEM}" + local_host="{LOCAL_HOST}" + local_user="{LOCAL_USER}" + local_distro="{LOCAL_DISTRO}" + template_builtin "{input_file}" "{output_file}" + """ + run = runner(command=['bash'], inp=script) + assert run.success + assert run.err == '' + assert output_file.read() == EXPECTED diff --git a/test/test_unit_template_j2.py b/test/test_unit_template_j2.py new file mode 100644 index 0000000..e511576 --- /dev/null +++ b/test/test_unit_template_j2.py @@ -0,0 +1,99 @@ +"""Unit tests: template_j2cli & template_envtpl""" +import pytest + +LOCAL_CLASS = "j2_Test+@-!^Class" +LOCAL_SYSTEM = "j2_Test+@-!^System" +LOCAL_HOST = "j2_Test+@-!^Host" +LOCAL_USER = "j2_Test+@-!^User" +LOCAL_DISTRO = "j2_Test+@-!^Distro" +TEMPLATE = f''' +start of template +j2 class = >{{{{YADM_CLASS}}}}< +j2 os = >{{{{YADM_OS}}}}< +j2 host = >{{{{YADM_HOSTNAME}}}}< +j2 user = >{{{{YADM_USER}}}}< +j2 distro = >{{{{YADM_DISTRO}}}}< +{{%- if YADM_CLASS == "wrongclass1" %}} +wrong class 1 +{{%- endif %}} +{{%- if YADM_CLASS == "{LOCAL_CLASS}" %}} +Included section for class = {{{{YADM_CLASS}}}} ({{{{YADM_CLASS}}}} repeated) +{{%- endif %}} +{{%- if YADM_CLASS == "wrongclass2" %}} +wrong class 2 +{{%- endif %}} +{{%- if YADM_OS == "wrongos1" %}} +wrong os 1 +{{%- endif %}} +{{%- if YADM_OS == "{LOCAL_SYSTEM}" %}} +Included section for os = {{{{YADM_OS}}}} ({{{{YADM_OS}}}} repeated) +{{%- endif %}} +{{%- if YADM_OS == "wrongos2" %}} +wrong os 2 +{{%- endif %}} +{{%- if YADM_HOSTNAME == "wronghost1" %}} +wrong host 1 +{{%- endif %}} +{{%- if YADM_HOSTNAME == "{LOCAL_HOST}" %}} +Included section for host = {{{{YADM_HOSTNAME}}}} ({{{{YADM_HOSTNAME}}}} again) +{{%- endif %}} +{{%- if YADM_HOSTNAME == "wronghost2" %}} +wrong host 2 +{{%- endif %}} +{{%- if YADM_USER == "wronguser1" %}} +wrong user 1 +{{%- endif %}} +{{%- if YADM_USER == "{LOCAL_USER}" %}} +Included section for user = {{{{YADM_USER}}}} ({{{{YADM_USER}}}} repeated) +{{%- endif %}} +{{%- if YADM_USER == "wronguser2" %}} +wrong user 2 +{{%- endif %}} +{{%- if YADM_DISTRO == "wrongdistro1" %}} +wrong distro 1 +{{%- endif %}} +{{%- if YADM_DISTRO == "{LOCAL_DISTRO}" %}} +Included section for distro = {{{{YADM_DISTRO}}}} ({{{{YADM_DISTRO}}}} again) +{{%- endif %}} +{{%- if YADM_DISTRO == "wrongdistro2" %}} +wrong distro 2 +{{%- endif %}} +end of template +''' +EXPECTED = f''' +start of template +j2 class = >{LOCAL_CLASS}< +j2 os = >{LOCAL_SYSTEM}< +j2 host = >{LOCAL_HOST}< +j2 user = >{LOCAL_USER}< +j2 distro = >{LOCAL_DISTRO}< +Included section for class = {LOCAL_CLASS} ({LOCAL_CLASS} repeated) +Included section for os = {LOCAL_SYSTEM} ({LOCAL_SYSTEM} repeated) +Included section for host = {LOCAL_HOST} ({LOCAL_HOST} again) +Included section for user = {LOCAL_USER} ({LOCAL_USER} repeated) +Included section for distro = {LOCAL_DISTRO} ({LOCAL_DISTRO} again) +end of template +''' + + +@pytest.mark.parametrize('processor', ('j2cli', 'envtpl')) +def test_template_j2(runner, yadm, tmpdir, processor): + """Test processing by j2cli & envtpl""" + + input_file = tmpdir.join('input') + input_file.write(TEMPLATE, ensure=True) + output_file = tmpdir.join('output') + + script = f""" + YADM_TEST=1 source {yadm} + local_class="{LOCAL_CLASS}" + local_system="{LOCAL_SYSTEM}" + local_host="{LOCAL_HOST}" + local_user="{LOCAL_USER}" + local_distro="{LOCAL_DISTRO}" + template_{processor} "{input_file}" "{output_file}" + """ + run = runner(command=['bash'], inp=script) + assert run.success + assert run.err == '' + assert output_file.read() == EXPECTED diff --git a/test/utils.py b/test/utils.py index 0b52b37..00a0fdd 100644 --- a/test/utils.py +++ b/test/utils.py @@ -3,6 +3,7 @@ This module holds values/functions common to multiple tests. """ +import re import os ALT_FILE1 = 'test_alt' @@ -55,7 +56,7 @@ def create_alt_files(paths, suffix, # Do not test directory support for jinja alternates test_paths = [new_file1, new_file2] test_names = [ALT_FILE1, ALT_FILE2] - if suffix != '##yadm.j2': + if not re.match(r'##(t|template|yadm)\.', suffix): test_paths += [new_dir] test_names += [ALT_DIR] @@ -69,6 +70,22 @@ def create_alt_files(paths, suffix, _create_encrypt(encrypt, test_names, suffix, paths, exclude) +def parse_alt_output(output, linked=True): + """Parse output of 'alt', and return list of linked files""" + regex = r'Creating (.+) from template (.+)$' + if linked: + regex = r'Linking (.+) to (.+)$' + parsed_list = dict() + for line in output.splitlines(): + match = re.match(regex, line) + if match: + if linked: + parsed_list[match.group(2)] = match.group(1) + else: + parsed_list[match.group(1)] = match.group(2) + return parsed_list.values() + + def _create_includefiles(includefile, paths, test_paths): if includefile: for dpath in INCLUDE_DIRS: diff --git a/yadm b/yadm index 145d249..f7cab19 100755 --- a/yadm +++ b/yadm @@ -39,6 +39,8 @@ FULL_COMMAND="" GPG_PROGRAM="gpg" GIT_PROGRAM="git" +AWK_PROGRAM="awk" +J2CLI_PROGRAM="j2" ENVTPL_PROGRAM="envtpl" LSB_RELEASE_PROGRAM="lsb_release" @@ -128,6 +130,225 @@ function main() { } + +# ****** Alternate Processing ****** + +function score_file() { + target="$1" + filename="${target%%##*}" + conditions="${target#*##}" + score=0 + IFS=',' read -ra fields <<< "$conditions" + for field in "${fields[@]}"; do + label=${field%%.*} + value=${field#*.} + score=$((score + 1000)) + # default condition + if [[ "$label" =~ ^(default)$ ]]; then + score=$((score + 0)) + # variable conditions + elif [[ "$label" =~ ^(o|os)$ ]]; then + if [ "$value" = "$local_system" ]; then + score=$((score + 1)) + else + score=0 + return + fi + elif [[ "$label" =~ ^(c|class)$ ]]; then + if [ "$value" = "$local_class" ]; then + score=$((score + 2)) + else + score=0 + return + fi + elif [[ "$label" =~ ^(h|hostname)$ ]]; then + if [ "$value" = "$local_host" ]; then + score=$((score + 4)) + else + score=0 + return + fi + elif [[ "$label" =~ ^(u|user)$ ]]; then + if [ "$value" = "$local_user" ]; then + score=$((score + 8)) + else + score=0 + return + fi + # templates + elif [[ "$label" =~ ^(t|template|yadm)$ ]]; then + score=0 + cmd=$(choose_template_cmd "$value") + if [ -n "$cmd" ]; then + record_template "$filename" "$cmd" "$target" + else + debug "No supported template processor for template $target" + [ -n "$loud" ] && echo "No supported template processor for template $target" + fi + return 0 + # unsupported values + else + score=0 + return + fi + done + + record_score "$score" "$filename" "$target" +} + +function record_score() { + score="$1" + filename="$2" + target="$3" + + # record nothing if the score is zero + [ "$score" -eq 0 ] && return + + # search for the index of this filename, to see if we already are tracking it + index=-1 + for search_index in "${!alt_filenames[@]}"; do + if [ "${alt_filenames[$search_index]}" = "$filename" ]; then + index="$search_index" + break + fi + done + # if we don't find an existing index, create one by appending to the array + if [ "$index" -eq -1 ]; then + alt_filenames+=("$filename") + # set index to the last index (newly created one) + for index in "${!alt_filenames[@]}"; do :; done + # and set its initial score to zero + alt_scores[$index]=0 + fi + + # record nothing if a template command is registered for this file + [ "${alt_template_cmds[$index]+isset}" ] && return + + # record higher scoring targets + if [ "$score" -gt "${alt_scores[$index]}" ]; then + alt_scores[$index]="$score" + alt_targets[$index]="$target" + fi + +} + +function record_template() { + filename="$1" + cmd="$2" + target="$3" + + # search for the index of this filename, to see if we already are tracking it + index=-1 + for search_index in "${!alt_filenames[@]}"; do + if [ "${alt_filenames[$search_index]}" = "$filename" ]; then + index="$search_index" + break + fi + done + # if we don't find an existing index, create one by appending to the array + if [ "$index" -eq -1 ]; then + alt_filenames+=("$filename") + # set index to the last index (newly created one) + for index in "${!alt_filenames[@]}"; do :; done + fi + + # record the template command, last one wins + alt_template_cmds[$index]="$cmd" + alt_targets[$index]="$target" + +} + +function choose_template_cmd() { + kind="$1" + + if [ "$kind" = "builtin" ] || [ "$kind" = "" ] && awk_available; then + echo "template_builtin" + elif [ "$kind" = "j2cli" ] || [ "$kind" = "j2" ] && j2cli_available; then + echo "template_j2cli" + elif [ "$kind" = "envtpl" ] || [ "$kind" = "j2" ] && envtpl_available; then + echo "template_envtpl" + else + return # this "kind" of template is not supported + fi + +} + +# ****** Template Processors ****** + +function template_builtin() { + input="$1" + output="$2" + + awk_pgm=$(cat << "EOF" +# built-in template processor +BEGIN { + c["CLASS"] = class + c["OS"] = os + c["HOSTNAME"] = host + c["USER"] = user + c["DISTRO"] = distro + valid = conditions() + end = "^YADM_END$" + skip = "^YADM_(IF|END$)" +} +{ replace_vars() } # variable replacements +$0 ~ valid, $0 ~ end { if ($0 ~ skip) next } # valid conditional blocks +/^YADM_IF/, $0 ~ end { next } # invalid conditional blocks +{ print } + +function replace_vars() { + for (label in c) { + gsub(("YADM_" label), c[label]) + } +} + +function conditions() { + pattern = "^(" + for (label in c) { + value = c[label] + gsub(/[\\.^$(){}\[\]|*+?]/, "\\\\&", value) + pattern = sprintf("%sYADM_IF +%s *= *\"%s\"|", pattern, label, value) + } + sub(/\|$/,")",pattern) + return pattern +} +EOF + ) + + "$AWK_PROGRAM" \ + -v class="$local_class" \ + -v os="$local_system" \ + -v host="$local_host" \ + -v user="$local_user" \ + -v distro="$local_distro" \ + "$awk_pgm" \ + "$input" > "$output" +} + +function template_j2cli() { + input="$1" + output="$2" + + YADM_CLASS="$local_class" \ + YADM_OS="$local_system" \ + YADM_HOSTNAME="$local_host" \ + YADM_USER="$local_user" \ + YADM_DISTRO="$local_distro" \ + "$J2CLI_PROGRAM" "$input" -o "$output" +} + +function template_envtpl() { + input="$1" + output="$2" + + YADM_CLASS="$local_class" \ + YADM_OS="$local_system" \ + YADM_HOSTNAME="$local_host" \ + YADM_USER="$local_user" \ + YADM_DISTRO="$local_distro" \ + "$ENVTPL_PROGRAM" --keep-template "$input" -o "$output" +} + # ****** yadm Commands ****** function alt() { @@ -140,6 +361,7 @@ function alt() { local local_system local local_host local local_user + local local_distro set_local_alt_values # only be noisy if the "alt" command was run directly @@ -220,12 +442,54 @@ function set_local_alt_values() { local_user=$(id -u -n) fi + local_distro="$(query_distro)" + } function alt_future_linking() { + local alt_scores + local alt_filenames + local alt_targets + local alt_template_cmds + alt_scores=() + alt_filenames=() + alt_targets=() + alt_template_cmds=() - return + for alt_path in $(for tracked in "${tracked_files[@]}"; do printf "%s\n" "$tracked" "${tracked%/*}"; done | LC_ALL=C sort -u) "${ENCRYPT_INCLUDE_FILES[@]}"; do + alt_path="$YADM_WORK/$alt_path" + if [[ "$alt_path" =~ .\#\#. ]]; then + if [ -e "$alt_path" ] ; then + score_file "$alt_path" + fi + fi + done + + for index in "${!alt_filenames[@]}"; do + filename="${alt_filenames[$index]}" + target="${alt_targets[$index]}" + template_cmd="${alt_template_cmds[$index]}" + if [ -n "$template_cmd" ]; then + # a template is defined, process the template + debug "Creating $filename from template $target" + [ -n "$loud" ] && echo "Creating $filename from template $target" + "$template_cmd" "$target" "$filename" + elif [ -n "$target" ]; then + # a link target is defined, create symlink + debug "Linking $target to $filename" + [ -n "$loud" ] && echo "Linking $target to $filename" + if [ "$do_copy" -eq 1 ]; then + if [ -L "$filename" ]; then + rm -f "$filename" + fi + cp -f "$target" "$filename" + else + ln -nfs "$target" "$filename" + alt_linked+=("$target") + fi + fi + done } @@ -290,7 +554,7 @@ function alt_past_linking() { YADM_OS="$local_system" \ YADM_HOSTNAME="$local_host" \ YADM_USER="$local_user" \ - YADM_DISTRO=$(query_distro) \ + YADM_DISTRO="$local_distro" \ "$ENVTPL_PROGRAM" --keep-template "$tracked_file" -o "$real_file" else debug "envtpl not available, not creating $real_file from template $tracked_file" @@ -1204,6 +1468,14 @@ function bootstrap_available() { [ -f "$YADM_BOOTSTRAP" ] && [ -x "$YADM_BOOTSTRAP" ] && return return 1 } +function awk_available() { + command -v "$AWK_PROGRAM" >/dev/null 2>&1 && return + return 1 +} +function j2cli_available() { + command -v "$J2CLI_PROGRAM" >/dev/null 2>&1 && return + return 1 +} function envtpl_available() { command -v "$ENVTPL_PROGRAM" >/dev/null 2>&1 && return return 1