diff --git a/.travis.yml b/.travis.yml index e340cd2..5683e96 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,6 +3,6 @@ language: minimal services: - docker before_install: - - docker pull yadm/testbed:2020-01-20 + - docker pull yadm/testbed:2020-07-08 script: - - docker run -t --rm -v "$PWD:/yadm:ro" yadm/testbed:2020-01-20 + - docker run -t --rm -v "$PWD:/yadm:ro" yadm/testbed:2020-07-08 diff --git a/Dockerfile b/Dockerfile index 469fe54..f686108 100644 --- a/Dockerfile +++ b/Dockerfile @@ -36,6 +36,10 @@ RUN pip3 install \ yamllint==1.17.0 \ ; +RUN \ + curl https://raw.githubusercontent.com/jirutka/esh/v0.3.0/esh > /usr/local/bin/esh && \ + chmod +x /usr/local/bin/esh + # Create a flag to identify when running inside the yadm testbed RUN touch /.yadmtestbed diff --git a/Makefile b/Makefile index c9509a1..e7895be 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,5 @@ PYTESTS = $(wildcard test/test_*.py) +IMAGE = yadm/testbed:2020-07-08 .PHONY: all all: @@ -112,7 +113,7 @@ testhost: require-docker --hostname testhost \ --rm -it \ -v "/tmp/testhost:/bin/yadm:ro" \ - yadm/testbed:2020-01-20 \ + $(IMAGE) \ bash -l .PHONY: scripthost @@ -129,7 +130,7 @@ scripthost: require-docker --rm -it \ -v "$$PWD/script.gz:/script.gz:rw" \ -v "/tmp/testhost:/bin/yadm:ro" \ - yadm/testbed:2020-01-20 \ + $(IMAGE) \ bash -c "script /tmp/script -q -c 'bash -l'; gzip < /tmp/script > /script.gz" @echo @echo "Script saved to $$PWD/script.gz" diff --git a/docker-compose.yml b/docker-compose.yml index 831a062..60458d5 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -4,4 +4,4 @@ services: testbed: volumes: - .:/yadm:ro - image: yadm/testbed:2020-01-20 + image: yadm/testbed:2020-07-08 diff --git a/test/test_alt.py b/test/test_alt.py index 359f32d..96c3129 100644 --- a/test/test_alt.py +++ b/test/test_alt.py @@ -126,7 +126,7 @@ def test_alt_conditions( @pytest.mark.usefixtures('ds1_copy') @pytest.mark.parametrize( - 'kind', ['default', '', None, 'envtpl', 'j2cli', 'j2']) + 'kind', ['default', '', None, 'envtpl', 'j2cli', 'j2', 'esh']) @pytest.mark.parametrize('label', ['t', 'template', 'yadm', ]) def test_alt_templates( runner, paths, kind, label): diff --git a/test/test_unit_record_score.py b/test/test_unit_record_score.py index 525c967..78596e1 100644 --- a/test/test_unit_record_score.py +++ b/test/test_unit_record_score.py @@ -112,3 +112,30 @@ def test_existing_template(runner, yadm): assert 'SCORES:1\n' in run.out assert 'TARGETS:testtgt\n' in run.out assert 'SOURCES:\n' in run.out + + +def test_config_first(runner, yadm): + """Verify YADM_CONFIG is always processed first""" + + config = 'yadm_config_file' + script = f""" + YADM_TEST=1 source {yadm} + {INIT_VARS} + YADM_CONFIG={config} + record_score "1" "tgt_before" "src_before" + record_template "tgt_tmp" "cmd_tmp" "src_tmp" + record_score "2" "{config}" "src_config" + record_score "3" "tgt_after" "src_after" + {REPORT_RESULTS} + echo "CMD_VALUE:${{alt_template_cmds[@]}}" + echo "CMD_INDEX:${{!alt_template_cmds[@]}}" + """ + run = runner(command=['bash'], inp=script) + assert run.success + assert run.err == '' + assert 'SIZE:3\n' in run.out + assert 'SCORES:2 1 3\n' in run.out + assert f'TARGETS:{config} tgt_before tgt_tmp tgt_after\n' in run.out + assert 'SOURCES:src_config src_before src_tmp src_after\n' in run.out + assert 'CMD_VALUE:cmd_tmp\n' in run.out + assert 'CMD_INDEX:2\n' in run.out diff --git a/test/test_unit_template_esh.py b/test/test_unit_template_esh.py new file mode 100644 index 0000000..c290b5f --- /dev/null +++ b/test/test_unit_template_esh.py @@ -0,0 +1,114 @@ +"""Unit tests: template_esh""" + +LOCAL_CLASS = "esh_Test+@-!^Class" +LOCAL_SYSTEM = "esh_Test+@-!^System" +LOCAL_HOST = "esh_Test+@-!^Host" +LOCAL_USER = "esh_Test+@-!^User" +LOCAL_DISTRO = "esh_Test+@-!^Distro" +TEMPLATE = f''' +start of template +esh class = ><%=$YADM_CLASS%>< +esh os = ><%=$YADM_OS%>< +esh host = ><%=$YADM_HOSTNAME%>< +esh user = ><%=$YADM_USER%>< +esh distro = ><%=$YADM_DISTRO%>< +<% if [ "$YADM_CLASS" = "wrongclass1" ]; then -%> +wrong class 1 +<% fi -%> +<% if [ "$YADM_CLASS" = "{LOCAL_CLASS}" ]; then -%> +Included section for class = <%=$YADM_CLASS%> (<%=$YADM_CLASS%> repeated) +<% fi -%> +<% if [ "$YADM_CLASS" = "wrongclass2" ]; then -%> +wrong class 2 +<% fi -%> +<% if [ "$YADM_OS" = "wrongos1" ]; then -%> +wrong os 1 +<% fi -%> +<% if [ "$YADM_OS" = "{LOCAL_SYSTEM}" ]; then -%> +Included section for os = <%=$YADM_OS%> (<%=$YADM_OS%> repeated) +<% fi -%> +<% if [ "$YADM_OS" = "wrongos2" ]; then -%> +wrong os 2 +<% fi -%> +<% if [ "$YADM_HOSTNAME" = "wronghost1" ]; then -%> +wrong host 1 +<% fi -%> +<% if [ "$YADM_HOSTNAME" = "{LOCAL_HOST}" ]; then -%> +Included section for host = <%=$YADM_HOSTNAME%> (<%=$YADM_HOSTNAME%> again) +<% fi -%> +<% if [ "$YADM_HOSTNAME" = "wronghost2" ]; then -%> +wrong host 2 +<% fi -%> +<% if [ "$YADM_USER" = "wronguser1" ]; then -%> +wrong user 1 +<% fi -%> +<% if [ "$YADM_USER" = "{LOCAL_USER}" ]; then -%> +Included section for user = <%=$YADM_USER%> (<%=$YADM_USER%> repeated) +<% fi -%> +<% if [ "$YADM_USER" = "wronguser2" ]; then -%> +wrong user 2 +<% fi -%> +<% if [ "$YADM_DISTRO" = "wrongdistro1" ]; then -%> +wrong distro 1 +<% fi -%> +<% if [ "$YADM_DISTRO" = "{LOCAL_DISTRO}" ]; then -%> +Included section for distro = <%=$YADM_DISTRO%> (<%=$YADM_DISTRO%> again) +<% fi -%> +<% if [ "$YADM_DISTRO" = "wrongdistro2" ]; then -%> +wrong distro 2 +<% fi -%> +end of template +''' +EXPECTED = f''' +start of template +esh class = >{LOCAL_CLASS}< +esh os = >{LOCAL_SYSTEM}< +esh host = >{LOCAL_HOST}< +esh user = >{LOCAL_USER}< +esh distro = >{LOCAL_DISTRO}< +Included section for class = {LOCAL_CLASS} ({LOCAL_CLASS} repeated) +Included section for os = {LOCAL_SYSTEM} ({LOCAL_SYSTEM} repeated) +Included section for host = {LOCAL_HOST} ({LOCAL_HOST} again) +Included section for user = {LOCAL_USER} ({LOCAL_USER} repeated) +Included section for distro = {LOCAL_DISTRO} ({LOCAL_DISTRO} again) +end of template +''' + + +def test_template_esh(runner, yadm, tmpdir): + """Test processing by esh""" + + input_file = tmpdir.join('input') + input_file.write(TEMPLATE, ensure=True) + output_file = tmpdir.join('output') + + script = f""" + YADM_TEST=1 source {yadm} + local_class="{LOCAL_CLASS}" + local_system="{LOCAL_SYSTEM}" + local_host="{LOCAL_HOST}" + local_user="{LOCAL_USER}" + local_distro="{LOCAL_DISTRO}" + template_esh "{input_file}" "{output_file}" + """ + run = runner(command=['bash'], inp=script) + assert run.success + assert run.err == '' + assert output_file.read().strip() == str(EXPECTED).strip() + + +def test_source(runner, yadm, tmpdir): + """Test YADM_SOURCE""" + + input_file = tmpdir.join('input') + input_file.write('<%= $YADM_SOURCE %>', ensure=True) + output_file = tmpdir.join('output') + + script = f""" + YADM_TEST=1 source {yadm} + template_esh "{input_file}" "{output_file}" + """ + run = runner(command=['bash'], inp=script) + assert run.success + assert run.err == '' + assert output_file.read().strip() == str(input_file) diff --git a/yadm b/yadm index 28bfce2..4b43e4f 100755 --- a/yadm +++ b/yadm @@ -45,6 +45,7 @@ GIT_CRYPT_PROGRAM="git-crypt" TRANSCRYPT_PROGRAM="transcrypt" J2CLI_PROGRAM="j2" ENVTPL_PROGRAM="envtpl" +ESH_PROGRAM="esh" LSB_RELEASE_PROGRAM="lsb_release" OS_RELEASE="/etc/os-release" @@ -250,11 +251,28 @@ function record_score() { done # if we don't find an existing index, create one by appending to the array if [ "$index" -eq -1 ]; then - alt_targets+=("$tgt") - # set index to the last index (newly created one) - for index in "${!alt_targets[@]}"; do :; done - # and set its initial score to zero - alt_scores[$index]=0 + # $YADM_CONFIG must be processed first, in case other templates lookup yadm configurations + if [ "$tgt" = "$YADM_CONFIG" ]; then + alt_targets=("$tgt" "${alt_targets[@]}") + alt_sources=("$src" "${alt_sources[@]}") + alt_scores=(0 "${alt_scores[@]}") + index=0 + # increase the index of any existing alt_template_cmds + new_cmds=() + for cmd_index in "${!alt_template_cmds[@]}"; do + new_cmds[$((cmd_index+1))]="${alt_template_cmds[$cmd_index]}" + done + alt_template_cmds=() + for cmd_index in "${!new_cmds[@]}"; do + alt_template_cmds[$cmd_index]="${new_cmds[$cmd_index]}" + done + else + alt_targets+=("$tgt") + # set index to the last index (newly created one) + for index in "${!alt_targets[@]}"; do :; done + # and set its initial score to zero + alt_scores[$index]=0 + fi fi # record nothing if a template command is registered for this file @@ -299,6 +317,8 @@ function choose_template_cmd() { if [ "$kind" = "default" ] || [ "$kind" = "" ] && awk_available; then echo "template_default" + elif [ "$kind" = "esh" ] && esh_available; then + echo "template_esh" elif [ "$kind" = "j2cli" ] || [ "$kind" = "j2" ] && j2cli_available; then echo "template_j2cli" elif [ "$kind" = "envtpl" ] || [ "$kind" = "j2" ] && envtpl_available; then @@ -407,6 +427,22 @@ function template_envtpl() { [ -f "$temp_file" ] && mv -f "$temp_file" "$output" } +function template_esh() { + input="$1" + output="$2" + temp_file="${output}.$$.$RANDOM" + + "$ESH_PROGRAM" -o "$temp_file" "$input" \ + YADM_CLASS="$local_class" \ + YADM_OS="$local_system" \ + YADM_HOSTNAME="$local_host" \ + YADM_USER="$local_user" \ + YADM_DISTRO="$local_distro" \ + YADM_SOURCE="$input" + + [ -f "$temp_file" ] && mv -f "$temp_file" "$output" +} + # ****** yadm Commands ****** function alt() { @@ -1938,6 +1974,10 @@ function envtpl_available() { command -v "$ENVTPL_PROGRAM" &> /dev/null && return return 1 } +function esh_available() { + command -v "$ESH_PROGRAM" &> /dev/null && return + return 1 +} function readlink_available() { command -v "readlink" &> /dev/null && return return 1 diff --git a/yadm.1 b/yadm.1 index 0e48376..37dfacc 100644 --- a/yadm.1 +++ b/yadm.1 @@ -637,6 +637,15 @@ upon which is available on most *nix systems. To use this processor, specify the value of "default" or just leave the value off (e.g. "##template"). .TP +.B ESH +ESH is a template processor written in POSIX compliant shell. It allows +executing shell commands within templates. This can be used to reference your +own configurations within templates, for example: + + <% yadm config mysection.myconfig %> + +To use the ESH template processor, specify the value of "esh" +.TP .B j2cli To use the j2cli Jinja template processor, specify the value of "j2" or "j2cli". @@ -654,7 +663,7 @@ to create or overwrite files. During processing, the following variables are available in the template: - Default Jinja Description + Default Jinja or ESH Description ------------- ------------- -------------------------- yadm.class YADM_CLASS Locally defined yadm class yadm.distro YADM_DISTRO lsb_release -si @@ -702,6 +711,16 @@ would look like: config=dev-whatever {% endif -%} +An equivalent ESH templated named +.I whatever##template.esh +would look like: + + <% if [ "$YADM_USER" = "harvey" ]; then -%> + config=<%= $YADM_CLASS %>-<%= $YADM_OS %> + <% else -%> + config=dev-whatever + <% fi -%> + .SH ENCRYPTION It can be useful to manage confidential files, like SSH or GPG keys, across