Merge pull request #220

.travis.yml

@@ -3,6 +3,6 @@ language: minimal
 services:
   - docker
 before_install:
-  - docker pull yadm/testbed:2020-01-20
+  - docker pull yadm/testbed:2020-07-08
 script:
-  - docker run -t --rm -v "$PWD:/yadm:ro" yadm/testbed:2020-01-20
+  - docker run -t --rm -v "$PWD:/yadm:ro" yadm/testbed:2020-07-08

Dockerfile

@@ -36,6 +36,10 @@ RUN pip3 install \
       yamllint==1.17.0 \
     ;
 
+RUN \
+  curl https://raw.githubusercontent.com/jirutka/esh/v0.3.0/esh > /usr/local/bin/esh && \
+  chmod +x /usr/local/bin/esh
+
 # Create a flag to identify when running inside the yadm testbed
 RUN touch /.yadmtestbed
 

Makefile

@@ -1,4 +1,5 @@
 PYTESTS = $(wildcard test/test_*.py)
+IMAGE = yadm/testbed:2020-07-08
 
 .PHONY: all
 all:
@@ -112,7 +113,7 @@ testhost: require-docker
 		--hostname testhost \
 		--rm -it \
 		-v "/tmp/testhost:/bin/yadm:ro" \
-		yadm/testbed:2020-01-20 \
+		$(IMAGE) \
 		bash -l
 
 .PHONY: scripthost
@@ -129,7 +130,7 @@ scripthost: require-docker
 		--rm -it \
 		-v "$$PWD/script.gz:/script.gz:rw" \
 		-v "/tmp/testhost:/bin/yadm:ro" \
-		yadm/testbed:2020-01-20 \
+		$(IMAGE) \
 		bash -c "script /tmp/script -q -c 'bash -l'; gzip < /tmp/script > /script.gz"
 	@echo
 	@echo "Script saved to $$PWD/script.gz"

docker-compose.yml

@@ -4,4 +4,4 @@ services:
   testbed:
     volumes:
       - .:/yadm:ro
-    image: yadm/testbed:2020-01-20
+    image: yadm/testbed:2020-07-08

test/test_alt.py

@@ -126,7 +126,7 @@ def test_alt_conditions(
 
 @pytest.mark.usefixtures('ds1_copy')
 @pytest.mark.parametrize(
-    'kind', ['default', '', None, 'envtpl', 'j2cli', 'j2'])
+    'kind', ['default', '', None, 'envtpl', 'j2cli', 'j2', 'esh'])
 @pytest.mark.parametrize('label', ['t', 'template', 'yadm', ])
 def test_alt_templates(
         runner, paths, kind, label):

test/test_unit_record_score.py

@@ -112,3 +112,30 @@ def test_existing_template(runner, yadm):
     assert 'SCORES:1\n' in run.out
     assert 'TARGETS:testtgt\n' in run.out
     assert 'SOURCES:\n' in run.out
+
+
+def test_config_first(runner, yadm):
+    """Verify YADM_CONFIG is always processed first"""
+
+    config = 'yadm_config_file'
+    script = f"""
+        YADM_TEST=1 source {yadm}
+        {INIT_VARS}
+        YADM_CONFIG={config}
+        record_score "1" "tgt_before" "src_before"
+        record_template "tgt_tmp" "cmd_tmp" "src_tmp"
+        record_score "2" "{config}"   "src_config"
+        record_score "3" "tgt_after"  "src_after"
+        {REPORT_RESULTS}
+        echo "CMD_VALUE:${{alt_template_cmds[@]}}"
+        echo "CMD_INDEX:${{!alt_template_cmds[@]}}"
+    """
+    run = runner(command=['bash'], inp=script)
+    assert run.success
+    assert run.err == ''
+    assert 'SIZE:3\n' in run.out
+    assert 'SCORES:2 1 3\n' in run.out
+    assert f'TARGETS:{config} tgt_before tgt_tmp tgt_after\n' in run.out
+    assert 'SOURCES:src_config src_before src_tmp src_after\n' in run.out
+    assert 'CMD_VALUE:cmd_tmp\n' in run.out
+    assert 'CMD_INDEX:2\n' in run.out

test/test_unit_template_esh.py

@@ -0,0 +1,114 @@
+"""Unit tests: template_esh"""
+
+LOCAL_CLASS = "esh_Test+@-!^Class"
+LOCAL_SYSTEM = "esh_Test+@-!^System"
+LOCAL_HOST = "esh_Test+@-!^Host"
+LOCAL_USER = "esh_Test+@-!^User"
+LOCAL_DISTRO = "esh_Test+@-!^Distro"
+TEMPLATE = f'''
+start of template
+esh class  = ><%=$YADM_CLASS%><
+esh os     = ><%=$YADM_OS%><
+esh host   = ><%=$YADM_HOSTNAME%><
+esh user   = ><%=$YADM_USER%><
+esh distro = ><%=$YADM_DISTRO%><
+<% if [ "$YADM_CLASS" = "wrongclass1" ]; then -%>
+wrong class 1
+<% fi -%>
+<% if [ "$YADM_CLASS" = "{LOCAL_CLASS}" ]; then -%>
+Included section for class = <%=$YADM_CLASS%> (<%=$YADM_CLASS%> repeated)
+<% fi -%>
+<% if [ "$YADM_CLASS" = "wrongclass2" ]; then -%>
+wrong class 2
+<% fi -%>
+<% if [ "$YADM_OS" = "wrongos1" ]; then -%>
+wrong os 1
+<% fi -%>
+<% if [ "$YADM_OS" = "{LOCAL_SYSTEM}" ]; then -%>
+Included section for os = <%=$YADM_OS%> (<%=$YADM_OS%> repeated)
+<% fi -%>
+<% if [ "$YADM_OS" = "wrongos2" ]; then -%>
+wrong os 2
+<% fi -%>
+<% if [ "$YADM_HOSTNAME" = "wronghost1" ]; then -%>
+wrong host 1
+<% fi -%>
+<% if [ "$YADM_HOSTNAME" = "{LOCAL_HOST}" ]; then -%>
+Included section for host = <%=$YADM_HOSTNAME%> (<%=$YADM_HOSTNAME%> again)
+<% fi -%>
+<% if [ "$YADM_HOSTNAME" = "wronghost2" ]; then -%>
+wrong host 2
+<% fi -%>
+<% if [ "$YADM_USER" = "wronguser1" ]; then -%>
+wrong user 1
+<% fi -%>
+<% if [ "$YADM_USER" = "{LOCAL_USER}" ]; then -%>
+Included section for user = <%=$YADM_USER%> (<%=$YADM_USER%> repeated)
+<% fi -%>
+<% if [ "$YADM_USER" = "wronguser2" ]; then -%>
+wrong user 2
+<% fi -%>
+<% if [ "$YADM_DISTRO" = "wrongdistro1" ]; then -%>
+wrong distro 1
+<% fi -%>
+<% if [ "$YADM_DISTRO" = "{LOCAL_DISTRO}" ]; then -%>
+Included section for distro = <%=$YADM_DISTRO%> (<%=$YADM_DISTRO%> again)
+<% fi -%>
+<% if [ "$YADM_DISTRO" = "wrongdistro2" ]; then -%>
+wrong distro 2
+<% fi -%>
+end of template
+'''
+EXPECTED = f'''
+start of template
+esh class  = >{LOCAL_CLASS}<
+esh os     = >{LOCAL_SYSTEM}<
+esh host   = >{LOCAL_HOST}<
+esh user   = >{LOCAL_USER}<
+esh distro = >{LOCAL_DISTRO}<
+Included section for class = {LOCAL_CLASS} ({LOCAL_CLASS} repeated)
+Included section for os = {LOCAL_SYSTEM} ({LOCAL_SYSTEM} repeated)
+Included section for host = {LOCAL_HOST} ({LOCAL_HOST} again)
+Included section for user = {LOCAL_USER} ({LOCAL_USER} repeated)
+Included section for distro = {LOCAL_DISTRO} ({LOCAL_DISTRO} again)
+end of template
+'''
+
+
+def test_template_esh(runner, yadm, tmpdir):
+    """Test processing by esh"""
+
+    input_file = tmpdir.join('input')
+    input_file.write(TEMPLATE, ensure=True)
+    output_file = tmpdir.join('output')
+
+    script = f"""
+        YADM_TEST=1 source {yadm}
+        local_class="{LOCAL_CLASS}"
+        local_system="{LOCAL_SYSTEM}"
+        local_host="{LOCAL_HOST}"
+        local_user="{LOCAL_USER}"
+        local_distro="{LOCAL_DISTRO}"
+        template_esh "{input_file}" "{output_file}"
+    """
+    run = runner(command=['bash'], inp=script)
+    assert run.success
+    assert run.err == ''
+    assert output_file.read().strip() == str(EXPECTED).strip()
+
+
+def test_source(runner, yadm, tmpdir):
+    """Test YADM_SOURCE"""
+
+    input_file = tmpdir.join('input')
+    input_file.write('<%= $YADM_SOURCE %>', ensure=True)
+    output_file = tmpdir.join('output')
+
+    script = f"""
+        YADM_TEST=1 source {yadm}
+        template_esh "{input_file}" "{output_file}"
+    """
+    run = runner(command=['bash'], inp=script)
+    assert run.success
+    assert run.err == ''
+    assert output_file.read().strip() == str(input_file)

yadm

@@ -45,6 +45,7 @@ GIT_CRYPT_PROGRAM="git-crypt"
 TRANSCRYPT_PROGRAM="transcrypt"
 J2CLI_PROGRAM="j2"
 ENVTPL_PROGRAM="envtpl"
+ESH_PROGRAM="esh"
 LSB_RELEASE_PROGRAM="lsb_release"
 
 OS_RELEASE="/etc/os-release"
@@ -250,12 +251,29 @@ function record_score() {
   done
   # if we don't find an existing index, create one by appending to the array
   if [ "$index" -eq -1 ]; then
+    # $YADM_CONFIG must be processed first, in case other templates lookup yadm configurations
+    if [ "$tgt" = "$YADM_CONFIG" ]; then
+        alt_targets=("$tgt" "${alt_targets[@]}")
+        alt_sources=("$src" "${alt_sources[@]}")
+        alt_scores=(0 "${alt_scores[@]}")
+        index=0
+        # increase the index of any existing alt_template_cmds
+        new_cmds=()
+        for cmd_index in "${!alt_template_cmds[@]}"; do
+            new_cmds[$((cmd_index+1))]="${alt_template_cmds[$cmd_index]}"
+        done
+        alt_template_cmds=()
+        for cmd_index in "${!new_cmds[@]}"; do
+            alt_template_cmds[$cmd_index]="${new_cmds[$cmd_index]}"
+        done
+    else
         alt_targets+=("$tgt")
         # set index to the last index (newly created one)
         for index in "${!alt_targets[@]}"; do :; done
         # and set its initial score to zero
         alt_scores[$index]=0
      fi
+  fi
 
   # record nothing if a template command is registered for this file
   [ "${alt_template_cmds[$index]+isset}" ] && return
@@ -299,6 +317,8 @@ function choose_template_cmd() {
 
   if [ "$kind" = "default" ] || [ "$kind" = "" ] && awk_available; then
     echo "template_default"
+  elif [ "$kind" = "esh" ] && esh_available; then
+    echo "template_esh"
   elif [ "$kind" = "j2cli" ] || [ "$kind" = "j2" ] && j2cli_available; then
     echo "template_j2cli"
   elif [ "$kind" = "envtpl" ] || [ "$kind" = "j2" ] && envtpl_available; then
@@ -407,6 +427,22 @@ function template_envtpl() {
   [ -f "$temp_file" ] && mv -f "$temp_file" "$output"
 }
 
+function template_esh() {
+  input="$1"
+  output="$2"
+  temp_file="${output}.$$.$RANDOM"
+
+  "$ESH_PROGRAM" -o "$temp_file" "$input" \
+  YADM_CLASS="$local_class"   \
+  YADM_OS="$local_system"     \
+  YADM_HOSTNAME="$local_host" \
+  YADM_USER="$local_user"     \
+  YADM_DISTRO="$local_distro" \
+  YADM_SOURCE="$input"
+
+  [ -f "$temp_file" ] && mv -f "$temp_file" "$output"
+}
+
 # ****** yadm Commands ******
 
 function alt() {
@@ -1938,6 +1974,10 @@ function envtpl_available() {
   command -v "$ENVTPL_PROGRAM" &> /dev/null && return
   return 1
 }
+function esh_available() {
+  command -v "$ESH_PROGRAM" &> /dev/null && return
+  return 1
+}
 function readlink_available() {
   command -v "readlink" &> /dev/null && return
   return 1

yadm.1

@@ -637,6 +637,15 @@ upon
 which is available on most *nix systems. To use this processor,
 specify the value of "default" or just leave the value off (e.g. "##template").
 .TP
+.B ESH
+ESH is a template processor written in POSIX compliant shell. It allows
+executing shell commands within templates. This can be used to reference your
+own configurations within templates, for example:
+
+  <% yadm config mysection.myconfig %>
+
+To use the ESH template processor, specify the value of "esh"
+.TP
 .B j2cli
 To use the j2cli Jinja template processor, specify the value of "j2"  or
 "j2cli".
@@ -654,7 +663,7 @@ to create or overwrite files.
 
 During processing, the following variables are available in the template:
 
- Default         Jinja           Description
+ Default         Jinja or ESH    Description
  -------------   -------------   --------------------------
  yadm.class      YADM_CLASS      Locally defined yadm class
  yadm.distro     YADM_DISTRO     lsb_release -si
@@ -702,6 +711,16 @@ would look like:
   config=dev-whatever
   {% endif -%}
 
+An equivalent ESH templated named
+.I whatever##template.esh
+would look like:
+
+  <% if [ "$YADM_USER" = "harvey" ]; then -%>
+  config=<%= $YADM_CLASS %>-<%= $YADM_OS %>
+  <% else -%>
+  config=dev-whatever
+  <% fi -%>
+
 .SH ENCRYPTION
 
 It can be useful to manage confidential files, like SSH or GPG keys, across