psi-jack
/
yadm
mirror of
1
0
Fork 0
Code Issues Releases Wiki Activity

Merge pull request #220

This commit is contained in:
Tim Byrne 2020-07-08 16:17:24 -05:00
parent 6d8494a35f ed4a4a5fbd
commit 6654e29c62
No known key found for this signature in database
GPG Key ID: 14DB4FC2465A4B12
9 changed files with 217 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
.travis.yml
View File

@ -3,6 +3,6 @@ language: minimal
services:
- docker
before_install:
- docker pull yadm/testbed:2020-01-20
- docker pull yadm/testbed:2020-07-08
script:
- docker run -t --rm -v "$PWD:/yadm:ro" yadm/testbed:2020-01-20
- docker run -t --rm -v "$PWD:/yadm:ro" yadm/testbed:2020-07-08

4
Dockerfile
View File

@ -36,6 +36,10 @@ RUN pip3 install \
yamllint==1.17.0 \
;
RUN \
curl https://raw.githubusercontent.com/jirutka/esh/v0.3.0/esh > /usr/local/bin/esh && \
chmod +x /usr/local/bin/esh
# Create a flag to identify when running inside the yadm testbed
RUN touch /.yadmtestbed

5
Makefile
View File

@ -1,4 +1,5 @@
PYTESTS = $(wildcard test/test_*.py)
IMAGE = yadm/testbed:2020-07-08
.PHONY: all
all:
@ -112,7 +113,7 @@ testhost: require-docker
--hostname testhost \
--rm -it \
-v "/tmp/testhost:/bin/yadm:ro" \
yadm/testbed:2020-01-20 \
$(IMAGE) \
bash -l
.PHONY: scripthost
@ -129,7 +130,7 @@ scripthost: require-docker
--rm -it \
-v "$$PWD/script.gz:/script.gz:rw" \
-v "/tmp/testhost:/bin/yadm:ro" \
yadm/testbed:2020-01-20 \
$(IMAGE) \
bash -c "script /tmp/script -q -c 'bash -l'; gzip < /tmp/script > /script.gz"
@echo
@echo "Script saved to $$PWD/script.gz"

2
docker-compose.yml
View File

@ -4,4 +4,4 @@ services:
testbed:
volumes:
- .:/yadm:ro
image: yadm/testbed:2020-01-20
image: yadm/testbed:2020-07-08

2
test/test_alt.py
View File

@ -126,7 +126,7 @@ def test_alt_conditions(
@pytest.mark.usefixtures('ds1_copy')
@pytest.mark.parametrize(
'kind', ['default', '', None, 'envtpl', 'j2cli', 'j2'])
'kind', ['default', '', None, 'envtpl', 'j2cli', 'j2', 'esh'])
@pytest.mark.parametrize('label', ['t', 'template', 'yadm', ])
def test_alt_templates(
runner, paths, kind, label):

27
test/test_unit_record_score.py
View File

@ -112,3 +112,30 @@ def test_existing_template(runner, yadm):
assert 'SCORES:1\n' in run.out
assert 'TARGETS:testtgt\n' in run.out
assert 'SOURCES:\n' in run.out
def test_config_first(runner, yadm):
"""Verify YADM_CONFIG is always processed first"""
config = 'yadm_config_file'
script = f"""
YADM_TEST=1 source {yadm}
{INIT_VARS}
YADM_CONFIG={config}
record_score "1" "tgt_before" "src_before"
record_template "tgt_tmp" "cmd_tmp" "src_tmp"
record_score "2" "{config}" "src_config"
record_score "3" "tgt_after" "src_after"
{REPORT_RESULTS}
echo "CMD_VALUE:${{alt_template_cmds[@]}}"
echo "CMD_INDEX:${{!alt_template_cmds[@]}}"
"""
run = runner(command=['bash'], inp=script)
assert run.success
assert run.err == ''
assert 'SIZE:3\n' in run.out
assert 'SCORES:2 1 3\n' in run.out
assert f'TARGETS:{config} tgt_before tgt_tmp tgt_after\n' in run.out
assert 'SOURCES:src_config src_before src_tmp src_after\n' in run.out
assert 'CMD_VALUE:cmd_tmp\n' in run.out
assert 'CMD_INDEX:2\n' in run.out

114
test/test_unit_template_esh.py Normal file
View File

@ -0,0 +1,114 @@
"""Unit tests: template_esh"""
LOCAL_CLASS = "esh_Test+@-!^Class"
LOCAL_SYSTEM = "esh_Test+@-!^System"
LOCAL_HOST = "esh_Test+@-!^Host"
LOCAL_USER = "esh_Test+@-!^User"
LOCAL_DISTRO = "esh_Test+@-!^Distro"
TEMPLATE = f'''
start of template
esh class = ><%=$YADM_CLASS%><
esh os = ><%=$YADM_OS%><
esh host = ><%=$YADM_HOSTNAME%><
esh user = ><%=$YADM_USER%><
esh distro = ><%=$YADM_DISTRO%><
<% if [ "$YADM_CLASS" = "wrongclass1" ]; then -%>
wrong class 1
<% fi -%>
<% if [ "$YADM_CLASS" = "{LOCAL_CLASS}" ]; then -%>
Included section for class = <%=$YADM_CLASS%> (<%=$YADM_CLASS%> repeated)
<% fi -%>
<% if [ "$YADM_CLASS" = "wrongclass2" ]; then -%>
wrong class 2
<% fi -%>
<% if [ "$YADM_OS" = "wrongos1" ]; then -%>
wrong os 1
<% fi -%>
<% if [ "$YADM_OS" = "{LOCAL_SYSTEM}" ]; then -%>
Included section for os = <%=$YADM_OS%> (<%=$YADM_OS%> repeated)
<% fi -%>
<% if [ "$YADM_OS" = "wrongos2" ]; then -%>
wrong os 2
<% fi -%>
<% if [ "$YADM_HOSTNAME" = "wronghost1" ]; then -%>
wrong host 1
<% fi -%>
<% if [ "$YADM_HOSTNAME" = "{LOCAL_HOST}" ]; then -%>
Included section for host = <%=$YADM_HOSTNAME%> (<%=$YADM_HOSTNAME%> again)
<% fi -%>
<% if [ "$YADM_HOSTNAME" = "wronghost2" ]; then -%>
wrong host 2
<% fi -%>
<% if [ "$YADM_USER" = "wronguser1" ]; then -%>
wrong user 1
<% fi -%>
<% if [ "$YADM_USER" = "{LOCAL_USER}" ]; then -%>
Included section for user = <%=$YADM_USER%> (<%=$YADM_USER%> repeated)
<% fi -%>
<% if [ "$YADM_USER" = "wronguser2" ]; then -%>
wrong user 2
<% fi -%>
<% if [ "$YADM_DISTRO" = "wrongdistro1" ]; then -%>
wrong distro 1
<% fi -%>
<% if [ "$YADM_DISTRO" = "{LOCAL_DISTRO}" ]; then -%>
Included section for distro = <%=$YADM_DISTRO%> (<%=$YADM_DISTRO%> again)
<% fi -%>
<% if [ "$YADM_DISTRO" = "wrongdistro2" ]; then -%>
wrong distro 2
<% fi -%>
end of template
'''
EXPECTED = f'''
start of template
esh class = >{LOCAL_CLASS}<
esh os = >{LOCAL_SYSTEM}<
esh host = >{LOCAL_HOST}<
esh user = >{LOCAL_USER}<
esh distro = >{LOCAL_DISTRO}<
Included section for class = {LOCAL_CLASS} ({LOCAL_CLASS} repeated)
Included section for os = {LOCAL_SYSTEM} ({LOCAL_SYSTEM} repeated)
Included section for host = {LOCAL_HOST} ({LOCAL_HOST} again)
Included section for user = {LOCAL_USER} ({LOCAL_USER} repeated)
Included section for distro = {LOCAL_DISTRO} ({LOCAL_DISTRO} again)
end of template
'''
def test_template_esh(runner, yadm, tmpdir):
"""Test processing by esh"""
input_file = tmpdir.join('input')
input_file.write(TEMPLATE, ensure=True)
output_file = tmpdir.join('output')
script = f"""
YADM_TEST=1 source {yadm}
local_class="{LOCAL_CLASS}"
local_system="{LOCAL_SYSTEM}"
local_host="{LOCAL_HOST}"
local_user="{LOCAL_USER}"
local_distro="{LOCAL_DISTRO}"
template_esh "{input_file}" "{output_file}"
"""
run = runner(command=['bash'], inp=script)
assert run.success
assert run.err == ''
assert output_file.read().strip() == str(EXPECTED).strip()
def test_source(runner, yadm, tmpdir):
"""Test YADM_SOURCE"""
input_file = tmpdir.join('input')
input_file.write('<%= $YADM_SOURCE %>', ensure=True)
output_file = tmpdir.join('output')
script = f"""
YADM_TEST=1 source {yadm}
template_esh "{input_file}" "{output_file}"
"""
run = runner(command=['bash'], inp=script)
assert run.success
assert run.err == ''
assert output_file.read().strip() == str(input_file)

50
yadm
View File

@ -45,6 +45,7 @@ GIT_CRYPT_PROGRAM="git-crypt"
TRANSCRYPT_PROGRAM="transcrypt"
J2CLI_PROGRAM="j2"
ENVTPL_PROGRAM="envtpl"
ESH_PROGRAM="esh"
LSB_RELEASE_PROGRAM="lsb_release"
OS_RELEASE="/etc/os-release"
@ -250,11 +251,28 @@ function record_score() {
done
# if we don't find an existing index, create one by appending to the array
if [ "$index" -eq -1 ]; then
alt_targets+=("$tgt")
# set index to the last index (newly created one)
for index in "${!alt_targets[@]}"; do :; done
# and set its initial score to zero
alt_scores[$index]=0
# $YADM_CONFIG must be processed first, in case other templates lookup yadm configurations
if [ "$tgt" = "$YADM_CONFIG" ]; then
alt_targets=("$tgt" "${alt_targets[@]}")
alt_sources=("$src" "${alt_sources[@]}")
alt_scores=(0 "${alt_scores[@]}")
index=0
# increase the index of any existing alt_template_cmds
new_cmds=()
for cmd_index in "${!alt_template_cmds[@]}"; do
new_cmds[$((cmd_index+1))]="${alt_template_cmds[$cmd_index]}"
done
alt_template_cmds=()
for cmd_index in "${!new_cmds[@]}"; do
alt_template_cmds[$cmd_index]="${new_cmds[$cmd_index]}"
done
else
alt_targets+=("$tgt")
# set index to the last index (newly created one)
for index in "${!alt_targets[@]}"; do :; done
# and set its initial score to zero
alt_scores[$index]=0
fi
fi
# record nothing if a template command is registered for this file
@ -299,6 +317,8 @@ function choose_template_cmd() {
if [ "$kind" = "default" ] || [ "$kind" = "" ] && awk_available; then
echo "template_default"
elif [ "$kind" = "esh" ] && esh_available; then
echo "template_esh"
elif [ "$kind" = "j2cli" ] || [ "$kind" = "j2" ] && j2cli_available; then
echo "template_j2cli"
elif [ "$kind" = "envtpl" ] || [ "$kind" = "j2" ] && envtpl_available; then
@ -407,6 +427,22 @@ function template_envtpl() {
[ -f "$temp_file" ] && mv -f "$temp_file" "$output"
}
function template_esh() {
input="$1"
output="$2"
temp_file="${output}.$$.$RANDOM"
"$ESH_PROGRAM" -o "$temp_file" "$input" \
YADM_CLASS="$local_class" \
YADM_OS="$local_system" \
YADM_HOSTNAME="$local_host" \
YADM_USER="$local_user" \
YADM_DISTRO="$local_distro" \
YADM_SOURCE="$input"
[ -f "$temp_file" ] && mv -f "$temp_file" "$output"
}
# ****** yadm Commands ******
function alt() {
@ -1938,6 +1974,10 @@ function envtpl_available() {
command -v "$ENVTPL_PROGRAM" &> /dev/null && return
return 1
}
function esh_available() {
command -v "$ESH_PROGRAM" &> /dev/null && return
return 1
}
function readlink_available() {
command -v "readlink" &> /dev/null && return
return 1

21
yadm.1
View File

@ -637,6 +637,15 @@ upon
which is available on most *nix systems. To use this processor,
specify the value of "default" or just leave the value off (e.g. "##template").
.TP
.B ESH
ESH is a template processor written in POSIX compliant shell. It allows
executing shell commands within templates. This can be used to reference your
own configurations within templates, for example:
<% yadm config mysection.myconfig %>
To use the ESH template processor, specify the value of "esh"
.TP
.B j2cli
To use the j2cli Jinja template processor, specify the value of "j2" or
"j2cli".
@ -654,7 +663,7 @@ to create or overwrite files.
During processing, the following variables are available in the template:
Default Jinja Description
Default Jinja or ESH Description
------------- ------------- --------------------------
yadm.class YADM_CLASS Locally defined yadm class
yadm.distro YADM_DISTRO lsb_release -si
@ -702,6 +711,16 @@ would look like:
config=dev-whatever
{% endif -%}
An equivalent ESH templated named
.I whatever##template.esh
would look like:
<% if [ "$YADM_USER" = "harvey" ]; then -%>
config=<%= $YADM_CLASS %>-<%= $YADM_OS %>
<% else -%>
config=dev-whatever
<% fi -%>
.SH ENCRYPTION
It can be useful to manage confidential files, like SSH or GPG keys, across