From a479b70d8a977649067851e0d577baf07a71b189 Mon Sep 17 00:00:00 2001 From: Jan Schulz Date: Sat, 25 Mar 2017 19:26:10 +0100 Subject: [PATCH 1/2] Add jinja processing to alt command With the new functionality, when the 'alt' command is called (or automatically triggered), any file with a name ending in '##yadm_tmpl' is treated as a jinja template. The template is processed by envtpl and the result is written to a file without the '##yadm_tmpl' name. The variables passed into the template processing are YADM_CLASS YADM_OS YADM_HOSTNAME YADM_USER These variables are set according to the normal rules for CLASS, OS, HOSTNAME, and USER during the alt processing. --- test/113_accept_jinja_alt.bats | 151 +++++++++++++++++++++++++++++++++ test/common.bash | 2 + yadm | 56 ++++++++---- yadm.1 | 40 ++++++++- 4 files changed, 233 insertions(+), 16 deletions(-) create mode 100644 test/113_accept_jinja_alt.bats diff --git a/test/113_accept_jinja_alt.bats b/test/113_accept_jinja_alt.bats new file mode 100644 index 0000000..ec1d440 --- /dev/null +++ b/test/113_accept_jinja_alt.bats @@ -0,0 +1,151 @@ +load common +load_fixtures +status=;output=; #; populated by bats run() + +IN_REPO=(alt* "dir one") +export TEST_TREE_WITH_ALT=1 + + +setup() { + destroy_tmp + build_repo "${IN_REPO[@]}" +} + + +function test_alt() { + local alt_type="$1" + local test_overwrite="$2" + local auto_alt="$3" + + #; detemine test parameters + case $alt_type in + base) + real_name="alt-jinja" + file_content_match="-${T_SYS}-${T_HOST}-${T_USER}" + ;; + override_all) + real_name="alt-jinja" + file_content_match="custom_class-custom_system-custom_host-custom_user" + ;; + esac + + if [ "$test_overwrite" = "true" ] ; then + #; create incorrect links (to overwrite) + echo "BAD_CONTENT" "$T_DIR_WORK/$real_name" + else + #; verify real file doesn't already exist + if [ -e "$T_DIR_WORK/$real_name" ] ; then + echo "ERROR: real file already exists before running yadm" + return 1 + fi + fi + + #; configure yadm.auto_alt=false + if [ "$auto_alt" = "false" ]; then + git config --file="$T_YADM_CONFIG" yadm.auto-alt false + fi + + #; run yadm (alt or status) + if [ -z "$auto_alt" ]; then + run "${T_YADM_Y[@]}" alt + #; validate status and output + if [ "$status" != 0 ] || [[ ! "$output" =~ Creating.+$real_name ]]; then + echo "OUTPUT:$output" + echo "ERROR: Could not confirm status and output of alt command" + return 1; + fi + else + #; running any passed through Git command should trigger auto-alt + run "${T_YADM_Y[@]}" status + if [ -n "$auto_alt" ] && [[ "$output" =~ Creating.+$real_name ]]; then + echo "ERROR: Reporting of jinja processing should not happen" + return 1 + fi + fi + + #; validate link content + if [[ "$alt_type" =~ none ]] || [ "$auto_alt" = "false" ]; then + #; no real file should be present + if [ -L "$T_DIR_WORK/$real_name" ] ; then + echo "ERROR: Real file should not exist" + return 1 + fi + else + #; correct real file should be present + local file_content + file_content=$(cat "$T_DIR_WORK/$real_name") + if [ "$file_content" != "$file_content_match" ]; then + echo "file_content: ${file_content}" + echo "expected_content: ${file_content_match}" + echo "ERROR: Link content is not correct" + return 1 + fi + fi +} + +@test "Command 'alt' (select jinja)" { + echo " + When the command 'alt' is provided + and file matches ##yadm_tmpl + Report jinja template processing + Verify that the correct content is written + Exit with 0 + " + + test_alt 'base' 'false' '' +} + +@test "Command 'auto-alt' (enabled)" { + echo " + When a command possibly changes the repo + and auto-alt is configured true + and file matches ##yadm_tmpl + automatically process alternates + report no linking (not loud) + Verify that the correct content is written + " + + test_alt 'base' 'false' 'true' +} + +@test "Command 'auto-alt' (disabled)" { + echo " + When a command possibly changes the repo + and auto-alt is configured false + and file matches ##yadm_tmpl + Report no jinja template processing + Verify no content + " + + test_alt 'base' 'false' 'false' +} + +@test "Command 'alt' (overwrite existing content)" { + echo " + When the command 'alt' is provided + and file matches ##yadm_tmpl + and the real file exists, and is wrong + Report jinja template processing + Verify that the correct content is written + Exit with 0 + " + + test_alt 'base' 'true' '' +} + +@test "Command 'alt' (overwritten settings)" { + echo " + When the command 'alt' is provided + and file matches ##yadm_tmpl + after setting local.* + Report jinja template processing + Verify that the correct content is written + Exit with 0 + " + + GIT_DIR="$T_DIR_REPO" git config local.os custom_system + GIT_DIR="$T_DIR_REPO" git config local.user custom_user + GIT_DIR="$T_DIR_REPO" git config local.host custom_host + GIT_DIR="$T_DIR_REPO" git config local.class custom_class + test_alt 'override_all' 'false' '' +} diff --git a/test/common.bash b/test/common.bash index 03bc7c2..c3360ee 100644 --- a/test/common.bash +++ b/test/common.bash @@ -202,8 +202,10 @@ function create_worktree() { make_parents "$DIR_WORKTREE/$f" echo "$f" > "$DIR_WORKTREE/$f" done + echo "{{ YADM_CLASS }}-{{ YADM_OS }}-{{ YADM_HOSTNAME }}-{{ YADM_USER }}" > "$DIR_WORKTREE/alt-jinja##yadm_tmpl" fi + if [ ! -z "$TEST_TREE_WITH_WILD" ] ; then #; wildcard test data - yes this is a big mess :( #; none diff --git a/yadm b/yadm index 526aae4..7583a8d 100755 --- a/yadm +++ b/yadm @@ -114,30 +114,32 @@ function alt() { require_repo - match_class="$(config local.class)" - if [ -z "$match_class" ] ; then + local_class="$(config local.class)" + if [ -z "$local_class" ] ; then match_class="%" + else + match_class="$local_class" fi match_class="(%|$match_class)" - match_system="$(config local.os)" - if [ -z "$match_system" ] ; then - match_system=$(uname -s) + local_system="$(config local.os)" + if [ -z "$local_system" ] ; then + local_system=$(uname -s) fi - match_system="(%|$match_system)" + match_system="(%|$local_system)" - match_host="$(config local.host)" - if [ -z "$match_host" ] ; then - match_host=$(hostname) - match_host=${match_host%%.*} #; trim any domain from hostname + local_host="$(config local.host)" + if [ -z "$local_host" ] ; then + local_host=$(hostname) + local_host=${local_host%%.*} #; trim any domain from hostname fi - match_host="(%|$match_host)" + match_host="(%|$local_host)" - match_user="$(config local.user)" - if [ -z "$match_user" ] ; then - match_user=$(id -u -n) + local_user="$(config local.user)" + if [ -z "$local_user" ] ; then + local_user=$(id -u -n) fi - match_user="(%|$match_user)" + match_user="(%|$local_user)" #; regex for matching "##CLASS.SYSTEM.HOSTNAME.USER" match1="^(.+)##(()|$match_system|$match_system\.$match_host|$match_system\.$match_host\.$match_user)$" @@ -193,6 +195,30 @@ function alt() { done done + #; loop over all "tracked" files + #; for every file which is a *##yadm_tmpl create a real file + local IFS=$'\n' + local match="^(.+)##yadm_tmpl" + local envtpl_bin=$(which envtpl) + for tracked_file in $("$GIT_PROGRAM" ls-files | sort) $(cat "$YADM_ENCRYPT" 2>/dev/null); do + tracked_file="$YADM_WORK/$tracked_file" + if [ -e "$tracked_file" ] ; then + if [[ $tracked_file =~ $match ]] ; then + if [[ -z "$envtpl_bin" ]]; then + debug "'envtpl' (pip install envtpl) not available, not creating $real_file from template $tracked_file" + [ -n "$loud" ] && echo "'envtpl' (pip install envtpl) not available, not creating $real_file from template $tracked_file" + else + real_file="${BASH_REMATCH[1]}" + debug "Creating $real_file from template $tracked_file" + [ -n "$loud" ] && echo "Creating $real_file from template $tracked_file" + YADM_CLASS="$local_class" YADM_OS="$local_system" \ + YADM_HOSTNAME="$local_host" YADM_USER="$local_user" \ + $envtpl_bin < $tracked_file > $real_file + fi + fi + fi + done + } function bootstrap() { diff --git a/yadm.1 b/yadm.1 index c0e8db7..d97ada2 100644 --- a/yadm.1 +++ b/yadm.1 @@ -89,7 +89,8 @@ Instead use the command (see below). .TP .B alt -Create symbolic links for any managed files matching the naming rules describe in the ALTERNATES section. +Create symbolic links and process jinja templates for any managed files matching the naming rules describe +in the ALTERNATES section. It is usually unnecessary to run this command, as .B yadm automatically processes alternates by default. @@ -473,6 +474,43 @@ using the configuration options and .BR local.user . +If +.BR envtpl +( +.BR pip\ install\ envtpl +) is available, you can also create +.B jinja +templates (http://jinja.pocoo.org/) which will transformed into real files. +.B yadm +will treat files ending in + + ##yadm_tmpl + +as jinja templates. During processing, the following variables are +set according to the above rules: + + YADM_CLASS + YADM_OS + YADM_HOSTNAME + YADM_USER + +E.g. a file 'whatever##yadm_tmpl' with the following content + + {% if YADM_USER == 'harvey' -%} + config={{YADM_CLASS}}-{{ YADM_OS }} + {% else -%} + config=dev-whatever + {% endif -%} + +would output a file with the follwing content, if the username would be 'harvey' + + config=work-Linux + +and the following otherwise: + + config=dev-whatever + + .SH ENCRYPTION It can be useful to manage confidential files, like SSH or GPG keys, across multiple systems. However, doing so would put plain text data into a Git From ca849b50de9bef58c2607ac018d413365b5dc523 Mon Sep 17 00:00:00 2001 From: Jan Schulz Date: Sat, 25 Mar 2017 19:31:46 +0100 Subject: [PATCH 2/2] Autogenerate yadm.md after new jinja functionality --- yadm.md | 117 ++++++++++++++++++++++++++++++++++++-------------------- 1 file changed, 75 insertions(+), 42 deletions(-) diff --git a/yadm.md b/yadm.md index 8e0d767..6610a21 100644 --- a/yadm.md +++ b/yadm.md @@ -50,21 +50,22 @@ The config command is not passed directly through. Instead use the gitconfig command (see below). - alt Create symbolic links for any managed files matching the naming - rules describe in the ALTERNATES section. It is usually unnec- - essary to run this command, as yadm automatically processes - alternates by default. This automatic behavior can be disabled - by setting the configuration yadm.auto-alt to "false". + alt Create symbolic links and process jinja templates for any man- + aged files matching the naming rules describe in the ALTERNATES + section. It is usually unnecessary to run this command, as yadm + automatically processes alternates by default. This automatic + behavior can be disabled by setting the configuration yadm.auto- + alt to "false". bootstrap Execute $HOME/.yadm/bootstrap if it exists. clone url Clone a remote repository for tracking dotfiles. After the con- - tents of the remote repository have been fetched, a "merge" of - origin/master is attempted. If there are conflicting files - already present in the work-tree, this merge will fail and - instead a "reset" of origin/master will be done, followed by a + tents of the remote repository have been fetched, a "merge" of + origin/master is attempted. If there are conflicting files + already present in the work-tree, this merge will fail and + instead a "reset" of origin/master will be done, followed by a "stash". This "stash" operation will preserve the original data. You can review the stashed conflicts by running the command @@ -238,7 +239,7 @@ to have an automated way of choosing an alternate version of a file for a different operating system, host, or user. yadm implements a feature which will automatically create a symbolic link to the appropriate ver- - sion of a file, as long as you follow a specific naming convention. + sion of a file, as long as you follow a specific naming convention. yadm can detect files with names ending in any of the following: ## @@ -250,10 +251,10 @@ ##OS.HOSTNAME ##OS.HOSTNAME.USER - If there are any files managed by yadm's repository, or listed in + If there are any files managed by yadm's repository, or listed in $HOME/.yadm/encrypt, which match this naming convention, symbolic links - will be created for the most appropriate version. This may best be - demonstrated by example. Assume the following files are managed by + will be created for the most appropriate version. This may best be + demonstrated by example. Assume the following files are managed by yadm's repository: - $HOME/path/example.txt## @@ -275,7 +276,7 @@ $HOME/path/example.txt -> $HOME/path/example.txt##Darwin - Since the hostname doesn't match any of the managed files, the more + Since the hostname doesn't match any of the managed files, the more generic version is chosen. If running on a Linux server named "host4", the link will be: @@ -293,73 +294,105 @@ If no "##" version exists and no files match the current CLASS/OS/HOST- NAME/USER, then no link will be created. - Links are also created for directories named this way, as long as they + Links are also created for directories named this way, as long as they have at least one yadm managed file within them. - CLASS must be manually set using yadm config local.class . OS - is determined by running uname -s, HOSTNAME by running hostname, and - USER by running id -u -n. yadm will automatically create these links + CLASS must be manually set using yadm config local.class . OS + is determined by running uname -s, HOSTNAME by running hostname, and + USER by running id -u -n. yadm will automatically create these links by default. This can be disabled using the yadm.auto-alt configuration. Even if disabled, links can be manually created by running yadm alt. - It is possible to use "%" as a "wildcard" in place of CLASS, OS, HOST- - NAME, or USER. For example, The following file could be linked for any + It is possible to use "%" as a "wildcard" in place of CLASS, OS, HOST- + NAME, or USER. For example, The following file could be linked for any host when the user is "harvey". $HOME/path/example.txt##%.%.harvey - CLASS is a special value which is stored locally on each host (inside - the local repository). To use alternate symlinks using CLASS, you must - set the value of class using the configuration local.class. This is + CLASS is a special value which is stored locally on each host (inside + the local repository). To use alternate symlinks using CLASS, you must + set the value of class using the configuration local.class. This is set like any other yadm configuration with the yadm config command. The following sets the CLASS to be "Work". yadm config local.class Work - Similarly, the values of OS, HOSTNAME, and USER can be manually over- - ridden using the configuration options local.os, local.hostname, and + Similarly, the values of OS, HOSTNAME, and USER can be manually over- + ridden using the configuration options local.os, local.hostname, and local.user. + If envtpl ( pip install envtpl ) is available, you can also create + jinja templates (http://jinja.pocoo.org/) which will transformed into + real files. yadm will treat files ending in + + ##yadm_tmpl + + as jinja templates. During processing, the following variables are set + according to the above rules: + + YADM_CLASS + YADM_OS + YADM_HOSTNAME + YADM_USER + + E.g. a file 'whatever##yadm_tmpl' with the following content + + {% if YADM_USER == 'harvey' -%} + config={{YADM_CLASS}}-{{ YADM_OS }} + {% else -%} + config=dev-whatever + {% endif -%} + + would output a file with the follwing content, if the username would be + 'harvey' + + config=work-Linux + + and the following otherwise: + + config=dev-whatever + + ## ENCRYPTION - It can be useful to manage confidential files, like SSH or GPG keys, - across multiple systems. However, doing so would put plain text data - into a Git repository, which often resides on a public system. yadm - implements a feature which can make it easy to encrypt and decrypt a - set of files so the encrypted version can be maintained in the Git - repository. This feature will only work if the gpg(1) command is + It can be useful to manage confidential files, like SSH or GPG keys, + across multiple systems. However, doing so would put plain text data + into a Git repository, which often resides on a public system. yadm + implements a feature which can make it easy to encrypt and decrypt a + set of files so the encrypted version can be maintained in the Git + repository. This feature will only work if the gpg(1) command is available. - To use this feature, a list of patterns must be created and saved as - $HOME/.yadm/encrypt. This list of patterns should be relative to the + To use this feature, a list of patterns must be created and saved as + $HOME/.yadm/encrypt. This list of patterns should be relative to the configured work-tree (usually $HOME). For example: .ssh/*.key .gnupg/*.gpg The yadm encrypt command will find all files matching the patterns, and - prompt for a password. Once a password has confirmed, the matching - files will be encrypted and saved as $HOME/.yadm/files.gpg. The pat- - terns and files.gpg should be added to the yadm repository so they are + prompt for a password. Once a password has confirmed, the matching + files will be encrypted and saved as $HOME/.yadm/files.gpg. The pat- + terns and files.gpg should be added to the yadm repository so they are available across multiple systems. To decrypt these files later, or on another system run yadm decrypt and - provide the correct password. After files are decrypted, permissions + provide the correct password. After files are decrypted, permissions are automatically updated as described in the PERMISSIONS section. - Symmetric encryption is used by default, but asymmetric encryption may + Symmetric encryption is used by default, but asymmetric encryption may be enabled using the yadm.gpg-recipient configuration. - NOTE: It is recommended that you use a private repository when keeping + NOTE: It is recommended that you use a private repository when keeping confidential files, even though they are encrypted. ## PERMISSIONS - When files are checked out of a Git repository, their initial permis- + When files are checked out of a Git repository, their initial permis- sions are dependent upon the user's umask. This can result in confiden- tial files with lax permissions. To prevent this, yadm will automatically update the permissions of con- - fidential files. The "group" and "others" permissions will be removed + fidential files. The "group" and "others" permissions will be removed from the following files: - $HOME/.yadm/files.gpg @@ -371,7 +404,7 @@ - The GPG directory and files, .gnupg/* yadm will automatically update permissions by default. This can be dis- - abled using the yadm.auto-perms configuration. Even if disabled, per- + abled using the yadm.auto-perms configuration. Even if disabled, per- missions can be manually updated by running yadm perms. The SSH direc- tory processing can be disabled using the yadm.ssh-perms configuration.