Rewrite default template to handle nested ifs, != and env vars in if

The awk script now performs all processing in the BEGIN block using an
implementation that is capable of handling if statements which contain nested
if statments (fixes #436). To make nested ifs look better, if, else and endif
lines can now have optional whitespace before {%.

Includes are now handled in the same way as the main file which means that
included files can both include other files and have if statements in addition
to variables (fixes #406). Include lines can now also have optional whitespace
before {%.

All variables are handled in the same way now so it's now possible to use env
variables in if statements (fixes #488).

Also add support for != in addition to == (fixes #358). Thus it's now
e.g. possible to check if a variable is set (#477) by doing:

{% if yadm.class != ""%}
Class is set to {{ yadm.class }}
{% endif %}

A non-existing yadm or env variable is now replaced with the empty string.

test/test_unit_template_default.py

@@ -1,4 +1,5 @@
 """Unit tests: template_default"""
+
 import os
 
 FILE_MODE = 0o754
@@ -12,6 +13,7 @@ LOCAL_HOST = "default_Test+@-!^Host"
 LOCAL_USER = "default_Test+@-!^User"
 LOCAL_DISTRO = "default_Test+@-!^Distro"
 LOCAL_DISTRO_FAMILY = "default_Test+@-!^Family"
+ENV_VAR = "default_Test+@-!^Env"
 TEMPLATE = f"""
 start of template
 default class         = >{{{{yadm.class}}}}<
@@ -30,6 +32,9 @@ Included section from else
 {{% if yadm.class == "wrongclass1" %}}
 wrong class 1
 {{% endif %}}
+{{% if yadm.class != "wronglcass" %}}
+Included section from !=
+{{%     endif\t\t  %}}
 {{% if yadm.class == "{LOCAL_CLASS}" %}}
 Included section for class = {{{{yadm.class}}}} ({{{{yadm.class}}}} repeated)
 Multiple lines
@@ -97,6 +102,13 @@ Included section for distro_family = \
 {{% if yadm.distro_family == "wrongfamily2" %}}
 wrong family 2
 {{% endif %}}
+{{% if env.VAR == "{ENV_VAR}" %}}
+Included section for env.VAR = {{{{env.VAR}}}} ({{{{env.VAR}}}} again)
+{{% endif %}}
+{{% if env.VAR == "wrongenvvar" %}}
+wrong env.VAR
+{{% endif %}}
+yadm.no_such_var="{{{{ yadm.no_such_var }}}}" and env.NO_SUCH_VAR="{{{{ env.NO_SUCH_VAR }}}}"
 end of template
 """
 EXPECTED = f"""
@@ -111,6 +123,7 @@ default distro_family = >{LOCAL_DISTRO_FAMILY}<
 classes = >{LOCAL_CLASS2}
 {LOCAL_CLASS}<
 Included section from else
+Included section from !=
 Included section for class = {LOCAL_CLASS} ({LOCAL_CLASS} repeated)
 Multiple lines
 Included section for second class
@@ -121,6 +134,8 @@ Included section for user = {LOCAL_USER} ({LOCAL_USER} repeated)
 Included section for distro = {LOCAL_DISTRO} ({LOCAL_DISTRO} again)
 Included section for distro_family = \
 {LOCAL_DISTRO_FAMILY} ({LOCAL_DISTRO_FAMILY} again)
+Included section for env.VAR = {ENV_VAR} ({ENV_VAR} again)
+yadm.no_such_var="" and env.NO_SUCH_VAR=""
 end of template
 """
 
@@ -154,6 +169,42 @@ Include basic again:
 basic
 """
 
+TEMPLATE_NESTED_IFS = """\
+{% if yadm.user == "me" %}
+    print1
+  {% if yadm.user == "me" %}
+    print2
+  {% else %}
+    no print1
+  {% endif %}
+{% else %}
+  {% if yadm.user == "me" %}
+    no print2
+  {% else %}
+    no print3
+  {% endif %}
+{% endif %}
+{% if yadm.user != "me" %}
+    no print4
+  {% if yadm.user == "me" %}
+    no print5
+  {% else %}
+    no print6
+  {% endif %}
+{% else %}
+  {% if yadm.user == "me" %}
+    print3
+  {% else %}
+    no print7
+  {% endif %}
+{% endif %}
+"""
+EXPECTED_NESTED_IFS = """\
+    print1
+    print2
+    print3
+"""
+
 
 def test_template_default(runner, yadm, tmpdir):
     """Test template_default"""
@@ -182,7 +233,7 @@ def test_template_default(runner, yadm, tmpdir):
         local_distro_family="{LOCAL_DISTRO_FAMILY}"
         template_default "{input_file}" "{output_file}"
     """
-    run = runner(command=["bash"], inp=script)
+    run = runner(command=["bash"], inp=script, env={"VAR": ENV_VAR})
     assert run.success
     assert run.err == ""
     assert output_file.read() == EXPECTED
@@ -243,12 +294,30 @@ def test_include(runner, yadm, tmpdir):
     assert os.stat(output_file).st_mode == os.stat(input_file).st_mode
 
 
+def test_nested_ifs(runner, yadm, tmpdir):
+    """Test nested if statements"""
+
+    input_file = tmpdir.join("input")
+    input_file.write(TEMPLATE_NESTED_IFS, ensure=True)
+    output_file = tmpdir.join("output")
+
+    script = f"""
+        YADM_TEST=1 source {yadm}
+        set_awk
+        local_user="me"
+        template_default "{input_file}" "{output_file}"
+    """
+    run = runner(command=["bash"], inp=script)
+    assert run.success
+    assert run.err == ""
+    assert output_file.read() == EXPECTED_NESTED_IFS
+
+
 def test_env(runner, yadm, tmpdir):
     """Test env"""
 
     input_file = tmpdir.join("input")
     input_file.write("{{env.PWD}}", ensure=True)
-    input_file.chmod(FILE_MODE)
     output_file = tmpdir.join("output")
 
     script = f"""

yadm

@@ -368,87 +368,110 @@ function template_default() {
   # the explicit "space + tab" character class used below is used because not
   # all versions of awk seem to support the POSIX character classes [[:blank:]]
   read -r -d '' awk_pgm << "EOF"
-# built-in default template processor
 BEGIN {
-  blank              = "[ 	]"
+  yadm["class"] = class
-  c["class"]         = class
+  yadm["classes"] = classes
-  c["classes"]       = classes
+  yadm["arch"] = arch
-  c["arch"]          = arch
+  yadm["os"] = os
-  c["os"]            = os
+  yadm["hostname"] = host
-  c["hostname"]      = host
+  yadm["user"] = user
-  c["user"]          = user
+  yadm["distro"] = distro
-  c["distro"]        = distro
+  yadm["distro_family"] = distro_family
-  c["distro_family"] = distro_family
+  yadm["source"] = source
-  c["source"]        = source
-  ifs                = "^{%" blank "*if"
-  els                = "^{%" blank "*else" blank "*%}$"
-  end                = "^{%" blank "*endif" blank "*%}$"
-  skp                = "^{%" blank "*(if|else|endif)"
-  vld                = conditions()
-  inc_start          = "^{%" blank "*include" blank "+\"?"
-  inc_end            = "\"?" blank "*%}$"
-  inc                = inc_start ".+" inc_end
-  prt                = 1
-  err                = 0
-}
-END { exit err }
-{ replace_vars() } # variable replacements
-$0 ~ vld, $0 ~ end {
-  if ($0 ~ vld || $0 ~ end) prt=1;
-  if ($0 ~ els) prt=0;
-  if ($0 ~ skp) next;
-}
-($0 ~ ifs && $0 !~ vld), $0 ~ end {
-  if ($0 ~ ifs && $0 !~ vld) prt=0;
-  if ($0 ~ els || $0 ~ end) prt=1;
-  if ($0 ~ skp) next;
-}
-{ if (!prt) next }
-$0 ~ inc {
-  file = $0
-  sub(inc_start, "", file)
-  sub(inc_end, "", file)
-  sub(/^[^\/].*$/, source_dir "/&", file)
 
-  while ((res = getline <file) > 0) {
+  VARIABLE = "(env|yadm)\\.[a-zA-Z0-9_]+"
-    replace_vars()
+
-    print
+  current = 0
-  }
+  filename[current] = ARGV[1]
-  if (res < 0) {
+  line[current] = 0
-    printf "%s:%d: error: could not read '%s'\n", FILENAME, NR, file | "cat 1>&2"
+
-    err = 1
+  level = 0
-  }
+  skip[level] = 0
-  close(file)
+
-  next
+  for (; current >= 0; --current) {
-}
+    while ((res = getline <filename[current]) > 0) {
-{ print }
+      ++line[current]
-function replace_vars() {
+      if ($0 ~ "^[ \t]*\\{%[ \t]*if[ \t]+" VARIABLE "[ \t]*[!=]=[ \t]*\".*\"[ \t]*%\\}$") {
-  for (label in c) {
+        if (skip[level]) { skip[++level] = 1; continue }
-    gsub(("{{" blank "*yadm\\." label blank "*}}"), c[label])
+
-  }
+        match($0, VARIABLE)
-  for (label in ENVIRON) {
+        lhs = substr($0, RSTART, RLENGTH)
-    gsub(("{{" blank "*env\\." label blank "*}}"), ENVIRON[label])
+        match($0, /[!=]=/)
-  }
+        op = substr($0, RSTART, RLENGTH)
-}
+        match($0, /".*"/)
-function condition_helper(label, value) {
+        rhs = replace_vars(substr($0, RSTART + 1, RLENGTH - 2))
-  gsub(/[\\.^$(){}\[\]|*+?]/, "\\\\&", value)
+
-  return sprintf("yadm\\.%s" blank "*==" blank "*\"%s\"", label, value)
+        if (lhs == "yadm.class") {
-}
+          lhs = "not" rhs
-function conditions() {
-  pattern = ifs blank "+("
-  for (label in c) {
-    if (label != "class") {
-      value = c[label]
-      pattern = sprintf("%s%s|", pattern, condition_helper(label, value));
-    }
-  }
           split(classes, cls_array, "\n")
           for (idx in cls_array) {
-    value = cls_array[idx]
+            if (rhs == cls_array[idx]) { lhs = rhs; break }
-    pattern = sprintf("%s%s|", pattern, condition_helper("class", value));
           }
-  sub(/\|$/, ")" blank "*%}$", pattern)
+        }
-  return pattern
+        else {
+          lhs = replace_vars("{{" lhs "}}")
+        }
+
+        if (op == "==") { skip[++level] = lhs != rhs }
+        else { skip[++level] = lhs == rhs }
+      }
+      else if (/^[ \t]*\{%[ \t]*else[ \t]*%\}$/) {
+        if (level == 0 || skip[level] < 0) { error("else without matching if") }
+        skip[level] = skip[level] ? skip[level - 1] : -1
+      }
+      else if (/^[ \t]*\{%[ \t]*endif[ \t]*%\}$/) {
+        if (--level < 0) { error("endif without matching if") }
+      }
+      else if (!skip[level]) {
+        $0 = replace_vars($0)
+        if (match($0, /^[ \t]*\{%[ \t]*include[ \t]+("[^"]+"|[^"]+)[ \t]*%\}$/)) {
+          include = $0
+          sub(/^[ \t]*\{%[ \t]*include[ \t]+"?/, "", include)
+          sub(/"?[ \t]*%\}$/, "", include)
+          if (index(include, "/") != 1) {
+            include = source_dir "/" include
+          }
+          filename[++current] = include
+          line[current] = 0
+        }
+        else { print }
+      }
+    }
+    if (res >= 0) { close(filename[current]) }
+    else if (current == 0) { error("could not read input file") }
+    else { --current; error("could not read include file '" filename[current + 1] "'") }
+  }
+  if (level > 0) {
+    current = 0
+    error("unterminated if")
+  }
+  exit 0
+}
+function error(text) {
+  printf "%s:%d: error: %s\n",
+    filename[current], line[current], text > "/dev/stderr"
+  exit 1
+}
+function replace_vars(input) {
+  output = ""
+  while (match(input, "\\{\\{[ \t]*" VARIABLE "[ \t]*\\}\\}")) {
+    if (RSTART > 1) {
+      output = output substr(input, 0, RSTART - 1)
+    }
+    data = substr(input, RSTART + 2, RLENGTH - 4)
+    input = substr(input, RSTART + RLENGTH)
+
+    gsub(/[ \t]+/, "", data)
+    split(data, fields, /\./)
+
+    if (fields[1] == "env") {
+      output = output ENVIRON[fields[2]]
+    }
+    else {
+      output = output yadm[fields[2]]
+    }
+  }
+  return output input
 }
 EOF