test harness: Run each test as a single subshell.

That allows tests to be completely independent of each other, so tests that
change global state — such as modify environment variables, define functions or
aliases, or hash commands — will not affect other tests, without needing an
explicit cleanup step.

This enables testing path-tilde-home with and without $HOME set, which is part
of issue #216.

While at it, convert the test harness to TAP.  This fixes issue #180 by adding
support for "not ok 42 # TODO" output.

This commit assumes that 'grep' supports POSIX-compliant -q and -v flags.

Patch-by: Matthew Martin <phy1729@gmail.com>

tests/README.md

@@ -6,9 +6,10 @@ Utility scripts for testing zsh-syntax-highlighting highlighters.
 The tests expect the highlighter directory to contain a `test-data` directory with test data files. See the [main highlighter](../highlighters/main/test-data) for examples.
 
 Each test should define the array parameter `$expected_region_highlight`.
-The value of that parameter is a list of `"$i $j $style"` strings.
+The value of that parameter is a list of `"$i $j $style [$todo]"` strings.
 Each string specifies the highlighting that `$BUFFER[$i,$j]` should have;
 that is, `$i` and `$j` specify a range, 1-indexed, inclusive of both endpoints.
+If `$todo` exists, the test point is marked as TODO (the failure of that test point will not fail the test), and `$todo` is used as the explanation.
 
 _Note_: `$region_highlight` uses the same `"$i $j $style"` syntax but interprets the indexes differently.
 
@@ -19,6 +20,12 @@ highlighting test
 
     zsh test-highlighting.zsh <HIGHLIGHTER NAME>
 
+All tests may be run with
+
+    make test
+
+which will run all highlighting tests and report results in [TAP](http://testanything.org/) format.
+
 
 performance test
 ----------------

tests/test-highlighting.zsh

@@ -47,87 +47,71 @@
   exit 1
 }
 
-local -a errors highlight_zone
-local -A observed_result
-local -A save_ZSH_HIGHLIGHT_STYLES
-integer something_failed=0
-local unused_highlight='bg=red,underline' # a style unused by anything else, for tests to use
-
 # Load the main script.
 . ${0:h:h}/zsh-syntax-highlighting.zsh
 
 # Activate the highlighter.
 ZSH_HIGHLIGHT_HIGHLIGHTERS=($1)
 
-# Cache a pristine set of styles.
-save_ZSH_HIGHLIGHT_STYLES=( "${(@kv)ZSH_HIGHLIGHT_STYLES}" )
-
-# Process each test data file in test data directory.
-for data_file in ${0:h:h}/highlighters/$1/test-data/*.zsh; do
+# Runs a highlighting test
+# $1: data file
+run_test() {
+  local -a highlight_zone
+  local unused_highlight='bg=red,underline' # a style unused by anything else, for tests to use
 
   # Load the data and prepare checking it.
-  PREBUFFER= BUFFER= ; expected_region_highlight=(); errors=()
-  echo -n "* ${data_file:t:r}: "
-  ZSH_HIGHLIGHT_STYLES=( "${(@kv)save_ZSH_HIGHLIGHT_STYLES}" )
-  . $data_file
+  PREBUFFER= BUFFER= ;
+  . "$1"
 
   # Check the data declares $PREBUFFER or $BUFFER.
-  if [[ ${#PREBUFFER} -eq 0 && ${#BUFFER} -eq 0 ]]; then
-    errors+=("Either 'PREBUFFER' or 'BUFFER' must be declared and non-blank")
-  else
-
-    # Check the data declares $expected_region_highlight.
-    if [[ ${#expected_region_highlight} -eq 0 ]]; then
-      errors+=("'expected_region_highlight' is not declared or empty.")
-    else
-
-      # Process the data.
-      region_highlight=()
-      _zsh_highlight
-
-      # Overlapping regions can be declared in region_highlight, so we first build an array of the
-      # observed highlighting.
-      observed_result=()
-      for i in {1..${#region_highlight}}; do
-        highlight_zone=${(z)region_highlight[$i]}
-        integer start=$highlight_zone[1] end=$highlight_zone[2]
-        if (( start < end )) # region_highlight ranges are half-open
-        then
-          (( --end )) # convert to closed range, like expected_region_highlight
-          (( ++start, ++end )) # region_highlight is 0-indexed; expected_region_highlight is 1-indexed
-          for j in {$start..$end}; do
-            observed_result[$j]=$highlight_zone[3]
-          done
-        else
-          # noop range; ignore.
-        fi
+  [[ -z $PREBUFFER && -z $BUFFER ]] && { echo >&2 "Bail out! Either 'PREBUFFER' or 'BUFFER' must be declared and non-blank"; return 1; }
+  # Check the data declares $expected_region_highlight.
+  (( ${#expected_region_highlight} == 0 )) && { echo >&2 "Bail out! 'expected_region_highlight' is not declared or empty."; return 1; }
+
+  # Process the data.
+  region_highlight=()
+  _zsh_highlight
+
+  # Overlapping regions can be declared in region_highlight, so we first build an array of the
+  # observed highlighting.
+  local -A observed_result
+  for i in {1..${#region_highlight}}; do
+    highlight_zone=${(z)region_highlight[$i]}
+    integer start=$highlight_zone[1] end=$highlight_zone[2]
+    if (( start < end )) # region_highlight ranges are half-open
+    then
+      (( --end )) # convert to closed range, like expected_region_highlight
+      (( ++start, ++end )) # region_highlight is 0-indexed; expected_region_highlight is 1-indexed
+      for j in {$start..$end}; do
+        observed_result[$j]=$highlight_zone[3]
       done
-
-      # Then we compare the observed result with the expected one.
-      for i in {1..${#expected_region_highlight}}; do
-        highlight_zone=${(z)expected_region_highlight[$i]}
-        for j in {$highlight_zone[1]..$highlight_zone[2]}; do
-          if [[ "$observed_result[$j]" != "$highlight_zone[3]" ]]; then
-            errors+=("'$BUFFER[$highlight_zone[1],$highlight_zone[2]]' [$highlight_zone[1],$highlight_zone[2]]: expected '$highlight_zone[3]', observed '$observed_result[$j]'.")
-            break
-          fi
-        done
-      done
-
+    else
+      # noop range; ignore.
     fi
-  fi
-
-  # Format result/errors.
-  if [[ ${#errors} -eq 0 ]]; then
-    echo "OK"
-  else
-    echo "KO"
-    (( something_failed=1 ))
-    for error in $errors; do
-      echo "   - $error"
+  done
+
+  # Then we compare the observed result with the expected one.
+  local todo
+  echo "1..${#expected_region_highlight}"
+  for i in {1..${#expected_region_highlight}}; do
+    highlight_zone=${(z)expected_region_highlight[$i]}
+    [[ -n "$highlight_zone[4]" ]] && todo=" # TODO $highlight_zone[4]"
+    for j in {$highlight_zone[1]..$highlight_zone[2]}; do
+      if [[ "$observed_result[$j]" != "$highlight_zone[3]" ]]; then
+        echo "not ok $i '$BUFFER[$highlight_zone[1],$highlight_zone[2]]' [$highlight_zone[1],$highlight_zone[2]]: expected '$highlight_zone[3]', observed '$observed_result[$j]'.$todo"
+        continue 2
+      fi
     done
-  fi
+    echo "ok $i$todo"
+  done
+}
 
+# Process each test data file in test data directory.
+integer something_failed=0
+for data_file in ${0:h:h}/highlighters/$1/test-data/*.zsh; do
+  echo "# ${data_file:t:r}"
+  (run_test "$data_file") | tee >(cat) | grep '^not ok' | grep -qv ' # TODO' && (( something_failed=1 ))
+  (( $pipestatus[1] )) && exit 2
 done
 
 exit $something_failed