From d99aa58aaaef6d9f416f338333277eeaec44ffdc Mon Sep 17 00:00:00 2001 From: Matthew Martin Date: Fri, 23 Oct 2015 03:27:05 +0000 Subject: [PATCH] test harness: Run each test as a single subshell. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit That allows tests to be completely independent of each other, so tests that change global state — such as modify environment variables, define functions or aliases, or hash commands — will not affect other tests, without needing an explicit cleanup step. This enables testing path-tilde-home with and without $HOME set, which is part of issue #216. While at it, convert the test harness to TAP. This fixes issue #180 by adding support for "not ok 42 # TODO" output. This commit assumes that 'grep' supports POSIX-compliant -q and -v flags. Patch-by: Matthew Martin --- tests/README.md | 9 ++- tests/test-highlighting.zsh | 118 ++++++++++++++++-------------------- 2 files changed, 59 insertions(+), 68 deletions(-) diff --git a/tests/README.md b/tests/README.md index 676dd64..15aacd8 100644 --- a/tests/README.md +++ b/tests/README.md @@ -6,9 +6,10 @@ Utility scripts for testing zsh-syntax-highlighting highlighters. The tests expect the highlighter directory to contain a `test-data` directory with test data files. See the [main highlighter](../highlighters/main/test-data) for examples. Each test should define the array parameter `$expected_region_highlight`. -The value of that parameter is a list of `"$i $j $style"` strings. +The value of that parameter is a list of `"$i $j $style [$todo]"` strings. Each string specifies the highlighting that `$BUFFER[$i,$j]` should have; that is, `$i` and `$j` specify a range, 1-indexed, inclusive of both endpoints. +If `$todo` exists, the test point is marked as TODO (the failure of that test point will not fail the test), and `$todo` is used as the explanation. _Note_: `$region_highlight` uses the same `"$i $j $style"` syntax but interprets the indexes differently. @@ -19,6 +20,12 @@ highlighting test zsh test-highlighting.zsh +All tests may be run with + + make test + +which will run all highlighting tests and report results in [TAP](http://testanything.org/) format. + performance test ---------------- diff --git a/tests/test-highlighting.zsh b/tests/test-highlighting.zsh index 83fde94..d855e2e 100755 --- a/tests/test-highlighting.zsh +++ b/tests/test-highlighting.zsh @@ -47,87 +47,71 @@ exit 1 } -local -a errors highlight_zone -local -A observed_result -local -A save_ZSH_HIGHLIGHT_STYLES -integer something_failed=0 -local unused_highlight='bg=red,underline' # a style unused by anything else, for tests to use - # Load the main script. . ${0:h:h}/zsh-syntax-highlighting.zsh # Activate the highlighter. ZSH_HIGHLIGHT_HIGHLIGHTERS=($1) -# Cache a pristine set of styles. -save_ZSH_HIGHLIGHT_STYLES=( "${(@kv)ZSH_HIGHLIGHT_STYLES}" ) - -# Process each test data file in test data directory. -for data_file in ${0:h:h}/highlighters/$1/test-data/*.zsh; do +# Runs a highlighting test +# $1: data file +run_test() { + local -a highlight_zone + local unused_highlight='bg=red,underline' # a style unused by anything else, for tests to use # Load the data and prepare checking it. - PREBUFFER= BUFFER= ; expected_region_highlight=(); errors=() - echo -n "* ${data_file:t:r}: " - ZSH_HIGHLIGHT_STYLES=( "${(@kv)save_ZSH_HIGHLIGHT_STYLES}" ) - . $data_file + PREBUFFER= BUFFER= ; + . "$1" # Check the data declares $PREBUFFER or $BUFFER. - if [[ ${#PREBUFFER} -eq 0 && ${#BUFFER} -eq 0 ]]; then - errors+=("Either 'PREBUFFER' or 'BUFFER' must be declared and non-blank") - else - - # Check the data declares $expected_region_highlight. - if [[ ${#expected_region_highlight} -eq 0 ]]; then - errors+=("'expected_region_highlight' is not declared or empty.") - else - - # Process the data. - region_highlight=() - _zsh_highlight - - # Overlapping regions can be declared in region_highlight, so we first build an array of the - # observed highlighting. - observed_result=() - for i in {1..${#region_highlight}}; do - highlight_zone=${(z)region_highlight[$i]} - integer start=$highlight_zone[1] end=$highlight_zone[2] - if (( start < end )) # region_highlight ranges are half-open - then - (( --end )) # convert to closed range, like expected_region_highlight - (( ++start, ++end )) # region_highlight is 0-indexed; expected_region_highlight is 1-indexed - for j in {$start..$end}; do - observed_result[$j]=$highlight_zone[3] - done - else - # noop range; ignore. - fi + [[ -z $PREBUFFER && -z $BUFFER ]] && { echo >&2 "Bail out! Either 'PREBUFFER' or 'BUFFER' must be declared and non-blank"; return 1; } + # Check the data declares $expected_region_highlight. + (( ${#expected_region_highlight} == 0 )) && { echo >&2 "Bail out! 'expected_region_highlight' is not declared or empty."; return 1; } + + # Process the data. + region_highlight=() + _zsh_highlight + + # Overlapping regions can be declared in region_highlight, so we first build an array of the + # observed highlighting. + local -A observed_result + for i in {1..${#region_highlight}}; do + highlight_zone=${(z)region_highlight[$i]} + integer start=$highlight_zone[1] end=$highlight_zone[2] + if (( start < end )) # region_highlight ranges are half-open + then + (( --end )) # convert to closed range, like expected_region_highlight + (( ++start, ++end )) # region_highlight is 0-indexed; expected_region_highlight is 1-indexed + for j in {$start..$end}; do + observed_result[$j]=$highlight_zone[3] done - - # Then we compare the observed result with the expected one. - for i in {1..${#expected_region_highlight}}; do - highlight_zone=${(z)expected_region_highlight[$i]} - for j in {$highlight_zone[1]..$highlight_zone[2]}; do - if [[ "$observed_result[$j]" != "$highlight_zone[3]" ]]; then - errors+=("'$BUFFER[$highlight_zone[1],$highlight_zone[2]]' [$highlight_zone[1],$highlight_zone[2]]: expected '$highlight_zone[3]', observed '$observed_result[$j]'.") - break - fi - done - done - + else + # noop range; ignore. fi - fi - - # Format result/errors. - if [[ ${#errors} -eq 0 ]]; then - echo "OK" - else - echo "KO" - (( something_failed=1 )) - for error in $errors; do - echo " - $error" + done + + # Then we compare the observed result with the expected one. + local todo + echo "1..${#expected_region_highlight}" + for i in {1..${#expected_region_highlight}}; do + highlight_zone=${(z)expected_region_highlight[$i]} + [[ -n "$highlight_zone[4]" ]] && todo=" # TODO $highlight_zone[4]" + for j in {$highlight_zone[1]..$highlight_zone[2]}; do + if [[ "$observed_result[$j]" != "$highlight_zone[3]" ]]; then + echo "not ok $i '$BUFFER[$highlight_zone[1],$highlight_zone[2]]' [$highlight_zone[1],$highlight_zone[2]]: expected '$highlight_zone[3]', observed '$observed_result[$j]'.$todo" + continue 2 + fi done - fi + echo "ok $i$todo" + done +} +# Process each test data file in test data directory. +integer something_failed=0 +for data_file in ${0:h:h}/highlighters/$1/test-data/*.zsh; do + echo "# ${data_file:t:r}" + (run_test "$data_file") | tee >(cat) | grep '^not ok' | grep -qv ' # TODO' && (( something_failed=1 )) + (( $pipestatus[1] )) && exit 2 done exit $something_failed