test harness: Change cardinality check semantics

The cardinality check shall —

- if the test sets \$expected_mismatch, be XFail;
- elif any test points is XFail, be skipped;
- else, be expected to pass.

To test this, change «6 * 9» to «6 + 9» in test-data/arith1.zsh that
will be added in the after-next (grandchild) commit.
This commit is contained in:
Daniel Shahaf 2020-03-17 14:58:00 +00:00
parent ea2f1060f6
commit 61c1cfe99f
2 changed files with 22 additions and 10 deletions

View file

@ -30,9 +30,11 @@ need not match the order in `$region_highlight`.
4. 4.
Normally, tests fail if `$expected_region_highlight` and `$region_highlight` Normally, tests fail if `$expected_region_highlight` and `$region_highlight`
have different numbers of elements. Tests may set `$expected_mismatch` to an have different numbers of elements. To mark this check as expected to fail,
explanation string (like `$todo`) to avoid this and skip the cardinality check. tests may set `$expected_mismatch` to an explanation string (like `$todo`);
`$expected_mismatch` is set implicitly if the `$todo` component is present. this is useful when the only difference between actual and expected is that actual
has some additional, superfluous elements. This check is skipped if the
`$todo` component is present in any regular test point.
**Note**: `$region_highlight` uses the same `"$i $j $style"` syntax but **Note**: `$region_highlight` uses the same `"$i $j $style"` syntax but
interprets the indexes differently. interprets the indexes differently.

View file

@ -120,6 +120,7 @@ run_test_internal() {
# Load the data and prepare checking it. # Load the data and prepare checking it.
local BUFFER CURSOR MARK PENDING PREBUFFER REGION_ACTIVE WIDGET REPLY skip_test unsorted=0 local BUFFER CURSOR MARK PENDING PREBUFFER REGION_ACTIVE WIDGET REPLY skip_test unsorted=0
local expected_mismatch local expected_mismatch
local skip_mismatch
local -a expected_region_highlight region_highlight local -a expected_region_highlight region_highlight
. "$srcdir"/"$1" . "$srcdir"/"$1"
@ -155,11 +156,11 @@ run_test_internal() {
local todo= local todo=
if (( $+expected_highlight_zone[4] )); then if (( $+expected_highlight_zone[4] )); then
todo="# TODO $expected_highlight_zone[4]" todo="# TODO $expected_highlight_zone[4]"
: ${expected_mismatch:="cardinality check disabled whilst regular test points are expected to fail"} skip_mismatch="cardinality check disabled whilst regular test points are expected to fail"
fi fi
if ! (( $+region_highlight[i] )); then if ! (( $+region_highlight[i] )); then
print -r -- "not ok $i - unmatched expectation ($exp_start $exp_end $expected_highlight_zone[3])" \ print -r -- "not ok $i - unmatched expectation ($exp_start $exp_end $expected_highlight_zone[3])" \
"${expected_mismatch:+"# TODO ${(qqq)expected_mismatch}"}" "${skip_mismatch:+"# TODO ${(qqq)skip_mismatch}"}"
continue continue
fi fi
local -a highlight_zone; highlight_zone=( ${(z)region_highlight[i]} ) local -a highlight_zone; highlight_zone=( ${(z)region_highlight[i]} )
@ -183,18 +184,27 @@ run_test_internal() {
unset desc unset desc
done done
# If both $skip_mismatch and $expected_mismatch are set, that means the test
# has some XFail test points, _and_ explicitly sets $expected_mismatch as
# well. Explicit settings should have priority, so we ignore $skip_mismatch
# if $expected_mismatch is set.
if [[ -n $skip_mismatch && -z $expected_mismatch ]]; then
tap_escape $skip_mismatch; skip_mismatch=$REPLY
print "ok $i - cardinality check" "# SKIP $skip_mismatch"
else
local todo
if [[ -n $expected_mismatch ]]; then if [[ -n $expected_mismatch ]]; then
tap_escape $expected_mismatch; expected_mismatch=$REPLY tap_escape $expected_mismatch; expected_mismatch=$REPLY
print "ok $i - cardinality check" "# SKIP $expected_mismatch" todo="# TODO $expected_mismatch"
else fi
if (( $#expected_region_highlight == $#region_highlight )); then if (( $#expected_region_highlight == $#region_highlight )); then
print -r -- "ok $i - cardinality check" print -r -- "ok $i - cardinality check${todo:+ - }$todo"
else else
local details local details
details+="have $#expected_region_highlight expectations and $#region_highlight region_highlight entries: " details+="have $#expected_region_highlight expectations and $#region_highlight region_highlight entries: "
details+="«$(typeset_p expected_region_highlight)» «$(typeset_p region_highlight)»" details+="«$(typeset_p expected_region_highlight)» «$(typeset_p region_highlight)»"
tap_escape $details; details=$REPLY tap_escape $details; details=$REPLY
print -r -- "not ok $i - cardinality check - $details" print -r -- "not ok $i - cardinality check - $details${todo:+ - }$todo"
fi fi
fi fi
} }