From 9b64ad750f356f828c3cfa96bc42489b2f081d8a Mon Sep 17 00:00:00 2001
From: Daniel Shahaf <d.s@daniel.shahaf.name>
Date: Sat, 2 Jan 2016 21:22:01 +0000
Subject: [PATCH] tests: Add a 'print failures only' mode to 'make test',
 called 'make quiet-test'.

Fixes zsh-users/zsh-syntax-highlighting#262.

Currently, 'make quiet-test' uses Perl.  However, since it is considered a development
tool rather than a user-facing tool, users and downstream packages needn't install Perl.
Furthermore, even this dev-only dependency may be dropped in the future.

The only difference between tests/tap-filter here and the one in the issue is using
a `cat` subshell v. using 'undef $/; <STDIN>'.
---
 Makefile                    |  3 +++
 tests/README.md             |  3 +++
 tests/tap-filter            | 45 +++++++++++++++++++++++++++++++++++++
 tests/test-highlighting.zsh | 15 ++++++++++++-
 4 files changed, 65 insertions(+), 1 deletion(-)
 create mode 100755 tests/tap-filter

diff --git a/Makefile b/Makefile
index 0d01896..f454324 100644
--- a/Makefile
+++ b/Makefile
@@ -45,6 +45,9 @@ test:
 	done; \
 	exit $$result
 
+quiet-test:
+	$(MAKE) test QUIET=y
+
 perf:
 	@result=0; \
 	for test in highlighters/*; do \
diff --git a/tests/README.md b/tests/README.md
index faaed1e..2674aa7 100644
--- a/tests/README.md
+++ b/tests/README.md
@@ -38,6 +38,9 @@ All tests may be run with
     make test
 
 which will run all highlighting tests and report results in [TAP format][TAP].
+By default, the results of all tests will be printed; to show only "interesting"
+results (tests that failed but were expected to succeed, or vice-versa), run
+`make quiet-test` (or `make test QUIET=y`).
 
 [TAP]: http://testanything.org/
 
diff --git a/tests/tap-filter b/tests/tap-filter
new file mode 100755
index 0000000..6587808
--- /dev/null
+++ b/tests/tap-filter
@@ -0,0 +1,45 @@
+#!/usr/bin/env perl
+# -------------------------------------------------------------------------------------------------
+# Copyright (c) 2015 zsh-syntax-highlighting contributors
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without modification, are permitted
+# provided that the following conditions are met:
+#
+#  * Redistributions of source code must retain the above copyright notice, this list of conditions
+#    and the following disclaimer.
+#  * Redistributions in binary form must reproduce the above copyright notice, this list of
+#    conditions and the following disclaimer in the documentation and/or other materials provided
+#    with the distribution.
+#  * Neither the name of the zsh-syntax-highlighting contributors nor the names of its contributors
+#    may be used to endorse or promote products derived from this software without specific prior
+#    written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
+# IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
+# FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
+# IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+# OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+# -------------------------------------------------------------------------------------------------
+# vim: ft=perl sw=2 ts=2 et
+# -------------------------------------------------------------------------------------------------
+
+# This is a stdin-to-stdout filter that takes TAP output (such as 'make test')
+# on stdin and deletes lines pertaining to expected results.
+#
+# More specifically, if any of the test points in a test file either failed but
+# was expected to pass, or passed but was expected to fail, then emit that test
+# file's output; else, elide that test file's output.
+
+use v5.10.0;
+use warnings;
+use strict;
+
+undef $/; # slurp mode
+print for
+  grep { /^ok.*# TODO/m or /^not ok(?!.*# TODO)/m }
+    split /^(?=#)/m,
+      <STDIN>;
diff --git a/tests/test-highlighting.zsh b/tests/test-highlighting.zsh
index 5d35452..dc47989 100755
--- a/tests/test-highlighting.zsh
+++ b/tests/test-highlighting.zsh
@@ -132,10 +132,23 @@ run_test() {
   }
 }
 
+# Set up results_filter
+local results_filter
+if [[ $QUIET == y ]]; then
+  if type -w perl >/dev/null; then
+    results_filter=${0:A:h}/tap-filter
+  else
+    echo >&2 "Bail out! quiet mode not supported: perl not found"; exit 2
+  fi
+else
+  results_filter=cat
+fi
+[[ -n $results_filter ]] || { echo >&2 "Bail out! BUG setting \$results_filter"; exit 2 }
+
 # Process each test data file in test data directory.
 integer something_failed=0
 for data_file in ${0:h:h}/highlighters/$1/test-data/*.zsh; do
-  run_test "$data_file" | tee >(${0:A:h}/tap-colorizer.zsh) | grep -v '^not ok.*# TODO' | grep -q '^not ok\|^ok.*# TODO' && (( something_failed=1 ))
+  run_test "$data_file" | tee >($results_filter | ${0:A:h}/tap-colorizer.zsh) | grep -v '^not ok.*# TODO' | grep -q '^not ok\|^ok.*# TODO' && (( something_failed=1 ))
   (( $pipestatus[1] )) && exit 2
 done