git@vger.kernel.org mailing list mirror (one of many)
 help / color / mirror / code / Atom feed
From: "Ævar Arnfjörð Bjarmason" <avarab@gmail.com>
To: git@vger.kernel.org
Cc: "Junio C Hamano" <gitster@pobox.com>,
	"Derrick Stolee" <stolee@gmail.com>,
	"Ævar Arnfjörð Bjarmason" <avarab@gmail.com>
Subject: [PATCH v3 04/15] test-lib: add a --invert-exit-code switch
Date: Thu, 28 Jul 2022 01:13:32 +0200	[thread overview]
Message-ID: <patch-v3-04.15-f3144af2190-20220727T230800Z-avarab@gmail.com> (raw)
In-Reply-To: <cover-v3-00.15-00000000000-20220727T230800Z-avarab@gmail.com>

Add the ability to have those tests that fail return 0, and those
tests that succeed return 1. This is useful e.g. to run "--stress"
tests on tests that fail 99% of the time on some setup, i.e. to smoke
out the flaky run which yielded success.

In a subsequent commit a new SANITIZE=leak mode will make use of this.

Signed-off-by: Ævar Arnfjörð Bjarmason <avarab@gmail.com>
---
 t/t0000-basic.sh | 72 ++++++++++++++++++++++++++++++++++++++++++++++++
 t/test-lib.sh    | 45 ++++++++++++++++++++++++++++--
 2 files changed, 115 insertions(+), 2 deletions(-)

diff --git a/t/t0000-basic.sh b/t/t0000-basic.sh
index 17a268ccd1b..502b4bcf9ea 100755
--- a/t/t0000-basic.sh
+++ b/t/t0000-basic.sh
@@ -578,6 +578,78 @@ test_expect_success 'subtest: --run invalid range end' '
 	EOF_ERR
 '
 
+test_expect_success 'subtest: --invert-exit-code without --immediate' '
+	run_sub_test_lib_test_err full-pass \
+		--invert-exit-code &&
+	check_sub_test_lib_test_err full-pass \
+		<<-\EOF_OUT 3<<-EOF_ERR
+	ok 1 - passing test #1
+	ok 2 - passing test #2
+	ok 3 - passing test #3
+	# passed all 3 test(s)
+	1..3
+	# faking up non-zero exit with --invert-exit-code
+	EOF_OUT
+	EOF_ERR
+'
+
+test_expect_success 'subtest: --invert-exit-code with --immediate: all passed' '
+	run_sub_test_lib_test_err full-pass \
+		--invert-exit-code --immediate &&
+	check_sub_test_lib_test_err full-pass \
+		<<-\EOF_OUT 3<<-EOF_ERR
+	ok 1 - passing test #1
+	ok 2 - passing test #2
+	ok 3 - passing test #3
+	# passed all 3 test(s)
+	1..3
+	# faking up non-zero exit with --invert-exit-code
+	EOF_OUT
+	EOF_ERR
+'
+
+test_expect_success 'subtest: --invert-exit-code without --immediate: partial pass' '
+	run_sub_test_lib_test partial-pass \
+		--invert-exit-code &&
+	check_sub_test_lib_test partial-pass <<-\EOF
+	ok 1 - passing test #1
+	not ok 2 - # TODO induced breakage (--invert-exit-code): failing test #2
+	#	false
+	ok 3 - passing test #3
+	# failed 1 among 3 test(s)
+	1..3
+	# faked up failures as TODO & now exiting with 0 due to --invert-exit-code
+	EOF
+'
+
+test_expect_success 'subtest: --invert-exit-code with --immediate: partial pass' '
+	run_sub_test_lib_test partial-pass \
+		--invert-exit-code --immediate &&
+	check_sub_test_lib_test partial-pass \
+		<<-\EOF_OUT 3<<-EOF_ERR
+	ok 1 - passing test #1
+	not ok 2 - # TODO induced breakage (--invert-exit-code): failing test #2
+	#	false
+	1..2
+	# faked up failures as TODO & now exiting with 0 due to --invert-exit-code
+	EOF_OUT
+	EOF_ERR
+'
+
+test_expect_success 'subtest: --invert-exit-code --immediate: got a failure' '
+	run_sub_test_lib_test partial-pass \
+		--invert-exit-code --immediate &&
+	check_sub_test_lib_test_err partial-pass \
+		<<-\EOF_OUT 3<<-EOF_ERR
+	ok 1 - passing test #1
+	not ok 2 - # TODO induced breakage (--invert-exit-code): failing test #2
+	#	false
+	1..2
+	# faked up failures as TODO & now exiting with 0 due to --invert-exit-code
+	EOF_OUT
+	EOF_ERR
+'
+
 test_expect_success 'subtest: tests respect prerequisites' '
 	write_and_run_sub_test_lib_test prereqs <<-\EOF &&
 
diff --git a/t/test-lib.sh b/t/test-lib.sh
index 118720493bb..31213b5f95b 100644
--- a/t/test-lib.sh
+++ b/t/test-lib.sh
@@ -238,6 +238,9 @@ parse_option () {
 			;;
 		esac
 		;;
+	--invert-exit-code)
+		invert_exit_code=t
+		;;
 	*)
 		echo "error: unknown test option '$opt'" >&2; exit 1 ;;
 	esac
@@ -788,15 +791,31 @@ test_ok_ () {
 	finalize_test_case_output ok "$@"
 }
 
+_invert_exit_code_failure_end_blurb () {
+	say_color warn "# faked up failures as TODO & now exiting with 0 due to --invert-exit-code"
+}
+
 test_failure_ () {
 	failure_label=$1
 	test_failure=$(($test_failure + 1))
-	say_color error "not ok $test_count - $1"
+	local pfx=""
+	if test -n "$invert_exit_code" # && test -n "$HARNESS_ACTIVE"
+	then
+		pfx="# TODO induced breakage (--invert-exit-code):"
+	fi
+	say_color error "not ok $test_count - ${pfx:+$pfx }$1"
 	shift
 	printf '%s\n' "$*" | sed -e 's/^/#	/'
 	if test -n "$immediate"
 	then
 		say_color error "1..$test_count"
+		if test -n "$invert_exit_code"
+		then
+			finalize_test_output
+			_invert_exit_code_failure_end_blurb
+			GIT_EXIT_OK=t
+			exit 0
+		fi
 		_error_exit
 	fi
 	finalize_test_case_output failure "$failure_label" "$@"
@@ -1229,7 +1248,14 @@ test_done () {
 			esac
 		fi
 
-		if test -z "$debug" && test -n "$remove_trash"
+		if test -n "$stress" && test -n "$invert_exit_code"
+		then
+			# We're about to move our "$TRASH_DIRECTORY"
+			# to "$TRASH_DIRECTORY.stress-failed" if
+			# --stress is combined with
+			# --invert-exit-code.
+			say "with --stress and --invert-exit-code we're not removing '$TRASH_DIRECTORY'"
+		elif test -z "$debug" && test -n "$remove_trash"
 		then
 			test -d "$TRASH_DIRECTORY" ||
 			error "Tests passed but trash directory already removed before test cleanup; aborting"
@@ -1242,6 +1268,14 @@ test_done () {
 			} ||
 			error "Tests passed but test cleanup failed; aborting"
 		fi
+
+		if test -z "$skip_all" && test -n "$invert_exit_code"
+		then
+			say_color warn "# faking up non-zero exit with --invert-exit-code"
+			GIT_EXIT_OK=t
+			exit 1
+		fi
+
 		test_at_end_hook_
 
 		GIT_EXIT_OK=t
@@ -1254,6 +1288,13 @@ test_done () {
 			say "1..$test_count"
 		fi
 
+		if test -n "$invert_exit_code"
+		then
+			_invert_exit_code_failure_end_blurb
+			GIT_EXIT_OK=t
+			exit 0
+		fi
+
 		GIT_EXIT_OK=t
 		exit 1 ;;
 
-- 
2.37.1.1127.g4ecf1c08f67


  parent reply	other threads:[~2022-07-27 23:14 UTC|newest]

Thread overview: 57+ messages / expand[flat|nested]  mbox.gz  Atom feed  top
2022-07-19 21:05 [PATCH 00/10] leak test: add "check" test mode, mark leak-free tests Ævar Arnfjörð Bjarmason
2022-07-19 21:05 ` [PATCH 01/10] test-lib.sh: use $1, not $@ in test_known_broken_{ok,failure}_ Ævar Arnfjörð Bjarmason
2022-07-19 21:05 ` [PATCH 02/10] test-lib.sh: don't set GIT_EXIT_OK before calling test_atexit_handler Ævar Arnfjörð Bjarmason
2022-07-19 21:05 ` [PATCH 03/10] test-lib.sh: fix GIT_EXIT_OK logic errors, use BAIL_OUT Ævar Arnfjörð Bjarmason
2022-07-19 21:05 ` [PATCH 04/10] test-lib.sh: add a --invert-exit-code switch Ævar Arnfjörð Bjarmason
2022-07-19 21:05 ` [PATCH 05/10] t/README: reword the "GIT_TEST_PASSING_SANITIZE_LEAK" description Ævar Arnfjörð Bjarmason
2022-07-20  1:38   ` Derrick Stolee
2022-07-19 21:05 ` [PATCH 06/10] test-lib: add a SANITIZE=leak logging mode Ævar Arnfjörð Bjarmason
2022-07-19 21:05 ` [PATCH 07/10] test-lib.sh: add a GIT_TEST_PASSING_SANITIZE_LEAK=check mode Ævar Arnfjörð Bjarmason
2022-07-20  1:43   ` Derrick Stolee
2022-07-19 21:05 ` [PATCH 08/10] test-lib: have the "check" mode for SANITIZE=leak consider leak logs Ævar Arnfjörð Bjarmason
2022-07-20  1:47   ` Derrick Stolee
2022-07-19 21:05 ` [PATCH 09/10] leak tests: mark passing SANITIZE=leak tests as leak-free Ævar Arnfjörð Bjarmason
2022-07-20  1:50   ` Derrick Stolee
2022-07-19 21:05 ` [PATCH 10/10] log tests: don't use "exit 1" outside a sub-shell Ævar Arnfjörð Bjarmason
2022-07-20 17:11   ` Junio C Hamano
2022-07-20 21:21 ` [PATCH v2 00/14] leak test: add "check" test mode, mark leak-free tests Ævar Arnfjörð Bjarmason
2022-07-20 21:21   ` [PATCH v2 01/14] test-lib: use $1, not $@ in test_known_broken_{ok,failure}_ Ævar Arnfjörð Bjarmason
2022-07-20 21:21   ` [PATCH v2 02/14] test-lib: don't set GIT_EXIT_OK before calling test_atexit_handler Ævar Arnfjörð Bjarmason
2022-07-20 21:21   ` [PATCH v2 03/14] test-lib: fix GIT_EXIT_OK logic errors, use BAIL_OUT Ævar Arnfjörð Bjarmason
2022-07-20 21:21   ` [PATCH v2 04/14] test-lib: add a --invert-exit-code switch Ævar Arnfjörð Bjarmason
2022-07-20 21:21   ` [PATCH v2 05/14] t/README: reword the "GIT_TEST_PASSING_SANITIZE_LEAK" description Ævar Arnfjörð Bjarmason
2022-07-20 21:21   ` [PATCH v2 06/14] test-lib: add a SANITIZE=leak logging mode Ævar Arnfjörð Bjarmason
2022-07-20 21:21   ` [PATCH v2 07/14] t/Makefile: don't remove test-results in "clean-except-prove-cache" Ævar Arnfjörð Bjarmason
2022-07-20 21:21   ` [PATCH v2 08/14] tests: move copy/pasted PERL + Test::More checks to a lib-perl.sh Ævar Arnfjörð Bjarmason
2022-07-20 21:21   ` [PATCH v2 09/14] test-lib: simplify by removing test_external Ævar Arnfjörð Bjarmason
2022-07-20 21:21   ` [PATCH v2 10/14] test-lib: add a GIT_TEST_PASSING_SANITIZE_LEAK=check mode Ævar Arnfjörð Bjarmason
2022-07-20 21:21   ` [PATCH v2 11/14] test-lib: have the "check" mode for SANITIZE=leak consider leak logs Ævar Arnfjörð Bjarmason
2022-07-20 21:21   ` [PATCH v2 12/14] leak tests: mark passing SANITIZE=leak tests as leak-free Ævar Arnfjörð Bjarmason
2022-07-20 21:21   ` [PATCH v2 13/14] upload-pack: fix a memory leak in create_pack_file() Ævar Arnfjörð Bjarmason
2022-07-20 21:21   ` [PATCH v2 14/14] CI: use "GIT_TEST_SANITIZE_LEAK_LOG=true" in linux-leaks Ævar Arnfjörð Bjarmason
2022-07-27 23:13   ` [PATCH v3 00/15] leak test: add "check" test mode, mark leak-free tests Ævar Arnfjörð Bjarmason
2022-07-27 23:13     ` [PATCH v3 01/15] test-lib: use $1, not $@ in test_known_broken_{ok,failure}_ Ævar Arnfjörð Bjarmason
2022-07-27 23:13     ` [PATCH v3 02/15] test-lib: don't set GIT_EXIT_OK before calling test_atexit_handler Ævar Arnfjörð Bjarmason
2022-07-27 23:13     ` [PATCH v3 03/15] test-lib: fix GIT_EXIT_OK logic errors, use BAIL_OUT Ævar Arnfjörð Bjarmason
2022-07-27 23:13     ` Ævar Arnfjörð Bjarmason [this message]
2022-07-27 23:13     ` [PATCH v3 05/15] t/README: reword the "GIT_TEST_PASSING_SANITIZE_LEAK" description Ævar Arnfjörð Bjarmason
2022-07-27 23:13     ` [PATCH v3 06/15] test-lib: add a SANITIZE=leak logging mode Ævar Arnfjörð Bjarmason
2022-07-27 23:13     ` [PATCH v3 07/15] t/Makefile: don't remove test-results in "clean-except-prove-cache" Ævar Arnfjörð Bjarmason
2022-09-20 10:54       ` [PATCH] t/Makefile: remove 'test-results' on 'make clean' SZEDER Gábor
2022-09-20 19:51         ` Jeff King
2022-09-20 20:11           ` SZEDER Gábor
2022-09-20 20:42             ` Jeff King
2022-09-20 20:16         ` [PATCH v2] " SZEDER Gábor
2022-09-21  6:59           ` Ævar Arnfjörð Bjarmason
2022-09-21 17:49             ` Junio C Hamano
2022-09-21 17:52           ` Junio C Hamano
2022-09-26  9:08             ` Ævar Arnfjörð Bjarmason
2022-09-26 19:08               ` Junio C Hamano
2022-07-27 23:13     ` [PATCH v3 08/15] tests: move copy/pasted PERL + Test::More checks to a lib-perl.sh Ævar Arnfjörð Bjarmason
2022-07-27 23:13     ` [PATCH v3 09/15] test-lib: simplify by removing test_external Ævar Arnfjörð Bjarmason
2022-07-27 23:13     ` [PATCH v3 10/15] test-lib: add a GIT_TEST_PASSING_SANITIZE_LEAK=check mode Ævar Arnfjörð Bjarmason
2022-07-27 23:13     ` [PATCH v3 11/15] test-lib: have the "check" mode for SANITIZE=leak consider leak logs Ævar Arnfjörð Bjarmason
2022-07-27 23:13     ` [PATCH v3 12/15] leak tests: don't skip some tests under SANITIZE=leak Ævar Arnfjörð Bjarmason
2022-07-27 23:13     ` [PATCH v3 13/15] leak tests: mark passing SANITIZE=leak tests as leak-free Ævar Arnfjörð Bjarmason
2022-07-27 23:13     ` [PATCH v3 14/15] upload-pack: fix a memory leak in create_pack_file() Ævar Arnfjörð Bjarmason
2022-07-27 23:13     ` [PATCH v3 15/15] CI: use "GIT_TEST_SANITIZE_LEAK_LOG=true" in linux-leaks Ævar Arnfjörð Bjarmason

Reply instructions:

You may reply publicly to this message via plain-text email
using any one of the following methods:

* Save the following mbox file, import it into your mail client,
  and reply-to-all from there: mbox

  Avoid top-posting and favor interleaved quoting:
  https://en.wikipedia.org/wiki/Posting_style#Interleaved_style

  List information: http://vger.kernel.org/majordomo-info.html

* Reply using the --to, --cc, and --in-reply-to
  switches of git-send-email(1):

  git send-email \
    --in-reply-to=patch-v3-04.15-f3144af2190-20220727T230800Z-avarab@gmail.com \
    --to=avarab@gmail.com \
    --cc=git@vger.kernel.org \
    --cc=gitster@pobox.com \
    --cc=stolee@gmail.com \
    /path/to/YOUR_REPLY

  https://kernel.org/pub/software/scm/git/docs/git-send-email.html

* If your mail client supports setting the In-Reply-To header
  via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line before the message body.
Code repositories for project(s) associated with this public inbox

	https://80x24.org/mirrors/git.git

This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for read-only IMAP folder(s) and NNTP newsgroup(s).