[PATCH v2 7/7] wildmatch test: create & test files on disk in addition to in-memory

["Ævar Arnfjörð Bjarmason" <avarab@gmail.com>]

There has never been any full roundtrip testing of what git-ls-files
and other functions that use wildmatch() actually do, rather we've
been satisfied with just testing the underlying C function.

Due to git-ls-files and friends having their own codepaths before they
call wildmatch() there's sometimes differences in the behavior between
the two, and even when we test for those (as with
9e4e8a64c2 ("pathspec: die on empty strings as pathspec", 2017-06-06))
there was no one place where you can review how these two modes
differ.

Now there is. We now attempt to create a file called $haystack and
match $needle against it for each pair of $needle and $haystack that
we were passing to test-wildmatch.

If we can't create the file we skip the test. This ensures that we can
run this on all platforms and not maintain some infinitely growing
whitelist of e.g. platforms that don't support certain characters in
filenames.

As a result of doing this we can now see the cases where these two
ways of testing wildmatch differ:

 * Creating a file called 'a[]b' and running ls-files 'a[]b' will show
   that file, but wildmatch("a[]b", "a[]b") will not match

 * wildmatch() won't match a file called \ against \, but ls-files
   will.

 * `git --glob-pathspecs ls-files 'foo**'` will match a file
   'foo/bba/arr', but wildmatch won't, however pathmatch will.

   This seems like a bug to me, the two are otherwise equivalent as
   these tests show.

This also reveals the case discussed in 9e4e8a64c2 above, where '' is
now an error as far as ls-files is concerned, but wildmatch() itself
happily accepts it.

Signed-off-by: Ævar Arnfjörð Bjarmason <avarab@gmail.com>
---
 t/t3070-wildmatch.sh | 285 ++++++++++++++++++++++++++++++++++++++++++++++++---
 1 file changed, 273 insertions(+), 12 deletions(-)

diff --git a/t/t3070-wildmatch.sh b/t/t3070-wildmatch.sh
index 593b25b278..80d13b5b60 100755
--- a/t/t3070-wildmatch.sh
+++ b/t/t3070-wildmatch.sh
@@ -4,14 +4,95 @@ test_description='wildmatch tests'
 
 . ./test-lib.sh
 
+create_test_file() {
+	file=$1
+
+	case $file in
+	# `touch .` will succeed but obviously not do what we intend
+	# here.
+	".")
+		return 1
+		;;
+	# We cannot create a file with an empty filename.
+	"")
+		return 1
+		;;
+	# The tests that are testing that e.g. foo//bar is matched by
+	# foo/*/bar can't be tested on filesystems since there's no
+	# way we're getting a double slash.
+	*//*)
+		return 1
+		;;
+	# When testing the difference between foo/bar and foo/bar/ we
+	# can't test the latter.
+	*/)
+		return 1
+		;;
+	esac
+
+	# Turn foo/bar/baz into foo/bar to create foo/bar as a
+	# directory structure.
+	dirs=$(echo "$file" | sed -r 's!/[^/]+$!!')
+
+	# We touch "./$file" instead of "$file" because even an
+	# escaped "touch -- -" means get arguments from stdin.
+	if test "$file" != "$dirs"
+	then
+		mkdir -p -- "$dirs" &&
+		touch -- "./$file" &&
+		return 0
+	else
+		touch -- "./$file" &&
+		return 0
+	fi
+	return 1
+}
+
+wildtest_file_setup() {
+	test_when_finished "
+		rm -rf -- * &&
+		git reset
+	" &&
+	git add -A &&
+	>expect.err
+}
+
+wildtest_stdout_stderr_cmp() {
+	tr -d '\0' <actual.raw >actual &&
+	test_cmp expect.err actual.err &&
+	test_cmp expect actual
+}
+
 wildtest() {
-	match_w_glob=$1
-	match_w_globi=$2
-	match_w_pathmatch=$3
-	match_w_pathmatchi=$4
-	text=$5
-	pattern=$6
+	if test "$#" = 6
+	then
+		# When test-wildmatch and git ls-files produce the same
+		# result.
+		match_w_glob=$1
+		match_f_w_glob=$match_w_glob
+		match_w_globi=$2
+		match_f_w_globi=$match_w_globi
+		match_w_pathmatch=$3
+		match_f_w_pathmatch=$match_w_pathmatch
+		match_w_pathmatchi=$4
+		match_f_w_pathmatchi=$match_w_pathmatchi
+		text=$5
+		pattern=$6
+	elif test "$#" = 10
+	then
+		match_w_glob=$1
+		match_w_globi=$2
+		match_w_pathmatch=$3
+		match_w_pathmatchi=$4
+		match_f_w_glob=$5
+		match_f_w_globi=$6
+		match_f_w_pathmatch=$7
+		match_f_w_pathmatchi=$8
+		text=$9
+		pattern=$10
+	fi
 
+	# $1: Case sensitive glob match: test-wildmatch
 	if test "$match_w_glob" = 1
 	then
 		test_expect_success "wildmatch: match '$text' '$pattern'" "
@@ -26,6 +107,50 @@ wildtest() {
 		test_expect_success "PANIC: Test framework error. Unknown matches value $match_w_glob" 'false'
 	fi
 
+	# $1: Case sensitive glob match: ls-files
+	if test "$match_f_w_glob" = 'E'
+	then
+		if create_test_file "$text"
+		then
+			test_expect_success "wildmatch(ls): match dies on '$pattern' '$text'" "
+				wildtest_file_setup &&
+				printf '%s' '$text' >expect &&
+				test_must_fail git --glob-pathspecs ls-files -z -- '$pattern'
+			"
+		else
+			test_expect_failure "wildmatch(ls): match skip '$pattern' '$text'" 'false'
+		fi
+	elif test "$match_f_w_glob" = 1
+	then
+		if create_test_file "$text"
+		then
+			test_expect_success "wildmatch(ls): match '$pattern' '$text'" "
+				wildtest_file_setup &&
+				printf '%s' '$text' >expect &&
+				git --glob-pathspecs ls-files -z -- '$pattern' >actual.raw 2>actual.err &&
+				wildtest_stdout_stderr_cmp
+			"
+		else
+			test_expect_failure "wildmatch(ls): match skip '$pattern' '$text'" 'false'
+		fi
+	elif test "$match_f_w_glob" = 0
+	then
+		if create_test_file "$text"
+		then
+			test_expect_success "wildmatch(ls): no match '$pattern' '$text'" "
+				wildtest_file_setup &&
+				>expect &&
+				git --glob-pathspecs ls-files -z -- '$pattern' >actual.raw 2>actual.err &&
+				wildtest_stdout_stderr_cmp
+			"
+		else
+			test_expect_failure "wildmatch(ls): no match skip '$pattern' '$text'" 'false'
+		fi
+	else
+		test_expect_success "PANIC: Test framework error. Unknown matches value $match_f_w_glob" 'false'
+	fi
+
+	# $2: Case insensitive glob match: test-wildmatch
 	if test "$match_w_globi" = 1
 	then
 		test_expect_success "iwildmatch: match '$text' '$pattern'" "
@@ -40,6 +165,50 @@ wildtest() {
 		test_expect_success "PANIC: Test framework error. Unknown matches value $match_w_globi" 'false'
 	fi
 
+	# $2: Case insensitive glob match: ls-files
+	if test "$match_f_w_globi" = 'E'
+	then
+		if create_test_file "$text"
+		then
+			test_expect_success "wildmatch(ls): match dies on '$pattern' '$text'" "
+				wildtest_file_setup &&
+				printf '%s' '$text' >expect &&
+				test_must_fail git --glob-pathspecs --icase-pathspecs ls-files -z -- '$pattern'
+			"
+		else
+			test_expect_failure "wildmatch(ls): match skip '$pattern' '$text'" 'false'
+		fi
+	elif test "$match_f_w_globi" = 1
+	then
+		if create_test_file "$text"
+		then
+			test_expect_success "wildmatch(ls): match '$pattern' '$text'" "
+				wildtest_file_setup &&
+				printf '%s' '$text' >expect &&
+				git --glob-pathspecs --icase-pathspecs ls-files -z -- '$pattern' >actual.raw 2>actual.err &&
+				wildtest_stdout_stderr_cmp
+			"
+		else
+			test_expect_failure "wildmatch(ls): match skip '$pattern' '$text'" 'false'
+		fi
+	elif test "$match_f_w_globi" = 0
+	then
+		if create_test_file "$text"
+		then
+			test_expect_success "wildmatch(ls): no match '$pattern' '$text'" "
+				wildtest_file_setup &&
+				>expect &&
+				git --glob-pathspecs --icase-pathspecs ls-files -z -- '$pattern' >actual.raw 2>actual.err &&
+				wildtest_stdout_stderr_cmp
+			"
+		else
+			test_expect_failure "wildmatch(ls): no match skip '$pattern' '$text'" 'false'
+		fi
+	else
+		test_expect_success "PANIC: Test framework error. Unknown matches value $match_f_w_globi" 'false'
+	fi
+
+	# $3: Case sensitive path match: test-wildmatch
 	if test "$match_w_pathmatch" = 1
 	then
 		test_expect_success "pathmatch: match '$text' '$pattern'" "
@@ -54,9 +223,53 @@ wildtest() {
 		test_expect_success "PANIC: Test framework error. Unknown matches value $match_w_pathmatch" 'false'
 	fi
 
+	# $4: Case sensitive path match: ls-files
+	if test "$match_f_w_pathmatch" = 'E'
+	then
+		if create_test_file "$text"
+		then
+			test_expect_success "pathmatch(ls): match dies on '$pattern' '$text'" "
+				wildtest_file_setup &&
+				printf '%s' '$text' >expect &&
+				test_must_fail git ls-files -z -- '$pattern'
+			"
+		else
+			test_expect_failure "pathmatch(ls): match skip '$pattern' '$text'" 'false'
+		fi
+	elif test "$match_f_w_pathmatch" = 1
+	then
+		if create_test_file "$text"
+		then
+			test_expect_success "pathmatch(ls): match '$pattern' '$text'" "
+				wildtest_file_setup &&
+				printf '%s' '$text' >expect &&
+				git ls-files -z -- '$pattern' >actual.raw 2>actual.err &&
+				wildtest_stdout_stderr_cmp
+			"
+		else
+			test_expect_failure "pathmatch(ls): match skip '$pattern' '$text'" 'false'
+		fi
+	elif test "$match_f_w_pathmatch" = 0
+	then
+		if create_test_file "$text"
+		then
+			test_expect_success "pathmatch(ls): no match '$pattern' '$text'" "
+				wildtest_file_setup &&
+				>expect &&
+				git ls-files -z -- '$pattern' >actual.raw 2>actual.err &&
+				wildtest_stdout_stderr_cmp
+			"
+		else
+			test_expect_failure "pathmatch(ls): no match skip '$pattern' '$text'" 'false'
+		fi
+	else
+		test_expect_success "PANIC: Test framework error. Unknown matches value $match_f_w_pathmatch" 'false'
+	fi
+
+	# $4: Case insensitive path match: test-wildmatch
 	if test "$match_w_pathmatchi" = 1
 	then
-		test_expect_success "ipathmatch:    match '$text' '$pattern'" "
+		test_expect_success "ipathmatch: match '$text' '$pattern'" "
 			test-wildmatch ipathmatch '$text' '$pattern'
 		"
 	elif test "$match_w_pathmatchi" = 0
@@ -67,6 +280,49 @@ wildtest() {
 	else
 		test_expect_success "PANIC: Test framework error. Unknown matches value $match_w_pathmatchi" 'false'
 	fi
+
+	# $4: Case insensitive path match: ls-files
+	if test "$match_f_w_pathmatchi" = 'E'
+	then
+		if create_test_file "$text"
+		then
+			test_expect_success "ipathmatch(ls): match dies on '$pattern' '$text'" "
+				wildtest_file_setup &&
+				printf '%s' '$text' >expect &&
+				test_must_fail git --icase-pathspecs ls-files -z -- '$pattern'
+			"
+		else
+			test_expect_failure "pathmatch(ls): match skip '$pattern' '$text'" 'false'
+		fi
+	elif test "$match_f_w_pathmatchi" = 1
+	then
+		if create_test_file "$text"
+		then
+			test_expect_success "ipathmatch(ls): match '$pattern' '$text'" "
+				wildtest_file_setup &&
+				printf '%s' '$text' >expect &&
+				git --icase-pathspecs ls-files -z -- '$pattern' >actual.raw 2>actual.err &&
+				wildtest_stdout_stderr_cmp
+			"
+		else
+			test_expect_failure "pathmatch(ls): match skip '$pattern' '$text'" 'false'
+		fi
+	elif test "$match_f_w_pathmatchi" = 0
+	then
+		if create_test_file "$text"
+		then
+			test_expect_success "ipathmatch(ls): no match '$pattern' '$text'" "
+				wildtest_file_setup &&
+				>expect &&
+				git ls-files -z -- '$pattern' >actual.raw 2>actual.err &&
+				wildtest_stdout_stderr_cmp
+			"
+		else
+			test_expect_failure "pathmatch(ls): no match skip '$pattern' '$text'" 'false'
+		fi
+	else
+		test_expect_success "PANIC: Test framework error. Unknown matches value $match_f_w_pathmatchi" 'false'
+	fi
 }
 
 # Basic wildmatch features
@@ -135,7 +391,8 @@ wildtest 1 1 1 1 'acrt' 'a[c-c]rt'
 wildtest 0 0 0 0 ']' '[!]-]'
 wildtest 1 1 1 1 'a' '[!]-]'
 wildtest 0 0 0 0 '' '\'
-wildtest 0 0 0 0 '\' '\'
+wildtest 0 0 0 0 \
+	 1 1 1 1 '\' '\'
 wildtest 0 0 0 0 'XXX/\' '*/\'
 wildtest 1 1 1 1 'XXX/\' '*/\\'
 wildtest 1 1 1 1 'foo' 'foo'
@@ -149,7 +406,8 @@ wildtest 1 1 1 1 '[ab]' '[[:digit]ab]'
 wildtest 1 1 1 1 '[ab]' '[\[:]ab]'
 wildtest 1 1 1 1 '?a?b' '\??\?b'
 wildtest 1 1 1 1 'abc' '\a\b\c'
-wildtest 0 0 0 0 'foo' ''
+wildtest 0 0 0 0 \
+	 E E E E 'foo' ''
 wildtest 1 1 1 1 'foo/bar/baz/to' '**/t[o]'
 
 # Character class tests
@@ -179,8 +437,10 @@ wildtest 1 1 1 1 ']' '[\]]'
 wildtest 0 0 0 0 '\]' '[\]]'
 wildtest 0 0 0 0 '\' '[\]]'
 wildtest 0 0 0 0 'ab' 'a[]b'
-wildtest 0 0 0 0 'a[]b' 'a[]b'
-wildtest 0 0 0 0 'ab[' 'ab['
+wildtest 0 0 0 0 \
+	 1 1 1 1 'a[]b' 'a[]b'
+wildtest 0 0 0 0 \
+	 1 1 1 1 'ab[' 'ab['
 wildtest 0 0 0 0 'ab' '[!'
 wildtest 0 0 0 0 'ab' '[-'
 wildtest 1 1 1 1 '-' '[-]'
@@ -248,7 +508,8 @@ wildtest 1 1 1 1 foo/bar 'foo/*'
 wildtest 0 0 1 1 foo/bba/arr 'foo/*'
 wildtest 1 1 1 1 foo/bba/arr 'foo/**'
 wildtest 0 0 1 1 foo/bba/arr 'foo*'
-wildtest 0 0 1 1 foo/bba/arr 'foo**'
+wildtest 0 0 1 1 \
+	 1 1 1 1 foo/bba/arr 'foo**'
 wildtest 0 0 1 1 foo/bba/arr 'foo/*arr'
 wildtest 0 0 1 1 foo/bba/arr 'foo/**arr'
 wildtest 0 0 0 0 foo/bba/arr 'foo/*z'
-- 
2.15.1.424.g9478a66081