[PATCH v5 8/8] use smudgeToFile filter in recursive merge

[Joey Hess <joeyh@joeyh.name>]

Recursive merge updates the work tree and so should use the smudgeToFile
filter.

At this point, smudgeToFile is run by everything that updates work
tree files.

Signed-off-by: Joey Hess <joeyh@joeyh.name>
---
 merge-recursive.c     | 53 ++++++++++++++++++++++++++++++++++++++++-----------
 t/t0021-conversion.sh | 16 +++++++++++++++-
 2 files changed, 57 insertions(+), 12 deletions(-)

diff --git a/merge-recursive.c b/merge-recursive.c
index a4a1195..5fe3f50 100644
--- a/merge-recursive.c
+++ b/merge-recursive.c
@@ -758,6 +758,7 @@ static void update_file_flags(struct merge_options *o,
 		enum object_type type;
 		void *buf;
 		unsigned long size;
+		int isreg;
 
 		if (S_ISGITLINK(mode)) {
 			/*
@@ -774,22 +775,16 @@ static void update_file_flags(struct merge_options *o,
 			die(_("cannot read object %s '%s'"), oid_to_hex(oid), path);
 		if (type != OBJ_BLOB)
 			die(_("blob expected for %s '%s'"), oid_to_hex(oid), path);
-		if (S_ISREG(mode)) {
-			struct strbuf strbuf = STRBUF_INIT;
-			if (convert_to_working_tree(path, buf, size, &strbuf)) {
-				free(buf);
-				size = strbuf.len;
-				buf = strbuf_detach(&strbuf, NULL);
-			}
-		}
 
 		if (make_room_for_path(o, path) < 0) {
 			update_wd = 0;
 			free(buf);
 			goto update_index;
 		}
-		if (S_ISREG(mode) || (!has_symlinks && S_ISLNK(mode))) {
+		isreg = S_ISREG(mode);
+		if (isreg || (!has_symlinks && S_ISLNK(mode))) {
 			int fd;
+			int smudge_to_file;
 			if (mode & 0100)
 				mode = 0777;
 			else
@@ -797,8 +792,44 @@ static void update_file_flags(struct merge_options *o,
 			fd = open(path, O_WRONLY | O_TRUNC | O_CREAT, mode);
 			if (fd < 0)
 				die_errno(_("failed to open '%s'"), path);
-			write_in_full(fd, buf, size);
-			close(fd);
+
+			smudge_to_file = can_smudge_to_file(path);
+			if (smudge_to_file) {
+				close(fd);
+				fd = convert_to_working_tree_filter_to_file(path, path, buf, size);
+				if (fd < 0) {
+					/*
+					 * smudgeToFile filter failed;
+					 * continue with regular file
+					 * creation.
+					 */
+					smudge_to_file = 0;
+					fd = open(path, O_WRONLY | O_TRUNC | O_CREAT, mode);
+					if (fd < 0)
+						die_errno(_("failed to open '%s'"), path);
+				}
+				else {
+					close(fd);
+				}
+			}
+
+			/*
+			 * Not an else of above if (smudge_to_file) because
+			 * the smudgeToFile filter may fail and in that case
+			 * this is run to recover.
+			 */
+			if (!smudge_to_file) {
+				if (isreg) {
+					struct strbuf strbuf = STRBUF_INIT;
+					if (convert_to_working_tree(path, buf, size, &strbuf)) {
+						free(buf);
+						size = strbuf.len;
+						buf = strbuf_detach(&strbuf, NULL);
+					}
+				}
+				write_in_full(fd, buf, size);
+				close(fd);
+			}
 		} else if (S_ISLNK(mode)) {
 			char *lnk = xmemdupz(buf, size);
 			safe_create_leading_directories_const(path);
diff --git a/t/t0021-conversion.sh b/t/t0021-conversion.sh
index 42b28aa..64b2b8f 100755
--- a/t/t0021-conversion.sh
+++ b/t/t0021-conversion.sh
@@ -334,10 +334,24 @@ test_expect_success 'recovery from failure of smudgeToFile filter that deletes t
 	test_cmp test fstest.t
 '
 
+test_expect_success 'smudgeToFile filter is used in merge' '
+	test_config filter.rot13.smudgeToFile ./rot13-to-file.sh &&
+
+	git commit -m "added fstest.t" fstest.t &&
+	git checkout -b old &&
+	git reset --hard HEAD^ &&
+	git merge master &&
+	git checkout master &&
+
+	test -e rot13-to-file.ran &&
+	rm -f rot13-to-file.ran &&
+
+	test_cmp test fstest.t
+'
+
 test_expect_success 'smudgeToFile filter is used by git am' '
 	test_config filter.rot13.smudgeToFile ./rot13-to-file.sh &&
 
-	git commit fstest.t -m "added fstest.t" &&
 	git format-patch HEAD^ --stdout >fstest.patch &&
 	git reset --hard HEAD^ &&
 	git am fstest.patch &&
-- 
2.8.1