truncate: Work around trailing slash bug in truncate() on AIX

[Bruno Haible <bruno@clisp.org>]

On AIX 7.2, I see this test failure:

FAIL: test-truncate
===================

../../gltests/test-truncate.c:95: assertion 'truncate (BASE "file/", 0) == -1' failed
FAIL test-truncate (exit status: 134)

Yet another function which need a trailing slash workaround. This patch fixes it.

Here, the usual idiom with '#undef truncate' does not work, because AIX
does '#define truncate truncate64' and we really need to call truncate64
not the truncate function which takes only a 32-bit offset.


2021-01-05  Bruno Haible  <bruno@clisp.org>

	truncate: Work around trailing slash bug in truncate() on AIX 7.2.
	* m4/truncate.m4 (gl_FUNC_TRUNCATE): Add a test whether truncate
	rejects trailing slashes. Set REPLACE_TRUNCATE and define
	TRUNCATE_TRAILING_SLASH_BUG if not.
	* lib/truncate.c (orig_truncate): New function.
	(truncate): Add alternative implementation when
	TRUNCATE_TRAILING_SLASH_BUG is defined.
	* modules/truncate (Depends-on): Add sys_stat, stat.

diff --git a/lib/truncate.c b/lib/truncate.c
index 143edf1..ec375d6 100644
--- a/lib/truncate.c
+++ b/lib/truncate.c
@@ -14,6 +14,10 @@
    You should have received a copy of the GNU General Public License along
    with this program; if not, see <https://www.gnu.org/licenses/>.  */
 
+/* If the user's config.h happens to include <unistd.h>, let it include only
+   the system's <unistd.h> here, so that orig_faccessat doesn't recurse to
+   rpl_faccessat.  */
+#define _GL_INCLUDING_UNISTD_H
 #include <config.h>
 
 /* Specification.  */
@@ -21,10 +25,38 @@
 
 #include <errno.h>
 #include <fcntl.h>
+#include <string.h>
+#include <sys/stat.h>
+#undef _GL_INCLUDING_UNISTD_H
+
+#if TRUNCATE_TRAILING_SLASH_BUG
+static int
+orig_truncate (const char *filename, off_t length)
+{
+  return truncate (filename, length);
+}
+#endif
+
+/* Write "unistd.h" here, not <unistd.h>, otherwise OSF/1 5.1 DTK cc
+   eliminates this include because of the preliminary #include <unistd.h>
+   above.  */
+#include "unistd.h"
 
 int
 truncate (const char *filename, off_t length)
 {
+#if TRUNCATE_TRAILING_SLASH_BUG
+  /* Use the original truncate(), but correct the trailing slash handling.  */
+  size_t len = strlen (filename);
+  if (len && filename[len - 1] == '/')
+    {
+      struct stat st;
+      if (stat (filename, &st) == 0)
+        errno = (S_ISDIR (st.st_mode) ? EISDIR : ENOTDIR);
+      return -1;
+    }
+  return orig_truncate (filename, length);
+#else
   int fd;
 
   if (length == 0)
@@ -48,4 +80,5 @@ truncate (const char *filename, off_t length)
     }
   close (fd);
   return 0;
+#endif
 }
diff --git a/m4/truncate.m4 b/m4/truncate.m4
index 1751a50..737e47f 100644
--- a/m4/truncate.m4
+++ b/m4/truncate.m4
@@ -1,4 +1,4 @@
-# truncate.m4 serial 2   -*- Autoconf -*-
+# truncate.m4 serial 3   -*- Autoconf -*-
 dnl Copyright (C) 2017-2021 Free Software Foundation, Inc.
 dnl This file is free software; the Free Software Foundation
 dnl gives unlimited permission to copy and/or distribute it,
@@ -7,6 +7,8 @@ dnl with or without modifications, as long as this notice is preserved.
 AC_DEFUN([gl_FUNC_TRUNCATE],
 [
   AC_REQUIRE([gl_UNISTD_H_DEFAULTS])
+  AC_REQUIRE([AC_CANONICAL_HOST])
+
   AC_CHECK_FUNCS_ONCE([truncate])
   dnl AC_CHECK_FUNC is not enough here, because when compiling for Android 4.4
   dnl or older with _FILE_OFFSET_BITS=64, truncate() is not declared.  There
@@ -15,7 +17,6 @@ AC_DEFUN([gl_FUNC_TRUNCATE],
   AC_CHECK_DECL([truncate], , , [[#include <unistd.h>]])
   if test $ac_cv_have_decl_truncate = yes; then
     m4_ifdef([gl_LARGEFILE], [
-      AC_REQUIRE([AC_CANONICAL_HOST])
       case "$host_os" in
         mingw*)
           dnl Native Windows, and Large File Support is requested.
@@ -29,6 +30,45 @@ AC_DEFUN([gl_FUNC_TRUNCATE],
     ], [
       :
     ])
+    if test $REPLACE_TRUNCATE = 0; then
+      dnl Check for AIX 7.2 bug with trailing slash.
+      AC_CACHE_CHECK([whether truncate rejects trailing slashes],
+        [gl_cv_func_truncate_works],
+        [echo foo > conftest.tmp
+         AC_RUN_IFELSE(
+           [AC_LANG_PROGRAM(
+              [[#include <unistd.h>
+              ]],
+              [[int result = 0;
+                if (!truncate ("conftest.tmp/", 2))
+                  result |= 1;
+                return result;
+              ]])
+           ],
+           [gl_cv_func_truncate_works=yes],
+           [gl_cv_func_truncate_works=no],
+           [case "$host_os" in
+                               # Guess yes on Linux systems.
+              linux-* | linux) gl_cv_func_truncate_works="guessing yes" ;;
+                               # Guess yes on glibc systems.
+              *-gnu* | gnu*)   gl_cv_func_truncate_works="guessing yes" ;;
+                               # Guess no on AIX systems.
+              aix*)            gl_cv_func_truncate_works="guessing no" ;;
+                               # If we don't know, obey --enable-cross-guesses.
+              *)               gl_cv_func_truncate_works="$gl_cross_guess_normal" ;;
+            esac
+           ])
+         rm -f conftest.tmp
+        ])
+      case "$gl_cv_func_truncate_works" in
+        *yes) ;;
+        *)
+           AC_DEFINE([TRUNCATE_TRAILING_SLASH_BUG], [1],
+             [Define to 1 if truncate mishandles trailing slash.])
+           REPLACE_TRUNCATE=1
+           ;;
+      esac
+    fi
   else
     HAVE_DECL_TRUNCATE=0
     if test $ac_cv_func_truncate = yes; then
diff --git a/modules/truncate b/modules/truncate
index b6fb377..82814fe 100644
--- a/modules/truncate
+++ b/modules/truncate
@@ -9,6 +9,8 @@ Depends-on:
 unistd
 sys_types
 largefile
+sys_stat
+stat            [test $REPLACE_TRUNCATE = 1]
 open            [test $HAVE_DECL_TRUNCATE = 0 || test $REPLACE_TRUNCATE = 1]
 ftruncate       [test $HAVE_DECL_TRUNCATE = 0 || test $REPLACE_TRUNCATE = 1]