[PATCH] xmalloca, etc.: avoid unlikely trap

[PATCH] xmalloca, etc.: avoid unlikely trap

[Paul Eggert]

* lib/xmalloca.h (xnmalloca):
Use size_t, not ptrdiff_t, to avoid implementation-defined
behavior (which could include a trap) if N is 0 and S exceeds
PTRDIFF_MAX.
* lib/xalloc-oversized.h: Adjust comment to match.
---
 ChangeLog              | 8 +++++++-
 lib/xalloc-oversized.h | 4 ++--
 lib/xmalloca.h         | 2 +-
 3 files changed, 10 insertions(+), 4 deletions(-)

diff --git a/ChangeLog b/ChangeLog
index 63471166c..f91f2e3df 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,7 +1,13 @@
 2021-04-24  Paul Eggert  <eggert@cs.ucla.edu>
 
-	xmalloca, etc.: fix some xalloc-oversized issues
+	xmalloca, etc.: avoid unlikely trap
 	* lib/malloca.h (nmalloca):
+	* lib/xmalloca.h (xnmalloca):
+	Use size_t, not ptrdiff_t, to avoid implementation-defined
+	behavior (which could include a trap) if N is 0 and S exceeds
+	PTRDIFF_MAX.
+	* lib/xalloc-oversized.h: Adjust comment to match.
+
 	* lib/xmalloca.h (xnmalloca): Convert S to ptrdiff_t to avoid
 	arithmetic overflow if N and S are both narrower than ptrdiff_t.
 	* lib/xalloc-oversized.h (xalloc_oversized):
diff --git a/lib/xalloc-oversized.h b/lib/xalloc-oversized.h
index 62f1ae94b..2619a2aab 100644
--- a/lib/xalloc-oversized.h
+++ b/lib/xalloc-oversized.h
@@ -38,8 +38,8 @@
 
    Warning: (xalloc_oversized (N, S) ? NULL : malloc (N * S)) can
    misbehave if N and S are both narrower than ptrdiff_t and size_t,
-   and can be rewritten as (xalloc_oversized (N, S) ?  NULL : malloc
-   (N * (ptrdiff_t) S)) or similarly with size_t.
+   and can be rewritten as (xalloc_oversized (N, S) ?  NULL
+   : malloc (N * (size_t) S)).
 
    This is a macro, not a function, so that it works even if an
    argument exceeds MAX (PTRDIFF_MAX, SIZE_MAX).  */
diff --git a/lib/xmalloca.h b/lib/xmalloca.h
index c7815f617..dbc648697 100644
--- a/lib/xmalloca.h
+++ b/lib/xmalloca.h
@@ -54,7 +54,7 @@ extern void * xmmalloca (size_t n);
 #if HAVE_ALLOCA
 /* Rely on xmalloca (SIZE_MAX) calling xalloc_die ().  */
 # define xnmalloca(n, s) \
-    xmalloca (xalloc_oversized (n, s) ? (size_t) (-1) : (n) * (ptrdiff_t) (s))
+    xmalloca (xalloc_oversized (n, s) ? (size_t) (-1) : (n) * (size_t) (s))
 #else
 # define xnmalloca(n, s) \
     xnmalloc (n, s)
-- 
2.27.0