[PATCH 5/8] xalloc-oversized: fix SIZE_MAX optimization bug

[Paul Eggert]

* lib/xalloc-oversized.h (xalloc_count_t): Remove; no longer
needed and was evidently error-prone anyway.
(xalloc_oversized): Omit some over-optimization that caused
SIZE_MAX to not be treated as too large (the Gnulib convention) on
unusual platforms where PTRDIFF_MAX == SIZE_MAX.  This change
should not affect typical platforms where PTRDIFF_MAX < SIZE_MAX.
When optimizing, simply use ptrdiff_t instead of xalloc_count_t.
---
 ChangeLog              |  9 +++++++++
 lib/xalloc-oversized.h | 17 ++++++-----------
 2 files changed, 15 insertions(+), 11 deletions(-)

diff --git a/ChangeLog b/ChangeLog
index 857b7048f..1d2607a0b 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,5 +1,14 @@
 2021-04-18  Paul Eggert  <eggert@cs.ucla.edu>
 
+       xalloc-oversized: fix SIZE_MAX optimization bug
+       * lib/xalloc-oversized.h (xalloc_count_t): Remove; no longer
+       needed and was evidently error-prone anyway.
+       (xalloc_oversized): Omit some over-optimization that caused
+       SIZE_MAX to not be treated as too large (the Gnulib convention) on
+       unusual platforms where PTRDIFF_MAX == SIZE_MAX.  This change
+       should not affect typical platforms where PTRDIFF_MAX < SIZE_MAX.
+       When optimizing, simply use ptrdiff_t instead of xalloc_count_t.
+
        xalloc: new function xreallocarray
        This effectively replaces xnmalloc, which perhaps should be deprecated.
        The name xreallocarray should be easier to remember now that
diff --git a/lib/xalloc-oversized.h b/lib/xalloc-oversized.h
index 3618c75cb..6437a5d8b 100644
--- a/lib/xalloc-oversized.h
+++ b/lib/xalloc-oversized.h
@@ -30,25 +30,20 @@
 #define __xalloc_oversized(n, s) \
   ((size_t) (PTRDIFF_MAX < SIZE_MAX ? PTRDIFF_MAX : SIZE_MAX - 1) / (s) < (n))
 
-#if PTRDIFF_MAX < SIZE_MAX
-typedef ptrdiff_t xalloc_count_t;
-#else
-typedef size_t xalloc_count_t;
-#endif
-
 /* Return 1 if an array of N objects, each of size S, cannot exist reliably
-   because its total size in bytes exceeds MIN (PTRDIFF_MAX, SIZE_MAX).
+   because its total size in bytes exceeds MIN (PTRDIFF_MAX, SIZE_MAX - 1).
    N must be nonnegative, S must be positive, and either N or S should be
    of type ptrdiff_t or size_t or wider.  This is a macro, not a function,
    so that it works even if an argument exceeds MAX (PTRDIFF_MAX, SIZE_MAX).  
*/
-#if 7 <= __GNUC__ && !defined __clang__
+#if 7 <= __GNUC__ && !defined __clang__ && PTRDIFF_MAX < SIZE_MAX
 # define xalloc_oversized(n, s) \
-   __builtin_mul_overflow_p (n, s, (xalloc_count_t) 1)
-#elif 5 <= __GNUC__ && !defined __ICC && !__STRICT_ANSI__
+   __builtin_mul_overflow_p (n, s, (ptrdiff_t) 1)
+#elif (5 <= __GNUC__ && !defined __ICC && !__STRICT_ANSI__ \
+       && PTRDIFF_MAX < SIZE_MAX)
 # define xalloc_oversized(n, s) \
    (__builtin_constant_p (n) && __builtin_constant_p (s) \
     ? __xalloc_oversized (n, s) \
-    : ({ xalloc_count_t __xalloc_count; \
+    : ({ ptrdiff_t __xalloc_count; \
          __builtin_mul_overflow (n, s, &__xalloc_count); }))
 
 /* Other compilers use integer division; this may be slower but is
-- 
2.27.0