crypto/{sha*,md5,sm3}-buffer: Ignore too old OpenSSL versions

[Bruno Haible]

Building a coreutils prerelease on AIX 7.3.1, I get this compilation error:

  CC       lib/libcoreutils_a-md5-stream.o
In file included from ../lib/md5-stream.c:28:
In file included from ../lib/md5.h:35:
In file included from /usr/include/openssl/md5.h:30:
/usr/include/openssl/macros.h:155:4: error: "The requested API level higher 
than the configured API compatibility level"
#  error "The requested API level higher than the configured API compatibility 
level"
   ^
1 error generated.

The problem here is that
  - Gnulib requires explicitly the OpenSSL 1.1.1 API, since
    https://lists.gnu.org/archive/html/bug-gnulib/2022-05/msg00015.html
  - On this AIX version, OpenSSL is preinstalled, with an
    <openssl/configuration.h> file that defines
      # define OPENSSL_CONFIGURED_API 10002
    That is, it is configured to offer the OpenSSL 1.0.2 API.
  - In <openssl/macros.h> there is a compile-time #error
    in case the application-requested OpenSSL API is higher than
    the configured one.

Maybe it would be possible that Gnulib requires only the OpenSSL 1.0.2 API?
But it sounds like a step backwards and/or a potential security problem.
I find it safer, in this case, to just ignore the installed OpenSSL version.
This patch does it.

Note that m4/gl-openssl.m4 still defines LIB_CRYPTO to non-empty in this case.
That's because each source file may have a different OpenSSL API requirement.
coreutils uses $(IGNORE_UNUSED_LIBRARIES_CFLAGS), to mitigate this missing
optimization.


2023-08-26  Bruno Haible  <bruno@clisp.org>

        crypto/{sha*,md5,sm3}-buffer: Ignore too old OpenSSL versions.
        * lib/sha1.h: If <openssl/macros.h> would give a compile-time error,
        undefine HAVE_OPENSSL_SHA1.
        * lib/sha256.h: If <openssl/macros.h> would give a compile-time error,
        undefine HAVE_OPENSSL_SHA256.
        * lib/sha512.h: If <openssl/macros.h> would give a compile-time error,
        undefine HAVE_OPENSSL_SHA512.
        * lib/md5.h: If <openssl/macros.h> would give a compile-time error,
        undefine HAVE_OPENSSL_MD5.
        * lib/sm3.h: If <openssl/macros.h> would give a compile-time error,
        undefine HAVE_OPENSSL_SM3.

diff --git a/lib/md5.h b/lib/md5.h
index 92dc603898..6ddf009148 100644
--- a/lib/md5.h
+++ b/lib/md5.h
@@ -32,7 +32,17 @@
 #  ifndef OPENSSL_API_COMPAT
 #   define OPENSSL_API_COMPAT 0x10101000L /* FIXME: Use OpenSSL 1.1+ API.  */
 #  endif
-#  include <openssl/md5.h>
+/* If <openssl/macros.h> would give a compile-time error, don't use OpenSSL.  
*/
+#  include <openssl/configuration.h>
+#  if (OPENSSL_CONFIGURED_API \
+       < (OPENSSL_API_COMPAT < 0x900000L ? OPENSSL_API_COMPAT : \
+          ((OPENSSL_API_COMPAT >> 28) & 0xF) * 10000 \
+          + ((OPENSSL_API_COMPAT >> 20) & 0xFF) * 100 \
+          + ((OPENSSL_API_COMPAT >> 12) & 0xFF)))
+#   undef HAVE_OPENSSL_MD5
+#  else
+#   include <openssl/md5.h>
+#  endif
 # endif
 
 #define MD5_DIGEST_SIZE 16
diff --git a/lib/sha1.h b/lib/sha1.h
index 854213a921..d5a6e72e2c 100644
--- a/lib/sha1.h
+++ b/lib/sha1.h
@@ -31,7 +31,17 @@
 #  ifndef OPENSSL_API_COMPAT
 #   define OPENSSL_API_COMPAT 0x10101000L /* FIXME: Use OpenSSL 1.1+ API.  */
 #  endif
-#  include <openssl/sha.h>
+/* If <openssl/macros.h> would give a compile-time error, don't use OpenSSL.  
*/
+#  include <openssl/configuration.h>
+#  if (OPENSSL_CONFIGURED_API \
+       < (OPENSSL_API_COMPAT < 0x900000L ? OPENSSL_API_COMPAT : \
+          ((OPENSSL_API_COMPAT >> 28) & 0xF) * 10000 \
+          + ((OPENSSL_API_COMPAT >> 20) & 0xFF) * 100 \
+          + ((OPENSSL_API_COMPAT >> 12) & 0xFF)))
+#   undef HAVE_OPENSSL_SHA1
+#  else
+#   include <openssl/sha.h>
+#  endif
 # endif
 
 # ifdef __cplusplus
diff --git a/lib/sha256.h b/lib/sha256.h
index 938b106706..508bce7de8 100644
--- a/lib/sha256.h
+++ b/lib/sha256.h
@@ -30,7 +30,17 @@
 #  ifndef OPENSSL_API_COMPAT
 #   define OPENSSL_API_COMPAT 0x10101000L /* FIXME: Use OpenSSL 1.1+ API.  */
 #  endif
-#  include <openssl/sha.h>
+/* If <openssl/macros.h> would give a compile-time error, don't use OpenSSL.  
*/
+#  include <openssl/configuration.h>
+#  if (OPENSSL_CONFIGURED_API \
+       < (OPENSSL_API_COMPAT < 0x900000L ? OPENSSL_API_COMPAT : \
+          ((OPENSSL_API_COMPAT >> 28) & 0xF) * 10000 \
+          + ((OPENSSL_API_COMPAT >> 20) & 0xFF) * 100 \
+          + ((OPENSSL_API_COMPAT >> 12) & 0xFF)))
+#   undef HAVE_OPENSSL_SHA256
+#  else
+#   include <openssl/sha.h>
+#  endif
 # endif
 
 # ifdef __cplusplus
diff --git a/lib/sha512.h b/lib/sha512.h
index f3465bc8a6..3259f1c7b8 100644
--- a/lib/sha512.h
+++ b/lib/sha512.h
@@ -30,7 +30,17 @@
 #  ifndef OPENSSL_API_COMPAT
 #   define OPENSSL_API_COMPAT 0x10101000L /* FIXME: Use OpenSSL 1.1+ API.  */
 #  endif
-#  include <openssl/sha.h>
+/* If <openssl/macros.h> would give a compile-time error, don't use OpenSSL.  
*/
+#  include <openssl/configuration.h>
+#  if (OPENSSL_CONFIGURED_API \
+       < (OPENSSL_API_COMPAT < 0x900000L ? OPENSSL_API_COMPAT : \
+          ((OPENSSL_API_COMPAT >> 28) & 0xF) * 10000 \
+          + ((OPENSSL_API_COMPAT >> 20) & 0xFF) * 100 \
+          + ((OPENSSL_API_COMPAT >> 12) & 0xFF)))
+#   undef HAVE_OPENSSL_SHA512
+#  else
+#   include <openssl/sha.h>
+#  endif
 # endif
 
 # ifdef __cplusplus
diff --git a/lib/sm3.h b/lib/sm3.h
index 28af4dff1c..f60efdfe9a 100644
--- a/lib/sm3.h
+++ b/lib/sm3.h
@@ -39,7 +39,17 @@
 #  ifndef OPENSSL_API_COMPAT
 #   define OPENSSL_API_COMPAT 0x10101000L /* FIXME: Use OpenSSL 1.1+ API.  */
 #  endif
-#  include <openssl/sm3.h>
+/* If <openssl/macros.h> would give a compile-time error, don't use OpenSSL.  
*/
+#  include <openssl/configuration.h>
+#  if (OPENSSL_CONFIGURED_API \
+       < (OPENSSL_API_COMPAT < 0x900000L ? OPENSSL_API_COMPAT : \
+          ((OPENSSL_API_COMPAT >> 28) & 0xF) * 10000 \
+          + ((OPENSSL_API_COMPAT >> 20) & 0xFF) * 100 \
+          + ((OPENSSL_API_COMPAT >> 12) & 0xFF)))
+#   undef HAVE_OPENSSL_SM3
+#  else
+#   include <openssl/sm3.h>
+#  endif
 # endif
 
 # ifdef __cplusplus