lib: Fix build against OpenSSL 4.1

X509_get_subject_name() and X509_NAME_ENTRY_get_data() return const
pointers in OpenSSL >= 4.1.

ASN1_STRING_length() is deprecated in 4.1 in favour of
ASN1_STRING_get_length(), which returns size_t rather than int.

Signed-off-by: Dimitri Staessens <dimitri@ouroboros.rocks>
Signed-off-by: Sander Vrijders <sander@ouroboros.rocks>

3 files changed, 29 insertions, 6 deletions

diff --git a/cmake/dependencies/crypt/openssl.cmake b/cmake/dependencies/crypt/openssl.cmake
index b0fa8d11..04451357 100644
--- a/cmake/dependencies/crypt/openssl.cmake
+++ b/cmake/dependencies/crypt/openssl.cmake
@@ -53,4 +53,11 @@ else()
   unset(HAVE_SLH CACHE)
 endif()
 
+if(OPENSSL_VERSION VERSION_GREATER_EQUAL "4.1.0")
+  set(HAVE_OPENSSL_4_1 TRUE CACHE INTERNAL
+    "OpenSSL 4.1 or later")
+else()
+  unset(HAVE_OPENSSL_4_1 CACHE)
+endif()
+
 # Secure memory options are in cmake/config/global.cmake
diff --git a/src/lib/config.h.in b/src/lib/config.h.in
index 62925a48..38d6f768 100644
--- a/src/lib/config.h.in
+++ b/src/lib/config.h.in
@@ -35,6 +35,7 @@
 #ifdef HAVE_OPENSSL
 #cmakedefine HAVE_ML
 #cmakedefine HAVE_SLH
+#cmakedefine HAVE_OPENSSL_4_1
 #define HAVE_ENCRYPTION
 #define SECMEM_MINSIZE           @SECMEM_MINSIZE@
 #endif
diff --git a/src/lib/crypt/openssl.c b/src/lib/crypt/openssl.c
index 8ec3b40f..1ee2318e 100644
--- a/src/lib/crypt/openssl.c
+++ b/src/lib/crypt/openssl.c
@@ -48,6 +48,7 @@
 #include <openssl/x509_vfy.h>
 
 #include <assert.h>
+#include <limits.h>
 #include <stdio.h>
 
 #define IS_EC_GROUP(str) (strcmp(str, "EC") == 0)
@@ -1608,12 +1609,26 @@ void openssl_free_key(EVP_PKEY * key)
         EVP_PKEY_free(key);
 }
 
+/* ASN1_STRING_length is deprecated in OpenSSL 4.1, and returns size_t */
+static int ossl_asn1_str_len(const ASN1_STRING * val)
+{
+#ifdef HAVE_OPENSSL_4_1
+        size_t len;
+
+        len = ASN1_STRING_get_length(val);
+
+        return len > INT_MAX ? -1 : (int) len;
+#else
+        return ASN1_STRING_length(val);
+#endif
+}
+
 int openssl_check_crt_name(void *       crt,
                            const char * name)
 {
         const unsigned char * cn;
-        ASN1_STRING *         val;
-        X509_NAME *           nm;
+        const ASN1_STRING *   val;
+        const X509_NAME *     nm;
         int                   idx;
         int                   len;
 
@@ -1627,7 +1642,7 @@ int openssl_check_crt_name(void *       crt,
 
         val = X509_NAME_ENTRY_get_data(X509_NAME_get_entry(nm, idx));
         cn  = ASN1_STRING_get0_data(val);
-        len = ASN1_STRING_length(val);
+        len = ossl_asn1_str_len(val);
 
         if (len < 0 || (size_t) len != strlen(name))
                 return -1;
@@ -1645,8 +1660,8 @@ int openssl_get_crt_name(void * crt,
                          char * name)
 {
         const unsigned char * cn;
-        ASN1_STRING *         val;
-        X509_NAME *           nm;
+        const ASN1_STRING *   val;
+        const X509_NAME *     nm;
         int                   idx;
         int                   len;
 
@@ -1660,7 +1675,7 @@ int openssl_get_crt_name(void * crt,
 
         val = X509_NAME_ENTRY_get_data(X509_NAME_get_entry(nm, idx));
         cn  = ASN1_STRING_get0_data(val);
-        len = ASN1_STRING_length(val);
+        len = ossl_asn1_str_len(val);
 
         if (len < 0)
                 return -1;