chg: dev: start treating openssl 3.0 and 1.1 as separate libraries

OpenSSL 3.0 has completely revamed the way applications are supposed to
interact with libcrypto with the old way gaining a hefty perfomance
penalty.

Almost every cryptographic functionality will have a pre-3.0 and
post-3.0 counterpart split by the preprocessor after the migration is
complete.

Instead of having a macros mess everywhere, start the long term plan of
having a cryptographic functionality layer with OpenSSL >=3.0 and
OpenSSL <3.0 as two separate backends.

Start this process by splitting the initialization function.

Merge branch 'aydin/libcrypto-version-split' into 'main'

See merge request isc-projects/bind9!10844

.gitlab-ci.yml

@@ -1126,7 +1126,7 @@ gcc:ossl3:sid:amd64:
   <<: *build_job
   variables:
     CC: gcc
-    CFLAGS: "${CFLAGS_COMMON} -DOPENSSL_NO_DEPRECATED=1 -DOPENSSL_API_COMPAT=30000"
+    CFLAGS: "${CFLAGS_COMMON}"
     # See https://gitlab.isc.org/isc-projects/bind9/-/issues/3444
     EXTRA_CONFIGURE: "-Doptimization=3 -Djemalloc=disabled -Dleak-detection=disabled"
     RUN_MESON_INSTALL: 1
@@ -1434,7 +1434,7 @@ tsan:stress:
 clang:bookworm:amd64:
   variables:
     CC: ${CLANG}
-    CFLAGS: "${CFLAGS_COMMON} -Wenum-conversion -DOPENSSL_API_COMPAT=10100"
+    CFLAGS: "${CFLAGS_COMMON} -Wenum-conversion"
     # See https://gitlab.isc.org/isc-projects/bind9/-/issues/3444
     EXTRA_CONFIGURE: "-Djemalloc=disabled -Dleak-detection=disabled"
     RUN_MESON_INSTALL: 1

lib/isc/crypto/meson.build

@@ -0,0 +1,16 @@
+# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
+#
+# SPDX-License-Identifier: MPL-2.0
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0.  If a copy of the MPL was not distributed with this
+# file, you can obtain one at https://mozilla.org/MPL/2.0/.
+#
+# See the COPYRIGHT file distributed with this work for additional
+# information regarding copyright ownership.
+
+isc_srcset.add(
+    when: 'HAVE_OPENSSL_3_API',
+    if_true: files('ossl3.c'),
+    if_false: files('ossl1.c'),
+)

lib/isc/crypto/ossl1.c

@@ -0,0 +1,222 @@
+/*
+ * Copyright (C) Internet Systems Consortium, Inc. ("ISC")
+ *
+ * SPDX-License-Identifier: MPL-2.0
+ *
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, you can obtain one at https://mozilla.org/MPL/2.0/.
+ *
+ * See the COPYRIGHT file distributed with this work for additional
+ * information regarding copyright ownership.
+ */
+
+#include <openssl/crypto.h>
+#include <openssl/err.h>
+#include <openssl/opensslv.h>
+#include <openssl/rand.h>
+#include <openssl/ssl.h>
+
+#include <isc/crypto.h>
+#include <isc/mem.h>
+#include <isc/tls.h>
+#include <isc/util.h>
+
+#ifndef LIBRESSL_VERSION_NUMBER
+static isc_mem_t *isc__crypto_mctx = NULL;
+#endif
+
+EVP_MD *isc__crypto_md5 = NULL;
+EVP_MD *isc__crypto_sha1 = NULL;
+EVP_MD *isc__crypto_sha224 = NULL;
+EVP_MD *isc__crypto_sha256 = NULL;
+EVP_MD *isc__crypto_sha384 = NULL;
+EVP_MD *isc__crypto_sha512 = NULL;
+
+#ifndef LIBRESSL_VERSION_NUMBER
+
+#if ISC_MEM_TRACKLINES
+/*
+ * We use the internal isc__mem API here, so we can pass the file and line
+ * arguments passed from OpenSSL >= 1.1.0 to our memory functions for better
+ * tracking of the OpenSSL allocations.  Without this, we would always just see
+ * isc__crypto_{malloc,realloc,free} in the tracking output, but with this in
+ * place we get to see the places in the OpenSSL code where the allocations
+ * happen.
+ */
+
+static void *
+isc__crypto_malloc_ex(size_t size, const char *file, int line) {
+	return isc__mem_allocate(isc__crypto_mctx, size, 0, __func__, file,
+				 (unsigned int)line);
+}
+
+static void *
+isc__crypto_realloc_ex(void *ptr, size_t size, const char *file, int line) {
+	return isc__mem_reallocate(isc__crypto_mctx, ptr, size, 0, __func__,
+				   file, (unsigned int)line);
+}
+
+static void
+isc__crypto_free_ex(void *ptr, const char *file, int line) {
+	if (ptr == NULL) {
+		return;
+	}
+	if (isc__crypto_mctx != NULL) {
+		isc__mem_free(isc__crypto_mctx, ptr, 0, __func__, file,
+			      (unsigned int)line);
+	}
+}
+
+#else /* ISC_MEM_TRACKLINES */
+
+static void *
+isc__crypto_malloc_ex(size_t size, const char *file, int line) {
+	UNUSED(file);
+	UNUSED(line);
+	return isc_mem_allocate(isc__crypto_mctx, size);
+}
+
+static void *
+isc__crypto_realloc_ex(void *ptr, size_t size, const char *file, int line) {
+	UNUSED(file);
+	UNUSED(line);
+	return isc_mem_reallocate(isc__crypto_mctx, ptr, size);
+}
+
+static void
+isc__crypto_free_ex(void *ptr, const char *file, int line) {
+	UNUSED(file);
+	UNUSED(line);
+	if (ptr == NULL) {
+		return;
+	}
+	if (isc__crypto_mctx != NULL) {
+		isc__mem_free(isc__crypto_mctx, ptr, 0);
+	}
+}
+
+#endif /* ISC_MEM_TRACKLINES */
+
+#endif /* !LIBRESSL_VERSION_NUMBER */
+
+#define md_register_algorithm(alg)                        \
+	do {                                              \
+		isc__crypto_##alg = UNCONST(EVP_##alg()); \
+		if (isc__crypto_##alg == NULL) {          \
+			ERR_clear_error();                \
+		}                                         \
+	} while (0)
+
+static isc_result_t
+register_algorithms(void) {
+	if (!isc_crypto_fips_mode()) {
+		md_register_algorithm(md5);
+	}
+
+	md_register_algorithm(sha1);
+	md_register_algorithm(sha224);
+	md_register_algorithm(sha256);
+	md_register_algorithm(sha384);
+	md_register_algorithm(sha512);
+
+	return ISC_R_SUCCESS;
+}
+
+#ifdef HAVE_FIPS_MODE
+bool
+isc_crypto_fips_mode(void) {
+	return FIPS_mode() != 0;
+}
+
+isc_result_t
+isc_crypto_fips_enable(void) {
+	if (isc_crypto_fips_mode()) {
+		return ISC_R_SUCCESS;
+	}
+
+	if (FIPS_mode_set(1) == 0) {
+		return isc_tlserr2result(ISC_LOGCATEGORY_GENERAL,
+					 ISC_LOGMODULE_CRYPTO, "FIPS_mode_set",
+					 ISC_R_CRYPTOFAILURE);
+	}
+
+	register_algorithms();
+
+	return ISC_R_SUCCESS;
+}
+#else
+bool
+isc_crypto_fips_mode(void) {
+	return false;
+}
+
+isc_result_t
+isc_crypto_fips_enable(void) {
+	return ISC_R_NOTIMPLEMENTED;
+}
+#endif /* HAVE_FIPS_MODE */
+
+#ifndef LIBRESSL_VERSION_NUMBER
+void
+isc__crypto_setdestroycheck(bool check) {
+	isc_mem_setdestroycheck(isc__crypto_mctx, check);
+}
+#else
+void
+isc__crypto_setdestroycheck(bool check) {
+	(void)check;
+}
+#endif /* !LIBRESSL_VERSION_NUMBER */
+
+void
+isc__crypto_initialize(void) {
+#ifndef LIBRESSL_VERSION_NUMBER
+	isc_mem_create("OpenSSL", &isc__crypto_mctx);
+	isc_mem_setdebugging(isc__crypto_mctx, 0);
+	isc_mem_setdestroycheck(isc__crypto_mctx, false);
+	/*
+	 * CRYPTO_set_mem_(_ex)_functions() returns 1 on success or 0 on
+	 * failure, which means OpenSSL already allocated some memory.  There's
+	 * nothing we can do about it.
+	 */
+	(void)CRYPTO_set_mem_functions(isc__crypto_malloc_ex,
+				       isc__crypto_realloc_ex,
+				       isc__crypto_free_ex);
+#endif /* LIBRESSL_VERSION_NUMBER */
+
+	/*
+	 * The OPENSSL_INIT_NO_ATEXIT flag was introduces with 3.0.0. Otherwise
+	 * it can only be found in the form of no-op such as with LibreSSL.
+	 *
+	 * https://github.com/openssl/openssl/commit/8f6a5c56c17aa89b80fef73875beec53aef1f2c8
+	 * https://github.com/libressl/openbsd/commit/da7b0f4bfa71c9b8be4c449be0da83036941e3a2
+	 */
+	RUNTIME_CHECK(OPENSSL_init_ssl(OPENSSL_INIT_LOAD_CONFIG, NULL) == 1);
+
+#ifdef ENABLE_FIPS_MODE
+	if (isc_crypto_fips_enable() != ISC_R_SUCCESS) {
+		ERR_clear_error();
+		FATAL_ERROR("Failed to toggle FIPS mode but is "
+			    "required for this build");
+	}
+#endif
+
+	register_algorithms();
+
+	/* Protect ourselves against unseeded PRNG */
+	if (RAND_status() != 1) {
+		FATAL_ERROR("OpenSSL pseudorandom number generator "
+			    "cannot be initialized (see the `PRNG not "
+			    "seeded' message in the OpenSSL FAQ)");
+	}
+}
+
+void
+isc__crypto_shutdown(void) {
+	OPENSSL_cleanup();
+
+#ifndef LIBRESSL_VERSION_NUMBER
+	isc_mem_detach(&isc__crypto_mctx);
+#endif /* LIBRESSL_VERSION_NUMBER */
+}

lib/isc/crypto/ossl3.c

@@ -14,93 +14,25 @@
 #include <openssl/crypto.h>
 #include <openssl/err.h>
 #include <openssl/evp.h>
+#include <openssl/provider.h>
 #include <openssl/rand.h>
 #include <openssl/ssl.h>
 
-#if OPENSSL_VERSION_NUMBER >= 0x30000000L
-#include <openssl/provider.h>
-#endif /* OPENSSL_VERSION_NUMBER >= 0x30000000L */
-
 #include <isc/crypto.h>
-#include <isc/log.h>
 #include <isc/mem.h>
 #include <isc/tls.h>
 #include <isc/util.h>
 
 static isc_mem_t *isc__crypto_mctx = NULL;
 
-#if OPENSSL_VERSION_NUMBER >= 0x30000000L
 static OSSL_PROVIDER *base = NULL, *fips = NULL;
-#endif /* OPENSSL_VERSION_NUMBER >= 0x30000000L */
 
-const EVP_MD *isc__crypto_md5 = NULL;
-const EVP_MD *isc__crypto_sha1 = NULL;
-const EVP_MD *isc__crypto_sha224 = NULL;
-const EVP_MD *isc__crypto_sha256 = NULL;
-const EVP_MD *isc__crypto_sha384 = NULL;
-const EVP_MD *isc__crypto_sha512 = NULL;
-
-#if OPENSSL_VERSION_NUMBER >= 0x30000000L
-#define md_register_algorithm(alg, algname)                            \
-	{                                                              \
-		REQUIRE(isc__crypto_##alg == NULL);                    \
-		isc__crypto_##alg = EVP_MD_fetch(NULL, algname, NULL); \
-		if (isc__crypto_##alg == NULL) {                       \
-			ERR_clear_error();                             \
-		}                                                      \
-	}
-
-#define md_unregister_algorithm(alg)                             \
-	{                                                        \
-		if (isc__crypto_##alg != NULL) {                 \
-			EVP_MD_free(UNCONST(isc__crypto_##alg)); \
-			isc__crypto_##alg = NULL;                \
-		}                                                \
-	}
-#else /* OPENSSL_VERSION_NUMBER >= 0x30000000L */
-#define md_register_algorithm(alg, algname)      \
-	{                                        \
-		isc__crypto_##alg = EVP_##alg(); \
-		if (isc__crypto_##alg == NULL) { \
-			ERR_clear_error();       \
-		}                                \
-	}
-#define md_unregister_algorithm(alg)
-#endif /* OPENSSL_VERSION_NUMBER >= 0x30000000L */
-
-static isc_result_t
-register_algorithms(void) {
-	if (!isc_crypto_fips_mode()) {
-		md_register_algorithm(md5, "MD5");
-	}
-
-	md_register_algorithm(sha1, "SHA1");
-	md_register_algorithm(sha224, "SHA224");
-	md_register_algorithm(sha256, "SHA256");
-	md_register_algorithm(sha384, "SHA384");
-	md_register_algorithm(sha512, "SHA512");
-
-	return ISC_R_SUCCESS;
-}
-
-static void
-unregister_algorithms(void) {
-	md_unregister_algorithm(sha512);
-	md_unregister_algorithm(sha384);
-	md_unregister_algorithm(sha256);
-	md_unregister_algorithm(sha224);
-	md_unregister_algorithm(sha1);
-	md_unregister_algorithm(md5);
-}
-
-#undef md_unregister_algorithm
-#undef md_register_algorithm
-
-#if !defined(LIBRESSL_VERSION_NUMBER) && OPENSSL_VERSION_NUMBER >= 0x30000000L
-/*
- * This was crippled with LibreSSL, so just skip it:
- * https://cvsweb.openbsd.org/src/lib/libcrypto/Attic/mem.c
- */
+EVP_MD *isc__crypto_md5 = NULL;
+EVP_MD *isc__crypto_sha1 = NULL;
+EVP_MD *isc__crypto_sha224 = NULL;
+EVP_MD *isc__crypto_sha256 = NULL;
+EVP_MD *isc__crypto_sha384 = NULL;
+EVP_MD *isc__crypto_sha512 = NULL;
 
 #if ISC_MEM_TRACKLINES
 /*
@@ -165,9 +97,51 @@ isc__crypto_free_ex(void *ptr, const char *file, int line) {
 
 #endif /* ISC_MEM_TRACKLINES */
 
-#endif /* !defined(LIBRESSL_VERSION_NUMBER) */
+#define md_register_algorithm(alg, algname)                            \
+	do {                                                           \
+		REQUIRE(isc__crypto_##alg == NULL);                    \
+		isc__crypto_##alg = EVP_MD_fetch(NULL, algname, NULL); \
+		if (isc__crypto_##alg == NULL) {                       \
+			ERR_clear_error();                             \
+		}                                                      \
+	} while (0)
+
+#define md_unregister_algorithm(alg)                    \
+	do {                                            \
+		if (isc__crypto_##alg != NULL) {        \
+			EVP_MD_free(isc__crypto_##alg); \
+			isc__crypto_##alg = NULL;       \
+		}                                       \
+	} while (0)
+
+static isc_result_t
+register_algorithms(void) {
+	if (!isc_crypto_fips_mode()) {
+		md_register_algorithm(md5, "MD5");
+	}
+
+	md_register_algorithm(sha1, "SHA1");
+	md_register_algorithm(sha224, "SHA224");
+	md_register_algorithm(sha256, "SHA256");
+	md_register_algorithm(sha384, "SHA384");
+	md_register_algorithm(sha512, "SHA512");
+
+	return ISC_R_SUCCESS;
+}
+
+static void
+unregister_algorithms(void) {
+	md_unregister_algorithm(sha512);
+	md_unregister_algorithm(sha384);
+	md_unregister_algorithm(sha256);
+	md_unregister_algorithm(sha224);
+	md_unregister_algorithm(sha1);
+	md_unregister_algorithm(md5);
+}
+
+#undef md_unregister_algorithm
+#undef md_register_algorithm
 
-#if defined(HAVE_EVP_DEFAULT_PROPERTIES_ENABLE_FIPS)
 bool
 isc_crypto_fips_mode(void) {
 	return EVP_default_properties_is_fips_enabled(NULL) != 0;
@@ -208,40 +182,6 @@ isc_crypto_fips_enable(void) {
 
 	return ISC_R_SUCCESS;
 }
-#elif defined(HAVE_FIPS_MODE)
-bool
-isc_crypto_fips_mode(void) {
-	return FIPS_mode() != 0;
-}
-
-isc_result_t
-isc_crypto_fips_enable(void) {
-	if (isc_crypto_fips_mode()) {
-		return ISC_R_SUCCESS;
-	}
-
-	if (FIPS_mode_set(1) == 0) {
-		return isc_tlserr2result(ISC_LOGCATEGORY_GENERAL,
-					 ISC_LOGMODULE_CRYPTO, "FIPS_mode_set",
-					 ISC_R_CRYPTOFAILURE);
-	}
-
-	unregister_algorithms();
-	register_algorithms();
-
-	return ISC_R_SUCCESS;
-}
-#else
-bool
-isc_crypto_fips_mode(void) {
-	return false;
-}
-
-isc_result_t
-isc_crypto_fips_enable(void) {
-	return ISC_R_NOTIMPLEMENTED;
-}
-#endif
 
 void
 isc__crypto_setdestroycheck(bool check) {
@@ -250,37 +190,22 @@ isc__crypto_setdestroycheck(bool check) {
 
 void
 isc__crypto_initialize(void) {
-	uint64_t opts = OPENSSL_INIT_LOAD_CONFIG;
+	constexpr uint64_t opts = OPENSSL_INIT_LOAD_CONFIG |
+				  OPENSSL_INIT_NO_ATEXIT;
 
 	isc_mem_create("OpenSSL", &isc__crypto_mctx);
 	isc_mem_setdebugging(isc__crypto_mctx, 0);
 	isc_mem_setdestroycheck(isc__crypto_mctx, false);
 
-#if !defined(LIBRESSL_VERSION_NUMBER) && OPENSSL_VERSION_NUMBER >= 0x30000000L
-	/*
-	 * CRYPTO_set_mem_(_ex)_functions() returns 1 on success or 0 on
-	 * failure, which means OpenSSL already allocated some memory.  There's
-	 * nothing we can do about it.
-	 */
 	(void)CRYPTO_set_mem_functions(isc__crypto_malloc_ex,
 				       isc__crypto_realloc_ex,
 				       isc__crypto_free_ex);
-#endif /* !defined(LIBRESSL_VERSION_NUMBER) && OPENSSL_VERSION_NUMBER >= \
-	  0x30000000L  */
-
-#if defined(OPENSSL_INIT_NO_ATEXIT)
-	/*
-	 * We call OPENSSL_cleanup() manually, in a correct order, thus disable
-	 * the automatic atexit() handler.
-	 */
-	opts |= OPENSSL_INIT_NO_ATEXIT;
-#endif
 
 	RUNTIME_CHECK(OPENSSL_init_ssl(opts, NULL) == 1);
 
 	register_algorithms();
 
-#if defined(ENABLE_FIPS_MODE)
+#ifdef ENABLE_FIPS_MODE
 	if (isc_crypto_fips_enable() != ISC_R_SUCCESS) {
 		ERR_clear_error();
 		FATAL_ERROR("Failed to toggle FIPS mode but is "
@@ -302,7 +227,6 @@ void
 isc__crypto_shutdown(void) {
 	unregister_algorithms();
 
-#if OPENSSL_VERSION_NUMBER >= 0x30000000L
 	if (base != NULL) {
 		OSSL_PROVIDER_unload(base);
 	}
@@ -310,7 +234,6 @@ isc__crypto_shutdown(void) {
 	if (fips != NULL) {
 		OSSL_PROVIDER_unload(fips);
 	}
-#endif /* OPENSSL_VERSION_NUMBER >= 0x30000000L */
 
 	OPENSSL_cleanup();
 

lib/isc/include/isc/crypto.h

@@ -17,12 +17,12 @@
 
 #include <isc/types.h>
 
-extern const EVP_MD *isc__crypto_md5;
-extern const EVP_MD *isc__crypto_sha1;
-extern const EVP_MD *isc__crypto_sha224;
-extern const EVP_MD *isc__crypto_sha256;
-extern const EVP_MD *isc__crypto_sha384;
-extern const EVP_MD *isc__crypto_sha512;
+extern EVP_MD *isc__crypto_md5;
+extern EVP_MD *isc__crypto_sha1;
+extern EVP_MD *isc__crypto_sha224;
+extern EVP_MD *isc__crypto_sha256;
+extern EVP_MD *isc__crypto_sha384;
+extern EVP_MD *isc__crypto_sha512;
 
 bool
 isc_crypto_fips_mode(void);

lib/isc/meson.build

@@ -19,6 +19,7 @@ endif
 # isc_inc += include_directories('include')
 isc_inc_p += include_directories('.')
 
+subdir('crypto')
 subdir('netmgr')
 
 isc_srcset.add(
@@ -71,7 +72,6 @@ isc_srcset.add(
         'base64.c',
         'commandline.c',
         'counter.c',
-        'crypto.c',
         'dir.c',
         'entropy.c',
         'errno.c',

meson.build

@@ -546,18 +546,31 @@ openssl_dep = [
     dependency('libssl', version: '>=1.1.1'),
 ]
 
-foreach fn, header : {
-    'EVP_default_properties_enable_fips': '#include <openssl/evp.h>',
-    'FIPS_mode': '#include <openssl/crypto.h>',
-}
-    if cc.has_function(fn, prefix: header, dependencies: openssl_dep)
-        config.set('HAVE_OPENSSL_FIPS_TOGGLE', 1)
-        config.set('HAVE_@0@'.format(fn.to_upper()), 1)
+config.set('OPENSSL_NO_DEPRECATED', true)
+
+have_fips_toggle = false
+
+# Forks such as LibreSSL do not care about keeping pkg-config version compatibility
+# with OpenSSL. Thus, we need to probe the provider header so see if it is before or
+# after OpenSSL 3.0.
+if cc.has_header('openssl/provider.h', dependencies: openssl_dep)
+    have_fips_toggle = true
+    config.set('OPENSSL_API_COMPAT', 30000)
+    config.set('HAVE_OPENSSL_3_API', true)
+else
+    config.set('OPENSSL_API_COMPAT', 10100)
+    if cc.has_function(
+        'FIPS_mode',
+        prefix: '#include <openssl/crypto.h>',
+        dependencies: openssl_dep,
+    )
+        have_fips_toggle = true
+        config.set('HAVE_FIPS_MODE', 1)
     endif
-endforeach
+endif
 
 fips_opt.require(
-    config.has('HAVE_OPENSSL_FIPS_TOGGLE'),
+    have_fips_toggle,
     error_message: 'OpenSSL FIPS mode requested but not available',
 )