Refactor the OpenSSL HMAC usage to use newer APIs

OpenSSL 3 deprecates the HMAC* family and associated APIs.

Rewrite portions of OpenSSL library usage code to use a newer
set of HMAC APIs.

configure.ac

@@ -626,10 +626,10 @@ AC_COMPILE_IFELSE(
 
 AC_CHECK_FUNCS([OPENSSL_init_ssl OPENSSL_init_crypto])
 AC_CHECK_FUNCS([CRYPTO_zalloc])
+AC_CHECK_FUNCS([EVP_PKEY_new_raw_private_key])
 AC_CHECK_FUNCS([EVP_CIPHER_CTX_new EVP_CIPHER_CTX_free])
 AC_CHECK_FUNCS([EVP_MD_CTX_new EVP_MD_CTX_free EVP_MD_CTX_reset EVP_MD_CTX_get0_md])
 AC_CHECK_FUNCS([ERR_get_error_all])
-AC_CHECK_FUNCS([HMAC_CTX_new HMAC_CTX_free HMAC_CTX_reset HMAC_CTX_get_md])
 AC_CHECK_FUNCS([SSL_read_ex SSL_peek_ex SSL_write_ex])
 AC_CHECK_FUNCS([BIO_read_ex BIO_write_ex])
 AC_CHECK_FUNCS([SSL_CTX_up_ref])

lib/isc/hmac.c

@@ -9,7 +9,7 @@
  * information regarding copyright ownership.
  */
 
-#include <openssl/hmac.h>
+#include <openssl/evp.h>
 #include <openssl/opensslv.h>
 
 #include <isc/assertions.h>
@@ -24,9 +24,9 @@
 
 isc_hmac_t *
 isc_hmac_new(void) {
-	HMAC_CTX *hmac = HMAC_CTX_new();
+	EVP_MD_CTX *hmac = EVP_MD_CTX_new();
 	RUNTIME_CHECK(hmac != NULL);
-	return ((struct hmac *)hmac);
+	return ((isc_hmac_t *)hmac);
 }
 
 void
@@ -35,23 +35,33 @@ isc_hmac_free(isc_hmac_t *hmac) {
 		return;
 	}
 
-	HMAC_CTX_free(hmac);
+	EVP_MD_CTX_free((EVP_MD_CTX *)hmac);
 }
 
 isc_result_t
-isc_hmac_init(isc_hmac_t *hmac, const void *key, size_t keylen,
+isc_hmac_init(isc_hmac_t *hmac, const void *key, const size_t keylen,
 	      const isc_md_type_t *md_type) {
+	EVP_PKEY *pkey;
+
 	REQUIRE(hmac != NULL);
 	REQUIRE(key != NULL);
+	REQUIRE(keylen <= INT_MAX);
 
 	if (md_type == NULL) {
 		return (ISC_R_NOTIMPLEMENTED);
 	}
 
-	if (HMAC_Init_ex(hmac, key, keylen, md_type, NULL) != 1) {
+	pkey = EVP_PKEY_new_raw_private_key(EVP_PKEY_HMAC, NULL, key, keylen);
+	if (pkey == NULL) {
 		return (ISC_R_CRYPTOFAILURE);
 	}
 
+	if (EVP_DigestSignInit(hmac, NULL, md_type, NULL, pkey) != 1) {
+		return (ISC_R_CRYPTOFAILURE);
+	}
+
+	EVP_PKEY_free(pkey);
+
 	return (ISC_R_SUCCESS);
 }
 
@@ -59,7 +69,7 @@ isc_result_t
 isc_hmac_reset(isc_hmac_t *hmac) {
 	REQUIRE(hmac != NULL);
 
-	if (HMAC_CTX_reset(hmac) != 1) {
+	if (EVP_MD_CTX_reset(hmac) != 1) {
 		return (ISC_R_CRYPTOFAILURE);
 	}
 
@@ -74,7 +84,7 @@ isc_hmac_update(isc_hmac_t *hmac, const unsigned char *buf, const size_t len) {
 		return (ISC_R_SUCCESS);
 	}
 
-	if (HMAC_Update(hmac, buf, len) != 1) {
+	if (EVP_DigestSignUpdate(hmac, buf, len) != 1) {
 		return (ISC_R_CRYPTOFAILURE);
 	}
 
@@ -84,13 +94,19 @@ isc_hmac_update(isc_hmac_t *hmac, const unsigned char *buf, const size_t len) {
 isc_result_t
 isc_hmac_final(isc_hmac_t *hmac, unsigned char *digest,
 	       unsigned int *digestlen) {
+	size_t len = 0;
+
 	REQUIRE(hmac != NULL);
 	REQUIRE(digest != NULL);
 
-	if (HMAC_Final(hmac, digest, digestlen) != 1) {
+	if (EVP_DigestSignFinal(hmac, digest, &len) != 1) {
 		return (ISC_R_CRYPTOFAILURE);
 	}
 
+	if (digestlen != NULL) {
+		*digestlen = (unsigned int)len;
+	}
+
 	return (ISC_R_SUCCESS);
 }
 
@@ -98,25 +114,25 @@ const isc_md_type_t *
 isc_hmac_get_md_type(isc_hmac_t *hmac) {
 	REQUIRE(hmac != NULL);
 
-	return (HMAC_CTX_get_md(hmac));
+	return (EVP_MD_CTX_get0_md(hmac));
 }
 
 size_t
 isc_hmac_get_size(isc_hmac_t *hmac) {
 	REQUIRE(hmac != NULL);
 
-	return ((size_t)EVP_MD_size(HMAC_CTX_get_md(hmac)));
+	return ((size_t)EVP_MD_CTX_size(hmac));
 }
 
 int
 isc_hmac_get_block_size(isc_hmac_t *hmac) {
 	REQUIRE(hmac != NULL);
 
-	return (EVP_MD_block_size(HMAC_CTX_get_md(hmac)));
+	return (EVP_MD_CTX_block_size(hmac));
 }
 
 isc_result_t
-isc_hmac(const isc_md_type_t *type, const void *key, const int keylen,
+isc_hmac(const isc_md_type_t *type, const void *key, const size_t keylen,
 	 const unsigned char *buf, const size_t len, unsigned char *digest,
 	 unsigned int *digestlen) {
 	isc_result_t res;

lib/isc/include/isc/hmac.h

@@ -41,7 +41,7 @@ typedef void isc_hmac_t;
  * (i.e. the length of the digest) will be written to the @digestlen.
  */
 isc_result_t
-isc_hmac(const isc_md_type_t *type, const void *key, const int keylen,
+isc_hmac(const isc_md_type_t *type, const void *key, const size_t keylen,
 	 const unsigned char *buf, const size_t len, unsigned char *digest,
 	 unsigned int *digestlen);
 
@@ -74,7 +74,7 @@ isc_hmac_free(isc_hmac_t *hmac);
  */
 
 isc_result_t
-isc_hmac_init(isc_hmac_t *hmac, const void *key, size_t keylen,
+isc_hmac_init(isc_hmac_t *hmac, const void *key, const size_t keylen,
 	      const isc_md_type_t *type);
 
 /**

lib/isc/openssl_shim.c

@@ -58,45 +58,6 @@ EVP_MD_CTX_reset(EVP_MD_CTX *ctx) {
 }
 #endif /* if !HAVE_EVP_MD_CTX_RESET */
 
-#if !HAVE_HMAC_CTX_NEW
-HMAC_CTX *
-HMAC_CTX_new(void) {
-	HMAC_CTX *ctx = OPENSSL_zalloc(sizeof(*ctx));
-	if (ctx != NULL) {
-		if (!HMAC_CTX_reset(ctx)) {
-			HMAC_CTX_free(ctx);
-			return (NULL);
-		}
-	}
-	return (ctx);
-}
-#endif /* if !HAVE_HMAC_CTX_NEW */
-
-#if !HAVE_HMAC_CTX_FREE
-void
-HMAC_CTX_free(HMAC_CTX *ctx) {
-	if (ctx != NULL) {
-		HMAC_CTX_cleanup(ctx);
-		OPENSSL_free(ctx);
-	}
-}
-#endif /* if !HAVE_HMAC_CTX_FREE */
-
-#if !HAVE_HMAC_CTX_RESET
-int
-HMAC_CTX_reset(HMAC_CTX *ctx) {
-	HMAC_CTX_cleanup(ctx);
-	return (1);
-}
-#endif /* if !HAVE_HMAC_CTX_RESET */
-
-#if !HAVE_HMAC_CTX_GET_MD
-const EVP_MD *
-HMAC_CTX_get_md(const HMAC_CTX *ctx) {
-	return (ctx->md);
-}
-#endif /* if !HAVE_HMAC_CTX_GET_MD */
-
 #if !HAVE_SSL_READ_EX
 int
 SSL_read_ex(SSL *ssl, void *buf, size_t num, size_t *readbytes) {

lib/isc/openssl_shim.h

@@ -27,6 +27,11 @@ CRYPTO_zalloc(size_t num, const char *file, int line);
 #define OPENSSL_zalloc(num) CRYPTO_zalloc(num, __FILE__, __LINE__)
 #endif
 
+#if !HAVE_EVP_PKEY_NEW_RAW_PRIVATE_KEY
+#define EVP_PKEY_new_raw_private_key(type, e, key, keylen) \
+	EVP_PKEY_new_mac_key(type, e, key, (int)(keylen))
+#endif /* if !HAVE_EVP_PKEY_NEW_RAW_PRIVATE_KEY */
+
 #if !HAVE_EVP_CIPHER_CTX_NEW
 EVP_CIPHER_CTX *
 EVP_CIPHER_CTX_new(void);
@@ -54,26 +59,6 @@ EVP_MD_CTX_reset(EVP_MD_CTX *ctx);
 #define EVP_MD_CTX_get0_md EVP_MD_CTX_md
 #endif /* if !HAVE_EVP_MD_CTX_GET0_MD */
 
-#if !HAVE_HMAC_CTX_NEW
-HMAC_CTX *
-HMAC_CTX_new(void);
-#endif /* if !HAVE_HMAC_CTX_NEW */
-
-#if !HAVE_HMAC_CTX_FREE
-void
-HMAC_CTX_free(HMAC_CTX *ctx);
-#endif /* if !HAVE_HMAC_CTX_FREE */
-
-#if !HAVE_HMAC_CTX_RESET
-int
-HMAC_CTX_reset(HMAC_CTX *ctx);
-#endif /* if !HAVE_HMAC_CTX_RESET */
-
-#if !HAVE_HMAC_CTX_GET_MD
-const EVP_MD *
-HMAC_CTX_get_md(const HMAC_CTX *ctx);
-#endif /* if !HAVE_HMAC_CTX_GET_MD */
-
 #if !HAVE_SSL_READ_EX
 int
 SSL_read_ex(SSL *ssl, void *buf, size_t num, size_t *readbytes);