Pass the digest buffer length to EVP_DigestSignFinal

OpenSSL 3.0.1 does not accept 0 as a digest buffer length when
calling EVP_DigestSignFinal as it now checks that the digest buffer
length is large enough for the digest.  Pass the digest buffer
length instead.

lib/dns/hmac_link.c

@@ -196,8 +196,8 @@ static inline isc_result_t
 hmac_sign(const dst_context_t *dctx, isc_buffer_t *sig) {
 	isc_hmac_t *ctx = dctx->ctxdata.hmac_ctx;
 	REQUIRE(ctx != NULL);
-	unsigned int digestlen;
 	unsigned char digest[ISC_MAX_MD_SIZE];
+	unsigned int digestlen = sizeof(digest);
 
 	if (isc_hmac_final(ctx, digest, &digestlen) != ISC_R_SUCCESS) {
 		return (DST_R_OPENSSLFAILURE);
@@ -219,8 +219,8 @@ hmac_sign(const dst_context_t *dctx, isc_buffer_t *sig) {
 static inline isc_result_t
 hmac_verify(const dst_context_t *dctx, const isc_region_t *sig) {
 	isc_hmac_t *ctx = dctx->ctxdata.hmac_ctx;
-	unsigned int digestlen;
 	unsigned char digest[ISC_MAX_MD_SIZE];
+	unsigned int digestlen = sizeof(digest);
 
 	REQUIRE(ctx != NULL);
 

lib/isc/hmac.c

@@ -94,18 +94,17 @@ isc_hmac_update(isc_hmac_t *hmac, const unsigned char *buf, const size_t len) {
 isc_result_t
 isc_hmac_final(isc_hmac_t *hmac, unsigned char *digest,
 	       unsigned int *digestlen) {
-	size_t len = 0;
-
 	REQUIRE(hmac != NULL);
 	REQUIRE(digest != NULL);
+	REQUIRE(digestlen != NULL);
+
+	size_t len = *digestlen;
 
 	if (EVP_DigestSignFinal(hmac, digest, &len) != 1) {
 		return (ISC_R_CRYPTOFAILURE);
 	}
 
-	if (digestlen != NULL) {
+	*digestlen = (unsigned int)len;
-		*digestlen = (unsigned int)len;
-	}
 
 	return (ISC_R_SUCCESS);
 }

lib/isc/include/isc/hmac.h

@@ -31,14 +31,15 @@ typedef void isc_hmac_t;
  * @buf: data to hash
  * @len: length of the data to hash
  * @digest: the output buffer
- * @digestlen: the length of the data written to @digest
+ * @digestlen: in: the length of @digest
+ *             out: the length of the data written to @digest
  *
  * This function computes the message authentication code using a digest type
  * @type with key @key which is @keylen bytes long from data in @buf which is
  * @len bytes long, and places the output into @digest, which must have space
- * for the hash function output (use ISC_MAX_MD_SIZE if unsure).  If the
+ * for the hash function output (use ISC_MAX_MD_SIZE if unsure). @digestlen
- * @digestlen parameter is not NULL then the number of bytes of data written
+ * is used to pass in the length of the digest buffer and returns the length
- * (i.e. the length of the digest) will be written to the @digestlen.
+ * of digest written to @digest.
  */
 isc_result_t
 isc_hmac(const isc_md_type_t *type, const void *key, const size_t keylen,
@@ -103,13 +104,14 @@ isc_hmac_update(isc_hmac_t *hmac, const unsigned char *buf, const size_t len);
  * isc_hmac_final:
  * @hmac: HMAC context
  * @digest: the output buffer
- * @digestlen: the length of the data written to @digest
+ * @digestlen: in: the length of @digest
+ *             out: the length of the data written to @digest
  *
  * This function retrieves the message authentication code from @hmac and places
- * it in @digest, which must have space for the hash function output.  If the
+ * it in @digest, which must have space for the hash function output. @digestlen
- * @digestlen parameter is not NULL then the number of bytes of data written
+ * is used to pass in the length of the digest buffer and returns the length
- * (i.e. the length of the digest) will be written to the @digestlen.  After
+ * of digest written to @digest.  After calling this function no additional
- * calling this function no additional calls to isc_hmac_update() can be made.
+ * calls to isc_hmac_update() can be made.
  */
 isc_result_t
 isc_hmac_final(isc_hmac_t *hmac, unsigned char *digest,

lib/isc/tests/hmac_test.c

@@ -99,7 +99,7 @@ isc_hmac_test(isc_hmac_t *hmac, const void *key, size_t keylen,
 	}
 
 	unsigned char digest[ISC_MAX_MD_SIZE];
-	unsigned int digestlen;
+	unsigned int digestlen = sizeof(digest);
 	assert_int_equal(isc_hmac_final(hmac, digest, &digestlen),
 			 ISC_R_SUCCESS);
 
@@ -198,7 +198,7 @@ isc_hmac_final_test(void **state) {
 	assert_non_null(hmac);
 
 	unsigned char digest[ISC_MAX_MD_SIZE];
-	unsigned int digestlen;
+	unsigned int digestlen = sizeof(digest);
 
 	/* Fail when message digest context is empty */
 	expect_assert_failure(isc_hmac_final(NULL, digest, &digestlen));
@@ -207,7 +207,8 @@ isc_hmac_final_test(void **state) {
 
 	assert_int_equal(isc_hmac_init(hmac, "", 0, ISC_MD_SHA512),
 			 ISC_R_SUCCESS);
-	assert_int_equal(isc_hmac_final(hmac, digest, NULL), ISC_R_SUCCESS);
+	/* Fail when the digest length pointer is empty */
+	expect_assert_failure(isc_hmac_final(hmac, digest, NULL));
 }
 
 static void

lib/isccc/cc.c

@@ -252,7 +252,7 @@ sign(unsigned char *data, unsigned int length, unsigned char *hmac,
 	isc_result_t result;
 	isccc_region_t source, target;
 	unsigned char digest[ISC_MAX_MD_SIZE];
-	unsigned int digestlen;
+	unsigned int digestlen = sizeof(digest);
 	unsigned char digestb64[HSHA_LENGTH + 4];
 
 	source.rstart = digest;
@@ -375,7 +375,7 @@ verify(isccc_sexpr_t *alist, unsigned char *data, unsigned int length,
 	isc_result_t result;
 	isccc_sexpr_t *_auth, *hmac;
 	unsigned char digest[ISC_MAX_MD_SIZE];
-	unsigned int digestlen;
+	unsigned int digestlen = sizeof(digest);
 	unsigned char digestb64[HSHA_LENGTH * 4];
 
 	/*