From c8bcf3a34e0beb6150478f7a21f43580ea67f968 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Timo=20Ter=C3=A4s?= <timo.teras@iki.fi>
Date: Mon, 26 Dec 2022 14:25:18 +0200
Subject: [PATCH] Refactor OpenSSL RSA exponent bits checking to a helper
 function

- Make it a separate opensslrsa_check_exponent_bits() function to
  clean up the code a bit
- Always use provider API first if using openssl 3.0, and fallback
  to EVP API for older openssl or if built with engine support
- Use RSA_get0_key() (with shim for openssl 1.0) to avoid memory
  allocations
---
 configure.ac              |  3 +-
 lib/dns/openssl_shim.h    |  8 +++++
 lib/dns/opensslrsa_link.c | 67 ++++++++++++++++++---------------------
 3 files changed, 41 insertions(+), 37 deletions(-)

diff --git a/configure.ac b/configure.ac
index c0a364e54b..4576c64da1 100644
--- a/configure.ac
+++ b/configure.ac
@@ -726,7 +726,8 @@ AC_CHECK_FUNCS([EVP_aes_128_ecb EVP_aes_192_ecb EVP_aes_256_ecb], [:],
 #
 # Check for OpenSSL 1.1.x/LibreSSL functions
 #
-AC_CHECK_FUNCS([DH_get0_key ECDSA_SIG_get0 RSA_set0_key EVP_PKEY_get0_EC_KEY])
+AC_CHECK_FUNCS([DH_get0_key ECDSA_SIG_get0 EVP_PKEY_get0_EC_KEY])
+AC_CHECK_FUNCS([RSA_set0_key EVP_PKEY_get0_RSA])
 
 AC_CHECK_FUNCS([TLS_server_method TLS_client_method])
 
diff --git a/lib/dns/openssl_shim.h b/lib/dns/openssl_shim.h
index e716b3a7a3..ad07638c3e 100644
--- a/lib/dns/openssl_shim.h
+++ b/lib/dns/openssl_shim.h
@@ -17,6 +17,7 @@
 #include <openssl/dh.h>
 #include <openssl/ecdsa.h>
 #include <openssl/err.h>
+#include <openssl/evp.h>
 #include <openssl/opensslv.h>
 #include <openssl/rsa.h>
 
@@ -27,6 +28,13 @@
 #define RSA_MAX_PUBEXP_BITS 35
 #endif /* ifndef RSA_MAX_PUBEXP_BITS */
 
+#if !HAVE_EVP_PKEY_GET0_RSA && OPENSSL_VERSION_NUMBER < 0x10100000L
+static inline const RSA *
+EVP_PKEY_get0_RSA(const EVP_PKEY *pkey) {
+	return (pkey->type == EVP_PKEY_RSA ? pkey->pkey.rsa : NULL);
+}
+#endif
+
 #if !HAVE_RSA_SET0_KEY && OPENSSL_VERSION_NUMBER < 0x30000000L
 int
 RSA_set0_key(RSA *r, BIGNUM *n, BIGNUM *e, BIGNUM *d);
diff --git a/lib/dns/opensslrsa_link.c b/lib/dns/opensslrsa_link.c
index e6813ed3e1..94f4f45731 100644
--- a/lib/dns/opensslrsa_link.c
+++ b/lib/dns/opensslrsa_link.c
@@ -17,19 +17,17 @@
 #include <stdbool.h>
 
 #include <openssl/bn.h>
+#include <openssl/err.h>
+#include <openssl/objects.h>
 #include <openssl/opensslv.h>
-#if OPENSSL_VERSION_NUMBER >= 0x30000000L && OPENSSL_API_LEVEL >= 30000
-#include <openssl/core_names.h>
-#endif
+#include <openssl/rsa.h>
 #if !defined(OPENSSL_NO_ENGINE) && OPENSSL_API_LEVEL < 30000
 #include <openssl/engine.h>
 #endif /* if !defined(OPENSSL_NO_ENGINE) && OPENSSL_API_LEVEL < 30000 */
-#include <openssl/err.h>
-#include <openssl/objects.h>
-#if OPENSSL_VERSION_NUMBER >= 0x30000000L && OPENSSL_API_LEVEL >= 30000
+#if OPENSSL_VERSION_NUMBER >= 0x30000000L
+#include <openssl/core_names.h>
 #include <openssl/param_build.h>
 #endif
-#include <openssl/rsa.h>
 
 #include <isc/mem.h>
 #include <isc/result.h>
@@ -177,44 +175,41 @@ opensslrsa_sign(dst_context_t *dctx, isc_buffer_t *sig) {
 	return (ISC_R_SUCCESS);
 }
 
+static bool
+opensslrsa_check_exponent_bits(EVP_PKEY *pkey, int maxbits) {
+	/* Always use the new API first with OpenSSL 3.x. */
+#if OPENSSL_VERSION_NUMBER >= 0x30000000L
+	BIGNUM *e = NULL;
+	if (EVP_PKEY_get_bn_param(pkey, OSSL_PKEY_PARAM_RSA_E, &e) == 1) {
+		int bits = BN_num_bits(e);
+		BN_free(e);
+		return (bits < maxbits);
+	}
+#endif
+	/* Use old API for the OpenSSL ENGINE support, even with OpenSSL 3.x */
+#if OPENSSL_VERSION_NUMBER < 0x30000000L || OPENSSL_API_LEVEL < 30000
+	const RSA *rsa = EVP_PKEY_get0_RSA(pkey);
+	if (rsa != NULL) {
+		const BIGNUM *ce = NULL;
+		RSA_get0_key(rsa, NULL, &ce, NULL);
+		if (ce != NULL) {
+			return (BN_num_bits(ce) < maxbits);
+		}
+	}
+#endif
+	return (false);
+}
+
 static isc_result_t
 opensslrsa_verify2(dst_context_t *dctx, int maxbits, const isc_region_t *sig) {
 	dst_key_t *key = dctx->key;
 	int status = 0;
-#if OPENSSL_VERSION_NUMBER < 0x30000000L || OPENSSL_API_LEVEL < 30000
-	RSA *rsa;
-	const BIGNUM *e = NULL;
-#else
-	BIGNUM *e = NULL;
-#endif /* OPENSSL_VERSION_NUMBER < 0x30000000L || OPENSSL_API_LEVEL < 30000 */
 	EVP_MD_CTX *evp_md_ctx = dctx->ctxdata.evp_md_ctx;
 	EVP_PKEY *pkey = key->keydata.pkey;
-	int bits;
 
 	REQUIRE(opensslrsa_valid_key_alg(dctx->key->key_alg));
 
-#if OPENSSL_VERSION_NUMBER < 0x30000000L || OPENSSL_API_LEVEL < 30000
-	rsa = EVP_PKEY_get1_RSA(pkey);
-	if (rsa == NULL) {
-		return (dst__openssl_toresult(DST_R_OPENSSLFAILURE));
-	}
-	RSA_get0_key(rsa, NULL, &e, NULL);
-	if (e == NULL) {
-		RSA_free(rsa);
-		return (dst__openssl_toresult(DST_R_VERIFYFAILURE));
-	}
-	bits = BN_num_bits(e);
-	RSA_free(rsa);
-#else
-	EVP_PKEY_get_bn_param(pkey, OSSL_PKEY_PARAM_RSA_E, &e);
-	if (e == NULL) {
-		return (dst__openssl_toresult(DST_R_VERIFYFAILURE));
-	}
-	bits = BN_num_bits(e);
-	BN_free(e);
-#endif /* OPENSSL_VERSION_NUMBER < 0x30000000L || OPENSSL_API_LEVEL < 30000 */
-
-	if (bits > maxbits && maxbits != 0) {
+	if (maxbits != 0 && !opensslrsa_check_exponent_bits(pkey, maxbits)) {
 		return (DST_R_VERIFYFAILURE);
 	}