From 8d093a6b66f4c68be3f5cba9aa19ddcbaa00dcba Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Ayd=C4=B1n=20Mercan?= <aydin@isc.org>
Date: Wed, 4 Dec 2024 13:11:45 +0300
Subject: [PATCH] disable deterministic ecdsa for fips builds

FIPS 186-5 [1] allows the usage deterministic ECDSA (Section 6.3) which
is compabile with RFC 6979 [2] but OpenSSL seems to follow FIPS 186-4
(Section 6.3) [3] which only allows for random k values, failing
k value generation for OpenSSL >=3.2. [4]

Fix signing by not using deterministic ECDSA when FIPS mode is active.

[1]: https://nvlpubs.nist.gov/nistpubs/FIPS/NIST.FIPS.186-5.pdf
[2]: https://datatracker.ietf.org/doc/html/rfc6979
[3]: https://nvlpubs.nist.gov/nistpubs/FIPS/NIST.FIPS.186-4.pdf
[4]: https://github.com/openssl/openssl/blob/85f17585b0d8b55b335f561e2862db14a20b1e64/crypto/ec/ecdsa_ossl.c#L201-L207
---
 lib/dns/opensslecdsa_link.c | 10 +++++++---
 tests/dns/dst_test.c        |  7 ++++++-
 2 files changed, 13 insertions(+), 4 deletions(-)

diff --git a/lib/dns/opensslecdsa_link.c b/lib/dns/opensslecdsa_link.c
index be3a8a43ff..74cab51ec2 100644
--- a/lib/dns/opensslecdsa_link.c
+++ b/lib/dns/opensslecdsa_link.c
@@ -26,6 +26,7 @@
 #include <openssl/param_build.h>
 #endif
 
+#include <isc/fips.h>
 #include <isc/mem.h>
 #include <isc/result.h>
 #include <isc/safe.h>
@@ -706,9 +707,12 @@ opensslecdsa_createctx(dst_key_t *key, dst_context_t *dctx) {
 		}
 
 #if OPENSSL_VERSION_NUMBER >= 0x30200000L
-		ret = opensslecdsa_set_deterministic(pctx, dctx->key->key_alg);
-		if (ret != ISC_R_SUCCESS) {
-			goto err;
+		if (!isc_fips_mode()) {
+			ret = opensslecdsa_set_deterministic(
+				pctx, dctx->key->key_alg);
+			if (ret != ISC_R_SUCCESS) {
+				goto err;
+			}
 		}
 #endif /* OPENSSL_VERSION_NUMBER >= 0x30200000L */
 
diff --git a/tests/dns/dst_test.c b/tests/dns/dst_test.c
index 79c7fb5170..2121e3883e 100644
--- a/tests/dns/dst_test.c
+++ b/tests/dns/dst_test.c
@@ -31,6 +31,7 @@
 #include <cmocka.h>
 
 #include <isc/file.h>
+#include <isc/fips.h>
 #include <isc/hex.h>
 #include <isc/result.h>
 #include <isc/stdio.h>
@@ -467,7 +468,11 @@ ISC_RUN_TEST_IMPL(ecdsa_determinism_test) {
 	dst_context_destroy(&ctx);
 
 #if OPENSSL_VERSION_NUMBER >= 0x30200000L
-	assert_memory_equal(sigbuf1->base, sigbuf2->base, siglen);
+	if (isc_fips_mode()) {
+		assert_memory_not_equal(sigbuf1->base, sigbuf2->base, siglen);
+	} else {
+		assert_memory_equal(sigbuf1->base, sigbuf2->base, siglen);
+	}
 #else
 	assert_memory_not_equal(sigbuf1->base, sigbuf2->base, siglen);
 #endif