refactor(oauth-provider): refuse unrotatable clients before verifying the assertion

verifyClientAssertion takes an `assertClient` precondition that runs on the looked-up client before any key is fetched or the jti is consumed. Key rotation uses it, so a jwks_uri or trusted client is refused without a JWKS fetch, a signature check, or spending the assertion.

.changeset/oauth-provider-rotation-precheck.md

@@ -0,0 +1,5 @@
+---
+"@better-auth/oauth-provider": patch
+---
+
+`POST /oauth2/client/rotate-keys` now refuses a client it cannot rotate, such as one that publishes a `jwks_uri` or a trusted client configured in code, before it fetches the client's keys or uses up the client assertion.

packages/oauth-provider/src/client-key-rotation.test.ts

@@ -428,6 +428,36 @@ describe.each(adapters)("client key rotation ($name)", async ({
 		trustedClients.delete(clientId);
 	});
 
+	// The memory adapter does not enforce the primary key the jti tombstone relies on.
+	it.skipIf(name === "memory adapter")(
+		"refuses a client it cannot rotate before spending the assertion",
+		async () => {
+			const oldKey = await createKey("old");
+			const newKey = await createKey("new");
+			const clientId = await createClient([oldKey]);
+			const assertion = await sign(clientId, oldKey, rotateURL);
+
+			trustedClients.add(clientId);
+			const refused = await rotate({
+				clientId,
+				signer: oldKey,
+				next: [newKey],
+				assertion,
+			});
+			trustedClients.delete(clientId);
+			expect(refused.status).toBe(400);
+
+			// The refusal came before verification, so the jti was never consumed.
+			const accepted = await rotate({
+				clientId,
+				signer: oldKey,
+				next: [newKey],
+				assertion,
+			});
+			expect(accepted.status).toBe(200);
+		},
+	);
+
 	it("rejects a disabled client", async () => {
 		const oldKey = await createKey("old");
 		const clientId = await createClient([oldKey]);

packages/oauth-provider/src/client-key-rotation.ts

@@ -49,9 +49,11 @@ export async function rotateClientKeysEndpoint(
 		body.client_assertion_type,
 		body.client_id,
 		`${ctx.context.baseURL}/oauth2/client/rotate-keys`,
-		{ endpointAudienceOnly: true },
+		{
+			endpointAudienceOnly: true,
+			assertClient: (candidate) => assertRotatable(opts, candidate),
+		},
 	);
-	assertRotatable(opts, client);
 
 	const updated = await ctx.context.adapter.incrementOne<SchemaClient<Scope[]>>(
 		{

packages/oauth-provider/src/utils/client-assertion.ts

@@ -460,6 +460,10 @@ export async function consumeClientAssertion(
  * an assertion addressed to `expectedAudience` itself verifies. Endpoints that
  * change client credentials use it, so an assertion minted for the token
  * endpoint or the issuer cannot be redirected to them.
+ *
+ * `assertClient` runs on the looked-up client before any key is fetched or the
+ * jti is consumed, so a caller can refuse a client it does not serve without
+ * spending the assertion or reaching out to a `jwks_uri`.
  */
 export async function verifyClientAssertion(
 	ctx: GenericEndpointContext,
@@ -468,7 +472,10 @@ export async function verifyClientAssertion(
 	clientAssertionType: string,
 	clientIdHint?: string,
 	expectedAudience?: string,
-	options?: { endpointAudienceOnly?: boolean },
+	options?: {
+		endpointAudienceOnly?: boolean;
+		assertClient?: (client: SchemaClient<Scope[]>) => void;
+	},
 ): Promise<{ clientId: string; client: SchemaClient<Scope[]> }> {
 	if (clientAssertionType !== CLIENT_ASSERTION_TYPE) {
 		throw new APIError("BAD_REQUEST", {
@@ -542,6 +549,7 @@ export async function verifyClientAssertion(
 			error: "invalid_client",
 		});
 	}
+	options?.assertClient?.(client);
 
 	// Fetch JWKS and verify signature + claims
 	const jwks = await fetchClientJwks(ctx, opts, client);