fix(rate-limit): fallback to IP when keyGenerator returns empty or throws

docs/content/docs/concepts/rate-limit.mdx

@@ -85,7 +85,7 @@ export const auth = betterAuth({
 ```
 
 <Callout>
-The request path is automatically appended to the returned value.
+The request path is automatically appended to the returned value. If the function returns an empty value or throws an error, the IP address will be used as fallback.
 </Callout>
 
 ### Rate Limit Window

packages/better-auth/src/api/rate-limiter/index.ts

@@ -141,22 +141,21 @@ export async function onRequestRateLimit(req: Request, ctx: AuthContext) {
 	);
 	let window = ctx.rateLimit.window;
 	let max = ctx.rateLimit.max;
-	let key: string = "";
+	let key: string | null = null;
 
 	if (ctx.options.rateLimit?.keyGenerator) {
 		try {
 			const generatedKey = await ctx.options.rateLimit.keyGenerator(req);
 
-			if (!generatedKey?.length) {
-				return;
+			if (generatedKey?.length) {
+				key = generatedKey;
 			}
-
-			key = generatedKey;
 		} catch (e) {
 			ctx.logger.error("Error generating rate limit key", e);
-			return;
 		}
-	} else {
+	}
+
+	if (!key) {
 		const ip = getIp(req, ctx.options);
 
 		if (!ip) {

packages/better-auth/src/api/rate-limiter/rate-limiter.test.ts

@@ -233,7 +233,7 @@ describe("should work with custom keyGenerator", async () => {
 				max: 3,
 				keyGenerator: async (request) => {
 					const customId = request.headers.get("x-custom-id");
-					return customId || "unknown";
+					return customId || "";
 				},
 			},
 			secondaryStorage: {
@@ -270,6 +270,45 @@ describe("should work with custom keyGenerator", async () => {
 
 		expect(store.has("user-123/sign-in/email")).toBe(true);
 	});
+
+	it("should fallback to use IP when keyGenerator returns empty", async () => {
+		const store = new Map<string, string>();
+		const { client, testUser } = await getTestInstance({
+			rateLimit: {
+				enabled: true,
+				window: 10,
+				max: 3,
+				keyGenerator: async () => {
+					return "";
+				},
+			},
+			secondaryStorage: {
+				set(key, value) {
+					store.set(key, value);
+				},
+				get(key) {
+					return store.get(key) || null;
+				},
+				delete(key) {
+					store.delete(key);
+				},
+			},
+		});
+
+		for (let i = 0; i < 4; i++) {
+			const response = await client.signIn.email({
+				email: testUser.email,
+				password: testUser.password,
+			});
+			if (i >= 3) {
+				expect(response.error?.status).toBe(429);
+			} else {
+				expect(response.error).toBeNull();
+			}
+		}
+
+		expect(store.has("127.0.0.1/sign-in/email")).toBe(true);
+	});
 });
 
 describe("should work in development/test environment", () => {