KV quota fixes: edge caching, compare scraper gate, kv-cache hardening

db/kv-cache.ts

@@ -1,4 +1,4 @@
-import { env } from "cloudflare:workers";
+import { env, waitUntil } from "cloudflare:workers";
 
 function getKV(): KVNamespace {
 	return env.KV;
@@ -8,23 +8,71 @@ function getKV(): KVNamespace {
 // isn't run after a reseed. Matches the 1-week edge cache window.
 const DEFAULT_TTL = 604800;
 
+// Per-isolate hot cache in front of KV. Every kv.get() counts against
+// the 100k/day free-plan read quota — hit or miss — so repeat lookups
+// of the same key (nav tags, category lists, sitemap slugs) within an
+// isolate's lifetime should never reach KV at all.
+const MEM_TTL_MS = 5 * 60 * 1000;
+const MEM_MAX_ENTRIES = 256;
+const memCache = new Map<string, { value: unknown; expires: number }>();
+
+function memGet(key: string): unknown {
+	const entry = memCache.get(key);
+	if (!entry) return undefined;
+	if (Date.now() > entry.expires) {
+		memCache.delete(key);
+		return undefined;
+	}
+	return entry.value;
+}
+
+function memSet(key: string, value: unknown): void {
+	if (memCache.size >= MEM_MAX_ENTRIES) {
+		// Map iterates in insertion order, so the first key is the oldest
+		const oldest = memCache.keys().next().value;
+		if (oldest !== undefined) memCache.delete(oldest);
+	}
+	memCache.set(key, { value, expires: Date.now() + MEM_TTL_MS });
+}
+
 export async function kvCached<T>(
 	key: string,
 	fn: () => Promise<T>,
 	opts?: { ttl?: number },
 ): Promise<T> {
+	const hot = memGet(key);
+	if (hot !== undefined) return hot as T;
+
 	const kv = getKV();
-	const cached = await kv.get(key, "json");
-	if (cached !== null) return cached as T;
+	// KV throws once the daily free-plan quota is exhausted — fall
+	// through to the database instead of failing the page.
+	let cached: T | null = null;
+	try {
+		cached = await kv.get(key, "json");
+	} catch {}
+	if (cached !== null) {
+		memSet(key, cached);
+		return cached;
+	}
 
 	const result = await fn();
 	// Null/undefined reads back as a cache miss, so storing it only
 	// burns a KV write on every lookup of a nonexistent slug.
 	if (result != null) {
-		// Fire-and-forget write — don't block the response
-		kv.put(key, JSON.stringify(result), {
-			expirationTtl: opts?.ttl ?? DEFAULT_TTL,
-		}).catch(() => {});
+		memSet(key, result);
+		// A bare floating promise is cancelled when the request context
+		// ends, so the write must be kept alive via waitUntil to ever land.
+		// Log failures — a silently broken write path leaves every request
+		// paying for a cache that never fills.
+		waitUntil(
+			kv
+				.put(key, JSON.stringify(result), {
+					expirationTtl: opts?.ttl ?? DEFAULT_TTL,
+				})
+				.catch((err) => {
+					console.error(`KV put failed for ${key}:`, err);
+				}),
+		);
 	}
 	return result;
 }

src/routes/-worker-entry.ts

@@ -44,6 +44,10 @@ const ONE_DAY = 86400;
 
 const cacheRules: CacheRule[] = [
 	{ pattern: /^\/search/, header: "no-store" },
+	// Server-fn GET responses (catalog queries keyed by their payload in
+	// the query string) are deterministic between reseeds, and the reseed
+	// flow already purges the edge cache
+	{ pattern: /^\/_serverFn\//, header: `public, s-maxage=${ONE_DAY}` },
 	{
 		pattern: /^\/sitemap.*\.xml$|^\/robots\.txt$/,
 		header: `public, s-maxage=${ONE_WEEK}`,
@@ -75,6 +79,7 @@ function robotsTxt(): Response {
 	const body = `User-agent: *
 Allow: /
 Disallow: /search
+Disallow: /compare
 
 Sitemap: ${SITE_URL}/sitemap.xml
 `;
@@ -363,10 +368,68 @@ async function handleIconProxy(request: Request): Promise<Response> {
 	}
 }
 
+// ─── Scraper Gate ───────────────────────────────────────────────────
+
+// /compare pages are 25k noindexed, robots-disallowed thin pages, yet
+// scraper fleets on residential proxies (fake browser UAs) crawl them
+// hard enough to exhaust the daily KV read quota. Real users only ever
+// fetch this HTML via a hard navigation — client-side nav goes through
+// /_serverFn — and every browser engine since ~2021 sends
+// Sec-Fetch-Mode: navigate on navigations, which scrapers and crawlers
+// don't. Refuse everything else before SSR spends KV reads on it.
+function isCompareScrape(request: Request, pathname: string): boolean {
+	if (!pathname.startsWith("/compare/")) return false;
+	if (request.method !== "GET") return false;
+	const accept = request.headers.get("Accept") ?? "";
+	const secFetchMode = request.headers.get("Sec-Fetch-Mode");
+	return !(accept.includes("text/html") && secFetchMode === "navigate");
+}
+
 // ─── Worker Entry ───────────────────────────────────────────────────
 
+async function renderResponse(
+	request: Request,
+	pathname: string,
+	cacheHeader: string | null,
+): Promise<Response> {
+	// robots.txt + sitemaps
+	if (pathname === "/robots.txt") {
+		return robotsTxt();
+	}
+
+	const sitemapHandler = sitemapHandlers[pathname];
+	if (sitemapHandler) {
+		return sitemapHandler();
+	}
+
+	// Pass through to TanStack Start
+	const response = await tanstackFetch(request);
+
+	// Comparison pages stay usable in-app but are kept out of search
+	// indexes — 25k thin auto-generated pages triggered a site-wide
+	// quality suppression (see GSC "Crawled - currently not indexed").
+	const noindex = pathname.startsWith("/compare/");
+
+	if ((cacheHeader && response.status === 200) || noindex) {
+		const newResponse = new Response(response.body, response);
+		if (cacheHeader && response.status === 200) {
+			newResponse.headers.set("Cache-Control", cacheHeader);
+		}
+		if (noindex) {
+			newResponse.headers.set("X-Robots-Tag", "noindex");
+		}
+		return newResponse;
+	}
+
+	return response;
+}
+
 export default {
-	async fetch(request: Request, _env: Env): Promise<Response> {
+	async fetch(
+		request: Request,
+		_env: Env,
+		ctx: ExecutionContext,
+	): Promise<Response> {
 		const url = new URL(request.url);
 		const { pathname } = url;
 
@@ -381,39 +444,42 @@ export default {
 			return Response.redirect(target.toString(), 301);
 		}
 
-		// robots.txt + sitemaps
-		if (pathname === "/robots.txt") {
-			return robotsTxt();
-		}
-
-		const sitemapHandler = sitemapHandlers[pathname];
-		if (sitemapHandler) {
-			return sitemapHandler();
+		if (isCompareScrape(request, pathname)) {
+			return new Response("Forbidden", { status: 403 });
 		}
 
 		// Icon proxy (keeps its own Cache API usage)
 		if (pathname === "/icon") {
 			return handleIconProxy(request);
 		}
 
-		// Pass through to TanStack Start
-		const response = await tanstackFetch(request);
-
+		// Serve repeat GETs from the Cloudflare edge cache. Cache-Control
+		// headers alone do nothing for Worker-generated responses — they
+		// only enter the CDN cache through an explicit Cache API put.
 		const cacheHeader = getCacheHeader(pathname);
-		// Comparison pages stay usable in-app but are kept out of search
-		// indexes — 25k thin auto-generated pages triggered a site-wide
-		// quality suppression (see GSC "Crawled - currently not indexed").
-		const noindex = pathname.startsWith("/compare/");
-
-		if ((cacheHeader && response.status === 200) || noindex) {
-			const newResponse = new Response(response.body, response);
-			if (cacheHeader && response.status === 200) {
-				newResponse.headers.set("Cache-Control", cacheHeader);
-			}
-			if (noindex) {
-				newResponse.headers.set("X-Robots-Tag", "noindex");
+		const edgeCacheable =
+			request.method === "GET" &&
+			cacheHeader !== null &&
+			cacheHeader !== "no-store";
+		const cache = (caches as unknown as { default: Cache }).default;
+
+		if (edgeCacheable) {
+			const hit = await cache.match(request.url);
+			if (hit) {
+				const response = new Response(hit.body, hit);
+				response.headers.set("X-Edge-Cache", "HIT");
+				return response;
 			}
-			return newResponse;
+		}
+
+		const response = await renderResponse(request, pathname, cacheHeader);
+
+		if (
+			edgeCacheable &&
+			response.status === 200 &&
+			!response.headers.has("Set-Cookie")
+		) {
+			ctx.waitUntil(cache.put(request.url, response.clone()).catch(() => {}));
 		}
 
 		return response;