diff --git a/db/kv-cache.ts b/db/kv-cache.ts index e3ed39a..ceaea24 100644 --- a/db/kv-cache.ts +++ b/db/kv-cache.ts @@ -1,4 +1,4 @@ -import { env } from "cloudflare:workers"; +import { env, waitUntil } from "cloudflare:workers"; function getKV(): KVNamespace { return env.KV; @@ -8,23 +8,71 @@ function getKV(): KVNamespace { // isn't run after a reseed. Matches the 1-week edge cache window. const DEFAULT_TTL = 604800; +// Per-isolate hot cache in front of KV. Every kv.get() counts against +// the 100k/day free-plan read quota — hit or miss — so repeat lookups +// of the same key (nav tags, category lists, sitemap slugs) within an +// isolate's lifetime should never reach KV at all. +const MEM_TTL_MS = 5 * 60 * 1000; +const MEM_MAX_ENTRIES = 256; +const memCache = new Map(); + +function memGet(key: string): unknown { + const entry = memCache.get(key); + if (!entry) return undefined; + if (Date.now() > entry.expires) { + memCache.delete(key); + return undefined; + } + return entry.value; +} + +function memSet(key: string, value: unknown): void { + if (memCache.size >= MEM_MAX_ENTRIES) { + // Map iterates in insertion order, so the first key is the oldest + const oldest = memCache.keys().next().value; + if (oldest !== undefined) memCache.delete(oldest); + } + memCache.set(key, { value, expires: Date.now() + MEM_TTL_MS }); +} + export async function kvCached( key: string, fn: () => Promise, opts?: { ttl?: number }, ): Promise { + const hot = memGet(key); + if (hot !== undefined) return hot as T; + const kv = getKV(); - const cached = await kv.get(key, "json"); - if (cached !== null) return cached as T; + // KV throws once the daily free-plan quota is exhausted — fall + // through to the database instead of failing the page. + let cached: T | null = null; + try { + cached = await kv.get(key, "json"); + } catch {} + if (cached !== null) { + memSet(key, cached); + return cached; + } const result = await fn(); // Null/undefined reads back as a cache miss, so storing it only // burns a KV write on every lookup of a nonexistent slug. if (result != null) { - // Fire-and-forget write — don't block the response - kv.put(key, JSON.stringify(result), { - expirationTtl: opts?.ttl ?? DEFAULT_TTL, - }).catch(() => {}); + memSet(key, result); + // A bare floating promise is cancelled when the request context + // ends, so the write must be kept alive via waitUntil to ever land. + // Log failures — a silently broken write path leaves every request + // paying for a cache that never fills. + waitUntil( + kv + .put(key, JSON.stringify(result), { + expirationTtl: opts?.ttl ?? DEFAULT_TTL, + }) + .catch((err) => { + console.error(`KV put failed for ${key}:`, err); + }), + ); } return result; } diff --git a/src/routes/-worker-entry.ts b/src/routes/-worker-entry.ts index ba67b1c..119985a 100644 --- a/src/routes/-worker-entry.ts +++ b/src/routes/-worker-entry.ts @@ -44,6 +44,10 @@ const ONE_DAY = 86400; const cacheRules: CacheRule[] = [ { pattern: /^\/search/, header: "no-store" }, + // Server-fn GET responses (catalog queries keyed by their payload in + // the query string) are deterministic between reseeds, and the reseed + // flow already purges the edge cache + { pattern: /^\/_serverFn\//, header: `public, s-maxage=${ONE_DAY}` }, { pattern: /^\/sitemap.*\.xml$|^\/robots\.txt$/, header: `public, s-maxage=${ONE_WEEK}`, @@ -75,6 +79,7 @@ function robotsTxt(): Response { const body = `User-agent: * Allow: / Disallow: /search +Disallow: /compare Sitemap: ${SITE_URL}/sitemap.xml `; @@ -363,10 +368,68 @@ async function handleIconProxy(request: Request): Promise { } } +// ─── Scraper Gate ─────────────────────────────────────────────────── + +// /compare pages are 25k noindexed, robots-disallowed thin pages, yet +// scraper fleets on residential proxies (fake browser UAs) crawl them +// hard enough to exhaust the daily KV read quota. Real users only ever +// fetch this HTML via a hard navigation — client-side nav goes through +// /_serverFn — and every browser engine since ~2021 sends +// Sec-Fetch-Mode: navigate on navigations, which scrapers and crawlers +// don't. Refuse everything else before SSR spends KV reads on it. +function isCompareScrape(request: Request, pathname: string): boolean { + if (!pathname.startsWith("/compare/")) return false; + if (request.method !== "GET") return false; + const accept = request.headers.get("Accept") ?? ""; + const secFetchMode = request.headers.get("Sec-Fetch-Mode"); + return !(accept.includes("text/html") && secFetchMode === "navigate"); +} + // ─── Worker Entry ─────────────────────────────────────────────────── +async function renderResponse( + request: Request, + pathname: string, + cacheHeader: string | null, +): Promise { + // robots.txt + sitemaps + if (pathname === "/robots.txt") { + return robotsTxt(); + } + + const sitemapHandler = sitemapHandlers[pathname]; + if (sitemapHandler) { + return sitemapHandler(); + } + + // Pass through to TanStack Start + const response = await tanstackFetch(request); + + // Comparison pages stay usable in-app but are kept out of search + // indexes — 25k thin auto-generated pages triggered a site-wide + // quality suppression (see GSC "Crawled - currently not indexed"). + const noindex = pathname.startsWith("/compare/"); + + if ((cacheHeader && response.status === 200) || noindex) { + const newResponse = new Response(response.body, response); + if (cacheHeader && response.status === 200) { + newResponse.headers.set("Cache-Control", cacheHeader); + } + if (noindex) { + newResponse.headers.set("X-Robots-Tag", "noindex"); + } + return newResponse; + } + + return response; +} + export default { - async fetch(request: Request, _env: Env): Promise { + async fetch( + request: Request, + _env: Env, + ctx: ExecutionContext, + ): Promise { const url = new URL(request.url); const { pathname } = url; @@ -381,14 +444,8 @@ export default { return Response.redirect(target.toString(), 301); } - // robots.txt + sitemaps - if (pathname === "/robots.txt") { - return robotsTxt(); - } - - const sitemapHandler = sitemapHandlers[pathname]; - if (sitemapHandler) { - return sitemapHandler(); + if (isCompareScrape(request, pathname)) { + return new Response("Forbidden", { status: 403 }); } // Icon proxy (keeps its own Cache API usage) @@ -396,24 +453,33 @@ export default { return handleIconProxy(request); } - // Pass through to TanStack Start - const response = await tanstackFetch(request); - + // Serve repeat GETs from the Cloudflare edge cache. Cache-Control + // headers alone do nothing for Worker-generated responses — they + // only enter the CDN cache through an explicit Cache API put. const cacheHeader = getCacheHeader(pathname); - // Comparison pages stay usable in-app but are kept out of search - // indexes — 25k thin auto-generated pages triggered a site-wide - // quality suppression (see GSC "Crawled - currently not indexed"). - const noindex = pathname.startsWith("/compare/"); - - if ((cacheHeader && response.status === 200) || noindex) { - const newResponse = new Response(response.body, response); - if (cacheHeader && response.status === 200) { - newResponse.headers.set("Cache-Control", cacheHeader); - } - if (noindex) { - newResponse.headers.set("X-Robots-Tag", "noindex"); + const edgeCacheable = + request.method === "GET" && + cacheHeader !== null && + cacheHeader !== "no-store"; + const cache = (caches as unknown as { default: Cache }).default; + + if (edgeCacheable) { + const hit = await cache.match(request.url); + if (hit) { + const response = new Response(hit.body, hit); + response.headers.set("X-Edge-Cache", "HIT"); + return response; } - return newResponse; + } + + const response = await renderResponse(request, pathname, cacheHeader); + + if ( + edgeCacheable && + response.status === 200 && + !response.headers.has("Set-Cookie") + ) { + ctx.waitUntil(cache.put(request.url, response.clone()).catch(() => {})); } return response;