negative-cache 404s, materialize desktop-only flag

db/kv-cache.ts

@@ -56,8 +56,6 @@ export async function kvCached<T>(
 	}
 
 	const result = await fn();
-	// Null/undefined reads back as a cache miss, so storing it only
-	// burns a KV write on every lookup of a nonexistent slug.
 	if (result != null) {
 		memSet(key, result);
 		// A bare floating promise is cancelled when the request context
@@ -73,6 +71,13 @@ export async function kvCached<T>(
 					console.error(`KV put failed for ${key}:`, err);
 				}),
 		);
+	} else {
+		// Negative results stay out of KV: a stored null reads back as a
+		// miss anyway, and the 1k/day free-plan write quota is too scarce
+		// to spend on scraper probes of nonexistent slugs. The isolate
+		// cache absorbs repeat lookups here; the edge cache holds the
+		// rendered 404 (see -worker-entry.ts).
+		memSet(key, null);
 	}
 	return result;
 }

db/queries.ts

@@ -4,7 +4,7 @@
 // Imported by page loaders (via server functions) and /api/* routes.
 // ═══════════════════════════════════════════════════════════════════
 
-import { and, desc, eq, inArray, like, notInArray, or, sql } from "drizzle-orm";
+import { and, desc, eq, inArray, like, or, sql } from "drizzle-orm";
 import type { DrizzleDB, TursoClient } from "./client";
 import type { SourceType, TagType } from "./schema";
 import {
@@ -22,25 +22,9 @@ import {
 
 // ─── Desktop-Only Filter ─────────────────────────────────────────────
 // Desktop-only = has at least one desktop platform tag but NO mobile tag.
-// Single query with conditional aggregation instead of 4 nested subqueries.
-
-function desktopOnlyAppIds(db: DrizzleDB) {
-	return db
-		.select({ appId: appTags.appId })
-		.from(appTags)
-		.innerJoin(tags, eq(appTags.tagId, tags.id))
-		.where(
-			or(
-				inArray(tags.slug, ["desktop", "linux", "macos", "windows"]),
-				inArray(tags.slug, ["android", "ios"]),
-			),
-		)
-		.groupBy(appTags.appId)
-		.having(
-			sql`sum(case when ${tags.slug} in ('desktop','linux','macos','windows') then 1 else 0 end) > 0
-			and sum(case when ${tags.slug} in ('android','ios') then 1 else 0 end) = 0`,
-		);
-}
+// Precomputed into apps.desktop_only at import time
+// (db/seed/lib/desktop-only.ts) — evaluating it here as a subquery
+// scanned the whole app_tags join on every list-query cache miss.
 
 // ─── Slim List Helper ────────────────────────────────────────────────
 // List pages only need card-relevant fields. Loading full sources
@@ -139,7 +123,7 @@ export async function listApps(
 	const conditions: ReturnType<typeof eq>[] = [];
 
 	// Exclude desktop-only apps
-	conditions.push(notInArray(apps.id, desktopOnlyAppIds(db)));
+	conditions.push(eq(apps.desktopOnly, false));
 
 	if (tagSlugs?.length) {
 		const matchingTagIds = db
@@ -372,12 +356,7 @@ export async function searchApps(db: DrizzleDB, query: string) {
 		db
 			.select(appCardColumns)
 			.from(apps)
-			.where(
-				and(
-					like(apps.name, pattern),
-					notInArray(apps.id, desktopOnlyAppIds(db)),
-				),
-			)
+			.where(and(like(apps.name, pattern), eq(apps.desktopOnly, false)))
 			.limit(20)
 			.orderBy(apps.name),
 		db
@@ -400,7 +379,7 @@ export async function getRecentApps(db: DrizzleDB) {
 	const rows = await db
 		.select(appCardColumns)
 		.from(apps)
-		.where(notInArray(apps.id, desktopOnlyAppIds(db)))
+		.where(eq(apps.desktopOnly, false))
 		.orderBy(sql`${apps.createdAt} desc`)
 		.limit(20);
 

db/schema.ts

@@ -30,6 +30,13 @@ export const apps = sqliteTable("apps", {
 	license: text("license"), // e.g. "GPL-3.0", "MIT", "Apache-2.0"
 	websiteUrl: text("website_url"),
 	repositoryUrl: text("repository_url"),
+	// Has a desktop platform tag but no mobile tag. Derived from app_tags
+	// at import time (db/seed/lib/desktop-only.ts) so hot list queries
+	// filter on a column instead of re-scanning the app_tags join on
+	// every request.
+	desktopOnly: integer("desktop_only", { mode: "boolean" })
+		.notNull()
+		.default(false),
 	...timestamps,
 });
 

db/seed/import.ts

@@ -13,6 +13,7 @@ import { proprietaryApps as proprietaryAppSeeds } from "./data/proprietary-apps"
 import { fdroidAntiFeatureMap, fdroidCategoryMap, tagSeeds } from "./data/tags";
 import { webApps as webAppSeeds } from "./data/web-apps";
 import { deduplicateApps } from "./lib/dedup";
+import { updateDesktopOnlyFlags } from "./lib/desktop-only";
 import { generateId } from "./lib/id";
 import { sanitizeDescription } from "./lib/sanitize";
 import { slugify } from "./lib/slugify";
@@ -516,6 +517,12 @@ async function updateTagCounts() {
 	console.log(`  ${result.rows[0].total} tags with apps`);
 }
 
+async function updateDesktopFlags() {
+	console.log("Updating desktop-only flags...");
+	const total = await updateDesktopOnlyFlags(client);
+	console.log(`  ${total} desktop-only apps flagged`);
+}
+
 async function main() {
 	console.log("\nSeed import");
 	console.log("═".repeat(50));
@@ -536,6 +543,7 @@ async function main() {
 	await upsertProprietaryApps();
 	await upsertAlternatives();
 	await updateTagCounts();
+	await updateDesktopFlags();
 
 	console.log(`\n${"═".repeat(50)}`);
 	console.log("Import complete:");

db/seed/lib/desktop-only.ts

@@ -0,0 +1,29 @@
+import type { Client } from "@libsql/client";
+
+// Desktop-only = has at least one desktop platform tag but NO mobile
+// tag — the same predicate the old desktopOnlyAppIds subquery in
+// db/queries.ts evaluated per request. Computed once here instead, so
+// list queries filter on apps.desktop_only rather than re-scanning the
+// whole app_tags join.
+export async function updateDesktopOnlyFlags(client: Client) {
+	await client.execute(`
+		UPDATE apps SET desktop_only = (
+			EXISTS (
+				SELECT 1 FROM app_tags at
+				JOIN tags t ON t.id = at.tag_id
+				WHERE at.app_id = apps.id
+					AND t.slug IN ('desktop', 'linux', 'macos', 'windows')
+			)
+			AND NOT EXISTS (
+				SELECT 1 FROM app_tags at
+				JOIN tags t ON t.id = at.tag_id
+				WHERE at.app_id = apps.id
+					AND t.slug IN ('android', 'ios')
+			)
+		)
+	`);
+	const result = await client.execute(
+		"SELECT COUNT(*) AS total FROM apps WHERE desktop_only = 1",
+	);
+	return Number(result.rows[0].total);
+}

package.json

@@ -20,6 +20,7 @@
 		"seed:import": "tsx db/seed/import.ts && pnpm cache:purge",
 		"seed:enrich": "tsx db/seed/enrich.ts && pnpm cache:purge",
 		"cache:purge": "tsx scripts/cache-purge.ts",
+		"db:backfill-desktop-only": "tsx scripts/backfill-desktop-only.ts",
 		"seed:comparisons": "tsx db/seed/generate-comparisons.ts",
 		"seed:embeddings": "tsx db/seed/generate-embeddings.ts"
 	},

scripts/backfill-desktop-only.ts

@@ -0,0 +1,23 @@
+import "dotenv/config";
+import { createClient } from "@libsql/client";
+import { updateDesktopOnlyFlags } from "../db/seed/lib/desktop-only";
+
+// One-off backfill for the apps.desktop_only column on an existing
+// database. Run `pnpm db:push` first so the column exists. Subsequent
+// reseeds keep the flag current via seed:import.
+
+if (!process.env.TURSO_DATABASE_URL || !process.env.TURSO_AUTH_TOKEN) {
+	console.error(
+		"Missing TURSO_DATABASE_URL or TURSO_AUTH_TOKEN environment variables",
+	);
+	process.exit(1);
+}
+
+const client = createClient({
+	url: process.env.TURSO_DATABASE_URL,
+	authToken: process.env.TURSO_AUTH_TOKEN,
+});
+
+const total = await updateDesktopOnlyFlags(client);
+console.log(`${total} desktop-only apps flagged`);
+client.close();

src/routes/-worker-entry.ts

@@ -42,6 +42,13 @@ type CacheRule = { pattern: RegExp; header: string };
 const ONE_WEEK = 604800;
 const ONE_DAY = 86400;
 
+// Rendered 404s are edge-cached briefly on every route: scrapers probe
+// nonexistent slugs and junk paths, and each uncached 404 costs a full
+// SSR plus Turso round trips, because negative results never enter KV
+// (see db/kv-cache.ts). Kept short so a slug added by the next reseed
+// isn't masked for long.
+const NOT_FOUND_TTL = 3600;
+
 const cacheRules: CacheRule[] = [
 	{ pattern: /^\/search/, header: "no-store" },
 	// Server-fn GET responses (catalog queries keyed by their payload in
@@ -456,11 +463,12 @@ export default {
 		// Serve repeat GETs from the Cloudflare edge cache. Cache-Control
 		// headers alone do nothing for Worker-generated responses — they
 		// only enter the CDN cache through an explicit Cache API put.
+		// 200s are only stored on routes with an explicit cache rule;
+		// 404s are stored on every route (with NOT_FOUND_TTL), so the
+		// match below must also run for routes without a rule.
 		const cacheHeader = getCacheHeader(pathname);
 		const edgeCacheable =
-			request.method === "GET" &&
-			cacheHeader !== null &&
-			cacheHeader !== "no-store";
+			request.method === "GET" && cacheHeader !== "no-store";
 		const cache = (caches as unknown as { default: Cache }).default;
 
 		if (edgeCacheable) {
@@ -474,12 +482,18 @@ export default {
 
 		const response = await renderResponse(request, pathname, cacheHeader);
 
-		if (
-			edgeCacheable &&
-			response.status === 200 &&
-			!response.headers.has("Set-Cookie")
-		) {
-			ctx.waitUntil(cache.put(request.url, response.clone()).catch(() => {}));
+		if (edgeCacheable && !response.headers.has("Set-Cookie")) {
+			if (cacheHeader !== null && response.status === 200) {
+				ctx.waitUntil(cache.put(request.url, response.clone()).catch(() => {}));
+			} else if (response.status === 404) {
+				const negative = new Response(response.body, response);
+				negative.headers.set(
+					"Cache-Control",
+					`public, s-maxage=${NOT_FOUND_TTL}`,
+				);
+				ctx.waitUntil(cache.put(request.url, negative.clone()).catch(() => {}));
+				return negative;
+			}
 		}
 
 		return response;