diff --git a/db/kv-cache.ts b/db/kv-cache.ts index ceaea24..f2026ff 100644 --- a/db/kv-cache.ts +++ b/db/kv-cache.ts @@ -56,8 +56,6 @@ export async function kvCached( } const result = await fn(); - // Null/undefined reads back as a cache miss, so storing it only - // burns a KV write on every lookup of a nonexistent slug. if (result != null) { memSet(key, result); // A bare floating promise is cancelled when the request context @@ -73,6 +71,13 @@ export async function kvCached( console.error(`KV put failed for ${key}:`, err); }), ); + } else { + // Negative results stay out of KV: a stored null reads back as a + // miss anyway, and the 1k/day free-plan write quota is too scarce + // to spend on scraper probes of nonexistent slugs. The isolate + // cache absorbs repeat lookups here; the edge cache holds the + // rendered 404 (see -worker-entry.ts). + memSet(key, null); } return result; } diff --git a/db/queries.ts b/db/queries.ts index 8933d87..2268cee 100644 --- a/db/queries.ts +++ b/db/queries.ts @@ -4,7 +4,7 @@ // Imported by page loaders (via server functions) and /api/* routes. // ═══════════════════════════════════════════════════════════════════ -import { and, desc, eq, inArray, like, notInArray, or, sql } from "drizzle-orm"; +import { and, desc, eq, inArray, like, or, sql } from "drizzle-orm"; import type { DrizzleDB, TursoClient } from "./client"; import type { SourceType, TagType } from "./schema"; import { @@ -22,25 +22,9 @@ import { // ─── Desktop-Only Filter ───────────────────────────────────────────── // Desktop-only = has at least one desktop platform tag but NO mobile tag. -// Single query with conditional aggregation instead of 4 nested subqueries. - -function desktopOnlyAppIds(db: DrizzleDB) { - return db - .select({ appId: appTags.appId }) - .from(appTags) - .innerJoin(tags, eq(appTags.tagId, tags.id)) - .where( - or( - inArray(tags.slug, ["desktop", "linux", "macos", "windows"]), - inArray(tags.slug, ["android", "ios"]), - ), - ) - .groupBy(appTags.appId) - .having( - sql`sum(case when ${tags.slug} in ('desktop','linux','macos','windows') then 1 else 0 end) > 0 - and sum(case when ${tags.slug} in ('android','ios') then 1 else 0 end) = 0`, - ); -} +// Precomputed into apps.desktop_only at import time +// (db/seed/lib/desktop-only.ts) — evaluating it here as a subquery +// scanned the whole app_tags join on every list-query cache miss. // ─── Slim List Helper ──────────────────────────────────────────────── // List pages only need card-relevant fields. Loading full sources @@ -139,7 +123,7 @@ export async function listApps( const conditions: ReturnType[] = []; // Exclude desktop-only apps - conditions.push(notInArray(apps.id, desktopOnlyAppIds(db))); + conditions.push(eq(apps.desktopOnly, false)); if (tagSlugs?.length) { const matchingTagIds = db @@ -372,12 +356,7 @@ export async function searchApps(db: DrizzleDB, query: string) { db .select(appCardColumns) .from(apps) - .where( - and( - like(apps.name, pattern), - notInArray(apps.id, desktopOnlyAppIds(db)), - ), - ) + .where(and(like(apps.name, pattern), eq(apps.desktopOnly, false))) .limit(20) .orderBy(apps.name), db @@ -400,7 +379,7 @@ export async function getRecentApps(db: DrizzleDB) { const rows = await db .select(appCardColumns) .from(apps) - .where(notInArray(apps.id, desktopOnlyAppIds(db))) + .where(eq(apps.desktopOnly, false)) .orderBy(sql`${apps.createdAt} desc`) .limit(20); diff --git a/db/schema.ts b/db/schema.ts index 9ab308a..f9ff01f 100644 --- a/db/schema.ts +++ b/db/schema.ts @@ -30,6 +30,13 @@ export const apps = sqliteTable("apps", { license: text("license"), // e.g. "GPL-3.0", "MIT", "Apache-2.0" websiteUrl: text("website_url"), repositoryUrl: text("repository_url"), + // Has a desktop platform tag but no mobile tag. Derived from app_tags + // at import time (db/seed/lib/desktop-only.ts) so hot list queries + // filter on a column instead of re-scanning the app_tags join on + // every request. + desktopOnly: integer("desktop_only", { mode: "boolean" }) + .notNull() + .default(false), ...timestamps, }); diff --git a/db/seed/import.ts b/db/seed/import.ts index 56a4c91..6b366ec 100644 --- a/db/seed/import.ts +++ b/db/seed/import.ts @@ -13,6 +13,7 @@ import { proprietaryApps as proprietaryAppSeeds } from "./data/proprietary-apps" import { fdroidAntiFeatureMap, fdroidCategoryMap, tagSeeds } from "./data/tags"; import { webApps as webAppSeeds } from "./data/web-apps"; import { deduplicateApps } from "./lib/dedup"; +import { updateDesktopOnlyFlags } from "./lib/desktop-only"; import { generateId } from "./lib/id"; import { sanitizeDescription } from "./lib/sanitize"; import { slugify } from "./lib/slugify"; @@ -516,6 +517,12 @@ async function updateTagCounts() { console.log(` ${result.rows[0].total} tags with apps`); } +async function updateDesktopFlags() { + console.log("Updating desktop-only flags..."); + const total = await updateDesktopOnlyFlags(client); + console.log(` ${total} desktop-only apps flagged`); +} + async function main() { console.log("\nSeed import"); console.log("═".repeat(50)); @@ -536,6 +543,7 @@ async function main() { await upsertProprietaryApps(); await upsertAlternatives(); await updateTagCounts(); + await updateDesktopFlags(); console.log(`\n${"═".repeat(50)}`); console.log("Import complete:"); diff --git a/db/seed/lib/desktop-only.ts b/db/seed/lib/desktop-only.ts new file mode 100644 index 0000000..0c4fe98 --- /dev/null +++ b/db/seed/lib/desktop-only.ts @@ -0,0 +1,29 @@ +import type { Client } from "@libsql/client"; + +// Desktop-only = has at least one desktop platform tag but NO mobile +// tag — the same predicate the old desktopOnlyAppIds subquery in +// db/queries.ts evaluated per request. Computed once here instead, so +// list queries filter on apps.desktop_only rather than re-scanning the +// whole app_tags join. +export async function updateDesktopOnlyFlags(client: Client) { + await client.execute(` + UPDATE apps SET desktop_only = ( + EXISTS ( + SELECT 1 FROM app_tags at + JOIN tags t ON t.id = at.tag_id + WHERE at.app_id = apps.id + AND t.slug IN ('desktop', 'linux', 'macos', 'windows') + ) + AND NOT EXISTS ( + SELECT 1 FROM app_tags at + JOIN tags t ON t.id = at.tag_id + WHERE at.app_id = apps.id + AND t.slug IN ('android', 'ios') + ) + ) + `); + const result = await client.execute( + "SELECT COUNT(*) AS total FROM apps WHERE desktop_only = 1", + ); + return Number(result.rows[0].total); +} diff --git a/package.json b/package.json index 4dd9408..cdf8060 100644 --- a/package.json +++ b/package.json @@ -20,6 +20,7 @@ "seed:import": "tsx db/seed/import.ts && pnpm cache:purge", "seed:enrich": "tsx db/seed/enrich.ts && pnpm cache:purge", "cache:purge": "tsx scripts/cache-purge.ts", + "db:backfill-desktop-only": "tsx scripts/backfill-desktop-only.ts", "seed:comparisons": "tsx db/seed/generate-comparisons.ts", "seed:embeddings": "tsx db/seed/generate-embeddings.ts" }, diff --git a/scripts/backfill-desktop-only.ts b/scripts/backfill-desktop-only.ts new file mode 100644 index 0000000..ffe0305 --- /dev/null +++ b/scripts/backfill-desktop-only.ts @@ -0,0 +1,23 @@ +import "dotenv/config"; +import { createClient } from "@libsql/client"; +import { updateDesktopOnlyFlags } from "../db/seed/lib/desktop-only"; + +// One-off backfill for the apps.desktop_only column on an existing +// database. Run `pnpm db:push` first so the column exists. Subsequent +// reseeds keep the flag current via seed:import. + +if (!process.env.TURSO_DATABASE_URL || !process.env.TURSO_AUTH_TOKEN) { + console.error( + "Missing TURSO_DATABASE_URL or TURSO_AUTH_TOKEN environment variables", + ); + process.exit(1); +} + +const client = createClient({ + url: process.env.TURSO_DATABASE_URL, + authToken: process.env.TURSO_AUTH_TOKEN, +}); + +const total = await updateDesktopOnlyFlags(client); +console.log(`${total} desktop-only apps flagged`); +client.close(); diff --git a/src/routes/-worker-entry.ts b/src/routes/-worker-entry.ts index 119985a..fa6f271 100644 --- a/src/routes/-worker-entry.ts +++ b/src/routes/-worker-entry.ts @@ -42,6 +42,13 @@ type CacheRule = { pattern: RegExp; header: string }; const ONE_WEEK = 604800; const ONE_DAY = 86400; +// Rendered 404s are edge-cached briefly on every route: scrapers probe +// nonexistent slugs and junk paths, and each uncached 404 costs a full +// SSR plus Turso round trips, because negative results never enter KV +// (see db/kv-cache.ts). Kept short so a slug added by the next reseed +// isn't masked for long. +const NOT_FOUND_TTL = 3600; + const cacheRules: CacheRule[] = [ { pattern: /^\/search/, header: "no-store" }, // Server-fn GET responses (catalog queries keyed by their payload in @@ -456,11 +463,12 @@ export default { // Serve repeat GETs from the Cloudflare edge cache. Cache-Control // headers alone do nothing for Worker-generated responses — they // only enter the CDN cache through an explicit Cache API put. + // 200s are only stored on routes with an explicit cache rule; + // 404s are stored on every route (with NOT_FOUND_TTL), so the + // match below must also run for routes without a rule. const cacheHeader = getCacheHeader(pathname); const edgeCacheable = - request.method === "GET" && - cacheHeader !== null && - cacheHeader !== "no-store"; + request.method === "GET" && cacheHeader !== "no-store"; const cache = (caches as unknown as { default: Cache }).default; if (edgeCacheable) { @@ -474,12 +482,18 @@ export default { const response = await renderResponse(request, pathname, cacheHeader); - if ( - edgeCacheable && - response.status === 200 && - !response.headers.has("Set-Cookie") - ) { - ctx.waitUntil(cache.put(request.url, response.clone()).catch(() => {})); + if (edgeCacheable && !response.headers.has("Set-Cookie")) { + if (cacheHeader !== null && response.status === 200) { + ctx.waitUntil(cache.put(request.url, response.clone()).catch(() => {})); + } else if (response.status === 404) { + const negative = new Response(response.body, response); + negative.headers.set( + "Cache-Control", + `public, s-maxage=${NOT_FOUND_TTL}`, + ); + ctx.waitUntil(cache.put(request.url, negative.clone()).catch(() => {})); + return negative; + } } return response;