diff --git a/.env.example b/.env.example index 46e9a931..d400282f 100644 --- a/.env.example +++ b/.env.example @@ -48,14 +48,17 @@ DATABASE_URL="postgresql://postgres:postgres@localhost:5432/dossier" # Defaults for Settings → Job search; a choice saved in the app overrides them. # Source: jsearch (LinkedIn, Indeed and more through Google for Jobs; needs a # key from OpenWeb Ninja or RapidAPI), adzuna (needs an app ID and key), -# arbeitsagentur (Germany only, no key; unofficial, see below) or off. Off -# until you choose. +# firecrawl (one web search across LinkedIn, Indeed, Naukri, Glassdoor, +# Greenhouse, Lever and company sites at once; needs a key from +# firecrawl.dev — this fork, see README), arbeitsagentur (Germany only, no +# key; unofficial, see below) or off. Off until you choose. # DOSSIER_JOBS_SOURCE=jsearch # DOSSIER_JSEARCH_KEY= # Where the JSearch key is from: openwebninja (default) or rapidapi. # DOSSIER_JSEARCH_HOST=openwebninja # DOSSIER_ADZUNA_APP_ID= # DOSSIER_ADZUNA_KEY= +# DOSSIER_FIRECRAWL_KEY= # The Bundesagentur für Arbeit has no official API: Dossier uses the access # its own job search uses, without its permission, and its terms of use don't # allow automated access. Whoever chooses it does so on their own diff --git a/.gitignore b/.gitignore index 7580fe6a..3ac9d86c 100644 --- a/.gitignore +++ b/.gitignore @@ -40,6 +40,9 @@ yarn-error.log* .env* !.env.example +# local port overrides for running a second dev stack side by side +compose.override.yml + # vercel .vercel diff --git a/README.md b/README.md index 502e44fc..59ad7f4b 100644 --- a/README.md +++ b/README.md @@ -23,6 +23,19 @@ It's for one person on a small server: one password, your data in your own Postg **Get started:** [run it on your computer](#run-it-on-your-computer) in a few minutes, or [self-host it](#self-hosting) on a server. The [installation guide](https://github.com/AienTech/dossier/wiki) covers every way: Docker with or without Compose, Node.js, a plain Ubuntu server, CapRover, Coolify and Dokploy. +## This fork + +This is a fork of [AienTech/dossier](https://github.com/AienTech/dossier) with changes made for my own job search: + +- **JSearch's 10 s timeout was too tight.** A live request against OpenWeb Ninja's JSearch API took 10.3 s to answer, so every search lost the race against Dossier's hard-coded 10-second limit and came back "JSearch isn't answering right now" even with a working key. [`src/lib/job-search/http.ts`](src/lib/job-search/http.ts) now waits 25 s. +- **Blocked companies.** Settings → Job search has a new "Blocked companies" list: Find jobs, the job agent and the Claude connector never show, score or draft an application for a posting that names one of these as the company, or mentions it in the title or description — which also catches a staffing listing for a role placed there. See [`src/lib/job-search/blocklist.ts`](src/lib/job-search/blocklist.ts). +- **Firecrawl, a fourth job source.** JSearch and Adzuna each index one source; Firecrawl instead runs two web searches — one biased toward LinkedIn, Indeed, Naukri, Glassdoor, Monster, Greenhouse, Lever, Ashby, Workday and SmartRecruiters, one wide open with no site restriction at all for whatever that list misses (a smaller board, a company's own careers page) — merged into one page, the way actually googling "data engineer jobs in Bengaluru" would, without Dossier maintaining a scraper per site (which breaks the moment a site's markup changes, and risks its terms of use). Get a key at [firecrawl.dev](https://www.firecrawl.dev/app/api-keys) and choose it in Settings → Job search. See [`src/lib/job-search/firecrawl.ts`](src/lib/job-search/firecrawl.ts). +- **Falls back across sources.** Save a key for more than one source (JSearch, Adzuna and Firecrawl each keep their own, independent of which is "chosen") and a fresh search tries the chosen one first, then moves on to the next with a key saved if it's out of quota, down, or doesn't cover that country — instead of just failing. A kept search's "Show more" always stays on the exact source it started with. See [`searchWithFallback` in `src/lib/job-search/find.ts`](src/lib/job-search/find.ts). +- **Hyperlinks on the resume.** Email, website and LinkedIn in the header were plain text; they're links now (`mailto:`, `tel:` and `https://`), styled the same muted, underline-free way as a publication's citation link. See [`src/lib/resume-content.ts`](src/lib/resume-content.ts) and [`Contact`/`ContactLine` in `document-page.tsx`](src/components/documents/document-page.tsx). +- **One-page resumes already existed** — it just wasn't on by default. Settings → Documents → Resume length was already "Up to two pages"; nothing to fix, just set it to "One page" if that's what you want (upstream feature, not this fork's). + +Everything else — the profile, resumes, cover letters, tracking, the job agent, the Claude connector — is upstream Dossier, unchanged. [Build and run this fork](#build-and-run-this-fork) below; [upstream's own instructions](#run-it-on-your-computer) pull their published image instead, without these changes. + ## Demo ![Dossier in use: the dashboard, dragging a card on the board, a new application from a pasted posting, refining the tailored resume in the chat, the cover letter, and switching resume themes](.github/images/demo.gif) @@ -33,7 +46,7 @@ It's for one person on a small server: one password, your data in your own Postg - **Profile:** basics, skills and languages, experience, projects, education, publications and references, in the order you choose, with a strength meter that counts only what a profile needs. Projects (side, open-source, course or thesis work) show what you can do where your job titles don't yet. Publications are cited in your own style, with bold and italics, and go on resumes word for word. Start it from an existing resume (PDF, Word or Markdown) if you like. - **Base resume:** twelve themes and a theme builder, previewed live and printed to PDF on the server. Duplicate themes, or share them as files. -- **Find jobs:** search postings near your city, or anywhere you choose, through JSearch (LinkedIn, Indeed and more) or Adzuna. Save the good ones, dismiss the rest, score any of them against your profile, and start an application from a posting with one click. +- **Find jobs:** search postings near your city, or anywhere you choose, through JSearch (LinkedIn, Indeed and more), Adzuna, or Firecrawl (this fork: one web search across LinkedIn, Indeed, Naukri, Glassdoor, Greenhouse, Lever and company sites at once — see [This fork](#this-fork)). Save the good ones, dismiss the rest, score any of them against your profile, and start an application from a posting with one click. Block a company in Settings → Job search (this fork) and no search shows it again. - **Startups:** startups that are hiring, from YC startups' job posts and Hacker News' monthly “Who is hiring?” thread, kept to the ones that hire where you live or remotely from your country and ranked against your profile, with the address each one publishes for applications. Dossier reads their sites politely and follows their robots.txt. Start an application to one with a click; you decide whom to write to. - **Job agent (optional):** once a day it runs one search for you, skips what you've seen, scores the new postings against your profile and keeps the best on the Agent screen, with its reasons. It can prepare tailored drafts too, which wait for your review. It never sends or applies. - **New application:** paste a posting; Dossier pulls out the details and requirements, matches them against your profile, and writes a tailored resume and cover letter in the tone you pick. @@ -103,6 +116,30 @@ The first start downloads the image (about 1.7 GB) and sets up the database. Ope `docker compose stop` stops it and `docker compose start` starts it again, with your data kept. It's only reachable from your own computer. The [installation guide's Docker Compose page](https://github.com/AienTech/dossier/wiki/Run-locally-with-Docker-Compose) has more: another port, your time zone, upgrades and removing it. +## Build and run this fork + +The steps above pull upstream's published image, without this fork's changes. To build this fork's own image instead: + +```bash +git clone https://github.com/rahulrathore95/dossier.git +cd dossier +docker build -t dossier-fork:latest . +``` + +This runs the same `Dockerfile` the published image is built from (the `production` stage, built by default), so the result is a drop-in replacement for `ghcr.io/aientech/dossier`. Then run it with [`compose.production.yml`](compose.production.yml), pointing `DOSSIER_IMAGE` at the image you just built instead of the published one: + +```bash +mkdir -p ~/dossier && cd ~/dossier +cp /path/to/this/repo/compose.production.yml compose.yml +echo "DOSSIER_IMAGE=dossier-fork:latest" > .env +echo "POSTGRES_PASSWORD=$(openssl rand -hex 24)" >> .env +echo "DOSSIER_SECRET=$(openssl rand -base64 32)" >> .env +docker run --rm -it dossier-fork:latest hash-password # add the printed line to .env as DOSSIER_PASSWORD=… +docker compose up -d +``` + +Open [http://localhost:3000](http://localhost:3000) and sign in. Rebuilding after a code change is the same `docker build` command, then `docker compose up -d` again to pick it up. [Choosing a job source](https://github.com/AienTech/dossier/wiki/Choosing-a-job-source) covers JSearch and Adzuna; for Firecrawl, get a key at [firecrawl.dev](https://www.firecrawl.dev/app/api-keys) and enter it in **Settings → Job search**, or set `DOSSIER_FIRECRAWL_KEY` the same way as the other sources' keys. + ## Self-hosting To put Dossier on a server, on your own domain with HTTPS, follow the [installation guide](https://github.com/AienTech/dossier/wiki#self-host-on-a-server): a plain Ubuntu server with Docker Compose and Caddy (or Node.js, without Docker), CapRover, Coolify or Dokploy. Give the server at least 2 vCPUs and 2 GB of memory (see [Performance and sizing](#performance-and-sizing)). diff --git a/compose.production.yml b/compose.production.yml index a44504c1..9581d4e4 100644 --- a/compose.production.yml +++ b/compose.production.yml @@ -51,6 +51,7 @@ services: DOSSIER_JSEARCH_HOST: ${DOSSIER_JSEARCH_HOST:-} DOSSIER_ADZUNA_APP_ID: ${DOSSIER_ADZUNA_APP_ID:-} DOSSIER_ADZUNA_KEY: ${DOSSIER_ADZUNA_KEY:-} + DOSSIER_FIRECRAWL_KEY: ${DOSSIER_FIRECRAWL_KEY:-} # Gmail and Calendar: a Google OAuth client (or enter it in Settings → Google). DOSSIER_GOOGLE_CLIENT_ID: ${DOSSIER_GOOGLE_CLIENT_ID:-} DOSSIER_GOOGLE_CLIENT_SECRET: ${DOSSIER_GOOGLE_CLIENT_SECRET:-} diff --git a/prisma/migrations/20261028120000_firecrawl_source/migration.sql b/prisma/migrations/20261028120000_firecrawl_source/migration.sql new file mode 100644 index 00000000..0ac9163a --- /dev/null +++ b/prisma/migrations/20261028120000_firecrawl_source/migration.sql @@ -0,0 +1,2 @@ +-- AlterEnum +ALTER TYPE "PostingSource" ADD VALUE 'FIRECRAWL'; diff --git a/prisma/schema.prisma b/prisma/schema.prisma index 12c5fba5..16d79c35 100644 --- a/prisma/schema.prisma +++ b/prisma/schema.prisma @@ -575,10 +575,11 @@ model Note { // ─── Job search (E14) ──────────────────────────────────────────────────────── -/// The job sources Find jobs can search (E14-1 to E14-3). +/// The job sources Find jobs can search (E14-1 to E14-3, E14-7). enum PostingSource { JSEARCH ADZUNA + FIRECRAWL ARBEITSAGENTUR } diff --git a/src/app/(app)/settings/actions.db.test.ts b/src/app/(app)/settings/actions.db.test.ts index a0145fa7..cc6a379c 100644 --- a/src/app/(app)/settings/actions.db.test.ts +++ b/src/app/(app)/settings/actions.db.test.ts @@ -158,7 +158,9 @@ describe("job search settings (E14-4)", () => { source: "JSEARCH", jsearch: { host: "openwebninja", keyState: "none" }, adzuna: { appId: "", keyState: "none" }, + firecrawl: { keyState: "none" }, arbeitsagentur: { offered: true, acceptedAt: null }, + blockedCompanies: [], }, }); await saveJobSource({ source: "OFF" }); diff --git a/src/app/(app)/settings/actions.ts b/src/app/(app)/settings/actions.ts index 4133cee4..41a64140 100644 --- a/src/app/(app)/settings/actions.ts +++ b/src/app/(app)/settings/actions.ts @@ -140,17 +140,19 @@ export const saveJobSearchDetails = action( }, ); +const JOB_SOURCE_KEY_FIELD = { JSEARCH: "jsearchKeyEnc", ADZUNA: "adzunaKeyEnc", FIRECRAWL: "firecrawlKeyEnc" } as const; + /** Saves a source's key, encrypted for job sources only, or removes it. The key never comes back to the browser. */ export const saveJobSourceKey = action( z.object({ - source: z.enum(["JSEARCH", "ADZUNA"]), + source: z.enum(["JSEARCH", "ADZUNA", "FIRECRAWL"]), change: z.discriminatedUnion("action", [ z.object({ action: z.literal("set"), value: z.string().trim().min(1).max(1000) }), z.object({ action: z.literal("clear") }), ]), }), async ({ source, change }) => { - const field = source === "JSEARCH" ? "jsearchKeyEnc" : "adzunaKeyEnc"; + const field = JOB_SOURCE_KEY_FIELD[source]; const current = await loadSettings(); await saveSettings({ [field]: applySecretChange(JOB_SOURCE_KEY, current[field], change) }); return loadView(); @@ -171,6 +173,34 @@ export const testJobSource = action(z.object({}), async () => { } }); +/** The most companies Settings → Job search keeps blocked at once; past that, ask the user to trim the list rather than letting it grow without bound. */ +const MAX_BLOCKED_COMPANIES = 50; + +/** + * Adds a company to the blocklist (this fork, E14-8): no search shows, scores + * or drafts for it again. Case-insensitive against what's already there, so + * "Best Buy" twice stays one entry. + */ +export const blockCompany = action( + z.object({ name: z.string().trim().min(1, "Type a company name first.").max(200) }), + async ({ name }) => { + const { blockedCompanies } = await loadSettings(); + if (blockedCompanies.some((entry) => entry.toLocaleLowerCase() === name.toLocaleLowerCase())) return { blockedCompanies }; + if (blockedCompanies.length >= MAX_BLOCKED_COMPANIES) throw new ActionError(`You can block up to ${MAX_BLOCKED_COMPANIES} companies; remove one first.`); + const next = [...blockedCompanies, name]; + await saveSettings({ blockedCompanies: next }); + return { blockedCompanies: next }; + }, +); + +/** Removes a company from the blocklist. */ +export const unblockCompany = action(z.object({ name: z.string().min(1) }), async ({ name }) => { + const { blockedCompanies } = await loadSettings(); + const next = blockedCompanies.filter((entry) => entry !== name); + await saveSettings({ blockedCompanies: next }); + return { blockedCompanies: next }; +}); + /** * Saves the settings that are a plain choice or switch, each as it changes * (E10-2, E10-3). Only the fields given are touched. diff --git a/src/app/(app)/settings/job-search-section.test.tsx b/src/app/(app)/settings/job-search-section.test.tsx index d74b9681..c929d232 100644 --- a/src/app/(app)/settings/job-search-section.test.tsx +++ b/src/app/(app)/settings/job-search-section.test.tsx @@ -4,7 +4,7 @@ import { beforeEach, describe, expect, it, vi } from "vitest"; import type { JobSearchSettingsView } from "@/lib/job-search/sources"; -import { saveJobSearchDetails, saveJobSource, saveJobSourceKey, testJobSource } from "./actions"; +import { blockCompany, saveJobSearchDetails, saveJobSource, saveJobSourceKey, testJobSource, unblockCompany } from "./actions"; import { JobSearchSection } from "./job-search-section"; vi.mock("./actions", () => ({ @@ -12,6 +12,8 @@ vi.mock("./actions", () => ({ saveJobSearchDetails: vi.fn(), saveJobSourceKey: vi.fn(), testJobSource: vi.fn(), + blockCompany: vi.fn(), + unblockCompany: vi.fn(), })); vi.mock("@/components/toast", () => ({ toast: vi.fn(), toastResult: (result: unknown) => result })); @@ -19,7 +21,9 @@ const off: JobSearchSettingsView = { source: "OFF", jsearch: { host: "openwebninja", keyState: "none" }, adzuna: { appId: "", keyState: "none" }, + firecrawl: { keyState: "none" }, arbeitsagentur: { offered: true, acceptedAt: null }, + blockedCompanies: [], }; const ACCEPTED = "2026-10-23T09:00:00.000Z"; const withSource = (view: Partial): JobSearchSettingsView => ({ ...off, ...view }); @@ -32,6 +36,10 @@ beforeEach(() => { })); vi.mocked(saveJobSourceKey).mockReset().mockResolvedValue({ ok: true, data: withSource({ source: "JSEARCH", jsearch: { host: "openwebninja", keyState: "set" } }) }); vi.mocked(testJobSource).mockReset().mockResolvedValue({ ok: true, data: { connected: true, seconds: 0.8 } }); + vi.mocked(blockCompany) + .mockReset() + .mockImplementation(async ({ name }) => ({ ok: true, data: { blockedCompanies: [name] } })); + vi.mocked(unblockCompany).mockReset().mockResolvedValue({ ok: true, data: { blockedCompanies: [] } }); }); describe("JobSearchSection", () => { @@ -41,6 +49,7 @@ describe("JobSearchSection", () => { expect(screen.getAllByRole("radio").map((radio) => radio.getAttribute("aria-label"))).toEqual([ "JSearch", "Adzuna", + "Firecrawl", "Bundesagentur für Arbeit", "Off", ]); @@ -126,7 +135,27 @@ describe("JobSearchSection", () => { it("leaves the Arbeitsagentur out when the server doesn't offer it", () => { render(); - expect(screen.getAllByRole("radio").map((radio) => radio.getAttribute("aria-label"))).toEqual(["JSearch", "Adzuna", "Off"]); + expect(screen.getAllByRole("radio").map((radio) => radio.getAttribute("aria-label"))).toEqual(["JSearch", "Adzuna", "Firecrawl", "Off"]); + }); + + it("asks Firecrawl for a key, like JSearch and Adzuna", async () => { + vi.mocked(saveJobSourceKey).mockResolvedValue({ ok: true, data: withSource({ source: "FIRECRAWL", firecrawl: { keyState: "set" } }) }); + render(); + expect(screen.getByRole("link", { name: "Firecrawl" })).toHaveAttribute("href", "https://www.firecrawl.dev/app/api-keys"); + await userEvent.type(screen.getByLabelText("API key"), "fc_secret"); + await userEvent.tab(); + expect(saveJobSourceKey).toHaveBeenCalledWith({ source: "FIRECRAWL", change: { action: "set", value: "fc_secret" } }); + }); + + it("blocks and unblocks a company, kept regardless of the chosen source", async () => { + render(); + await userEvent.type(screen.getByLabelText("Block a company"), "Best Buy{Enter}"); + expect(blockCompany).toHaveBeenCalledWith({ name: "Best Buy" }); + expect(await screen.findByText("Best Buy")).toBeInTheDocument(); + + await userEvent.click(screen.getByRole("button", { name: "Unblock Best Buy" })); + expect(unblockCompany).toHaveBeenCalledWith({ name: "Best Buy" }); + await waitFor(() => expect(screen.queryByText("Best Buy")).not.toBeInTheDocument()); }); it("shows keys from the environment and keys that can't be read", () => { diff --git a/src/app/(app)/settings/job-search-section.tsx b/src/app/(app)/settings/job-search-section.tsx index 49d3f9a8..77b6ebd2 100644 --- a/src/app/(app)/settings/job-search-section.tsx +++ b/src/app/(app)/settings/job-search-section.tsx @@ -1,20 +1,21 @@ "use client"; import { RadioGroup } from "radix-ui"; -import { useId, useState } from "react"; +import { useId, useState, type KeyboardEvent } from "react"; import { ArbeitsagenturNote } from "@/components/arbeitsagentur-note"; import { useTrackSave } from "@/components/save-tracker"; -import { toastResult } from "@/components/toast"; +import { toast, toastResult } from "@/components/toast"; import { Button } from "@/components/ui/button"; import { Card } from "@/components/ui/card"; +import { RemovableChip } from "@/components/ui/chip"; import { Input } from "@/components/ui/input"; import { SegmentedControl } from "@/components/ui/segmented-control"; import { JOB_SOURCES, type JobSource, type JSearchHost } from "@/lib/job-search/search"; import { JOB_KEY_UNREADABLE, JSEARCH_KEY_URLS, SOURCE_INFO, type JobKeyState, type JobSearchSettingsView } from "@/lib/job-search/sources"; import { cn } from "@/lib/cn"; -import { saveJobSearchDetails, saveJobSource, saveJobSourceKey, testJobSource } from "./actions"; +import { blockCompany, saveJobSearchDetails, saveJobSource, saveJobSourceKey, testJobSource, unblockCompany } from "./actions"; import { SettingsSectionIntro } from "./settings-screen"; type Choice = JobSource | "OFF"; @@ -75,7 +76,7 @@ export function JobSearchSection({ initial }: { initial: JobSearchSettingsView } apply(toastResult(await track(saveJobSearchDetails({ adzunaAppId: appId })))); }; - const saveKey = async (keySource: "JSEARCH" | "ADZUNA", change: { action: "set"; value: string } | { action: "clear" }) => { + const saveKey = async (keySource: "JSEARCH" | "ADZUNA" | "FIRECRAWL", change: { action: "set"; value: string } | { action: "clear" }) => { changed(); return apply(toastResult(await track(saveJobSourceKey({ source: keySource, change })))); }; @@ -165,6 +166,15 @@ export function JobSearchSection({ initial }: { initial: JobSearchSettingsView } /> )} + {source === "FIRECRAWL" && ( + saveKey("FIRECRAWL", change)} + /> + )} {source === "ARBEITSAGENTUR" && ( <> @@ -183,10 +193,97 @@ export function JobSearchSection({ initial }: { initial: JobSearchSettingsView } )} + + ); } +/** + * Blocked companies (this fork, E14-8): Find jobs, the job agent and the + * Claude connector never show, score or draft for a posting from one of + * these, by any source. Kept regardless of which source is chosen, or Off, + * so the list is ready before a source is. + */ +function BlockedCompaniesCard({ initial }: { initial: readonly string[] }) { + const track = useTrackSave(); + const [blocked, setBlocked] = useState(initial); + const [draft, setDraft] = useState(""); + + async function add() { + const name = draft.trim(); + if (!name) return; + if (blocked.some((entry) => entry.toLocaleLowerCase() === name.toLocaleLowerCase())) { + setDraft(""); + toast(`“${name}” is already blocked.`); + return; + } + setDraft(""); + const before = blocked; + setBlocked((current) => [...current, name]); + const result = await track(blockCompany({ name })); + if (!result.ok) { + setBlocked(before); + setDraft(name); + toast(result.error); + return; + } + setBlocked(result.data.blockedCompanies); + } + + async function remove(name: string) { + const before = blocked; + setBlocked((current) => current.filter((entry) => entry !== name)); + const result = await track(unblockCompany({ name })); + if (!result.ok) { + setBlocked(before); + toast(result.error); + return; + } + setBlocked(result.data.blockedCompanies); + } + + const onKeyDown = (event: KeyboardEvent) => { + if (event.key !== "Enter") return; + event.preventDefault(); + void add(); + }; + + return ( + +
+

+ Blocked companies (optional) +

+ {blocked.length} blocked +
+

+ A past or current employer, or one you never want to apply to again. No search shows, scores or drafts for a posting that names one of these as + the company, or mentions it in the title or description — which also catches a staffing listing for a role placed there. +

+ {blocked.length > 0 && ( +
    + {blocked.map((name) => ( +
  • + void remove(name)}> + {name} + +
    • + ))} +
+ )} + setDraft(event.target.value)} + onKeyDown={onKeyDown} + enterKeyHint="done" + /> + + ); +} + function HostChoice({ host, onChange }: { host: JSearchHost; onChange: (host: JSearchHost) => void }) { const labelId = useId(); return ( diff --git a/src/app/api/mcp/route.test.ts b/src/app/api/mcp/route.test.ts index d466e749..2914b345 100644 --- a/src/app/api/mcp/route.test.ts +++ b/src/app/api/mcp/route.test.ts @@ -1,7 +1,7 @@ import { beforeEach, describe, expect, it, vi } from "vitest"; vi.mock("@/lib/env", () => ({ env: { DOSSIER_SECRET: "s".repeat(40), DOSSIER_PASSWORD: "argon2id:m=19456,t=2,p=1:c2FsdA:aGFzaA" } })); -const settings = vi.hoisted(() => ({ connectorEnabledAt: "2026-10-23T09:00:00.000Z" as string | null })); +const settings = vi.hoisted(() => ({ connectorEnabledAt: "2026-10-23T09:00:00.000Z" as string | null, blockedCompanies: [] as string[] })); vi.mock("@/lib/settings-data", () => ({ loadSettings: async () => settings })); const db = vi.hoisted(() => ({ postingSearch: { findUnique: vi.fn() }, postingScore: { upsert: vi.fn() } })); vi.mock("@/lib/prisma", () => ({ prisma: db })); diff --git a/src/components/documents/__snapshots__/resume-document.test.tsx.snap b/src/components/documents/__snapshots__/resume-document.test.tsx.snap index b859abf2..f6233808 100644 --- a/src/components/documents/__snapshots__/resume-document.test.tsx.snap +++ b/src/components/documents/__snapshots__/resume-document.test.tsx.snap @@ -23,17 +23,31 @@ exports[`ResumeDocument > renders the demo profile in Atlas 1`] = `

- - jane.doe@example.com - + + jane.doe@example.com + + · - - +31 6 1234 5678 - + + +31 6 1234 5678 + + · renders the demo profile in Atlas 1`] = ` Rotterdam, Netherlands · - - example.com - + + example.com + + · - - linkedin.com/in/janedoe-example - + + linkedin.com/in/janedoe-example + +

renders the demo profile in Chronicle 1`] = `

- - jane.doe@example.com - + + jane.doe@example.com + + · - - +31 6 1234 5678 - + + +31 6 1234 5678 + + · renders the demo profile in Chronicle 1`] = ` Rotterdam, Netherlands · - - example.com - + + example.com + + · - - linkedin.com/in/janedoe-example - + + linkedin.com/in/janedoe-example + +

renders the demo profile in Classic 1`] = `

- - jane.doe@example.com - + + jane.doe@example.com + + · - - +31 6 1234 5678 - + + +31 6 1234 5678 + + · renders the demo profile in Classic 1`] = ` Rotterdam, Netherlands · - - example.com - + + example.com + + · - - linkedin.com/in/janedoe-example - + + linkedin.com/in/janedoe-example + +

renders the demo profile in Console 1`] = ` class="_contactList_fa1ef7" >
  • - - jane.doe@example.com - + + jane.doe@example.com + +
  • - - +31 6 1234 5678 - + + +31 6 1234 5678 + +
  • renders the demo profile in Console 1`] = `
  • - - example.com - + + example.com + +
  • - - linkedin.com/in/janedoe-example - + + linkedin.com/in/janedoe-example + +
    • @@ -1678,17 +1790,31 @@ exports[`ResumeDocument > renders the demo profile in Editorial 1`] = `

    - - jane.doe@example.com - + + jane.doe@example.com + + · - - +31 6 1234 5678 - + + +31 6 1234 5678 + + · renders the demo profile in Editorial 1`] = ` Rotterdam, Netherlands · - - example.com - + + example.com + + · - - linkedin.com/in/janedoe-example - + + linkedin.com/in/janedoe-example + +

    renders the demo profile in Folio 1`] = `

    - - jane.doe@example.com - + + jane.doe@example.com + + · - - +31 6 1234 5678 - + + +31 6 1234 5678 + + · renders the demo profile in Folio 1`] = ` Rotterdam, Netherlands · - - example.com - + + example.com + + · - - linkedin.com/in/janedoe-example - + + linkedin.com/in/janedoe-example + +

    renders the demo profile in Ledger 1`] = ` class="_contactList_fa1ef7" >
  • - - jane.doe@example.com - + + jane.doe@example.com + +
  • - - +31 6 1234 5678 - + + +31 6 1234 5678 + +
  • renders the demo profile in Ledger 1`] = `
  • - - example.com - + + example.com + +
  • - - linkedin.com/in/janedoe-example - + + linkedin.com/in/janedoe-example + +
    • @@ -2923,17 +3119,31 @@ exports[`ResumeDocument > renders the demo profile in Minimal 1`] = `

    - - jane.doe@example.com - + + jane.doe@example.com + + · - - +31 6 1234 5678 - + + +31 6 1234 5678 + + · renders the demo profile in Minimal 1`] = ` Rotterdam, Netherlands · - - example.com - + + example.com + + · - - linkedin.com/in/janedoe-example - + + linkedin.com/in/janedoe-example + +

    renders the demo profile in Modern 1`] = ` class="_contactList_fa1ef7" >
  • - - jane.doe@example.com - + + jane.doe@example.com + +
  • - - +31 6 1234 5678 - + + +31 6 1234 5678 + +
  • renders the demo profile in Modern 1`] = `
  • - - example.com - + + example.com + +
  • - - linkedin.com/in/janedoe-example - + + linkedin.com/in/janedoe-example + +
    • @@ -3758,17 +4010,31 @@ exports[`ResumeDocument > renders the demo profile in Signal 1`] = `

    - - jane.doe@example.com - + + jane.doe@example.com + + · - - +31 6 1234 5678 - + + +31 6 1234 5678 + + · renders the demo profile in Signal 1`] = ` Rotterdam, Netherlands · - - example.com - + + example.com + + · - - linkedin.com/in/janedoe-example - + + linkedin.com/in/janedoe-example + +

    renders the demo profile in Studio 1`] = ` class="_contactList_fa1ef7" >
  • - - jane.doe@example.com - + + jane.doe@example.com + +
  • - - +31 6 1234 5678 - + + +31 6 1234 5678 + +
  • renders the demo profile in Studio 1`] = `
  • - - example.com - + + example.com + +
  • - - linkedin.com/in/janedoe-example - + + linkedin.com/in/janedoe-example + +
    • @@ -4609,18 +4917,32 @@ exports[`ResumeDocument > renders the demo profile in Technical 1`] = ` class="_contactList_fa1ef7" >
  • - - jane.doe@example.com - + + jane.doe@example.com + +
  • - - +31 6 1234 5678 - + + +31 6 1234 5678 + +
  • renders the demo profile in Technical 1`] = `
  • - - example.com - + + example.com + +
  • - - linkedin.com/in/janedoe-example - + + linkedin.com/in/janedoe-example + +
    • diff --git a/src/components/documents/document-page.tsx b/src/components/documents/document-page.tsx index 569f4840..4c13267d 100644 --- a/src/components/documents/document-page.tsx +++ b/src/components/documents/document-page.tsx @@ -3,7 +3,7 @@ import { Fragment, type CSSProperties, type ReactNode } from "react"; import type { PaperSize } from "@/generated/prisma/enums"; import { cn } from "@/lib/cn"; import { hasRightToLeft, type TextDirection } from "@/lib/document-language"; -import { CONTACT_SEPARATOR } from "@/lib/resume-content"; +import { CONTACT_SEPARATOR, type ContactField } from "@/lib/resume-content"; import { PAPER_SIZES, SIDEBAR, type ResolvedTheme } from "@/lib/themes"; import styles from "./document.module.css"; @@ -120,21 +120,33 @@ export function DocumentSection({ title, children }: { title: string; children: /** * A contact detail, kept left to right when it has no right-to-left letters * (email, phone, web address), so it reads right inside a Persian document. + * Linked when `href` is given (this fork: website, LinkedIn, email and + * phone), styled the same muted, underline-free way as a publication's link. */ -export function Contact({ children }: { children: string }) { - return {children}; +export function Contact({ children, href }: { children: string; href?: string | null }) { + const text = {children}; + return href ? ( + + {text} + + ) : ( + text + ); } -/** Contact details on one line, separated by " · ". */ -export function ContactLine({ items }: { items: string[] }) { +/** Contact details on one line, separated by " · "; a plain string has no link, a ContactField may. */ +export function ContactLine({ items }: { items: (string | ContactField)[] }) { return (

    - {items.map((item, index) => ( - - {index > 0 && CONTACT_SEPARATOR} - {item} - - ))} + {items.map((item, index) => { + const field = typeof item === "string" ? { text: item, href: null } : item; + return ( + + {index > 0 && CONTACT_SEPARATOR} + {field.text} + + ); + })}

    ); } diff --git a/src/components/documents/resume-document.test.tsx b/src/components/documents/resume-document.test.tsx index 79195a1e..b54c5ca0 100644 --- a/src/components/documents/resume-document.test.tsx +++ b/src/components/documents/resume-document.test.tsx @@ -22,7 +22,8 @@ describe("ResumeDocument", () => { it("puts contact details in the header of single-column layouts", () => { render(); - expect(screen.getByText("jane.doe@example.com").parentElement?.textContent).toBe( + // Email, website and LinkedIn are links now (this fork), so the contact line is the text's closest

    , not always its direct parent. + expect(screen.getByText("jane.doe@example.com").closest("p")?.textContent).toBe( "jane.doe@example.com · +31 6 1234 5678 · Rotterdam, Netherlands · example.com · linkedin.com/in/janedoe-example", ); expect(sectionTitles()).toEqual(["Profile", "Experience", "Projects", "Education", "Publications", "Skills", "Languages"]); diff --git a/src/components/documents/resume-document.tsx b/src/components/documents/resume-document.tsx index 91a8dc08..397ed84d 100644 --- a/src/components/documents/resume-document.tsx +++ b/src/components/documents/resume-document.tsx @@ -58,7 +58,7 @@ export function ResumeDocument({ theme, profile, paper, layout, language, render

      {content.contacts.map((item, index) => (
    • - {item} + {item.text}
      • ))}
    diff --git a/src/lib/agent/run.db.test.ts b/src/lib/agent/run.db.test.ts index 8152986e..c4ea39bf 100644 --- a/src/lib/agent/run.db.test.ts +++ b/src/lib/agent/run.db.test.ts @@ -38,6 +38,8 @@ const fake = vi.hoisted(() => ({ ai: true, failOn: null as string | null, onScore: (() => {}) as (title: string) => void, + // Blocked companies (this fork, E14-8): empty unless a test sets it. + blockedCompanies: [] as string[], })); vi.mock("@/lib/job-search", () => ({ searchPostings: vi.fn(async () => { @@ -49,6 +51,7 @@ vi.mock("@/lib/job-search/config", () => ({ loadJobSearchConfig: async () => ({ source: fake.source, saved: true, + blockedCompanies: fake.blockedCompanies, jsearch: { key: "jsearch-key", host: "openwebninja", keyState: "set" }, adzuna: { appId: "app", key: "adzuna-key", keyState: "set" }, }), @@ -103,7 +106,7 @@ const context = (abort?: AbortController): JobContext & { steps: string[] } => { beforeEach(async () => { await resetDatabase(prisma()); - Object.assign(fake, { postings: page, searches: 0, source: "JSEARCH", ai: true, failOn: null, onScore: () => {} }); + Object.assign(fake, { postings: page, searches: 0, source: "JSEARCH", ai: true, failOn: null, onScore: () => {}, blockedCompanies: [] }); await saveSettings({ agentEnabled: true, agentQuery: query }); }); afterAll(() => prisma().$disconnect()); @@ -128,6 +131,15 @@ describe("runAgentScan (E16-3)", () => { expect((await loadSettings()).agentRuns).toEqual([{ at: now.toISOString(), manual: false, message: "2 new postings are waiting for a look." }]); }); + it("never scores or drafts a posting from a blocked company (this fork, E14-8)", async () => { + fake.blockedCompanies = ["Company a"]; + const run = context(); + const result = await runAgentScan({ manual: false }, run, now); + expect(result).toMatchObject({ found: 2, fresh: 2, scored: 2 }); + const candidates = await prisma().agentCandidate.findMany(); + expect(candidates.map((candidate) => candidate.externalId).toSorted()).toEqual(["b", "c"]); + }); + it("scores the newest new postings first, up to the cap", async () => { await saveSettings({ agentMaxScores: 2 }); const scoredTitles: string[] = []; diff --git a/src/lib/agent/run.ts b/src/lib/agent/run.ts index 54424b7d..cbc58b95 100644 --- a/src/lib/agent/run.ts +++ b/src/lib/agent/run.ts @@ -9,9 +9,10 @@ import { isAiConfigured } from "@/lib/ai/config"; import { AiError } from "@/lib/ai/errors"; import { loadTaskOptions } from "@/lib/ai/task-options"; import { scorePosting } from "@/lib/ai/tasks"; -import { searchPostings } from "@/lib/job-search"; -import { credentialsFor, loadJobSearchConfig, requestContext } from "@/lib/job-search/config"; +import { compileBlocklist, isBlockedPosting } from "@/lib/job-search/blocklist"; +import { credentialsFor, loadJobSearchConfig } from "@/lib/job-search/config"; import { JobSearchError } from "@/lib/job-search/errors"; +import { searchWithFallback } from "@/lib/job-search/find"; import { postingKey, postingText, type FoundPosting, type SearchQuery } from "@/lib/job-search/search"; import { startJob, type JobContext } from "@/lib/jobs/runner"; import { logServerError } from "@/lib/log"; @@ -113,15 +114,14 @@ export async function runAgentScan({ manual }: { manual: boolean }, context: Job await context.step("Searching for new postings"); const config = await loadJobSearchConfig(); - const source = config.source!; - let postings: FoundPosting[]; - try { - // The first page only: the agent never asks for a second. - postings = (await searchPostings(source, credentialsFor(config, source), settings.agentQuery, null, requestContext(source))).postings; - } catch (error) { - if (error instanceof JobSearchError) return finish(stopped(error.message)); - throw error; - } + // The first page only: the agent never asks for a second. Falls back across every source with a key saved + // (this fork, E14-9) if the chosen one can't answer, except the Bundesagentur für Arbeit, which the agent + // never searches on its own (#223) whether it's chosen or not. + const answer = await searchWithFallback(config, config.source!, settings.agentQuery, null, { exclude: ["ARBEITSAGENTUR"] }); + if (!answer.ok) return finish(stopped(answer.message)); + // Blocked companies (this fork, E14-8): the agent runs unattended, so a blocked employer must never reach a score or an auto-draft. + const blocked = compileBlocklist(config.blockedCompanies); + const postings: FoundPosting[] = answer.postings.filter((posting) => !isBlockedPosting(posting, blocked)); const fresh = await newPostings(postings); const counts = { found: postings.length, fresh: fresh.length, skipped: postings.length - fresh.length, drafted: 0 }; diff --git a/src/lib/connector/tools.ts b/src/lib/connector/tools.ts index 3baab677..ca9ea046 100644 --- a/src/lib/connector/tools.ts +++ b/src/lib/connector/tools.ts @@ -10,6 +10,7 @@ import { profileForPrompt } from "@/lib/ai/tasks/prompt-parts"; import { scoreFrom } from "@/lib/ai/tasks/score-posting"; import { unitsSchema } from "@/lib/application-data"; import { fitBand } from "@/lib/fit"; +import { blockedBy, compileBlocklist } from "@/lib/job-search/blocklist"; import { countryCodeFor, countryName } from "@/lib/job-search/countries"; import { fullText } from "@/lib/job-search/find"; import { DEFAULT_RADIUS_KM, foundPostingSchema, POSTED_WITHIN, postingText, RADII_KM, type FoundPosting, type PostedWithin, type SearchQuery, type ShownPosting } from "@/lib/job-search/search"; @@ -21,6 +22,7 @@ import { loadSearch, runSearch, showMoreFor } from "@/lib/posting-searches"; import { foundFromPosting, storePosting } from "@/lib/postings"; import { prisma } from "@/lib/prisma"; import { loadProfile } from "@/lib/profile-data"; +import { loadSettings } from "@/lib/settings-data"; import { statusInfo } from "@/lib/status"; import { applyStatusChange } from "@/lib/status-changes"; import { appVersion } from "@/lib/version"; @@ -250,8 +252,10 @@ export function createDossierServer(origin: string): McpServer { const search = search_id ? await prisma.postingSearch.findUnique({ where: { id: search_id }, select: { postings: true } }) : null; if (search_id && !search) return failure("That search is no longer kept. Search again."); const kept = foundPostingSchema.array().catch([]).parse(search?.postings ?? []); + const blocklist = compileBlocklist((await loadSettings()).blockedCompanies); const saved: { number?: number; posting_id?: string; title: string; score: number; band: string }[] = []; const missing: (number | string)[] = []; + const blocked: { number?: number; posting_id?: string; company: string }[] = []; for (const entry of scores) { const row = entry.posting_id ? await prisma.posting.findUnique({ where: { id: entry.posting_id } }) : null; const posting = row ? foundFromPosting(row) : entry.number ? kept[entry.number - 1] : undefined; @@ -259,11 +263,22 @@ export function createDossierServer(origin: string): McpServer { missing.push(entry.posting_id ?? entry.number ?? "?"); continue; } + // Blocked companies (this fork, E14-8): search_jobs already filters these out, but a posting_id can point at one saved before it was blocked. + const matched = blockedBy(posting, blocklist); + if (matched) { + blocked.push({ number: entry.number, posting_id: entry.posting_id, company: matched }); + continue; + } const fit = scoreFrom({ score: entry.score, reasons: entry.reasons, redFlags: entry.red_flags }); await saveScore(posting, fit, WRITTEN_IN_CLAUDE); saved.push({ number: entry.number, posting_id: entry.posting_id, title: posting.title, score: fit.score, band: fitBand(fit.score) }); } - return text({ saved, not_found: missing.length > 0 ? missing : undefined, link: linkTo(search_id ? `/jobs?search=${search_id}` : "/jobs?view=saved") }); + return text({ + saved, + not_found: missing.length > 0 ? missing : undefined, + blocked: blocked.length > 0 ? blocked : undefined, + link: linkTo(search_id ? `/jobs?search=${search_id}` : "/jobs?view=saved"), + }); }, ); @@ -392,6 +407,9 @@ export function createDossierServer(origin: string): McpServer { async ({ details, resume, cover_letter, match, note, ...source }) => { const posting = await resolvePosting(source); if ("error" in posting) return failure(posting.error); + // Blocked companies (this fork, E14-8): search_jobs already keeps these out, but save_application can be reached straight from a link or pasted text. + const matched = blockedBy({ company: posting.company ?? "", title: posting.title ?? "", description: posting.text }, compileBlocklist((await loadSettings()).blockedCompanies)); + if (matched) return failure(`"${matched}" is on your blocked companies list (Settings → Job search). Dossier won't save an application for it; tell the user why, in case it's a mistake.`); const saved = await saveWrittenApplication({ posting: posting.text.slice(0, 100_000), postingUrl: posting.url.slice(0, 2000), @@ -428,6 +446,8 @@ export function createDossierServer(origin: string): McpServer { } const posting = await resolvePosting(source); if ("error" in posting) return failure(posting.error); + const matched = blockedBy({ company: posting.company ?? "", title: posting.title ?? "", description: posting.text }, compileBlocklist((await loadSettings()).blockedCompanies)); + if (matched) return failure(`"${matched}" is on your blocked companies list (Settings → Job search). Dossier won't draft an application for it; tell the user why, in case it's a mistake.`); const jobId = await startJob(connectorDraftJob, { posting: posting.text.slice(0, 100_000), postingUrl: posting.url.slice(0, 2000), note: note ?? "", postingId: posting.postingId }); return text({ draft_id: jobId, status: "started", next: "Call check_draft with this draft_id in a minute or so." }); }, diff --git a/src/lib/data-export.ts b/src/lib/data-export.ts index e2c7caef..703e0824 100644 --- a/src/lib/data-export.ts +++ b/src/lib/data-export.ts @@ -48,7 +48,7 @@ export const EXPORT_SCHEMA_VERSIONS: readonly number[] = [1, 2, 3, 4, 5]; /** The setting that holds the encrypted API key, which only this server can read. */ const API_KEY_SETTING = "apiKeyEnc"; /** The settings that hold encrypted keys, which only this server can read: never exported as they are. */ -export const SECRET_SETTINGS: readonly string[] = [API_KEY_SETTING, "jsearchKeyEnc", "adzunaKeyEnc"]; +export const SECRET_SETTINGS: readonly string[] = [API_KEY_SETTING, "jsearchKeyEnc", "adzunaKeyEnc", "firecrawlKeyEnc"]; /** The settings that belong to this install rather than its data (the usage ping's and its consent, the update check's, the Google client and the browser's time zone): never exported, and kept by an import. */ export const INSTALL_SETTINGS: readonly string[] = [ "usagePingId", @@ -340,7 +340,7 @@ export const exportSchema = z.object({ /** The AI provider's key in plain text, only when the user asked for it. */ apiKey: text.optional(), /** The job sources' keys in plain text, likewise (E14-9). */ - jobSourceKeys: z.object({ jsearch: text.optional(), adzuna: text.optional() }).optional(), + jobSourceKeys: z.object({ jsearch: text.optional(), adzuna: text.optional(), firecrawl: text.optional() }).optional(), applications: z.array(application), /** Saved and dismissed postings (E14-9); absent in version 1 files. */ postings: z.array(posting).default([]), @@ -421,6 +421,7 @@ export async function buildExport( Object.entries({ jsearch: readable(stored("jsearchKeyEnc"), (value) => decryptSecret(JOB_SOURCE_KEY, value)), adzuna: readable(stored("adzunaKeyEnc"), (value) => decryptSecret(JOB_SOURCE_KEY, value)), + firecrawl: readable(stored("firecrawlKeyEnc"), (value) => decryptSecret(JOB_SOURCE_KEY, value)), }).filter(([, value]) => value), ); diff --git a/src/lib/data-import.ts b/src/lib/data-import.ts index 0f9b266c..e79f414a 100644 --- a/src/lib/data-import.ts +++ b/src/lib/data-import.ts @@ -82,6 +82,7 @@ export async function importExport(db: PrismaClient, data: DossierExport): Promi apiKeyEnc: apiKey ? encryptApiKey(apiKey) : null, jsearchKeyEnc: jobSourceKeys?.jsearch ? encryptSecret(JOB_SOURCE_KEY, jobSourceKeys.jsearch) : null, adzunaKeyEnc: jobSourceKeys?.adzuna ? encryptSecret(JOB_SOURCE_KEY, jobSourceKeys.adzuna) : null, + firecrawlKeyEnc: jobSourceKeys?.firecrawl ? encryptSecret(JOB_SOURCE_KEY, jobSourceKeys.firecrawl) : null, }; const keptKeys = [...SECRET_SETTINGS.filter((key) => !fromFile[key]), ...INSTALL_SETTINGS]; await db.$transaction( diff --git a/src/lib/env.test.ts b/src/lib/env.test.ts index 81556854..0c2565d6 100644 --- a/src/lib/env.test.ts +++ b/src/lib/env.test.ts @@ -68,6 +68,15 @@ describe("parseEnv", () => { expect(() => parseEnv({ ...valid, DOSSIER_AI_TIMEOUT_SECONDS: "1" })).toThrow(/at least 5/); }); + it("accepts a Firecrawl key, same as the other job sources' (this fork)", () => { + expect(parseEnv({ ...valid, DOSSIER_FIRECRAWL_KEY: "fc-test" }).DOSSIER_FIRECRAWL_KEY).toBe("fc-test"); + expect(parseEnv({ ...valid, DOSSIER_FIRECRAWL_KEY: "" }).DOSSIER_FIRECRAWL_KEY).toBeUndefined(); + }); + + it("accepts firecrawl as a job source (this fork)", () => { + expect(parseEnv({ ...valid, DOSSIER_JOBS_SOURCE: "firecrawl" }).DOSSIER_JOBS_SOURCE).toBe("firecrawl"); + }); + it("resolves a relative data directory", () => { const env = parseEnv({ ...valid, DOSSIER_DATA_DIR: "./data" }); expect(env.DOSSIER_DATA_DIR).toBe(path.resolve("data")); @@ -165,7 +174,7 @@ describe("job search variables (E14-4)", () => { it("rejects sources and hosts it doesn't know, without echoing keys", () => { expect(problems({ ...valid, DOSSIER_JOBS_SOURCE: "linkedin", DOSSIER_JSEARCH_HOST: "google", DOSSIER_JSEARCH_KEY: "ak_secret" })).toEqual([ - 'DOSSIER_JOBS_SOURCE must be one of jsearch, adzuna, arbeitsagentur, off (got "linkedin").', + 'DOSSIER_JOBS_SOURCE must be one of jsearch, adzuna, firecrawl, arbeitsagentur, off (got "linkedin").', 'DOSSIER_JSEARCH_HOST must be openwebninja or rapidapi (got "google").', ]); }); diff --git a/src/lib/env.ts b/src/lib/env.ts index 7bd9a88e..876597f8 100644 --- a/src/lib/env.ts +++ b/src/lib/env.ts @@ -11,7 +11,7 @@ const GENERATE_SECRET = "for example the output of `openssl rand -base64 32`"; const HASH_PASSWORD = "Make one with `docker run --rm -it ghcr.io/aientech/dossier hash-password`"; export const AI_PROVIDERS = ["local", "openai", "anthropic", "mistral", "lmstudio"] as const; -export const JOB_SOURCE_NAMES = ["jsearch", "adzuna", "arbeitsagentur", "off"] as const; +export const JOB_SOURCE_NAMES = ["jsearch", "adzuna", "firecrawl", "arbeitsagentur", "off"] as const; // An empty value (`DOSSIER_AI_KEY=` in .env) counts as unset. const unsetIfEmpty = (value: unknown) => @@ -122,6 +122,11 @@ const schema = z.object({ ), DOSSIER_ADZUNA_APP_ID: optional(z.string()), DOSSIER_ADZUNA_KEY: optional(z.string()), + // Firecrawl (E14-7, this fork): one key searches the open web for postings + // across LinkedIn, Indeed, Naukri, Glassdoor, Greenhouse, Lever and company + // sites at once, the way a person googling "data engineer jobs" would, so + // Dossier doesn't need a bespoke scraper per site. + DOSSIER_FIRECRAWL_KEY: optional(z.string()), // Off takes the Bundesagentur für Arbeit out everywhere (#223): it isn't an // official API, so a server can choose not to offer it. Unset means on. DOSSIER_JOBS_ARBEITSAGENTUR: optional( diff --git a/src/lib/job-search/adapter.ts b/src/lib/job-search/adapter.ts index 3813a208..80f0ac36 100644 --- a/src/lib/job-search/adapter.ts +++ b/src/lib/job-search/adapter.ts @@ -12,6 +12,7 @@ export type { JSearchHost }; export type SourceCredentials = { JSEARCH: { key: string; host: JSearchHost }; ADZUNA: { appId: string; key: string }; + FIRECRAWL: { key: string }; ARBEITSAGENTUR: Record; }; diff --git a/src/lib/job-search/blocklist.test.ts b/src/lib/job-search/blocklist.test.ts new file mode 100644 index 00000000..f913b8a6 --- /dev/null +++ b/src/lib/job-search/blocklist.test.ts @@ -0,0 +1,52 @@ +import { describe, expect, it } from "vitest"; + +import { blockedBy, compileBlocklist, isBlockedPosting } from "./blocklist"; + +describe("compileBlocklist", () => { + it("normalizes each name once, and drops one that normalizes to nothing", () => { + const blocked = compileBlocklist(["Best Buy", " ", "ProArch IT Solutions Pvt. Ltd."]); + expect(blocked).toEqual([ + { name: "Best Buy", normalized: "best buy" }, + { name: "ProArch IT Solutions Pvt. Ltd.", normalized: "proarch it solutions pvt ltd" }, + ]); + }); +}); + +describe("isBlockedPosting", () => { + it("is false with nothing blocked", () => { + expect(isBlockedPosting({ company: "Best Buy", title: "Data Engineer" }, [])).toBe(false); + }); + + it("matches the company field, case- and punctuation-insensitively", () => { + const blocked = compileBlocklist(["Best Buy"]); + expect(isBlockedPosting({ company: "Best Buy Co., Inc.", title: "Data Engineer" }, blocked)).toBe(true); + expect(isBlockedPosting({ company: "best buy", title: "Data Engineer" }, blocked)).toBe(true); + expect(isBlockedPosting({ company: "Target", title: "Data Engineer" }, blocked)).toBe(false); + }); + + it("matches a substring company name ('ProArch' blocks 'ProArch IT Solutions Pvt Ltd')", () => { + const blocked = compileBlocklist(["ProArch"]); + expect(isBlockedPosting({ company: "ProArch IT Solutions Pvt Ltd", title: "Sr. Data Engineer" }, blocked)).toBe(true); + }); + + it("catches a staffing listing that only names the company in the title or description", () => { + const blocked = compileBlocklist(["Best Buy"]); + expect(isBlockedPosting({ company: "Acme Staffing", title: "Data Engineer (role placed at Best Buy)" }, blocked)).toBe(true); + expect(isBlockedPosting({ company: "Acme Staffing", title: "Data Engineer", description: "Our client Best Buy is hiring." }, blocked)).toBe(true); + }); + + it("only matches a whole word, not part of a longer one", () => { + const blocked = compileBlocklist(["Best"]); + expect(isBlockedPosting({ company: "Bestbuy Inc", title: "Data Engineer" }, blocked)).toBe(false); + expect(isBlockedPosting({ company: "Best Logistics", title: "Data Engineer" }, blocked)).toBe(true); + }); +}); + +describe("blockedBy", () => { + it("names which blocked entry matched, or null", () => { + const blocked = compileBlocklist(["Best Buy", "ProArch"]); + expect(blockedBy({ company: "ProArch IT Solutions", title: "" }, blocked)).toBe("ProArch"); + expect(blockedBy({ company: "Acme Corp", title: "" }, blocked)).toBeNull(); + expect(blockedBy({ company: "Acme Corp", title: "" }, [])).toBeNull(); + }); +}); diff --git a/src/lib/job-search/blocklist.ts b/src/lib/job-search/blocklist.ts new file mode 100644 index 00000000..3bb0a333 --- /dev/null +++ b/src/lib/job-search/blocklist.ts @@ -0,0 +1,64 @@ +// Blocked companies (this fork, E14-8): postings Find jobs, the job agent +// and the Claude connector never show, score or let you draft for, by name +// — so a past or current employer, including a staffing role placed there, +// never comes back in a search. Settings → Job search keeps the list +// (settingsSchema.blockedCompanies in src/lib/settings.ts). +// +// A blocked name is matched against a posting's company, title AND +// description, not company alone: a staffing agency's own postings often +// name the client company only in the text ("role placed at Acme Corp"), +// and sources that don't give a separate company field at all (Firecrawl, +// see firecrawl.ts) only have the title and description to go on. This +// favours recall over precision on purpose — the point is never seeing the +// company again, not a perfectly worded match — so a blocked name can +// occasionally hide an unrelated posting that merely mentions it (e.g. "not +// an Acme Corp alumnus? no problem"). Keep blocked names specific for that +// reason: "Acme" blocks less precisely than "Acme Corporation". + +/** Case- and punctuation-insensitive, so "ProArch IT Solutions Pvt. Ltd." and "proarch it solutions" compare equal. */ +function normalize(value: string): string { + return value + .toLowerCase() + .normalize("NFKD") + .replace(/\p{Diacritic}/gu, "") + .replace(/[^\p{L}\p{N}]+/gu, " ") + .trim(); +} + +/** A blocked name kept as given (for the list in Settings) and normalized once (for fast repeated matching). */ +export type BlockedCompany = { name: string; normalized: string }; + +/** Settings' plain strings, normalized once, so searching many postings doesn't renormalize the list every time. */ +export function compileBlocklist(names: readonly string[]): BlockedCompany[] { + return names.map((name) => ({ name, normalized: normalize(name) })).filter((entry) => entry.normalized.length > 0); +} + +/** + * Whether `haystack` contains `needle` (already space-normalized, e.g. "best + * buy") as a run of whole words: "best buy" matches "...engineer best buy + * bengaluru...", but not "bestbuy" run together as one word. A short or + * common needle ("it", "data") still matches wherever it stands alone as a + * word, which is why the module note asks for specific blocked names. + */ +function containsWords(haystack: string, needle: string): boolean { + if (!needle) return false; + return new RegExp(`(?:^|\\s)${needle.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")}(?:\\s|$)`).test(` ${haystack} `); +} + +/** + * Whether a posting is from, or mentions, a blocked company: checked against + * its company, title and description (see the module note for why). + */ +export function isBlockedPosting(posting: { company: string; title: string; description?: string }, blocked: readonly BlockedCompany[]): boolean { + if (blocked.length === 0) return false; + const haystacks = [normalize(posting.company), normalize(posting.title), normalize(posting.description ?? "")]; + return blocked.some((entry) => haystacks.some((haystack) => containsWords(haystack, entry.normalized))); +} + +/** The reason a posting was kept out, for a server log; never the posting's own text. */ +export function blockedBy(posting: { company: string; title: string; description?: string }, blocked: readonly BlockedCompany[]): string | null { + if (blocked.length === 0) return null; + const haystacks = [normalize(posting.company), normalize(posting.title), normalize(posting.description ?? "")]; + const hit = blocked.find((entry) => haystacks.some((haystack) => containsWords(haystack, entry.normalized))); + return hit?.name ?? null; +} diff --git a/src/lib/job-search/config.test.ts b/src/lib/job-search/config.test.ts index 713477ce..dfda632b 100644 --- a/src/lib/job-search/config.test.ts +++ b/src/lib/job-search/config.test.ts @@ -12,7 +12,9 @@ const nothingSaved = { jsearchHost: "openwebninja" as const, adzunaAppId: null, adzunaKeyEnc: null, + firecrawlKeyEnc: null, arbeitsagenturAcceptedAt: null, + blockedCompanies: [], }; const ACCEPTED = "2026-10-23T09:00:00.000Z"; const noEnv = {}; @@ -25,10 +27,19 @@ describe("resolveJobSearchConfig", () => { source: "OFF", jsearch: { host: "openwebninja", keyState: "none" }, adzuna: { appId: "", keyState: "none" }, + firecrawl: { keyState: "none" }, arbeitsagentur: { offered: true, acceptedAt: null }, + blockedCompanies: [], }); }); + it("carries the blocked companies list through untouched", () => { + expect(resolveJobSearchConfig(nothingSaved, noEnv).blockedCompanies).toEqual([]); + const blocked = resolveJobSearchConfig({ ...nothingSaved, blockedCompanies: ["Best Buy", "ProArch"] }, noEnv); + expect(blocked.blockedCompanies).toEqual(["Best Buy", "ProArch"]); + expect(jobSearchView(blocked).blockedCompanies).toEqual(["Best Buy", "ProArch"]); + }); + it("uses the environment until something is saved", () => { const config = resolveJobSearchConfig(nothingSaved, { DOSSIER_JOBS_SOURCE: "jsearch", @@ -53,7 +64,9 @@ describe("resolveJobSearchConfig", () => { jsearchHost: "openwebninja" as const, adzunaAppId: "app-saved", adzunaKeyEnc: encryptSecret(JOB_SOURCE_KEY, "adz-saved"), + firecrawlKeyEnc: null, arbeitsagenturAcceptedAt: null, + blockedCompanies: ["Best Buy"], }; const config = resolveJobSearchConfig(saved, { DOSSIER_JOBS_SOURCE: "jsearch", DOSSIER_JSEARCH_KEY: "ak_env", DOSSIER_JSEARCH_HOST: "rapidapi" }); expect(config).toMatchObject({ @@ -61,6 +74,8 @@ describe("resolveJobSearchConfig", () => { saved: true, jsearch: { key: "ak_saved", host: "openwebninja", keyState: "set" }, adzuna: { appId: "app-saved", key: "adz-saved", keyState: "set" }, + firecrawl: { key: null, keyState: "none" }, + blockedCompanies: ["Best Buy"], }); expect(resolveJobSearchConfig({ ...saved, jobSource: "OFF" }, { DOSSIER_JOBS_SOURCE: "jsearch" })).toMatchObject({ source: null, saved: true }); }); @@ -86,6 +101,12 @@ describe("credentialsFor", () => { expect(credentialsFor({ ...config, arbeitsagentur: { offered: true, acceptedAt: ACCEPTED } }, "ARBEITSAGENTUR")).toEqual({}); }); + it("gives Firecrawl its own key, with no app ID to go with it", () => { + const firecrawl = resolveJobSearchConfig({ ...nothingSaved, firecrawlKeyEnc: encryptSecret(JOB_SOURCE_KEY, "fc_saved") }, noEnv); + expect(credentialsFor(firecrawl, "FIRECRAWL")).toEqual({ key: "fc_saved" }); + expect(() => credentialsFor(resolveJobSearchConfig(nothingSaved, noEnv), "FIRECRAWL")).toThrow("Add your Firecrawl key in Settings → Job search first."); + }); + it("says what's missing", () => { expect(() => credentialsFor(config, "ADZUNA")).toThrow("Add your Adzuna app ID and key in Settings → Job search first."); expect(() => credentialsFor(resolveJobSearchConfig(nothingSaved, noEnv), "JSEARCH")).toThrow("Add your JSearch key in Settings → Job search first."); diff --git a/src/lib/job-search/config.ts b/src/lib/job-search/config.ts index b9e9d71d..82d367ee 100644 --- a/src/lib/job-search/config.ts +++ b/src/lib/job-search/config.ts @@ -21,6 +21,9 @@ export type JobSearchConfig = { saved: boolean; jsearch: { key: string | null; host: JSearchHost; keyState: JobKeyState }; adzuna: { appId: string; key: string | null; keyState: JobKeyState }; + firecrawl: { key: string | null; keyState: JobKeyState }; + /** Companies no search ever shows a posting from (this fork, E14-8); see blocklist.ts. */ + blockedCompanies: readonly string[]; /** * The Bundesagentur für Arbeit (#223): whether this server offers it, and * when its note was accepted. Its searches wait until then. @@ -28,15 +31,25 @@ export type JobSearchConfig = { arbeitsagentur: { offered: boolean; acceptedAt: string | null }; }; -type SavedJobSettings = Pick; +type SavedJobSettings = Pick< + SettingsData, + "jobSource" | "jsearchKeyEnc" | "jsearchHost" | "adzunaAppId" | "adzunaKeyEnc" | "firecrawlKeyEnc" | "arbeitsagenturAcceptedAt" | "blockedCompanies" +>; type JobEnv = Pick< Env, - "DOSSIER_JOBS_SOURCE" | "DOSSIER_JSEARCH_KEY" | "DOSSIER_JSEARCH_HOST" | "DOSSIER_ADZUNA_APP_ID" | "DOSSIER_ADZUNA_KEY" | "DOSSIER_JOBS_ARBEITSAGENTUR" + | "DOSSIER_JOBS_SOURCE" + | "DOSSIER_JSEARCH_KEY" + | "DOSSIER_JSEARCH_HOST" + | "DOSSIER_ADZUNA_APP_ID" + | "DOSSIER_ADZUNA_KEY" + | "DOSSIER_FIRECRAWL_KEY" + | "DOSSIER_JOBS_ARBEITSAGENTUR" >; const FROM_ENV: Record, JobSource | null> = { jsearch: "JSEARCH", adzuna: "ADZUNA", + firecrawl: "FIRECRAWL", arbeitsagentur: "ARBEITSAGENTUR", off: null, }; @@ -71,6 +84,8 @@ export function resolveJobSearchConfig(saved: SavedJobSettings, source: JobEnv, appId: saved.adzunaAppId ?? source.DOSSIER_ADZUNA_APP_ID ?? "", ...keyOf(saved.adzunaKeyEnc, source.DOSSIER_ADZUNA_KEY, secret), }, + firecrawl: keyOf(saved.firecrawlKeyEnc, source.DOSSIER_FIRECRAWL_KEY, secret), + blockedCompanies: saved.blockedCompanies, arbeitsagentur: { offered, acceptedAt: saved.arbeitsagenturAcceptedAt }, }; } @@ -96,7 +111,9 @@ export function jobSearchView(config: JobSearchConfig): JobSearchSettingsView { source: config.source ?? "OFF", jsearch: { host: config.jsearch.host, keyState: config.jsearch.keyState }, adzuna: { appId: config.adzuna.appId, keyState: config.adzuna.keyState }, + firecrawl: { keyState: config.firecrawl.keyState }, arbeitsagentur: config.arbeitsagentur, + blockedCompanies: config.blockedCompanies, }; } @@ -129,6 +146,11 @@ export function credentialsFor(config: JobSearchConfig, sou if (!key || !appId) throw missing(keyState); return { appId, key } as SourceCredentials[S]; } + case "FIRECRAWL": { + const { key, keyState } = config.firecrawl; + if (!key) throw missing(keyState); + return { key } as SourceCredentials[S]; + } case "ARBEITSAGENTUR": { if (!config.arbeitsagentur.offered) throw jobSearchError("off", source); if (!config.arbeitsagentur.acceptedAt) throw jobSearchError("not-accepted", source); diff --git a/src/lib/job-search/find.ts b/src/lib/job-search/find.ts index bc2b89ce..de5aa877 100644 --- a/src/lib/job-search/find.ts +++ b/src/lib/job-search/find.ts @@ -1,33 +1,105 @@ import "server-only"; -import { credentialsFor, loadJobSearchConfig, requestContext } from "./config"; +import { compileBlocklist, isBlockedPosting } from "./blocklist"; +import { credentialsFor, loadJobSearchConfig, requestContext, type JobSearchConfig } from "./config"; import { JobSearchError } from "./errors"; import { postingDetails, searchPostings } from "./index"; -import type { FoundPosting, JobSource, SearchPage, SearchProblem, SearchQuery } from "./search"; +import { JOB_SOURCES, type FoundPosting, type JobSource, type SearchPage, type SearchProblem, type SearchQuery } from "./search"; +import { SOURCE_INFO } from "./sources"; // Asking a job source from Find jobs (E14-6): a page of postings, or what // went wrong in words for the screen. Kept searches are in // src/lib/posting-searches.ts. +export type SourceAnswer = ({ ok: true; source: JobSource } & SearchPage) | SearchProblem; + /** - * One page for `query` from `source`, the one chosen in Settings unless - * given (a kept search goes on with the source it ran on). + * The order to try sources in for a fresh search (this fork, E14-9): the + * one chosen in Settings first, then the rest in the order Settings lists + * them. */ -export async function askSource( +function fallbackOrder(primary: JobSource): JobSource[] { + return [primary, ...JOB_SOURCES.filter((source) => source !== primary)]; +} + +/** + * One page for `query`, trying every source with a key saved — starting + * from `primary` — until one answers (this fork, E14-9): one source's + * exhausted quota, an outage, or a country it doesn't cover no longer stops + * a search, it just moves on to the next. A source with no key saved (or, + * for the Bundesagentur für Arbeit, not yet switched on) is skipped without + * being asked at all. Shared by askSource below and the job agent's search + * step (src/lib/agent/run.ts), so both get the same resilience; the agent + * excludes the Bundesagentur für Arbeit itself (it never searches it, #223). + */ +export async function searchWithFallback( + config: JobSearchConfig, + primary: JobSource, query: SearchQuery, - { source, next = null }: { source?: JobSource; next?: string | null } = {}, -): Promise<({ ok: true; source: JobSource } & SearchPage) | SearchProblem> { + next: string | null = null, + { exclude = [] }: { exclude?: readonly JobSource[] } = {}, +): Promise { + // A source with no key saved (or, for the Bundesagentur für Arbeit, not yet switched on) was never asked, so it's + // not a failed attempt — it's skipped silently, the way it would be if it didn't exist. The one exception is the + // primary source itself: if nothing else is configured either, its own "add a key" message is the most useful + // thing to say, same as before there was anything to fall back to. + let primaryNotConfigured: SearchProblem | null = null; + const attempts: { source: JobSource; problem: SearchProblem }[] = []; + for (const candidate of fallbackOrder(primary)) { + if (exclude.includes(candidate)) continue; + let credentials; + try { + credentials = credentialsFor(config, candidate); + } catch (error) { + if (!(error instanceof JobSearchError)) throw error; + if (candidate === primary) primaryNotConfigured = { ok: false, problem: error.problem, message: error.message }; + continue; + } + try { + const page = await searchPostings(candidate, credentials, query, next, requestContext(candidate)); + return { ok: true, source: candidate, ...page }; + } catch (error) { + if (!(error instanceof JobSearchError)) throw error; + attempts.push({ source: candidate, problem: { ok: false, problem: error.problem, message: error.message } }); + } + } + if (attempts.length === 0) return primaryNotConfigured ?? { ok: false, problem: "off", message: "Choose where to search in Settings → Job search first." }; + if (attempts.length === 1) return attempts[0].problem; + return { + ok: false, + problem: attempts[0].problem.problem, + message: `None of your job sources could answer. ${attempts.map((a) => `${SOURCE_INFO[a.source].name}: ${a.problem.message}`).join(" ")}`, + }; +} + +/** + * One page for `query` from `source`, the one chosen in Settings unless + * given (a kept search goes on with the exact source it ran on — its `next` + * cursor only means something there, so a fresh search's fallback across + * sources doesn't apply to "Show more"). Postings from a blocked company + * (this fork, E14-8) never reach the caller — Find jobs, the job agent and + * the Claude connector's search_jobs all call this, so a blocked name stays + * out everywhere a search runs, not just on screen. + */ +export async function askSource(query: SearchQuery, { source, next = null }: { source?: JobSource; next?: string | null } = {}): Promise { const config = await loadJobSearchConfig(); - const from = source ?? config.source; - if (!from) return { ok: false, problem: "off", message: "Choose where to search in Settings → Job search first." }; if (!query.country) return { ok: false, problem: "no-country", message: "Choose a country to search in." }; - try { - const page = await searchPostings(from, credentialsFor(config, from), query, next, requestContext(from)); - return { ok: true, source: from, ...page }; - } catch (error) { - if (error instanceof JobSearchError) return { ok: false, problem: error.problem, message: error.message }; - throw error; + const blocked = compileBlocklist(config.blockedCompanies); + const withoutBlocked = (answer: SourceAnswer): SourceAnswer => + answer.ok ? { ...answer, postings: answer.postings.filter((posting) => !isBlockedPosting(posting, blocked)) } : answer; + + if (source) { + try { + const page = await searchPostings(source, credentialsFor(config, source), query, next, requestContext(source)); + return withoutBlocked({ ok: true, source, ...page }); + } catch (error) { + if (error instanceof JobSearchError) return { ok: false, problem: error.problem, message: error.message }; + throw error; + } } + + if (!config.source) return { ok: false, problem: "off", message: "Choose where to search in Settings → Job search first." }; + return withoutBlocked(await searchWithFallback(config, config.source, query, next)); } /** diff --git a/src/lib/job-search/firecrawl.test.ts b/src/lib/job-search/firecrawl.test.ts new file mode 100644 index 00000000..d07be071 --- /dev/null +++ b/src/lib/job-search/firecrawl.test.ts @@ -0,0 +1,174 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +import { json, refused, stubFetch } from "@/lib/ai/test-http"; + +const fetchPosting = vi.hoisted(() => vi.fn()); +vi.mock("@/lib/posting-fetch", () => ({ fetchPosting })); + +import { JobSearchError } from "./errors"; +import { companyFromAtsUrl, firecrawl, parseResultTitle, postingFromResult } from "./firecrawl"; +import { EMPTY_QUERY, type SearchQuery } from "./search"; + +const query: SearchQuery = { ...EMPTY_QUERY, keywords: "data engineer", place: "Bengaluru", country: "in", postedWithin: "week" }; +const creds = { key: "fc_test" }; + +beforeEach(() => { + vi.unstubAllGlobals(); + fetchPosting.mockReset(); +}); +afterEach(() => vi.unstubAllGlobals()); + +const answer = (web: unknown[]) => json({ success: true, data: { web } }); + +type SearchBody = { query: string; country?: string; location?: string; tbs?: string }; +const bodyOf = (request: { body: unknown }) => request.body as SearchBody; + +describe("Firecrawl search", () => { + it("POSTs two /v2/search requests per search: one job-board-biased, one wide open, both with the key, country and location", async () => { + const requests = stubFetch(() => answer([])); + await firecrawl.search(query, creds, null); + expect(requests).toHaveLength(2); + const [biased, open] = requests.map(bodyOf); + + for (const { url, method, headers } of requests) { + expect(method).toBe("POST"); + expect(`${url.origin}${url.pathname}`).toBe("https://api.firecrawl.dev/v2/search"); + expect(headers.get("authorization")).toBe("Bearer fc_test"); + } + expect(biased).toMatchObject({ country: "IN", location: "Bengaluru", tbs: "qdr:w" }); + expect(biased.query).toContain("data engineer jobs in Bengaluru"); + expect(biased.query).toContain("site:linkedin.com"); + expect(biased.query).toContain("site:naukri.com"); + + expect(open).toMatchObject({ country: "IN", location: "Bengaluru", tbs: "qdr:w" }); + expect(open.query).toBe("data engineer jobs in Bengaluru"); + expect(open.query).not.toContain("site:"); + }); + + it("widens a 3-day search to a week, since Google has no 3-day unit", async () => { + const requests = stubFetch(() => answer([])); + await firecrawl.search({ ...query, postedWithin: "3days" }, creds, null); + for (const request of requests) expect(bodyOf(request)).toMatchObject({ tbs: "qdr:w" }); + }); + + it("asks for no date range at all, and defaults the location to the whole country", async () => { + const requests = stubFetch(() => answer([])); + await firecrawl.search({ ...query, place: "", postedWithin: "any" }, creds, null); + for (const request of requests) { + expect(bodyOf(request)).not.toHaveProperty("tbs"); + expect(bodyOf(request)).toMatchObject({ location: "India" }); + } + }); + + it("never offers a next page: Firecrawl's search has no cursor", async () => { + stubFetch(() => answer([{ url: "https://example.com/1", title: "Data Engineer" }])); + const page = await firecrawl.search(query, creds, null); + expect(page.next).toBeNull(); + }); + + it("drops results with no usable link or title, and de-duplicates by link", async () => { + stubFetch(() => + answer([ + { url: "https://example.com/a", title: "Data Engineer" }, + { url: "javascript:alert(1)", title: "Data Engineer" }, + { url: "https://example.com/a", title: "Data Engineer (duplicate)" }, + { url: "https://example.com/b", title: "" }, + ]), + ); + const page = await firecrawl.search(query, creds, null); + expect(page.postings).toHaveLength(1); + expect(page.postings[0].url).toBe("https://example.com/a"); + }); + + it("says the source isn't answering when it can't be reached", async () => { + stubFetch(refused); + await expect(firecrawl.search(query, creds, null)).rejects.toMatchObject({ problem: "unavailable" }); + }); + + it("says the key was refused on 401", async () => { + stubFetch(() => json({ success: false }, { status: 401 })); + await expect(firecrawl.search(query, creds, null)).rejects.toMatchObject({ problem: "key" }); + }); + + it("says the source changed when success is false with a 200", async () => { + stubFetch(() => json({ success: false, data: { web: [] } })); + const error = await firecrawl.search(query, creds, null).catch((caught: unknown) => caught); + expect(error).toBeInstanceOf(JobSearchError); + expect(error).toMatchObject({ problem: "changed" }); + }); +}); + +describe("companyFromAtsUrl", () => { + it("reads the company from the path on each applicant-tracking system", () => { + expect(companyFromAtsUrl("https://job-boards.greenhouse.io/acmecorp/jobs/1234")).toBe("Acmecorp"); + expect(companyFromAtsUrl("https://jobs.lever.co/acme-corp/abc-123")).toBe("Acme Corp"); + expect(companyFromAtsUrl("https://jobs.ashbyhq.com/acmecorp/posting")).toBe("Acmecorp"); + expect(companyFromAtsUrl("https://jobs.smartrecruiters.com/Acmecorp/job")).toBe("Acmecorp"); + expect(companyFromAtsUrl("https://acmecorp.wd1.myworkdayjobs.com/en-US/careers/job/123")).toBe("Acmecorp"); + }); + + it("gives up on a link that isn't a known applicant-tracking system, or isn't a link at all", () => { + expect(companyFromAtsUrl("https://www.linkedin.com/jobs/view/123")).toBe(""); + expect(companyFromAtsUrl("not a url")).toBe(""); + }); +}); + +describe("parseResultTitle", () => { + it("splits 'Company hiring Role in Place' from LinkedIn-style titles", () => { + expect(parseResultTitle("Acme Corp hiring Senior Data Engineer in Bengaluru | LinkedIn")).toEqual({ + title: "Senior Data Engineer", + company: "Acme Corp", + }); + }); + + it("splits 'Role - Company - Place' from Indeed/Naukri-style titles", () => { + expect(parseResultTitle("Senior Data Engineer - Acme Corp - Bengaluru")).toEqual({ title: "Senior Data Engineer", company: "Bengaluru" }); + }); + + it("falls back to the whole title with no company when nothing matches", () => { + expect(parseResultTitle("Senior Data Engineer")).toEqual({ title: "Senior Data Engineer", company: "" }); + }); +}); + +describe("postingFromResult", () => { + it("prefers the ATS URL's company over a guess from the title", () => { + const posting = postingFromResult({ + url: "https://job-boards.greenhouse.io/acmecorp/jobs/1234", + title: "Senior Data Engineer", + description: "Build our data platform.", + markdown: null, + }); + expect(posting).toMatchObject({ source: "FIRECRAWL", title: "Senior Data Engineer", company: "Acmecorp", complete: false }); + expect(posting.externalId).toBe(posting.url); + }); + + it("uses the title guess when the link isn't a known ATS", () => { + const posting = postingFromResult({ + url: "https://www.linkedin.com/jobs/view/123", + title: "Acme Corp hiring Senior Data Engineer in Bengaluru | LinkedIn", + description: null, + markdown: "**Build** our data platform.", + }); + expect(posting).toMatchObject({ title: "Senior Data Engineer", company: "Acme Corp", description: "Build our data platform." }); + }); +}); + +describe("Firecrawl details and test", () => { + it("fetches the posting's own page for the rest of it, like Adzuna does", async () => { + fetchPosting.mockResolvedValueOnce({ ok: true, text: "The whole posting.", source: "page", url: "https://example.com/job" }); + const posting = postingFromResult({ url: "https://example.com/job", title: "Data Engineer", description: "snippet", markdown: null }); + expect(await firecrawl.details(posting, creds)).toBe("The whole posting."); + expect(fetchPosting).toHaveBeenCalledWith(posting.url); + + fetchPosting.mockResolvedValueOnce({ ok: false, error: "unreadable" }); + expect(await firecrawl.details(posting, creds)).toBeNull(); + expect(await firecrawl.details({ ...posting, url: "" }, creds)).toBeNull(); + }); + + it("tests the key with one small search", async () => { + const requests = stubFetch(() => answer([])); + await firecrawl.test(creds); + expect(requests).toHaveLength(1); + expect(requests[0].body).toMatchObject({ query: expect.stringContaining("developer jobs") }); + }); +}); diff --git a/src/lib/job-search/firecrawl.ts b/src/lib/job-search/firecrawl.ts new file mode 100644 index 00000000..c2bd71f2 --- /dev/null +++ b/src/lib/job-search/firecrawl.ts @@ -0,0 +1,236 @@ +import "server-only"; + +import { z } from "zod"; + +import { fetchPosting } from "@/lib/posting-fetch"; + +import type { JobSourceAdapter, RequestContext, SourceCredentials } from "./adapter"; +import { countryName } from "./countries"; +import { jobSearchError } from "./errors"; +import { requestJson } from "./http"; +import { fromMarkdown, plainText } from "./text"; +import { safeLink, type FoundPosting, type PostedWithin, type SearchPage, type SearchQuery } from "./search"; + +// Firecrawl (E14-7, this fork): a web search across the open web, the way a +// person googling "data engineer jobs in Bengaluru" would, rather than one +// job board's own index. It's the answer to "I don't want to miss postings +// that only show up on Naukri, or Greenhouse, or a company's own site" — +// without Dossier maintaining a scraper per site, which breaks the moment a +// site's markup changes and risks its terms of use the way the Bundesagentur +// für Arbeit source already owns up to (sources.ts). One page merges two +// searches (searchPage below): one biased toward the job boards and +// applicant-tracking systems in JOB_BOARD_DOMAINS (LinkedIn, Indeed, Naukri, +// Glassdoor, Monster, Greenhouse, Lever, Ashby, Workday, SmartRecruiters), +// one with no site restriction at all for whatever that fixed list misses. +// +// Trade-offs, on purpose: +// - Two Firecrawl credits a search, not one, for that reason. +// - The search only returns a title, a link and a short snippet — not a +// separate company field the way JSearch and Adzuna's APIs do. The +// company is pulled from the job board's own URL shape when the result is +// on an applicant-tracking system (reliable: the company is in the path), +// else guessed from the title's own wording (best-effort; see +// parseResultTitle below). Either way it can be "", which is honest rather +// than invented, and isBlockedPosting() (blocklist.ts) still catches a +// blocked company named in the title or snippet even when the company +// field comes up empty. +// - The full posting is fetched like Adzuna's (details() below, reusing +// src/lib/posting-fetch.ts) rather than spent on Firecrawl's own scrape +// for every result: scraping costs more credits than searching, and the +// guarded fetcher already does this safely for every other source. +// - Firecrawl's /search has no paging cursor, so Dossier asks for one +// generous page (PAGE_SIZE) per half and never offers "Show more" for +// this source (next is always null). + +const BASE = "https://api.firecrawl.dev/v2"; +const SEARCH_PATH = "/search"; +const PAGE_SIZE = 30; + +/** Job boards and applicant-tracking systems the search is biased toward, besides whatever else the open web turns up. */ +export const JOB_BOARD_DOMAINS = [ + "linkedin.com", + "indeed.com", + "naukri.com", + "glassdoor.com", + "monster.com", + "greenhouse.io", + "lever.co", + "ashbyhq.com", + "myworkdayjobs.com", + "smartrecruiters.com", + "wellfound.com", +] as const; + +/** Google's `tbs` date range closest to each "posted within" choice; Firecrawl passes it straight through. There's no 3-day unit, so that choice widens to a week rather than narrowing to a day, favouring a posting shown over one missed. */ +const TBS: Partial> = { day: "qdr:d", "3days": "qdr:w", week: "qdr:w", month: "qdr:m" }; + +const resultSchema = z.object({ + url: z.string().min(1), + title: z.string().nullish().catch(null), + description: z.string().nullish().catch(null), + markdown: z.string().nullish().catch(null), +}); +type Result = z.output; + +const responseSchema = z.object({ success: z.boolean(), data: z.object({ web: z.array(z.unknown()).nullish().catch([]) }).nullish().catch({ web: [] }) }); + +/** + * The company from an applicant-tracking system's own URL shape, which + * names it in the path or the subdomain: reliable where the search gives + * one, because the company owns that part of the address. + * + * https://job-boards.greenhouse.io/acmecorp/jobs/1234 -> "Acmecorp" + * https://jobs.lever.co/acmecorp/... -> "Acmecorp" + * https://jobs.ashbyhq.com/acmecorp/... -> "Acmecorp" + * https://acmecorp.wd1.myworkdayjobs.com/en-US/careers/job/… -> "Acmecorp" + * https://jobs.smartrecruiters.com/Acmecorp/... -> "Acmecorp" + */ +export function companyFromAtsUrl(url: string): string { + const parsed = URL.canParse(url) ? new URL(url) : null; + if (!parsed) return ""; + const host = parsed.hostname.toLowerCase(); + const segment = (index: number) => parsed.pathname.split("/").filter(Boolean)[index] ?? ""; + if (/(^|\.)(job-boards|jobs|boards)\.greenhouse\.io$/.test(host)) return titleCase(segment(0)); + if (/(^|\.)jobs\.lever\.co$/.test(host)) return titleCase(segment(0)); + if (/(^|\.)jobs\.ashbyhq\.com$/.test(host)) return titleCase(segment(0)); + if (/(^|\.)jobs\.smartrecruiters\.com$/.test(host)) return titleCase(segment(0)); + const workday = /^([a-z0-9-]+)\.[a-z0-9.]*myworkdayjobs\.com$/.exec(host); + if (workday) return titleCase(workday[1]); + return ""; +} + +function titleCase(slug: string): string { + return slug + .replace(/[-_]+/g, " ") + .trim() + .replace(/\w\S*/g, (word) => word[0].toUpperCase() + word.slice(1)); +} + +/** Suffixes a job board appends to its own page titles, trimmed before anything else is parsed out. */ +const SITE_SUFFIX = /\s*[|·]\s*(LinkedIn|Indeed(\.com)?|Naukri(\.com)?|Glassdoor|Monster(\.com)?|Greenhouse|Lever|Ashby|Workday|SmartRecruiters)\s*$/i; + +/** + * Best-effort split of an aggregator's page title into a role and a + * company, from the handful of phrasings LinkedIn, Indeed, Naukri and + * Glassdoor write their titles in ("Acme Corp hiring Data Engineer in + * Bengaluru", "Data Engineer - Acme Corp - Bengaluru"). Falls back to the + * whole title with no company, rather than guessing wrong — isBlockedPosting + * still has the full title and description to check against. + */ +export function parseResultTitle(raw: string): { title: string; company: string } { + const bare = plainText(raw).replace(SITE_SUFFIX, "").trim(); + const hiring = /^(.+?)\s+(?:is\s+)?hiring\b\s*(?:for\s+)?(?:an?\s+)?(.+)$/i.exec(bare); + if (hiring) { + const title = hiring[2].replace(/\s+\bin\b\s+[^,-]+(,.*)?$/i, "").trim(); + return { title: title || bare, company: hiring[1].trim() }; + } + const parts = bare.split(/\s+[-–—]\s+/).filter(Boolean); + if (parts.length >= 2) return { title: parts[0].trim(), company: parts[parts.length - 1].trim() }; + return { title: bare, company: "" }; +} + +export function postingFromResult(result: Result): FoundPosting { + const url = safeLink(result.url); + const parsed = parseResultTitle(result.title ?? ""); + const company = companyFromAtsUrl(url) || parsed.company; + const description = fromMarkdown(result.markdown) || plainText(result.description).slice(0, 100_000); + return { + source: "FIRECRAWL", + // Firecrawl doesn't hand back an id of its own; the link is the posting's stable identity. + externalId: url.slice(0, 500) || result.url.slice(0, 500), + title: parsed.title.slice(0, 500), + company: company.slice(0, 300), + place: "", + remote: null, + salary: "", + postedAt: null, + url: url.slice(0, 2000), + via: "", + description, + complete: false, + }; +} + +/** "data engineer jobs in Bengaluru remote (site:linkedin.com OR site:indeed.com OR …)": the job-board-biased half of the search. */ +function biasedSearchText(query: SearchQuery): string { + const sites = JOB_BOARD_DOMAINS.map((domain) => `site:${domain}`).join(" OR "); + return `${baseSearchText(query)} (${sites})`; +} + +/** "data engineer jobs in Bengaluru remote": the same query with no site restriction at all, for whatever the biased half's domain list misses — a smaller board, or a company's own careers page. */ +function openSearchText(query: SearchQuery): string { + return baseSearchText(query); +} + +function baseSearchText(query: SearchQuery): string { + const where = query.place ? ` in ${query.place}` : ""; + const remote = query.remoteOnly ? " remote" : ""; + return `${query.keywords} jobs${where}${remote}`; +} + +/** One Firecrawl /search call, parsed into postings. */ +async function runSearch(text: string, query: SearchQuery, key: string, context: RequestContext): Promise { + const url = new URL(`${context.baseUrl ?? BASE}${SEARCH_PATH}`); + const body: Record = { + query: text.slice(0, 500), + limit: PAGE_SIZE, + country: (query.country || "us").toUpperCase(), + location: query.place || countryName(query.country || "us"), + }; + const tbs = TBS[query.postedWithin]; + if (tbs) body.tbs = tbs; + + const answer = responseSchema.safeParse( + await requestJson("FIRECRAWL", url, { method: "POST", body, headers: { authorization: `Bearer ${key}` }, signal: context.signal }), + ); + if (!answer.success || answer.data.success === false) throw jobSearchError("changed", "FIRECRAWL", { detail: `${url.pathname}: unexpected answer` }); + + const postings: FoundPosting[] = []; + for (const item of answer.data.data?.web ?? []) { + const result = resultSchema.safeParse(item); + if (!result.success) continue; + const posting = postingFromResult(result.data); + if (posting.url && posting.title) postings.push(posting); + } + return postings; +} + +/** + * Two searches merged into one page (this fork): one biased toward the job + * boards in JOB_BOARD_DOMAINS, one with no site restriction at all, so a + * posting that's only on a smaller board or a company's own careers page — + * not in that fixed list — still turns up, the way actually googling it + * would. Costs two Firecrawl credits per Dossier search instead of one. + */ +async function searchPage(query: SearchQuery, { key }: SourceCredentials["FIRECRAWL"], _next: string | null, context: RequestContext = {}): Promise { + const [biased, open] = await Promise.all([ + runSearch(biasedSearchText(query), query, key, context), + runSearch(openSearchText(query), query, key, context), + ]); + + const seen = new Set(); + const postings: FoundPosting[] = []; + for (const posting of [...biased, ...open]) { + if (seen.has(posting.externalId)) continue; + seen.add(posting.externalId); + postings.push(posting); + } + // No cursor in Firecrawl's /search: one page, from both searches, is what there is. + return { postings, next: null }; +} + +export const firecrawl: JobSourceAdapter<"FIRECRAWL"> = { + source: "FIRECRAWL", + search: searchPage, + // The search only gives a snippet; the rest comes from the posting's own page, exactly as Adzuna's does. + details: async (posting) => { + if (!posting.url) return null; + const page = await fetchPosting(posting.url); + return page.ok ? page.text : null; + }, + // One search, not the usual two (searchPage runs both halves): the smallest request that shows the key works. + test: async ({ key }, context = {}) => { + const query: SearchQuery = { keywords: "developer", place: "", country: "us", radiusKm: 25, remoteOnly: false, postedWithin: "any" }; + await runSearch(openSearchText(query), query, key, context); + }, +}; diff --git a/src/lib/job-search/http.ts b/src/lib/job-search/http.ts index d57516a5..5bbd0d64 100644 --- a/src/lib/job-search/http.ts +++ b/src/lib/job-search/http.ts @@ -6,21 +6,36 @@ import type { JobSource } from "./search"; // Requests to job sources (E14-1). Each source has a fixed host, so unlike // fetching a posting (src/lib/posting-fetch.ts) there's no address to guard; // requests are still limited in time and size. - -export const REQUEST_TIMEOUT_MS = 10_000; +// +// This was a hard-coded 10 s until this fork: a live test against OpenWeb +// Ninja's JSearch took 10.3 s to answer, so every request lost the race and +// came back "isn't answering right now" even with a working key. 25 s gives +// a slower source, or a slower network, room to actually answer. (Not read +// from the environment on purpose: that would make every module that +// imports this one — including code that only registers job types, with no +// job source configured — depend on the whole app's environment validating +// cleanly, which it doesn't in many tests and needn't to run a job.) +export const REQUEST_TIMEOUT_MS = 25_000; const MAX_BYTES = 5 * 1024 * 1024; /** GETs JSON from a source, turning every failure into a JobSearchError. */ export async function requestJson( source: JobSource, url: URL, - { headers = {}, signal }: { headers?: Record; signal?: AbortSignal } = {}, + { + headers = {}, + signal, + method = "GET", + body, + }: { headers?: Record; signal?: AbortSignal; method?: "GET" | "POST"; body?: unknown } = {}, ): Promise { const timeout = AbortSignal.timeout(REQUEST_TIMEOUT_MS); let response: Response; try { response = await fetch(url, { - headers: { accept: "application/json", ...headers }, + method, + headers: { accept: "application/json", ...(body !== undefined ? { "content-type": "application/json" } : {}), ...headers }, + body: body !== undefined ? JSON.stringify(body) : undefined, signal: signal ? AbortSignal.any([signal, timeout]) : timeout, cache: "no-store", }); diff --git a/src/lib/job-search/index.ts b/src/lib/job-search/index.ts index 61901f21..7f5e9edb 100644 --- a/src/lib/job-search/index.ts +++ b/src/lib/job-search/index.ts @@ -7,6 +7,7 @@ import { cached, remember } from "./cache"; import { JobSearchError, jobSearchError } from "./errors"; import { adzuna } from "./adzuna"; import { arbeitsagentur } from "./arbeitsagentur"; +import { firecrawl } from "./firecrawl"; import { jsearch } from "./jsearch"; import { postingKey, type FoundPosting, type JobSource, type SearchPage, type SearchQuery } from "./search"; import { sourceCovers } from "./sources"; @@ -22,6 +23,7 @@ export { JobSearchError } from "./errors"; const ADAPTERS: { [S in JobSource]: JobSourceAdapter } = { JSEARCH: jsearch, ADZUNA: adzuna, + FIRECRAWL: firecrawl, ARBEITSAGENTUR: arbeitsagentur, }; diff --git a/src/lib/job-search/search.ts b/src/lib/job-search/search.ts index 5a50b6ee..5c24a13b 100644 --- a/src/lib/job-search/search.ts +++ b/src/lib/job-search/search.ts @@ -11,8 +11,8 @@ import { countryCodeFor } from "./countries"; // In this code "job" already means a background task (src/lib/jobs), so a // job ad is a "posting". -/** The job sources Dossier can search (E14-1 to E14-3), in the order Settings lists them; one is chosen there. */ -export const JOB_SOURCES = [PostingSource.JSEARCH, PostingSource.ADZUNA, PostingSource.ARBEITSAGENTUR] as const; +/** The job sources Dossier can search (E14-1 to E14-3, E14-7), in the order Settings lists them; one is chosen there. */ +export const JOB_SOURCES = [PostingSource.JSEARCH, PostingSource.ADZUNA, PostingSource.FIRECRAWL, PostingSource.ARBEITSAGENTUR] as const; export type JobSource = PostingSource; /** Where a JSearch key comes from: OpenWeb Ninja directly, or the RapidAPI marketplace. */ diff --git a/src/lib/job-search/sources.ts b/src/lib/job-search/sources.ts index 037ce461..38ff7aa0 100644 --- a/src/lib/job-search/sources.ts +++ b/src/lib/job-search/sources.ts @@ -8,8 +8,11 @@ export type JobSearchSettingsView = { source: JobSource | "OFF"; jsearch: { host: JSearchHost; keyState: JobKeyState }; adzuna: { appId: string; keyState: JobKeyState }; + firecrawl: { keyState: JobKeyState }; /** Whether this server offers the Bundesagentur für Arbeit, and when its note was accepted (#223). */ arbeitsagentur: { offered: boolean; acceptedAt: string | null }; + /** Companies no search shows a posting from (this fork, E14-8). */ + blockedCompanies: readonly string[]; }; export const JOB_KEY_UNREADABLE = "Your saved key can't be read. Please enter it again."; @@ -65,6 +68,16 @@ export const SOURCE_INFO: Record = { attributionLink: { home: "https://www.adzuna.co.uk", local: /^www\.adzuna\.(?:[a-z]{2,3}|co\.[a-z]{2}|com\.[a-z]{2})$/ }, countries: ["at", "au", "be", "br", "ca", "ch", "de", "es", "fr", "gb", "in", "it", "mx", "nl", "nz", "pl", "sg", "us", "za"], }, + FIRECRAWL: { + name: "Firecrawl", + description: "One web search across LinkedIn, Indeed, Naukri, Glassdoor, Greenhouse, Lever and company sites at once, the way googling would.", + tag: "Every country · pay-per-search key, a free trial to start", + keyUrl: "https://www.firecrawl.dev/app/api-keys", + quotaPeriod: null, + attribution: null, + attributionLink: null, + countries: null, + }, ARBEITSAGENTUR: { name: "Bundesagentur für Arbeit", description: "Germany's federal job board, with full postings. Unofficial: you choose it on your own responsibility.", diff --git a/src/lib/posting-searches.db.test.ts b/src/lib/posting-searches.db.test.ts index 448e4fd0..a12cbc74 100644 --- a/src/lib/posting-searches.db.test.ts +++ b/src/lib/posting-searches.db.test.ts @@ -23,6 +23,8 @@ const prisma = () => db.client!; const query = { ...EMPTY_QUERY, keywords: "product designer", place: "Hamburg", country: "de" }; const useJSearch = () => saveSettings({ jobSource: "JSEARCH", jsearchKeyEnc: encryptSecret(JOB_SOURCE_KEY, "ak_test") }); +const addFirecrawlKey = () => saveSettings({ firecrawlKeyEnc: encryptSecret(JOB_SOURCE_KEY, "fc_test") }); +const firecrawlAnswer = (web: unknown[] = []) => json({ success: true, data: { web } }); beforeEach(async () => { await resetDatabase(prisma()); @@ -70,6 +72,50 @@ describe("runSearch", () => { }); }); +describe("falling back across sources (this fork, E14-9)", () => { + it("moves on to the next source with a key saved when the chosen one is exhausted", async () => { + await useJSearch(); + await addFirecrawlKey(); + const requests = stubFetch((request) => + request.url.hostname === "api.openwebninja.com" ? json({}, { status: 429 }) : firecrawlAnswer([{ url: "https://example.com/job", title: "Designer" }]), + ); + const result = await runSearch(query); + expect(result).toEqual({ ok: true, id: expect.any(String) }); + const kept = await prisma().postingSearch.findUniqueOrThrow({ where: { id: result.ok ? result.id : "" } }); + // Firecrawl runs two searches per page (biased and open); JSearch was asked once, first. + expect(requests.filter((r) => r.url.hostname === "api.openwebninja.com")).toHaveLength(1); + expect(requests.filter((r) => r.url.hostname === "api.firecrawl.dev")).toHaveLength(2); + expect(kept.source).toBe("FIRECRAWL"); + }); + + it("says so, naming each one, when every configured source fails", async () => { + await useJSearch(); + await addFirecrawlKey(); + stubFetch((request) => (request.url.hostname === "api.openwebninja.com" ? json({}, { status: 429 }) : json({ success: false }, { status: 401 }))); + const result = await runSearch(query); + expect(result).toMatchObject({ + ok: false, + problem: "quota", + message: "None of your job sources could answer. JSearch: You've used up your JSearch searches for this month. Try again then, or move to a bigger plan. Firecrawl: Firecrawl didn't accept your key. Check it in Settings → Job search.", + }); + }); + + it("stops at the one source saved, same as before there was anything to fall back to", async () => { + await saveSettings({ jobSource: "JSEARCH" }); + expect(await runSearch(query)).toEqual({ ok: false, problem: "no-key", message: "Add your JSearch key in Settings → Job search first." }); + }); + + it("doesn't fall back for Show more: it stays on the search's own source", async () => { + await useJSearch(); + await addFirecrawlKey(); + stubFetch(() => json(jsearchV2Page)); + const run = await runSearch(query); + const id = run.ok ? run.id : ""; + stubFetch((request) => (request.url.hostname === "api.openwebninja.com" ? json({}, { status: 429 }) : firecrawlAnswer([]))); + expect(await showMoreFor(id)).toMatchObject({ ok: false, problem: "quota" }); + }); +}); + describe("loadSearch", () => { it("opens a kept search without asking the source, with where each posting stands now", async () => { await useJSearch(); diff --git a/src/lib/resume-content.test.ts b/src/lib/resume-content.test.ts index 347e1541..b892f181 100644 --- a/src/lib/resume-content.test.ts +++ b/src/lib/resume-content.test.ts @@ -37,6 +37,17 @@ describe("contactItems", () => { describe("resumeContent", () => { const profile = demoProfile(); + it("links the website, LinkedIn, email and phone (this fork); the rest has no link", () => { + const content = resumeContent(profile, { today }); + expect(content.contacts).toEqual([ + { text: "jane.doe@example.com", href: "mailto:jane.doe@example.com" }, + { text: "+31 6 1234 5678", href: "tel:+31612345678" }, + { text: "Rotterdam, Netherlands", href: null }, + { text: "example.com", href: "https://example.com/" }, + { text: "linkedin.com/in/janedoe-example", href: "https://linkedin.com/in/janedoe-example" }, + ]); + }); + it("builds the base resume from the profile", () => { const content = resumeContent(profile, { today }); expect(content.name).toBe("Jane Doe"); diff --git a/src/lib/resume-content.ts b/src/lib/resume-content.ts index b0c045b5..7ac3ad5a 100644 --- a/src/lib/resume-content.ts +++ b/src/lib/resume-content.ts @@ -11,8 +11,12 @@ export type ResumeContent = { headline: string; /** The profile photo's data URL, or null when the resume has none (#224). */ photo: string | null; - /** Email, phone, "City, Country", website, LinkedIn and "Age N", whichever are filled in. */ - contacts: string[]; + /** + * Email, phone, "City, Country", website, LinkedIn and "Age N", whichever + * are filled in. Website and LinkedIn link out (this fork); email and + * phone open the device's own app; the rest has no link. + */ + contacts: ContactField[]; summary: string; experience: { id: string; @@ -65,19 +69,34 @@ export type ResumeOptions = { /** The separator between contact details on one line. */ export const CONTACT_SEPARATOR = " · "; -/** Contact details in the order the resume header lists them. */ -export function contactItems(basics: Basics, today = new Date(), labels = ENGLISH_LABELS): string[] { +/** One contact detail, with where it links to when it's the kind that does (this fork). */ +export type ContactField = { text: string; href: string | null }; + +/** A telephone number as a "tel:" link opens it: digits and a leading "+" only. */ +const telHref = (phone: string) => `tel:${phone.replace(/[^\d+]/g, "")}`; + +/** Contact details in the order the resume header lists them, with a link where one makes sense. */ +function contactFields(basics: Basics, today: Date, labels: DocumentLabels): ContactField[] { const age = basics.showAge ? ageOn(basics.dob, today) : null; - return [ - basics.email, - basics.phone, - [basics.city, basics.country].map((part) => part.trim()).filter(Boolean).join(", "), - basics.website, - basics.linkedin, - age === null ? "" : ageLabel(age, labels), - ] - .map((item) => item.trim()) - .filter(Boolean); + const email = basics.email.trim(); + const phone = basics.phone.trim(); + const website = basics.website.trim(); + const linkedin = basics.linkedin.trim(); + return ( + [ + { text: email, href: email ? `mailto:${email}` : null }, + { text: phone, href: phone ? telHref(phone) : null }, + { text: [basics.city, basics.country].map((part) => part.trim()).filter(Boolean).join(", "), href: null }, + { text: website, href: website ? publicationHref(website) : null }, + { text: linkedin, href: linkedin ? publicationHref(linkedin) : null }, + { text: age === null ? "" : ageLabel(age, labels), href: null }, + ] satisfies ContactField[] + ).filter((item) => item.text); +} + +/** Contact details as plain text, for the cover letter's own sender block, which is free text the user can edit. */ +export function contactItems(basics: Basics, today = new Date(), labels = ENGLISH_LABELS): string[] { + return contactFields(basics, today, labels).map((item) => item.text); } /** A link as a resume shows it: "https://www.example.com/x/" is "example.com/x". */ @@ -138,7 +157,7 @@ export function resumeContent( name: basics.name.trim() || "Your name", headline: basics.headline.trim(), photo: profile.photo && (showPhoto ?? profile.photo.onResumes) ? profile.photo.src : null, - contacts: contactItems(basics, today, labels), + contacts: contactFields(basics, today, labels), summary: basics.summary.trim(), experience: profile.experience.map((entry) => experience(entry, entry.bullets)), projects: profile.projects.map((entry) => project(entry, entry.bullets)), diff --git a/src/lib/settings.test.ts b/src/lib/settings.test.ts index b59bccc7..038722ec 100644 --- a/src/lib/settings.test.ts +++ b/src/lib/settings.test.ts @@ -61,7 +61,9 @@ describe("stored settings", () => { jsearchHost: "openwebninja", adzunaAppId: null, adzunaKeyEnc: null, + firecrawlKeyEnc: null, arbeitsagenturAcceptedAt: null, + blockedCompanies: [], agentEnabled: false, agentQuery: { keywords: "", place: "", country: "", radiusKm: 25, remoteOnly: false, postedWithin: "any" }, agentHour: 7, @@ -110,11 +112,13 @@ describe("stored settings", () => { docLanguage: "Dutch", quietDays: 14, jobRetentionHours: 168, + blockedCompanies: ["Best Buy", "ProArch"], } satisfies Partial; const rows = Object.entries(saved).map(([key, value]) => ({ key, value: settingValue(value)! })); expect(rows).toContainEqual({ key: "strict", value: "false" }); expect(rows).toContainEqual({ key: "customLanguages", value: '["Farsi","Hebrew"]' }); expect(rows).toContainEqual({ key: "quietDays", value: "14" }); + expect(rows).toContainEqual({ key: "blockedCompanies", value: '["Best Buy","ProArch"]' }); expect(settingsFromRows(rows)).toMatchObject(saved); expect(settingValue(null)).toBeNull(); }); diff --git a/src/lib/settings.ts b/src/lib/settings.ts index 9745f374..6dc79497 100644 --- a/src/lib/settings.ts +++ b/src/lib/settings.ts @@ -153,8 +153,18 @@ export const settingsSchema = z.object({ jsearchHost: z.enum(JSEARCH_HOSTS).catch("openwebninja"), adzunaAppId: z.string().nullable().catch(null), adzunaKeyEnc: z.string().nullable().catch(null), + /** Firecrawl's own key (this fork, E14-7): one web search across every job board at once. */ + firecrawlKeyEnc: z.string().nullable().catch(null), /** When "I understand" was ticked under the Bundesagentur für Arbeit's note (#223); its searches wait for it. */ arbeitsagenturAcceptedAt: z.iso.datetime().nullable().catch(null), + /** + * Companies Find jobs, the job agent and the Claude connector never show + * or save a posting for (this fork, E14-8): past or current employers, so + * a search never resurfaces one by name. Matched against a posting's + * company, title and description, case-insensitively, so a staffing + * listing that only names the company in the description is still caught. + */ + blockedCompanies: stringList.catch([]), // Job agent (E16): one search, run once a day. Off until it's turned on. // It asks the source in Settings → Job search, with its key. agentEnabled: flag(false),