feat(seo): per-page meta descriptions and markdown-free titles

build.mjs

@@ -16,7 +16,7 @@ const SITE_URL = (
 ).replace(/\/?$/, "/");
 const SITE_NAME = "Refract";
 const SITE_DESCRIPTION =
-	"Refract — the open claim-history layer for public knowledge. Deterministic event stream of claims, sources, and disputes across revision histories.";
+	"A deterministic observation engine for revision histories. It reads a page's edits and emits a typed event for each change.";
 const SOURCE_REPOSITORY = "https://github.com/refract-org/refract";
 
 function assetVersion() {
@@ -65,7 +65,7 @@ function escapeMarkup(text) {
 		.replace(/'/g, "'");
 }
 
-function structuredData(title, slug) {
+function structuredData(title, description, slug) {
 	const url = slugHref(slug, SITE_URL);
 	const website = {
 		"@type": "WebSite",
@@ -79,7 +79,7 @@ function structuredData(title, slug) {
 			"@graph": [
 				{
 					...website,
-					description: SITE_DESCRIPTION,
+					description,
 					inLanguage: "en",
 					about: { "@id": `${SITE_URL}#source` },
 				},
@@ -99,7 +99,7 @@ function structuredData(title, slug) {
 				"@type": "TechArticle",
 				"@id": `${url}#article`,
 				headline: title,
-				description: SITE_DESCRIPTION,
+				description,
 				url,
 				mainEntityOfPage: url,
 				inLanguage: "en",
@@ -126,18 +126,17 @@ function structuredData(title, slug) {
 	};
 }
 
-function renderSeoHead(title, slug) {
+function renderSeoHead(title, description, slug) {
 	const url = escapeMarkup(slugHref(slug, SITE_URL));
 	// "<" is escaped so no title can close the script element early.
-	const jsonLd = JSON.stringify(structuredData(title, slug)).replace(
-		/</g,
-		"\\u003c",
-	);
+	const jsonLd = JSON.stringify(
+		structuredData(title, description, slug),
+	).replace(/</g, "\\u003c");
 	return `<link rel="canonical" href="${url}">
   <meta property="og:type" content="${slug === "index" ? "website" : "article"}">
   <meta property="og:site_name" content="${SITE_NAME}">
   <meta property="og:title" content="${escapeMarkup(title)}">
-  <meta property="og:description" content="${escapeMarkup(SITE_DESCRIPTION)}">
+  <meta property="og:description" content="${escapeMarkup(description)}">
   <meta property="og:url" content="${url}">
   <meta name="twitter:card" content="summary">
   <script type="application/ld+json">${jsonLd}</script>`;
@@ -179,15 +178,69 @@ function renderNav(currentSlug) {
 	return html;
 }
 
-function plainText(tokens) {
+function plainText(tokens, skipTypes = []) {
 	return tokens
 		.map((token) => {
-			if (token.tokens) return plainText(token.tokens);
+			if (skipTypes.includes(token.type)) return "";
+			if (token.tokens) return plainText(token.tokens, skipTypes);
 			return token.text ?? token.raw ?? "";
 		})
 		.join("");
 }
 
+// Inline text of a heading or paragraph with markdown, HTML tags and images
+// removed and whitespace collapsed.
+function inlineText(tokens) {
+	return plainText(tokens, ["html", "image"]).replace(/\s+/g, " ").trim();
+}
+
+function pageTitle(tokens, slug) {
+	const h1 = tokens.find((t) => t.type === "heading" && t.depth === 1);
+	return h1 ? inlineText(h1.tokens) : resolveTitle(slug);
+}
+
+const DESCRIPTION_LENGTH = 155;
+
+function truncateAtWord(text, max) {
+	if (text.length <= max) return text;
+	const cut = text.slice(0, max + 1);
+	const end = cut.lastIndexOf(" ");
+	return `${cut.slice(0, end > 0 ? end : max).replace(/[\s,;:—–-]+$/, "")}…`;
+}
+
+// A paragraph can stand in for the page when it is prose that ends a sentence.
+// That rules out lead-ins to the next block ("Use environment variables
+// instead:"), bold-label callouts ("**Note**: …", glossary-style
+// "**Term** — …"), parenthetical asides, "See [page]" pointers and images.
+function isSummaryParagraph(token, text) {
+	if (!/[.!?]["'”’)]*$/.test(text)) return false;
+	if (/^\(.*\)$/.test(text)) return false;
+	const [first, second] = token.tokens;
+	if (
+		first?.type === "strong" &&
+		second &&
+		(/:$/.test(first.text) || /^\s*[:—–]|^\s+-\s/.test(second.raw))
+	) {
+		return false;
+	}
+	return !(/^See\s/.test(text) && token.tokens.some((t) => t.type === "link"));
+}
+
+// The first prose paragraph after the H1, trimmed for a meta description.
+// Only top-level paragraphs count, so lists, tables, code blocks, blockquotes
+// and HTML blocks are skipped. Returns "" when the page has none.
+function pageSummary(tokens) {
+	const h1 = tokens.findIndex((t) => t.type === "heading" && t.depth === 1);
+	for (const token of tokens.slice(h1 + 1)) {
+		if (token.type !== "paragraph") continue;
+		const text = inlineText(token.tokens);
+		if (isSummaryParagraph(token, text)) {
+			return truncateAtWord(text, DESCRIPTION_LENGTH);
+		}
+	}
+	return "";
+}
+
 function slugifyHeading(text) {
 	return text
 		.toLowerCase()
@@ -197,28 +250,12 @@ function slugifyHeading(text) {
 		.replace(/\s/g, "-");
 }
 
-function getExcerpt(markdown) {
-	// Remove code blocks
-	let text = markdown.replace(/```[\s\S]*?```/g, "");
-	// Remove inline code tags
-	text = text.replace(/`([^`]+)`/g, "$1");
-	// Remove HTML tags
-	text = text.replace(/<[^>]*>/g, "");
-	// Remove markdown links
-	text = text.replace(/\[([^\]]+)\]\([^)]+\)/g, "$1");
-	// Remove headings
-	text = text.replace(/^#+\s+.+$/gm, "");
-	// Get first non-empty lines
-	const lines = text
-		.split("\n")
-		.map((l) => l.trim())
-		.filter(Boolean);
-	if (lines.length === 0) return "";
-	const firstLine = lines[0];
-	return firstLine.length > 140 ? `${firstLine.slice(0, 137)}...` : firstLine;
+// A title that already leads with the name doesn't repeat it as a suffix.
+function documentTitle(title) {
+	return title.startsWith(SITE_NAME) ? title : `${title} — ${SITE_NAME}`;
 }
 
-function wrapHTML(title, content, currentSlug, headings = []) {
+function wrapHTML(title, description, content, currentSlug, headings = []) {
 	let tocHtml = "";
 	if (headings.length > 0) {
 		tocHtml += `<div class="toc-title">On this page</div>`;
@@ -237,11 +274,11 @@ function wrapHTML(title, content, currentSlug, headings = []) {
 <head>
   <meta charset="UTF-8">
   <meta name="viewport" content="width=device-width, initial-scale=1.0">
-  <title>${title} — Refract</title>
+  <title>${documentTitle(title)}</title>
   <link rel="stylesheet" href="${BASE}style.css?v=${ASSET_VERSION}">
   <link rel="icon" type="image/svg+xml" href="data:image/svg+xml,<svg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 16 16'><style>path{fill:%2307090f}@media (prefers-color-scheme:dark){path{fill:%23e2e4ed}}</style><path d='M8 2 15 14H1z'/></svg>">
-  <meta name="description" content="${SITE_DESCRIPTION}">
-  ${renderSeoHead(title, currentSlug)}
+  <meta name="description" content="${escapeMarkup(description)}">
+  ${renderSeoHead(title, description, currentSlug)}
 </head>
 <body>
   <div id="progress" aria-hidden="true"></div>
@@ -250,7 +287,7 @@ function wrapHTML(title, content, currentSlug, headings = []) {
     <aside class="sidebar">
       <div class="sidebar-header">
         <a href="${BASE}" class="brand">Refract</a>
-        <p class="tagline">The open claim-history layer<br>for public knowledge.</p>
+        <p class="tagline">A deterministic observation engine<br>for revision histories.</p>
         <div class="sidebar-search">
           <button class="search-trigger" id="search-trigger" aria-label="Search documentation">
             <svg class="search-icon" viewBox="0 0 24 24" width="14" height="14" stroke="currentColor" stroke-width="2" fill="none" stroke-linecap="round" stroke-linejoin="round">
@@ -636,6 +673,7 @@ async function build() {
 	const { files, assets } = await collectFiles(DOCS_DIR);
 
 	const searchIndex = [];
+	const fallbackPages = [];
 	const renderer = new marked.Renderer();
 	let currentSourceDir = "";
 	let currentHeadingCounts = new Map();
@@ -678,9 +716,13 @@ async function build() {
 		currentHeadingCounts = new Map();
 		currentHeadings = [];
 		const body = marked.parse(raw, { renderer });
-		const h1Match = raw.match(/^#\s+(.+)/m);
-		const title = h1Match ? h1Match[1] : resolveTitle(file.slug);
-		const html = wrapHTML(title, body, file.slug, currentHeadings);
+		const tokens = marked.lexer(raw);
+		const title = pageTitle(tokens, file.slug);
+		// The home page keeps the site-wide positioning line.
+		const summary = file.slug === "index" ? "" : pageSummary(tokens);
+		if (file.slug !== "index" && !summary) fallbackPages.push(file.slug);
+		const description = summary || SITE_DESCRIPTION;
+		const html = wrapHTML(title, description, body, file.slug, currentHeadings);
 
 		if (file.slug === "index") {
 			await writeFile(join(DIST_DIR, "index.html"), html);
@@ -693,7 +735,7 @@ async function build() {
 		searchIndex.push({
 			title,
 			slug: file.slug,
-			excerpt: getExcerpt(raw),
+			excerpt: description,
 			headings: currentHeadings.map((h) => ({ text: h.text, id: h.id })),
 		});
 	}
@@ -719,6 +761,11 @@ async function build() {
 	console.log(
 		`Built ${files.length} pages and copied ${assets.length} assets to dist/`,
 	);
+	if (fallbackPages.length > 0) {
+		console.log(
+			`No prose paragraph to describe, so these pages use the site description: ${fallbackPages.sort().join(", ")}`,
+		);
+	}
 }
 
 build().catch((err) => {

docs/concepts.md

@@ -1,7 +1,7 @@
 # Concepts
 
-Refract is the open claim-history layer for public knowledge. It ingests revision histories
-from MediaWiki instances and produces a deterministic event stream showing how claims
+Refract is a deterministic observation engine for revision histories. It ingests them
+from MediaWiki instances and produces an event stream showing how claims
 enter, change, stabilize, and exit the public record — where they came from, what supported
 them, what challenged them, and when context altered their meaning.
 

docs/index.md

@@ -1,4 +1,4 @@
-# Refract: deterministic observation layer for model evaluation
+# Refract: deterministic observation engine for revision histories
 
 **Refract reveals how claims change across public revision histories — and gives AI researchers reproducible evidence for model evaluation.**
 

docs/why-refract.md

@@ -1,7 +1,7 @@
 # Why Refract?
 
-Refract is the open claim-history layer for public knowledge: it tells you what changed
-on a page, when, and in which revision.
+Refract is a deterministic observation engine for revision histories: it tells you what
+changed on a page, when, and in which revision.
 
 ## Who this is for
 

tests/seo.test.ts

@@ -8,6 +8,11 @@ import { beforeAll, describe, expect, it } from "vitest";
 const ROOT_DIR = resolve(__dirname, "..");
 const DIST_DIR = join(ROOT_DIR, "dist");
 const SITE_URL = "https://refract-org.github.io/refract-docs/";
+const SITE_DESCRIPTION =
+	"A deterministic observation engine for revision histories. It reads a page's edits and emits a typed event for each change.";
+// Pages that use the site description: the home page by design, and pages with
+// no prose paragraph to describe them (the build logs these).
+const SITE_DESCRIPTION_PAGES = ["glossary", "index"];
 const execFileAsync = promisify(execFile);
 
 type Page = { slug: string; title: string; html: string };
@@ -33,6 +38,27 @@ function metaContents(html: string, attr: string, key: string): string[] {
 	);
 }
 
+function descriptionOf(html: string): string {
+	return metaContents(html, "name", "description")[0] ?? "";
+}
+
+// Visible text of an HTML fragment: tags dropped, entities decoded,
+// whitespace collapsed.
+function textOf(fragment: string): string {
+	return decodeEntities(fragment.replace(/<[^>]+>/g, ""))
+		.replace(/\s+/g, " ")
+		.trim();
+}
+
+function stringsIn(value: unknown): string[] {
+	if (typeof value === "string") return [value];
+	if (Array.isArray(value)) return value.flatMap(stringsIn);
+	if (value && typeof value === "object") {
+		return Object.values(value).flatMap(stringsIn);
+	}
+	return [];
+}
+
 function canonicals(html: string): string[] {
 	return Array.from(
 		html.matchAll(/<link rel="canonical" href="([^"]*)">/g),
@@ -81,12 +107,80 @@ describe("technical SEO", () => {
 		}
 	});
 
-	it("leaves page titles and the meta description as they were", () => {
-		for (const { title, html } of pages) {
-			expect(html).toContain(`<title>${title} — Refract</title>`);
-			expect(metaContents(html, "name", "description")).toEqual([
-				"Refract — the open claim-history layer for public knowledge. Deterministic event stream of claims, sources, and disputes across revision histories.",
-			]);
+	it("titles each page with the visible text of its H1", () => {
+		for (const { slug, title, html } of pages) {
+			const h1 = html.match(/<h1 [^>]*>([\s\S]*?)<\/h1>/)?.[1] ?? "";
+			expect(title, slug).toBe(textOf(h1));
+			const documentTitle = title.startsWith("Refract")
+				? title
+				: `${title} — Refract`;
+			expect(html, slug).toContain(`<title>${documentTitle}</title>`);
+		}
+		const crossWiki = pages.find((page) =>
+			page.slug.endsWith("cross-wiki-diff"),
+		);
+		expect(crossWiki?.title).toBe(
+			"Tutorial: Cross-wiki comparison with refract diff",
+		);
+	});
+
+	it("keeps the site description on the home page", () => {
+		const home = pages.find((page) => page.slug === "index");
+		expect(descriptionOf(home?.html ?? "")).toBe(SITE_DESCRIPTION);
+	});
+
+	it("gives each doc page its own description, apart from listed fallbacks", () => {
+		const fallbacks = pages
+			.filter(({ html }) => descriptionOf(html) === SITE_DESCRIPTION)
+			.map(({ slug }) => slug)
+			.sort();
+		expect(fallbacks).toEqual(SITE_DESCRIPTION_PAGES);
+
+		const descriptions = pages
+			.filter(({ slug }) => !SITE_DESCRIPTION_PAGES.includes(slug))
+			.map(({ html }) => descriptionOf(html));
+		expect(new Set(descriptions).size).toBe(descriptions.length);
+	});
+
+	it("takes each description from the page's own text, cut at a word boundary", () => {
+		for (const { slug, html } of pages) {
+			if (SITE_DESCRIPTION_PAGES.includes(slug)) continue;
+			const description = descriptionOf(html);
+			const main = html.match(/<main class="content">([\s\S]*?)<\/main>/);
+			const body = textOf(main?.[1] ?? "");
+			const cut = description.endsWith("…");
+			const prefix = cut ? description.slice(0, -1) : description;
+			const at = body.indexOf(prefix);
+
+			expect(at, `${slug}: ${description}`).toBeGreaterThanOrEqual(0);
+			if (cut) {
+				expect(body[at + prefix.length], slug).toMatch(/[^\p{L}\p{N}]/u);
+			}
+		}
+	});
+
+	it("keeps every description to 160 characters or fewer", () => {
+		for (const { slug, html } of pages) {
+			const description = descriptionOf(html);
+			expect(description.length, `${slug}: ${description}`).toBeLessThanOrEqual(
+				160,
+			);
+			expect(description.length, slug).toBeGreaterThan(0);
+		}
+	});
+
+	it("leaves no markdown backticks or asterisks in titles or descriptions", () => {
+		for (const { slug, html } of pages) {
+			const texts = [
+				textOf(html.match(/<title>([\s\S]*?)<\/title>/)?.[1] ?? ""),
+				descriptionOf(html),
+				...metaContents(html, "property", "og:title"),
+				...metaContents(html, "property", "og:description"),
+				...stringsIn(jsonLdGraph(html)),
+			];
+			for (const text of texts) {
+				expect(text, slug).not.toMatch(/[`*]/);
+			}
 		}
 	});