feat(seo): canonical URLs, Open Graph tags, JSON-LD and sitemap.xml

build.mjs

@@ -9,6 +9,15 @@ const DOCS_DIR = join(__dirname, "docs");
 const DIST_DIR = join(__dirname, "dist");
 const ASSETS_DIR = join(__dirname, "assets");
 const BASE = process.env.BASE || "/refract-docs/";
+// Absolute address of the published site. Canonical URLs, og:url, JSON-LD
+// and the sitemap point here even when BASE is overridden for a local build.
+const SITE_URL = (
+	process.env.SITE_URL || "https://refract-org.github.io/refract-docs/"
+).replace(/\/?$/, "/");
+const SITE_NAME = "Refract";
+const SITE_DESCRIPTION =
+	"Refract — the open claim-history layer for public knowledge. Deterministic event stream of claims, sources, and disputes across revision histories.";
+const SOURCE_REPOSITORY = "https://github.com/refract-org/refract";
 
 function assetVersion() {
 	if (process.env.GITHUB_SHA) return process.env.GITHUB_SHA.slice(0, 12);
@@ -28,7 +37,7 @@ const ASSET_VERSION = assetVersion();
 
 let NAV = [];
 
-function resolveTitle(slug) {
+function resolveTitle(slug, fallback = basename(slug)) {
 	for (const item of NAV) {
 		if (item.slug === slug) return item.title;
 		if (item.children) {
@@ -37,11 +46,113 @@ function resolveTitle(slug) {
 			}
 		}
 	}
-	return basename(slug);
+	return fallback;
 }
 
-function slugHref(slug) {
-	return slug === "index" ? BASE : `${BASE}${slug}/`;
+// GitHub Pages serves dist/<slug>/index.html at <slug>/ and 301-redirects the
+// slash-less form, so the trailing-slash URL is the one to link and canonicalize.
+function slugHref(slug, base = BASE) {
+	return slug === "index" ? base : `${base}${slug}/`;
+}
+
+// Safe in HTML attribute values and in XML text alike.
+function escapeMarkup(text) {
+	return text
+		.replace(/&/g, "&amp;")
+		.replace(/</g, "&lt;")
+		.replace(/>/g, "&gt;")
+		.replace(/"/g, "&quot;")
+		.replace(/'/g, "&#39;");
+}
+
+function structuredData(title, slug) {
+	const url = slugHref(slug, SITE_URL);
+	const website = {
+		"@type": "WebSite",
+		"@id": `${SITE_URL}#website`,
+		name: SITE_NAME,
+		url: SITE_URL,
+	};
+	if (slug === "index") {
+		return {
+			"@context": "https://schema.org",
+			"@graph": [
+				{
+					...website,
+					description: SITE_DESCRIPTION,
+					inLanguage: "en",
+					about: { "@id": `${SITE_URL}#source` },
+				},
+				{
+					"@type": "SoftwareSourceCode",
+					"@id": `${SITE_URL}#source`,
+					name: SITE_NAME,
+					codeRepository: SOURCE_REPOSITORY,
+				},
+			],
+		};
+	}
+	return {
+		"@context": "https://schema.org",
+		"@graph": [
+			{
+				"@type": "TechArticle",
+				"@id": `${url}#article`,
+				headline: title,
+				description: SITE_DESCRIPTION,
+				url,
+				mainEntityOfPage: url,
+				inLanguage: "en",
+				isPartOf: website,
+			},
+			{
+				"@type": "BreadcrumbList",
+				itemListElement: [
+					{
+						"@type": "ListItem",
+						position: 1,
+						name: resolveTitle("index"),
+						item: SITE_URL,
+					},
+					{
+						"@type": "ListItem",
+						position: 2,
+						name: resolveTitle(slug, title),
+						item: url,
+					},
+				],
+			},
+		],
+	};
+}
+
+function renderSeoHead(title, slug) {
+	const url = escapeMarkup(slugHref(slug, SITE_URL));
+	// "<" is escaped so no title can close the script element early.
+	const jsonLd = JSON.stringify(structuredData(title, slug)).replace(
+		/</g,
+		"\\u003c",
+	);
+	return `<link rel="canonical" href="${url}">
+  <meta property="og:type" content="${slug === "index" ? "website" : "article"}">
+  <meta property="og:site_name" content="${SITE_NAME}">
+  <meta property="og:title" content="${escapeMarkup(title)}">
+  <meta property="og:description" content="${escapeMarkup(SITE_DESCRIPTION)}">
+  <meta property="og:url" content="${url}">
+  <meta name="twitter:card" content="summary">
+  <script type="application/ld+json">${jsonLd}</script>`;
+}
+
+function renderSitemap(slugs) {
+	const urls = slugs.map((slug) => slugHref(slug, SITE_URL)).sort();
+	const entries = urls
+		.map((url) => `  <url><loc>${escapeMarkup(url)}</loc></url>`)
+		.join("\n");
+	return `<?xml version="1.0" encoding="UTF-8"?>
+<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
+${entries}
+</urlset>
+`;
 }
 
 function renderNav(currentSlug) {
@@ -129,7 +240,8 @@ function wrapHTML(title, content, currentSlug, headings = []) {
   <title>${title} — Refract</title>
   <link rel="stylesheet" href="${BASE}style.css?v=${ASSET_VERSION}">
   <link rel="icon" type="image/svg+xml" href="data:image/svg+xml,<svg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 16 16'><style>path{fill:%2307090f}@media (prefers-color-scheme:dark){path{fill:%23e2e4ed}}</style><path d='M8 2 15 14H1z'/></svg>">
-  <meta name="description" content="Refract — the open claim-history layer for public knowledge. Deterministic event stream of claims, sources, and disputes across revision histories.">
+  <meta name="description" content="${SITE_DESCRIPTION}">
+  ${renderSeoHead(title, currentSlug)}
 </head>
 <body>
   <div id="progress" aria-hidden="true"></div>
@@ -590,6 +702,12 @@ async function build() {
 		join(DIST_DIR, "search-index.json"),
 		JSON.stringify(searchIndex, null, 2),
 	);
+	// No <lastmod>: the deploy checkout is shallow, so git history cannot date
+	// individual pages there.
+	await writeFile(
+		join(DIST_DIR, "sitemap.xml"),
+		renderSitemap(files.map((file) => file.slug)),
+	);
 
 	for (const asset of assets) {
 		const dest = join(DIST_DIR, asset.rel);

tests/seo.test.ts

@@ -0,0 +1,187 @@
+import { execFile } from "node:child_process";
+import { existsSync } from "node:fs";
+import { readFile } from "node:fs/promises";
+import { join, resolve } from "node:path";
+import { promisify } from "node:util";
+import { beforeAll, describe, expect, it } from "vitest";
+
+const ROOT_DIR = resolve(__dirname, "..");
+const DIST_DIR = join(ROOT_DIR, "dist");
+const SITE_URL = "https://refract-org.github.io/refract-docs/";
+const execFileAsync = promisify(execFile);
+
+type Page = { slug: string; title: string; html: string };
+type JsonLdNode = Record<string, unknown> & { "@type": string };
+
+function canonicalFor(slug: string): string {
+	return slug === "index" ? SITE_URL : `${SITE_URL}${slug}/`;
+}
+
+function decodeEntities(text: string): string {
+	return text
+		.replace(/&quot;/g, '"')
+		.replace(/&#39;/g, "'")
+		.replace(/&lt;/g, "<")
+		.replace(/&gt;/g, ">")
+		.replace(/&amp;/g, "&");
+}
+
+function metaContents(html: string, attr: string, key: string): string[] {
+	const pattern = new RegExp(`<meta ${attr}="${key}" content="([^"]*)">`, "g");
+	return Array.from(html.matchAll(pattern), (match) =>
+		decodeEntities(match[1]),
+	);
+}
+
+function canonicals(html: string): string[] {
+	return Array.from(
+		html.matchAll(/<link rel="canonical" href="([^"]*)">/g),
+		(match) => decodeEntities(match[1]),
+	);
+}
+
+function jsonLdGraph(html: string): JsonLdNode[] {
+	const blocks = Array.from(
+		html.matchAll(/<script type="application\/ld\+json">([\s\S]*?)<\/script>/g),
+		(match) => JSON.parse(match[1]),
+	);
+	expect(blocks).toHaveLength(1);
+	expect(blocks[0]["@context"]).toBe("https://schema.org");
+	return blocks[0]["@graph"];
+}
+
+function nodeOfType(graph: JsonLdNode[], type: string): JsonLdNode {
+	const node = graph.find((entry) => entry["@type"] === type);
+	expect(node, `missing ${type} node`).toBeDefined();
+	return node as JsonLdNode;
+}
+
+describe("technical SEO", () => {
+	const pages: Page[] = [];
+
+	beforeAll(async () => {
+		await execFileAsync(process.execPath, ["build.mjs"], { cwd: ROOT_DIR });
+		// The search index lists every rendered page; claims-e2e checks that.
+		const searchIndex = JSON.parse(
+			await readFile(join(DIST_DIR, "search-index.json"), "utf-8"),
+		) as Array<{ slug: string; title: string }>;
+		for (const { slug, title } of searchIndex) {
+			const file =
+				slug === "index"
+					? join(DIST_DIR, "index.html")
+					: join(DIST_DIR, slug, "index.html");
+			pages.push({ slug, title, html: await readFile(file, "utf-8") });
+		}
+		expect(pages.length).toBeGreaterThan(1);
+	});
+
+	it("gives every page one canonical URL, at the address it is served from", () => {
+		for (const { slug, html } of pages) {
+			expect(canonicals(html), slug).toEqual([canonicalFor(slug)]);
+		}
+	});
+
+	it("leaves page titles and the meta description as they were", () => {
+		for (const { title, html } of pages) {
+			expect(html).toContain(`<title>${title} — Refract</title>`);
+			expect(metaContents(html, "name", "description")).toEqual([
+				"Refract — the open claim-history layer for public knowledge. Deterministic event stream of claims, sources, and disputes across revision histories.",
+			]);
+		}
+	});
+
+	it("emits Open Graph and Twitter card tags that match the page", () => {
+		for (const { slug, title, html } of pages) {
+			const description = metaContents(html, "name", "description");
+			expect(metaContents(html, "property", "og:title"), slug).toEqual([title]);
+			expect(metaContents(html, "property", "og:description"), slug).toEqual(
+				description,
+			);
+			expect(metaContents(html, "property", "og:url"), slug).toEqual([
+				canonicalFor(slug),
+			]);
+			expect(metaContents(html, "property", "og:type"), slug).toEqual([
+				slug === "index" ? "website" : "article",
+			]);
+			expect(metaContents(html, "property", "og:site_name"), slug).toEqual([
+				"Refract",
+			]);
+			expect(metaContents(html, "name", "twitter:card"), slug).toEqual([
+				"summary",
+			]);
+		}
+	});
+
+	it("escapes titles in attributes and JSON-LD", () => {
+		const compare = pages.find((page) => page.slug === "compare");
+		expect(compare?.title).toContain("'");
+		expect(compare?.html).toContain("Wikipedia&#39;s page history");
+		for (const { slug, html } of pages) {
+			for (const [, json] of html.matchAll(
+				/<script type="application\/ld\+json">([\s\S]*?)<\/script>/g,
+			)) {
+				expect(json, slug).not.toContain("<");
+			}
+		}
+	});
+
+	it("describes the home page as a WebSite for the Refract source code", () => {
+		const home = pages.find((page) => page.slug === "index");
+		const graph = jsonLdGraph(home?.html ?? "");
+		const website = nodeOfType(graph, "WebSite");
+		const source = nodeOfType(graph, "SoftwareSourceCode");
+
+		expect(website).toMatchObject({ name: "Refract", url: SITE_URL });
+		expect(website.description).toBe(
+			metaContents(home?.html ?? "", "name", "description")[0],
+		);
+		expect(source.codeRepository).toBe(
+			"https://github.com/refract-org/refract",
+		);
+		expect(website.about).toEqual({ "@id": source["@id"] });
+	});
+
+	it("describes every doc page as a TechArticle with a breadcrumb trail", () => {
+		for (const { slug, title, html } of pages) {
+			if (slug === "index") continue;
+			const url = canonicalFor(slug);
+			const graph = jsonLdGraph(html);
+			const article = nodeOfType(graph, "TechArticle");
+			const breadcrumbs = nodeOfType(graph, "BreadcrumbList");
+
+			expect(article, slug).toMatchObject({
+				headline: title,
+				description: metaContents(html, "name", "description")[0],
+				url,
+				isPartOf: { "@type": "WebSite", url: SITE_URL },
+			});
+			expect(breadcrumbs.itemListElement, slug).toEqual([
+				expect.objectContaining({ position: 1, name: "Home", item: SITE_URL }),
+				expect.objectContaining({ position: 2, item: url }),
+			]);
+		}
+	});
+
+	it("lists every page once in sitemap.xml, with no guessed lastmod", async () => {
+		const sitemap = await readFile(join(DIST_DIR, "sitemap.xml"), "utf-8");
+		const locs = Array.from(sitemap.matchAll(/<loc>([^<]*)<\/loc>/g), (m) =>
+			decodeEntities(m[1]),
+		);
+
+		expect(sitemap.startsWith('<?xml version="1.0" encoding="UTF-8"?>')).toBe(
+			true,
+		);
+		expect(sitemap).toContain(
+			'<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">',
+		);
+		expect(sitemap).not.toContain("<lastmod>");
+		expect(new Set(locs).size).toBe(locs.length);
+		expect([...locs].sort()).toEqual(
+			pages.map((page) => canonicalFor(page.slug)).sort(),
+		);
+		for (const loc of locs) {
+			const diskPath = join(DIST_DIR, loc.slice(SITE_URL.length), "index.html");
+			expect(existsSync(diskPath), loc).toBe(true);
+		}
+	});
+});