From 0519023f07fd907fd86669d2d97560e2b2c2fb40 Mon Sep 17 00:00:00 2001 From: Kanav Jain Date: Thu, 8 Oct 2026 00:48:31 -0500 Subject: [PATCH 1/3] feat(seo): emit canonical URLs, Open Graph tags, JSON-LD and sitemap.xml Every built page now carries a canonical URL at the trailing-slash address GitHub Pages serves it from, Open Graph and twitter:card tags, and one JSON-LD block: WebSite plus SoftwareSourceCode on the home page, TechArticle plus BreadcrumbList on doc pages. The build also writes dist/sitemap.xml listing every page. It has no because the deploy checkout is shallow and cannot date pages from git history. Titles and the meta description are unchanged. Co-Authored-By: Claude Opus 5.5 --- build.mjs | 128 +++++++++++++++++++++++++++++-- tests/seo.test.ts | 187 ++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 310 insertions(+), 5 deletions(-) create mode 100644 tests/seo.test.ts diff --git a/build.mjs b/build.mjs index 72b2c50..5e30517 100644 --- a/build.mjs +++ b/build.mjs @@ -9,6 +9,15 @@ const DOCS_DIR = join(__dirname, "docs"); const DIST_DIR = join(__dirname, "dist"); const ASSETS_DIR = join(__dirname, "assets"); const BASE = process.env.BASE || "/refract-docs/"; +// Absolute address of the published site. Canonical URLs, og:url, JSON-LD +// and the sitemap point here even when BASE is overridden for a local build. +const SITE_URL = ( + process.env.SITE_URL || "https://refract-org.github.io/refract-docs/" +).replace(/\/?$/, "/"); +const SITE_NAME = "Refract"; +const SITE_DESCRIPTION = + "Refract — the open claim-history layer for public knowledge. Deterministic event stream of claims, sources, and disputes across revision histories."; +const SOURCE_REPOSITORY = "https://github.com/refract-org/refract"; function assetVersion() { if (process.env.GITHUB_SHA) return process.env.GITHUB_SHA.slice(0, 12); @@ -28,7 +37,7 @@ const ASSET_VERSION = assetVersion(); let NAV = []; -function resolveTitle(slug) { +function resolveTitle(slug, fallback = basename(slug)) { for (const item of NAV) { if (item.slug === slug) return item.title; if (item.children) { @@ -37,11 +46,113 @@ function resolveTitle(slug) { } } } - return basename(slug); + return fallback; } -function slugHref(slug) { - return slug === "index" ? BASE : `${BASE}${slug}/`; +// GitHub Pages serves dist//index.html at / and 301-redirects the +// slash-less form, so the trailing-slash URL is the one to link and canonicalize. +function slugHref(slug, base = BASE) { + return slug === "index" ? base : `${base}${slug}/`; +} + +// Safe in HTML attribute values and in XML text alike. +function escapeMarkup(text) { + return text + .replace(/&/g, "&") + .replace(//g, ">") + .replace(/"/g, """) + .replace(/'/g, "'"); +} + +function structuredData(title, slug) { + const url = slugHref(slug, SITE_URL); + const website = { + "@type": "WebSite", + "@id": `${SITE_URL}#website`, + name: SITE_NAME, + url: SITE_URL, + }; + if (slug === "index") { + return { + "@context": "https://schema.org", + "@graph": [ + { + ...website, + description: SITE_DESCRIPTION, + inLanguage: "en", + about: { "@id": `${SITE_URL}#source` }, + }, + { + "@type": "SoftwareSourceCode", + "@id": `${SITE_URL}#source`, + name: SITE_NAME, + codeRepository: SOURCE_REPOSITORY, + }, + ], + }; + } + return { + "@context": "https://schema.org", + "@graph": [ + { + "@type": "TechArticle", + "@id": `${url}#article`, + headline: title, + description: SITE_DESCRIPTION, + url, + mainEntityOfPage: url, + inLanguage: "en", + isPartOf: website, + }, + { + "@type": "BreadcrumbList", + itemListElement: [ + { + "@type": "ListItem", + position: 1, + name: resolveTitle("index"), + item: SITE_URL, + }, + { + "@type": "ListItem", + position: 2, + name: resolveTitle(slug, title), + item: url, + }, + ], + }, + ], + }; +} + +function renderSeoHead(title, slug) { + const url = escapeMarkup(slugHref(slug, SITE_URL)); + // "<" is escaped so no title can close the script element early. + const jsonLd = JSON.stringify(structuredData(title, slug)).replace( + / + + + + + + + `; +} + +function renderSitemap(slugs) { + const urls = slugs.map((slug) => slugHref(slug, SITE_URL)).sort(); + const entries = urls + .map((url) => ` ${escapeMarkup(url)}`) + .join("\n"); + return ` + +${entries} + +`; } function renderNav(currentSlug) { @@ -129,7 +240,8 @@ function wrapHTML(title, content, currentSlug, headings = []) { ${title} — Refract - + + ${renderSeoHead(title, currentSlug)} @@ -590,6 +702,12 @@ async function build() { join(DIST_DIR, "search-index.json"), JSON.stringify(searchIndex, null, 2), ); + // No : the deploy checkout is shallow, so git history cannot date + // individual pages there. + await writeFile( + join(DIST_DIR, "sitemap.xml"), + renderSitemap(files.map((file) => file.slug)), + ); for (const asset of assets) { const dest = join(DIST_DIR, asset.rel); diff --git a/tests/seo.test.ts b/tests/seo.test.ts new file mode 100644 index 0000000..2557a13 --- /dev/null +++ b/tests/seo.test.ts @@ -0,0 +1,187 @@ +import { execFile } from "node:child_process"; +import { existsSync } from "node:fs"; +import { readFile } from "node:fs/promises"; +import { join, resolve } from "node:path"; +import { promisify } from "node:util"; +import { beforeAll, describe, expect, it } from "vitest"; + +const ROOT_DIR = resolve(__dirname, ".."); +const DIST_DIR = join(ROOT_DIR, "dist"); +const SITE_URL = "https://refract-org.github.io/refract-docs/"; +const execFileAsync = promisify(execFile); + +type Page = { slug: string; title: string; html: string }; +type JsonLdNode = Record & { "@type": string }; + +function canonicalFor(slug: string): string { + return slug === "index" ? SITE_URL : `${SITE_URL}${slug}/`; +} + +function decodeEntities(text: string): string { + return text + .replace(/"/g, '"') + .replace(/'/g, "'") + .replace(/</g, "<") + .replace(/>/g, ">") + .replace(/&/g, "&"); +} + +function metaContents(html: string, attr: string, key: string): string[] { + const pattern = new RegExp(``, "g"); + return Array.from(html.matchAll(pattern), (match) => + decodeEntities(match[1]), + ); +} + +function canonicals(html: string): string[] { + return Array.from( + html.matchAll(//g), + (match) => decodeEntities(match[1]), + ); +} + +function jsonLdGraph(html: string): JsonLdNode[] { + const blocks = Array.from( + html.matchAll(/`; @@ -179,15 +178,69 @@ function renderNav(currentSlug) { return html; } -function plainText(tokens) { +function plainText(tokens, skipTypes = []) { return tokens .map((token) => { - if (token.tokens) return plainText(token.tokens); + if (skipTypes.includes(token.type)) return ""; + if (token.tokens) return plainText(token.tokens, skipTypes); return token.text ?? token.raw ?? ""; }) .join(""); } +// Inline text of a heading or paragraph with markdown, HTML tags and images +// removed and whitespace collapsed. +function inlineText(tokens) { + return plainText(tokens, ["html", "image"]).replace(/\s+/g, " ").trim(); +} + +function pageTitle(tokens, slug) { + const h1 = tokens.find((t) => t.type === "heading" && t.depth === 1); + return h1 ? inlineText(h1.tokens) : resolveTitle(slug); +} + +const DESCRIPTION_LENGTH = 155; + +function truncateAtWord(text, max) { + if (text.length <= max) return text; + const cut = text.slice(0, max + 1); + const end = cut.lastIndexOf(" "); + return `${cut.slice(0, end > 0 ? end : max).replace(/[\s,;:—–-]+$/, "")}…`; +} + +// A paragraph can stand in for the page when it is prose that ends a sentence. +// That rules out lead-ins to the next block ("Use environment variables +// instead:"), bold-label callouts ("**Note**: …", glossary-style +// "**Term** — …"), parenthetical asides, "See [page]" pointers and images. +function isSummaryParagraph(token, text) { + if (!/[.!?]["'”’)]*$/.test(text)) return false; + if (/^\(.*\)$/.test(text)) return false; + const [first, second] = token.tokens; + if ( + first?.type === "strong" && + second && + (/:$/.test(first.text) || /^\s*[:—–]|^\s+-\s/.test(second.raw)) + ) { + return false; + } + return !(/^See\s/.test(text) && token.tokens.some((t) => t.type === "link")); +} + +// The first prose paragraph after the H1, trimmed for a meta description. +// Only top-level paragraphs count, so lists, tables, code blocks, blockquotes +// and HTML blocks are skipped. Returns "" when the page has none. +function pageSummary(tokens) { + const h1 = tokens.findIndex((t) => t.type === "heading" && t.depth === 1); + for (const token of tokens.slice(h1 + 1)) { + if (token.type !== "paragraph") continue; + const text = inlineText(token.tokens); + if (isSummaryParagraph(token, text)) { + return truncateAtWord(text, DESCRIPTION_LENGTH); + } + } + return ""; +} + function slugifyHeading(text) { return text .toLowerCase() @@ -197,28 +250,7 @@ function slugifyHeading(text) { .replace(/\s/g, "-"); } -function getExcerpt(markdown) { - // Remove code blocks - let text = markdown.replace(/```[\s\S]*?```/g, ""); - // Remove inline code tags - text = text.replace(/`([^`]+)`/g, "$1"); - // Remove HTML tags - text = text.replace(/<[^>]*>/g, ""); - // Remove markdown links - text = text.replace(/\[([^\]]+)\]\([^)]+\)/g, "$1"); - // Remove headings - text = text.replace(/^#+\s+.+$/gm, ""); - // Get first non-empty lines - const lines = text - .split("\n") - .map((l) => l.trim()) - .filter(Boolean); - if (lines.length === 0) return ""; - const firstLine = lines[0]; - return firstLine.length > 140 ? `${firstLine.slice(0, 137)}...` : firstLine; -} - -function wrapHTML(title, content, currentSlug, headings = []) { +function wrapHTML(title, description, content, currentSlug, headings = []) { let tocHtml = ""; if (headings.length > 0) { tocHtml += `
On this page
`; @@ -240,8 +272,8 @@ function wrapHTML(title, content, currentSlug, headings = []) { ${title} — Refract - - ${renderSeoHead(title, currentSlug)} + + ${renderSeoHead(title, description, currentSlug)} @@ -636,6 +668,7 @@ async function build() { const { files, assets } = await collectFiles(DOCS_DIR); const searchIndex = []; + const fallbackPages = []; const renderer = new marked.Renderer(); let currentSourceDir = ""; let currentHeadingCounts = new Map(); @@ -678,9 +711,13 @@ async function build() { currentHeadingCounts = new Map(); currentHeadings = []; const body = marked.parse(raw, { renderer }); - const h1Match = raw.match(/^#\s+(.+)/m); - const title = h1Match ? h1Match[1] : resolveTitle(file.slug); - const html = wrapHTML(title, body, file.slug, currentHeadings); + const tokens = marked.lexer(raw); + const title = pageTitle(tokens, file.slug); + // The home page keeps the site-wide positioning line. + const summary = file.slug === "index" ? "" : pageSummary(tokens); + if (file.slug !== "index" && !summary) fallbackPages.push(file.slug); + const description = summary || SITE_DESCRIPTION; + const html = wrapHTML(title, description, body, file.slug, currentHeadings); if (file.slug === "index") { await writeFile(join(DIST_DIR, "index.html"), html); @@ -693,7 +730,7 @@ async function build() { searchIndex.push({ title, slug: file.slug, - excerpt: getExcerpt(raw), + excerpt: description, headings: currentHeadings.map((h) => ({ text: h.text, id: h.id })), }); } @@ -719,6 +756,11 @@ async function build() { console.log( `Built ${files.length} pages and copied ${assets.length} assets to dist/`, ); + if (fallbackPages.length > 0) { + console.log( + `No prose paragraph to describe, so these pages use the site description: ${fallbackPages.sort().join(", ")}`, + ); + } } build().catch((err) => { diff --git a/tests/seo.test.ts b/tests/seo.test.ts index 2557a13..3dab8b7 100644 --- a/tests/seo.test.ts +++ b/tests/seo.test.ts @@ -8,6 +8,11 @@ import { beforeAll, describe, expect, it } from "vitest"; const ROOT_DIR = resolve(__dirname, ".."); const DIST_DIR = join(ROOT_DIR, "dist"); const SITE_URL = "https://refract-org.github.io/refract-docs/"; +const SITE_DESCRIPTION = + "Refract — the open claim-history layer for public knowledge. Deterministic event stream of claims, sources, and disputes across revision histories."; +// Pages that use the site description: the home page by design, and pages with +// no prose paragraph to describe them (the build logs these). +const SITE_DESCRIPTION_PAGES = ["glossary", "index"]; const execFileAsync = promisify(execFile); type Page = { slug: string; title: string; html: string }; @@ -33,6 +38,27 @@ function metaContents(html: string, attr: string, key: string): string[] { ); } +function descriptionOf(html: string): string { + return metaContents(html, "name", "description")[0] ?? ""; +} + +// Visible text of an HTML fragment: tags dropped, entities decoded, +// whitespace collapsed. +function textOf(fragment: string): string { + return decodeEntities(fragment.replace(/<[^>]+>/g, "")) + .replace(/\s+/g, " ") + .trim(); +} + +function stringsIn(value: unknown): string[] { + if (typeof value === "string") return [value]; + if (Array.isArray(value)) return value.flatMap(stringsIn); + if (value && typeof value === "object") { + return Object.values(value).flatMap(stringsIn); + } + return []; +} + function canonicals(html: string): string[] { return Array.from( html.matchAll(//g), @@ -81,12 +107,77 @@ describe("technical SEO", () => { } }); - it("leaves page titles and the meta description as they were", () => { - for (const { title, html } of pages) { - expect(html).toContain(`${title} — Refract`); - expect(metaContents(html, "name", "description")).toEqual([ - "Refract — the open claim-history layer for public knowledge. Deterministic event stream of claims, sources, and disputes across revision histories.", - ]); + it("titles each page with the visible text of its H1", () => { + for (const { slug, title, html } of pages) { + const h1 = html.match(/

]*>([\s\S]*?)<\/h1>/)?.[1] ?? ""; + expect(title, slug).toBe(textOf(h1)); + expect(html, slug).toContain(`${title} — Refract`); + } + const crossWiki = pages.find((page) => + page.slug.endsWith("cross-wiki-diff"), + ); + expect(crossWiki?.title).toBe( + "Tutorial: Cross-wiki comparison with refract diff", + ); + }); + + it("keeps the site description on the home page", () => { + const home = pages.find((page) => page.slug === "index"); + expect(descriptionOf(home?.html ?? "")).toBe(SITE_DESCRIPTION); + }); + + it("gives each doc page its own description, apart from listed fallbacks", () => { + const fallbacks = pages + .filter(({ html }) => descriptionOf(html) === SITE_DESCRIPTION) + .map(({ slug }) => slug) + .sort(); + expect(fallbacks).toEqual(SITE_DESCRIPTION_PAGES); + + const descriptions = pages + .filter(({ slug }) => !SITE_DESCRIPTION_PAGES.includes(slug)) + .map(({ html }) => descriptionOf(html)); + expect(new Set(descriptions).size).toBe(descriptions.length); + }); + + it("takes each description from the page's own text, cut at a word boundary", () => { + for (const { slug, html } of pages) { + if (SITE_DESCRIPTION_PAGES.includes(slug)) continue; + const description = descriptionOf(html); + const main = html.match(/
([\s\S]*?)<\/main>/); + const body = textOf(main?.[1] ?? ""); + const cut = description.endsWith("…"); + const prefix = cut ? description.slice(0, -1) : description; + const at = body.indexOf(prefix); + + expect(at, `${slug}: ${description}`).toBeGreaterThanOrEqual(0); + if (cut) { + expect(body[at + prefix.length], slug).toMatch(/[^\p{L}\p{N}]/u); + } + } + }); + + it("keeps every description to 160 characters or fewer", () => { + for (const { slug, html } of pages) { + const description = descriptionOf(html); + expect(description.length, `${slug}: ${description}`).toBeLessThanOrEqual( + 160, + ); + expect(description.length, slug).toBeGreaterThan(0); + } + }); + + it("leaves no markdown backticks or asterisks in titles or descriptions", () => { + for (const { slug, html } of pages) { + const texts = [ + textOf(html.match(/([\s\S]*?)<\/title>/)?.[1] ?? ""), + descriptionOf(html), + ...metaContents(html, "property", "og:title"), + ...metaContents(html, "property", "og:description"), + ...stringsIn(jsonLdGraph(html)), + ]; + for (const text of texts) { + expect(text, slug).not.toMatch(/[`*]/); + } } }); From 6965eeca27c9f87fb98bccb7d7db19044f85e334 Mon Sep 17 00:00:00 2001 From: Kanav Jain <kanavj@gmail.com> Date: Thu, 8 Oct 2026 11:27:05 -0500 Subject: [PATCH 3/3] docs: describe Refract as a deterministic observation engine everywhere MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The site described Refract three ways: the home H1 said "observation layer for model evaluation", the meta description and sidebar said "the open claim-history layer for public knowledge", and AGENTS.md says to describe it as a deterministic observation engine, which is what the README leads with. The home H1, the site description, the sidebar tagline and the opening sentences of Why Refract and Concepts now use the README line. Those openings are the pages' meta descriptions now. The home <title> no longer appends "— Refract" to an H1 that already starts with the name. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --- build.mjs | 11 ++++++++--- docs/concepts.md | 4 ++-- docs/index.md | 2 +- docs/why-refract.md | 4 ++-- tests/seo.test.ts | 7 +++++-- 5 files changed, 18 insertions(+), 10 deletions(-) diff --git a/build.mjs b/build.mjs index ee08d76..a1dd053 100644 --- a/build.mjs +++ b/build.mjs @@ -16,7 +16,7 @@ const SITE_URL = ( ).replace(/\/?$/, "/"); const SITE_NAME = "Refract"; const SITE_DESCRIPTION = - "Refract — the open claim-history layer for public knowledge. Deterministic event stream of claims, sources, and disputes across revision histories."; + "A deterministic observation engine for revision histories. It reads a page's edits and emits a typed event for each change."; const SOURCE_REPOSITORY = "https://github.com/refract-org/refract"; function assetVersion() { @@ -250,6 +250,11 @@ function slugifyHeading(text) { .replace(/\s/g, "-"); } +// A title that already leads with the name doesn't repeat it as a suffix. +function documentTitle(title) { + return title.startsWith(SITE_NAME) ? title : `${title} — ${SITE_NAME}`; +} + function wrapHTML(title, description, content, currentSlug, headings = []) { let tocHtml = ""; if (headings.length > 0) { @@ -269,7 +274,7 @@ function wrapHTML(title, description, content, currentSlug, headings = []) { <head> <meta charset="UTF-8"> <meta name="viewport" content="width=device-width, initial-scale=1.0"> - <title>${title} — Refract + ${documentTitle(title)} @@ -282,7 +287,7 @@ function wrapHTML(title, description, content, currentSlug, headings = []) {