Skip to content
This repository was archived by the owner on Oct 13, 2026. It is now read-only.
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -16,4 +16,6 @@ jobs:
node-version: 22
cache: npm
- run: npm ci
- run: npm run lint
- run: npm test
- run: npm run build
64 changes: 46 additions & 18 deletions build.mjs
Original file line number Diff line number Diff line change
@@ -1,8 +1,10 @@
import { execFileSync } from "node:child_process";
import { realpathSync } from "node:fs";
import { cp, mkdir, readdir, readFile, rm, writeFile } from "node:fs/promises";
import { basename, dirname, join, posix } from "node:path";
import { fileURLToPath } from "node:url";
import { marked } from "marked";
import { CARD_HEIGHT, CARD_WIDTH, renderSocialCard } from "./social-card.mjs";

const __dirname = dirname(fileURLToPath(import.meta.url));
const DOCS_DIR = join(__dirname, "docs");
Expand All @@ -18,6 +20,10 @@ const SITE_NAME = "Refract";
const SITE_DESCRIPTION =
"A deterministic observation engine for revision histories. It reads a page's edits and emits a typed event for each change.";
const SOURCE_REPOSITORY = "https://github.com/refract-org/refract";
// One Open Graph card for every page, rendered at build time from the name
// and description above.
const SOCIAL_CARD_FILE = "social-card.png";
const SOCIAL_CARD_ALT = `${SITE_NAME}. ${SITE_DESCRIPTION}`;

function assetVersion() {
if (process.env.GITHUB_SHA) return process.env.GITHUB_SHA.slice(0, 12);
Expand Down Expand Up @@ -138,7 +144,11 @@ function renderSeoHead(title, description, slug) {
<meta property="og:title" content="${escapeMarkup(title)}">
<meta property="og:description" content="${escapeMarkup(description)}">
<meta property="og:url" content="${url}">
<meta name="twitter:card" content="summary">
<meta property="og:image" content="${escapeMarkup(`${SITE_URL}${SOCIAL_CARD_FILE}`)}">
<meta property="og:image:width" content="${CARD_WIDTH}">
<meta property="og:image:height" content="${CARD_HEIGHT}">
<meta property="og:image:alt" content="${escapeMarkup(SOCIAL_CARD_ALT)}">
<meta name="twitter:card" content="summary_large_image">
<script type="application/ld+json">${jsonLd}</script>`;
}

Expand Down Expand Up @@ -274,7 +284,7 @@ function wrapHTML(title, description, content, currentSlug, headings = []) {
<head>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>${documentTitle(title)}</title>
<title>${escapeMarkup(documentTitle(title))}</title>
<link rel="stylesheet" href="${BASE}style.css?v=${ASSET_VERSION}">
<link rel="icon" type="image/svg+xml" href="data:image/svg+xml,<svg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 16 16'><style>path{fill:%2307090f}@media (prefers-color-scheme:dark){path{fill:%23e2e4ed}}</style><path d='M8 2 15 14H1z'/></svg>">
<meta name="description" content="${escapeMarkup(description)}">
Expand Down Expand Up @@ -624,24 +634,27 @@ function wrapHTML(title, description, content, currentSlug, headings = []) {
</html>`;
}

function rewriteLink(href, sourceDir = "") {
// Resolves a markdown link against the source file's directory. A link to a
// page (`cli.md`, `cli`, `./cli/`, `../index.md`) becomes the page's
// trailing-slash URL, so it never 301-redirects. A path whose last segment has
// a file extension (`.svg`, `.png`, `.ipynb`) is an asset and keeps its form.
export function rewriteLink(href, sourceDir = "") {
if (!href) return href;
if (href.startsWith("http") || href.startsWith("#")) return href;
if (/^[a-z][a-z\d+.-]*:/i.test(href) || href.startsWith("#")) return href;

const [, rawPath = "", suffix = ""] = href.match(/^([^?#]*)([?#].*)?$/) ?? [];
if (!rawPath) return href;

let path = rawPath.replace(/\.md$/, "/");
if (sourceDir && !path.startsWith("/")) {
path = `${sourceDir}/${path}`;
const path = posix
.normalize(
rawPath.startsWith("/") ? rawPath : posix.join("/", sourceDir, rawPath),
)
.replace(/^\/+|\/+$/g, "");
if (posix.extname(path) && !path.endsWith(".md")) {
return `${BASE}${path}${suffix}`;
}

path = posix.normalize(path);
if (path === "." || path === "index") return `${BASE}${suffix}`;
if (path === "index/") return `${BASE}${suffix}`;

const normalizedPath = path.startsWith("/") ? path.slice(1) : path;
return `${BASE}${normalizedPath}${suffix}`;
const slug = path.replace(/\.md$/, "").replace(/(^|\/)index$/, "");
return `${slugHref(slug || "index")}${suffix}`;
}

async function collectFiles(dir, base = "") {
Expand Down Expand Up @@ -750,6 +763,14 @@ async function build() {
join(DIST_DIR, "sitemap.xml"),
renderSitemap(files.map((file) => file.slug)),
);
await writeFile(
join(DIST_DIR, SOCIAL_CARD_FILE),
renderSocialCard({
name: SITE_NAME,
description: SITE_DESCRIPTION,
address: SITE_URL.replace(/^https?:\/\//, "").replace(/\/$/, ""),
}),
);

for (const asset of assets) {
const dest = join(DIST_DIR, asset.rel);
Expand All @@ -768,7 +789,14 @@ async function build() {
}
}

build().catch((err) => {
console.error(err);
process.exit(1);
});
// Build only when run as a script, so tests can import rewriteLink. Both sides
// are real paths, so a symlinked checkout still builds.
if (
process.argv[1] &&
realpathSync(process.argv[1]) === fileURLToPath(import.meta.url)
) {
build().catch((err) => {
console.error(err);
process.exit(1);
});
}
43 changes: 29 additions & 14 deletions docs/compare.md
Original file line number Diff line number Diff line change
@@ -1,34 +1,49 @@
# Refract vs. Wikipedia's page history
# Wikipedia history tools compared

If you're evaluating whether Refract is worth adopting, this page compares what you
get from Wikipedia's built-in tools vs. what Refract adds.
This page compares tools for reading a Wikipedia article's history: the built-in page history, XTools, WikiBlame, Who Wrote That? and Refract.

Each answers a different question:

- the page history lists every edit with its editor, timestamp, edit summary and diff;
- XTools reports statistics about a page's edits and each editor's share of the current text;
- WikiBlame finds the revision in which a given text was inserted or removed;
- Who Wrote That? highlights who wrote each part of the current text;
- Refract reports each change between revisions as a typed event (sentences, citations, templates, reverts, sections), without editor identity.

## Wikipedia's page history

| Capability | Wikipedia UI | Refract |
Sources: [Help:Page history](https://en.wikipedia.org/wiki/Help:Page_history),
[Manual:Reverts](https://www.mediawiki.org/wiki/Manual:Reverts),
[API:Revisions](https://www.mediawiki.org/wiki/API:Revisions),
[Help:Watchlist](https://en.wikipedia.org/wiki/Help:Watchlist).

| Capability | Page history | Refract |
|---|---|---|
| **View a single revision diff** | Yes — click "prev" on any revision | Yes — every event carries `before`/`after` snapshots |
| **See who edited what** | Yes — username + timestamp per revision | No — Refract observes document change, not editor identity |
| **Find when a sentence first appeared** | Manual — search each revision sequentially | `refract claim "Page" --text "sentence"` → exact revision + timestamp |
| **Find when a sentence first appeared** | Not built in — on English Wikipedia, the history page's "Find addition/removal" link opens WikiBlame | `refract claim "Page" --text "sentence"` → exact revision + timestamp |
| **Track a sentence across its entire lifecycle** | Manual — follow the page history | Automatic — first seen, modified, removed, reintroduced, all timestamped |
| **Detect citation swapping** | Manual — compare each diff's reference section | `citation_replaced` event — `before`/`after` show the old and new source |
| **Detect edit wars** | Manual — look for back-and-forth in history | `revert_detected` + `edit_cluster_detected` — automatic structural detection |
| **Detect edit wars** | Partly — undos, rollbacks and manual reverts are tagged, and so are the edits they revert; spotting back-and-forth is manual | `revert_detected` + `edit_cluster_detected` — automatic structural detection |
| **Correlate article edits with talk page discussion** | Manual — check Talk tab separately | `talk_page_correlated` — Refract checks 7 days before / 3 days after each edit |
| **Compare the same topic across language editions** | Manual — open each wiki separately | `refract diff` — cross-wiki comparison with z-score outlier detection |
| **Query with SQL** | No | DuckDB: `SELECT "eventType", count(*) FROM 'events.jsonl' GROUP BY 1` |
| **Hash an export so others can check it** | No — screenshots are the only record | `refract export --manifest` → Merkle root over the event hashes; the same version over the same revision range reproduces it |
| **Automated monitoring** | No — you check manually | `refract cron` + `refract watch` → Slack, email, webhook alerts |
| **Hash an export so others can check it** | Per revision — each revision has a permanent link, and the API returns a SHA-1 of its content | `refract export --manifest` → Merkle root over the event hashes; the same version over the same revision range reproduces it |
| **Automated monitoring** | Watchlist, with optional email when any watched page changes | `refract cron` + `refract watch` → Slack, email, webhook alerts |
| **AI agent integration** | No | `refract mcp` → Claude Code, Cursor, VS Code can call Refract tools directly |

## Refract vs. other tools

| Tool | What it does | Refract's difference |
|---|---|---|
| **Wikipedia API** | Raw revision data | Refract adds deterministic analysis, event typing, provenance metadata |
| **WikiWho** | Editor-level authorship attribution | Refract tracks claim lifecycle, not editor attribution. Different question. |
| **WhoColor / WikiBlame** | Visual diff highlighting | Refract structures the data for querying, not just viewing |
| **Wikimedia Enterprise** | Bulk API access, commercial licensing | Refract is open-source, deterministic, and runs locally |
| **Internet Archive** | Historical snapshots | Refract produces structured, queryable event streams, not page captures |
| **[XTools](https://www.mediawiki.org/wiki/XTools)** | [Page History](https://www.mediawiki.org/wiki/XTools/Page_History): statistics about a page's edits, such as top editors and edits per year and month. [Authorship](https://www.mediawiki.org/wiki/XTools/Authorship): each editor's share of the current text, by character count. [Blame](https://www.mediawiki.org/wiki/XTools/Blame): the edits that added a given text. Authorship and Blame use WikiWho | Refract reports each change between revisions as a typed event and does not attribute text to editors |
| **[WikiBlame](https://en.wikipedia.org/wiki/User:Flominator/WikiBlame)** | Searches a page's revisions, by binary or linear search, for the one in which a given text was inserted or removed. Works on MediaWiki wikis | `refract claim` follows one sentence through every revision: first seen, modified, removed, reintroduced |
| **[Who Wrote That?](https://www.mediawiki.org/wiki/Who_Wrote_That%3F)** | Browser extension for Chrome and Firefox that highlights who wrote each part of an article's current text, using WikiWho | Refract does not identify editors; it reports what changed at each revision |
| **[WikiWho](https://www.mediawiki.org/wiki/WikiWho)** | Token-level provenance for about 70 Wikipedia editions: who added, removed or reinserted each token, and in which revision | Refract emits events for sentences, citations, templates, reverts and sections rather than tokens, and does not attribute editors |
| **[WhoColor](https://github.com/wikiwho/WhoColor)** | Userscript that colors article text by author, with conflict and age views, built on WikiWho | Refract structures the data for querying, not just viewing |
| **[Wikimedia Enterprise](https://meta.wikimedia.org/wiki/Wikimedia_Enterprise/FAQ)** | APIs that deliver Wikimedia project content: daily project snapshots, single articles on demand, and a real-time stream of updates. Free access, and paid plans with service-level agreements | Refract analyzes a page's past revisions, is open-source, and runs locally |
| **[MediaWiki API](https://www.mediawiki.org/wiki/API:Revisions)** | Raw revision data: metadata, content and a SHA-1 of each revision | Refract adds deterministic analysis, event typing, provenance metadata |
| **[Internet Archive](https://help.archive.org/help/wayback-machine-general-information/)** | Archived captures of web pages from its crawls | Refract produces structured, queryable event streams, not page captures |
| **Custom scrapers** | Ad-hoc revision analysis | Refract has 26 event types, deterministic hashing, and a published SDK |

## What Refract deliberately doesn't do
Expand Down Expand Up @@ -71,4 +86,4 @@ Refract's model evaluation capability — temporal leakage detection, provenance
- You need to see **who** made an edit
- You're browsing page history casually

Refract doesn't replace Wikipedia's UI. It adds capabilities that the UI can't provide — deterministic reproducibility, SQL queryability, hashed exports, and automated monitoring.
Refract doesn't replace the page history. It adds typed change events, SQL queries over them, exports that others can reproduce from the same revisions, and scheduled re-observation with Slack, email or webhook notifications.
2 changes: 1 addition & 1 deletion docs/mcp.md
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
# MCP — Model Context Protocol
# Wikipedia & MediaWiki revision-history MCP server

`refract mcp` starts a JSON-RPC server over stdio that exposes the Refract engine to AI agents via the Model Context Protocol. (`wikihistory mcp` also works.)

Expand Down
2 changes: 1 addition & 1 deletion docs/tutorials/wikipedia-history.md
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
# Tutorial: Track changes on a Wikipedia page
# Track changes to a Wikipedia article over time

## Goal

Expand Down
Loading
Loading