diff --git a/AGENTS.md b/AGENTS.md index 5c0420a..60d437d 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -6,7 +6,7 @@ This repository translates the Angular documentation site (angular.dev, the `ade - `i18n//src/**` mirrors `origin/adev/src/**`. A translated file `x.md` always has a sibling snapshot `x.en.md` (the English it was translated from) and starts with `` where `` is the git blob hash of that snapshot (`git hash-object x.en.md`). Same rule for app templates (`x.html` + `x.en.html`, marker ``) and TypeScript overlays (`x.ts` + `x.en.ts`, marker `// source: …`). - `i18n//LANGUAGE.md` holds the language rules (front matter is machine-checked), `glossary.json` the terminology (its `forbidden` renderings fail `pnpm check`), `dictionaries/*.json` the UI strings. Read them before translating anything. -- JSON overlays cannot carry a comment: their marker lives in `.source` next to them (`pnpm status --init` and `--mark` handle it). +- JSON and binary overlays (images, fonts, SVG) cannot carry a comment: their marker lives in `.source` next to them (`pnpm status --init` and `--mark` handle it). - `tools/` holds the scripts; `tools/adev-patches/` the code patches applied at build time; `.agents/skills/` the skills. ## Skills diff --git a/docs/translating.md b/docs/translating.md index acc8027..361e164 100644 --- a/docs/translating.md +++ b/docs/translating.md @@ -11,7 +11,7 @@ For every translated file there are two files side by side: | `i18n//src/content/guide/signals/overview.en.md` | The English page exactly as it was in upstream when you translated it (the *snapshot*). Refreshed by `pnpm update-origin`; never edit it by hand. | | `i18n//src/content/guide/signals/overview.md` | Your translation. First line: ``, the git blob hash of the snapshot (`git hash-object overview.en.md`). | -Same scheme for app templates (`.html` + `.en.html`, same marker) and TypeScript overlays (`.ts` + `.en.ts`, marker `// source: `). JSON files cannot carry a comment, so their marker lives in a sidecar: `versions.json` + `versions.en.json` + `versions.json.source`; `pnpm status --init` and `--mark` write it for you. +Same scheme for app templates (`.html` + `.en.html`, same marker) and TypeScript overlays (`.ts` + `.en.ts`, marker `// source: `). JSON files cannot carry a comment, so their marker lives in a sidecar: `versions.json` + `versions.en.json` + `versions.json.source`; `pnpm status --init` and `--mark` write it for you. Binary assets (images, fonts, SVG, PDF, video) work the same way: `logo.png` + `logo.en.png` + `logo.png.source`, bytes untouched. They have no diff class and no mechanical apply: replace the file by hand, then `--mark`. Why both: the snapshot gives everyone the exact English you worked from, so the next sync can show a readable diff; the hash tells the tooling whether your translation still matches the current snapshot. diff --git a/tools/commands/check.mjs b/tools/commands/check.mjs index bdd5cab..a17ec03 100644 --- a/tools/commands/check.mjs +++ b/tools/commands/check.mjs @@ -17,12 +17,12 @@ import pc from 'picocolors'; import {applyIndexHtmlPairs, checkPipelinePairs, extractNavLabels} from '../lib/adev.mjs'; import {command, langArg, resolveLang, restPositionals} from '../lib/cli.mjs'; import {git} from '../lib/exec.mjs'; -import {exists, readText} from '../lib/fs.mjs'; +import {exists, readBytes, readText} from '../lib/fs.mjs'; import {lintGlossary, writeGlossaryRules} from '../lib/glossary.mjs'; import {loadDictionaries, loadGlossary, loadLanguageRules, loadSite} from '../lib/languages.mjs'; import {ALERT_KEYWORDS, countAlerts, countCodeFences, countDocsElements, extractHeadings, extractLinkTargets, extractStepTitles, proseLines} from '../lib/markdown.mjs'; import {INDEX_HTML, NAVIGATION_FILE, adevSrc, langSrcDir, originDir, rootDir, toPosix} from '../lib/paths.mjs'; -import {blobHash, hasMarkerLine, isSnapshot, listSnapshots, markerStyle, readMarkerOf, sidecarOf, snapshotOf, stripMarker} from '../lib/snapshots.mjs'; +import {blobHash, hasMarkerLine, isSnapshot, listOverlayFiles, listSnapshots, looksLikeText, markerStyle, readMarkerOf, sidecarOf, snapshotOf, stripMarker} from '../lib/snapshots.mjs'; export default command({ meta: {name: 'check', description: 'Validate translations, dictionaries and language configuration'}, @@ -115,7 +115,18 @@ class Checker { return abs.startsWith(src) ? abs : path.join(src, p); }); } - return (await listSnapshots(lang)).map((s) => s.translated); + const snapshots = await listSnapshots(lang); + const paired = new Set(snapshots.map((s) => s.rel)); + const unpaired = []; + const upstreamSrc = adevSrc(originDir); + if (await exists(upstreamSrc)) { + for (const rel of await listOverlayFiles(lang)) { + if (paired.has(rel) || !(await exists(path.join(upstreamSrc, rel)))) continue; + const file = path.join(src, rel); + if (!markerStyle(file) || looksLikeText(await readBytes(file))) unpaired.push(file); + } + } + return [...snapshots.map((s) => s.translated), ...unpaired]; } // ------------------------------------------------------------------------------------------- @@ -134,10 +145,12 @@ class Checker { this.err(label, `no snapshot ${path.basename(snapshot)} next to it: the next sync would overwrite this translation`); return; } - const translated = await readText(file); - const english = await readText(snapshot); + const translatedBuf = await readBytes(file); + const englishBuf = await readBytes(snapshot); + const translated = translatedBuf.toString('utf8'); + const english = englishBuf.toString('utf8'); const marker = await readMarkerOf(file, translated); - const snapshotHash = blobHash(english); + const snapshotHash = blobHash(englishBuf); if (!marker) { const where = markerStyle(file) ? 'on the first line' : `in ${path.basename(sidecarOf(file))} next to it`; this.err(label, `no source marker ${where} (expected: source: ${snapshotHash}; pnpm status ${lang} --mark ${rel})`); @@ -149,6 +162,10 @@ class Checker { if ((await exists(adevSrc(originDir))) && !(await exists(upstream))) { this.warn(label, 'upstream no longer has this file: move or delete the pair'); } + if (!looksLikeText(englishBuf) || !looksLikeText(translatedBuf)) { + if (translatedBuf.equals(englishBuf)) this.warn(label, 'identical to the English snapshot: nothing translated yet'); + return; + } const ext = path.extname(file).toLowerCase(); const body = stripMarker(translated); diff --git a/tools/commands/status.mjs b/tools/commands/status.mjs index 83cddc9..963bcb7 100644 --- a/tools/commands/status.mjs +++ b/tools/commands/status.mjs @@ -16,10 +16,10 @@ import path from 'node:path'; import pc from 'picocolors'; import {command, langArg, resolveLang} from '../lib/cli.mjs'; import {applyMechanicalChange, formatDiff} from '../lib/diff.mjs'; -import {exists, readText} from '../lib/fs.mjs'; +import {exists, readBytes, readText} from '../lib/fs.mjs'; import {fail, log} from '../lib/log.mjs'; import {adevSrc, langSrcDir, originDir} from '../lib/paths.mjs'; -import {blobHash, copySnapshot, normalizeNewlines, readMarkerOf, snapshotOf, stripMarker, writeMarked} from '../lib/snapshots.mjs'; +import {blobHash, copySnapshot, looksLikeText, markerStyle, normalizeNewlines, readMarkerOf, snapshotOf, stripMarker, writeMarked} from '../lib/snapshots.mjs'; import {STATES, blobContent, computeStatus} from '../lib/status-core.mjs'; export default command({ @@ -68,6 +68,10 @@ async function applyMechanical({lang, src}, target) { log.error(`${rel}: need both the translation and its snapshot`); continue; } + if (!looksLikeText(await readBytes(snapshot))) { + log.error(`${rel}: binary file, nothing to apply mechanically: update it by hand, then run --mark`); + continue; + } const current = await readText(translated); const marker = await readMarkerOf(translated, current); const newEnglish = await readText(snapshot); @@ -195,7 +199,8 @@ async function init({src, args}, rel) { const translated = path.join(src, rel); const snapshot = snapshotOf(translated); if ((await exists(translated)) && !args.force) fail(`${path.relative(process.cwd(), translated)} exists; use --force to overwrite`); - const content = normalizeNewlines(await readText(upstream)); + const content = normalizeNewlines(await readBytes(upstream)); + if (markerStyle(translated) && !looksLikeText(content)) fail(`${rel} is binary and ${path.extname(rel)} has no sidecar marker: add the extension to MARKER_STYLES in tools/lib/snapshots.mjs`); await copySnapshot(upstream, snapshot); await writeMarked(translated, content, blobHash(content)); log.ok(`snapshot: ${path.relative(process.cwd(), snapshot)}`); @@ -206,8 +211,8 @@ async function mark({lang, src}, rel) { const translated = path.join(src, rel); const snapshot = snapshotOf(translated); if (!(await exists(translated)) || !(await exists(snapshot))) fail(`need both ${rel} and its .en snapshot under i18n/${lang}/src`); - const hash = blobHash(await readText(snapshot)); - await writeMarked(translated, stripMarker(await readText(translated)), hash); + const hash = blobHash(await readBytes(snapshot)); + await writeMarked(translated, markerStyle(translated) ? stripMarker(await readText(translated)) : await readBytes(translated), hash); log.ok(`marker set to ${hash.slice(0, 10)} on ${rel}. Run: pnpm check ${lang} ${path.relative(process.cwd(), translated)}`); } @@ -216,6 +221,10 @@ async function showDiff({src}, rel) { const snapshot = snapshotOf(translated); const upstream = path.join(adevSrc(originDir), rel); if (!(await exists(snapshot))) fail(`no snapshot for ${rel}; nothing translated yet (use --init)`); + if (!looksLikeText(await readBytes(snapshot))) { + log.warn(`${rel} is binary: no text diff. Compare it with origin/adev/src/${rel}, update the translation, then run --mark`); + return; + } const snapshotText = await readText(snapshot); const upstreamText = (await exists(upstream)) ? await readText(upstream) : null; if (upstreamText !== null && blobHash(upstreamText) !== blobHash(snapshotText)) { diff --git a/tools/lib/snapshots.mjs b/tools/lib/snapshots.mjs index 2d00f8d..b010d6f 100644 --- a/tools/lib/snapshots.mjs +++ b/tools/lib/snapshots.mjs @@ -57,6 +57,9 @@ const MARKER_STYLES = { '.js': {open: '// source: ', close: ''}, '.mjs': {open: '// source: ', close: ''}, '.json': null, // JSON cannot carry a comment; the marker lives in .source next to it + ...Object.fromEntries( + ['.png', '.jpg', '.jpeg', '.gif', '.webp', '.avif', '.ico', '.svg', '.woff', '.woff2', '.ttf', '.otf', '.pdf', '.mp4', '.webm'].map((ext) => [ext, null]), + ), }; /** Sidecar file holding the marker of a file that cannot carry one (JSON): `versions.json.source`. */ @@ -112,12 +115,13 @@ export async function readMarkerOf(file, content) { return m ? m[1] : null; } -/** Write a translated file with its marker set (in the content, or in the sidecar for JSON). */ +/** Write a translated file with its marker set (in the content, or in the sidecar for JSON and binary assets). */ export async function writeMarked(file, content, hash) { if (markerStyle(file)) { - await writeText(file, setMarker(file, content, hash)); + await writeText(file, setMarker(file, content.toString('utf8'), hash)); } else { - await writeText(file, content); + if (Buffer.isBuffer(content)) await writeBytes(file, content); + else await writeText(file, content); await writeText(sidecarOf(file), `${hash}\n`); } } diff --git a/tools/lib/status-core.mjs b/tools/lib/status-core.mjs index 5c62622..cda95c4 100644 --- a/tools/lib/status-core.mjs +++ b/tools/lib/status-core.mjs @@ -12,7 +12,7 @@ import { } from './adev.mjs'; import {loadDictionaries} from './languages.mjs'; import {INDEX_HTML, NAVIGATION_FILE, adevSrc, langSrcDir, originDir, rootDir} from './paths.mjs'; -import {blobHash, listSnapshots, readMarkerOf, stripMarker} from './snapshots.mjs'; +import {blobHash, listSnapshots, looksLikeText, readMarkerOf, stripMarker} from './snapshots.mjs'; export const STATES = { upToDate: 'up-to-date', @@ -90,10 +90,12 @@ export async function computeStatus(lang, {withClasses = true} = {}) { } async function describePair(rel, snap, originSrc, {withClasses}) { - const snapshotText = await readText(snap.snapshot); - const snapshotHash = blobHash(snapshotText); + const snapshotBuf = await readFile(snap.snapshot); + const snapshotText = snapshotBuf.toString('utf8'); + const snapshotHash = blobHash(snapshotBuf); const upstreamBuf = await readFile(path.join(originSrc, rel)); const upstreamHash = blobHash(upstreamBuf); + if (!looksLikeText(snapshotBuf)) withClasses = false; const hasTranslation = await exists(snap.translated); if (!hasTranslation) { return {rel, state: STATES.orphanSnapshot, snapshotHash}; diff --git a/tools/test/cli.test.mjs b/tools/test/cli.test.mjs index 2bd7659..4d1af5c 100644 --- a/tools/test/cli.test.mjs +++ b/tools/test/cli.test.mjs @@ -241,6 +241,14 @@ describe('translating a page', () => { let r = await run(['check', 'fr', lone]); assert.equal(r.code, 1); assert.match(r.stdout, /no snapshot NG0100.en.md next to it/); + r = await run(['check', 'fr']); + assert.equal(r.code, 1, 'the whole-language check (what CI runs) finds it too'); + assert.match(r.stdout, /no snapshot NG0100.en.md next to it/); + const binary = path.join(root, 'i18n/fr/src/content/reference/license.md'); + await write(binary, Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x00, 0x01])); + r = await run(['check', 'fr']); + assert.doesNotMatch(r.stdout, /license\.md/, 'binary overlays have no snapshot workflow yet: not selected'); + await (await import('node:fs/promises')).rm(binary); await write(lone.replace('.md', '.en.md'), await read(path.join(upstreamSrc, 'content/reference/errors/NG0100.md'))); r = await run(['check', 'fr', lone]); assert.equal(r.code, 1); @@ -317,6 +325,40 @@ describe('translating a page', () => { assert.equal(entry.state, 'up-to-date'); assert.equal(entry.overlay, true); }); + test('binary overlays carry their marker in a sidecar and keep their bytes', async () => { + const rel = 'assets/images/logo.png'; + const upstreamPng = Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a, 0x00, 0x00, 0x00, 0x0d, 0xff, 0x0d, 0x0a]); + const localizedPng = Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a, 0x00, 0x00, 0x00, 0x0d, 0xfe]); + const upstreamFile = path.join(upstreamSrc, rel); + const file = path.join(root, 'i18n/fr/src', rel); + const {readFile, rm} = await import('node:fs/promises'); + await write(upstreamFile, upstreamPng); + try { + assert.equal((await run(['status', 'fr', '--init', rel])).code, 0); + assert.ok((await readFile(file)).equals(upstreamPng), 'bytes untouched, no marker line'); + assert.ok((await readFile(file.replace('.png', '.en.png'))).equals(upstreamPng)); + assert.ok(await exists(`${file}.source`)); + await write(file, localizedPng); + let r = await run(['check', 'fr']); + assert.equal(r.code, 0, r.out); + let entry = JSON.parse((await run(['status', 'fr', '--json'])).stdout).files.find((f) => f.rel === rel); + assert.equal(entry.state, 'up-to-date'); + await rm(`${file}.source`); + r = await run(['check', 'fr', rel]); + assert.equal(r.code, 1); + assert.match(r.stdout, /no source marker in logo.png.source next to it/); + assert.equal((await run(['status', 'fr', '--mark', rel])).code, 0); + assert.ok((await readFile(file)).equals(localizedPng), '--mark leaves the bytes alone'); + assert.equal((await run(['check', 'fr', rel])).code, 0); + await write(file.replace('.png', '.en.png'), localizedPng); + entry = JSON.parse((await run(['status', 'fr', '--json'])).stdout).files.find((f) => f.rel === rel); + assert.equal(entry.state, 'stale-snapshot'); + assert.equal(entry.class, undefined, 'no diff class for binaries'); + assert.match((await run(['status', 'fr', '--diff', rel])).stdout, /is binary: no text diff/); + } finally { + for (const f of [upstreamFile, file, file.replace('.png', '.en.png'), `${file}.source`]) await rm(f, {force: true}); + } + }); }); describe('sync: upstream changes', () => {