Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion AGENTS.md
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ This repository translates the Angular documentation site (angular.dev, the `ade

- `i18n/<lang>/src/**` mirrors `origin/adev/src/**`. A translated file `x.md` always has a sibling snapshot `x.en.md` (the English it was translated from) and starts with `<!-- source: <hash> -->` where `<hash>` is the git blob hash of that snapshot (`git hash-object x.en.md`). Same rule for app templates (`x.html` + `x.en.html`, marker `<!-- source: … -->`) and TypeScript overlays (`x.ts` + `x.en.ts`, marker `// source: …`).
- `i18n/<lang>/LANGUAGE.md` holds the language rules (front matter is machine-checked), `glossary.json` the terminology (its `forbidden` renderings fail `pnpm check`), `dictionaries/*.json` the UI strings. Read them before translating anything.
- JSON overlays cannot carry a comment: their marker lives in `<file>.source` next to them (`pnpm status --init` and `--mark` handle it).
- JSON and binary overlays (images, fonts, SVG) cannot carry a comment: their marker lives in `<file>.source` next to them (`pnpm status --init` and `--mark` handle it).
- `tools/` holds the scripts; `tools/adev-patches/` the code patches applied at build time; `.agents/skills/` the skills.

## Skills
Expand Down
2 changes: 1 addition & 1 deletion docs/translating.md
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ For every translated file there are two files side by side:
| `i18n/<lang>/src/content/guide/signals/overview.en.md` | The English page exactly as it was in upstream when you translated it (the *snapshot*). Refreshed by `pnpm update-origin`; never edit it by hand. |
| `i18n/<lang>/src/content/guide/signals/overview.md` | Your translation. First line: `<!-- source: <hash> -->`, the git blob hash of the snapshot (`git hash-object overview.en.md`). |

Same scheme for app templates (`.html` + `.en.html`, same marker) and TypeScript overlays (`.ts` + `.en.ts`, marker `// source: <hash>`). JSON files cannot carry a comment, so their marker lives in a sidecar: `versions.json` + `versions.en.json` + `versions.json.source`; `pnpm status --init` and `--mark` write it for you.
Same scheme for app templates (`.html` + `.en.html`, same marker) and TypeScript overlays (`.ts` + `.en.ts`, marker `// source: <hash>`). JSON files cannot carry a comment, so their marker lives in a sidecar: `versions.json` + `versions.en.json` + `versions.json.source`; `pnpm status --init` and `--mark` write it for you. Binary assets (images, fonts, SVG, PDF, video) work the same way: `logo.png` + `logo.en.png` + `logo.png.source`, bytes untouched. They have no diff class and no mechanical apply: replace the file by hand, then `--mark`.

Why both: the snapshot gives everyone the exact English you worked from, so the next sync can show a readable diff; the hash tells the tooling whether your translation still matches the current snapshot.

Expand Down
29 changes: 23 additions & 6 deletions tools/commands/check.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -17,12 +17,12 @@ import pc from 'picocolors';
import {applyIndexHtmlPairs, checkPipelinePairs, extractNavLabels} from '../lib/adev.mjs';
import {command, langArg, resolveLang, restPositionals} from '../lib/cli.mjs';
import {git} from '../lib/exec.mjs';
import {exists, readText} from '../lib/fs.mjs';
import {exists, readBytes, readText} from '../lib/fs.mjs';
import {lintGlossary, writeGlossaryRules} from '../lib/glossary.mjs';
import {loadDictionaries, loadGlossary, loadLanguageRules, loadSite} from '../lib/languages.mjs';
import {ALERT_KEYWORDS, countAlerts, countCodeFences, countDocsElements, extractHeadings, extractLinkTargets, extractStepTitles, proseLines} from '../lib/markdown.mjs';
import {INDEX_HTML, NAVIGATION_FILE, adevSrc, langSrcDir, originDir, rootDir, toPosix} from '../lib/paths.mjs';
import {blobHash, hasMarkerLine, isSnapshot, listSnapshots, markerStyle, readMarkerOf, sidecarOf, snapshotOf, stripMarker} from '../lib/snapshots.mjs';
import {blobHash, hasMarkerLine, isSnapshot, listOverlayFiles, listSnapshots, looksLikeText, markerStyle, readMarkerOf, sidecarOf, snapshotOf, stripMarker} from '../lib/snapshots.mjs';

export default command({
meta: {name: 'check', description: 'Validate translations, dictionaries and language configuration'},
Expand Down Expand Up @@ -115,7 +115,18 @@ class Checker {
return abs.startsWith(src) ? abs : path.join(src, p);
});
}
return (await listSnapshots(lang)).map((s) => s.translated);
const snapshots = await listSnapshots(lang);
const paired = new Set(snapshots.map((s) => s.rel));
const unpaired = [];
const upstreamSrc = adevSrc(originDir);
if (await exists(upstreamSrc)) {
for (const rel of await listOverlayFiles(lang)) {
if (paired.has(rel) || !(await exists(path.join(upstreamSrc, rel)))) continue;
const file = path.join(src, rel);
if (!markerStyle(file) || looksLikeText(await readBytes(file))) unpaired.push(file);
}
}
return [...snapshots.map((s) => s.translated), ...unpaired];
}

// -------------------------------------------------------------------------------------------
Expand All @@ -134,10 +145,12 @@ class Checker {
this.err(label, `no snapshot ${path.basename(snapshot)} next to it: the next sync would overwrite this translation`);
return;
}
const translated = await readText(file);
const english = await readText(snapshot);
const translatedBuf = await readBytes(file);
const englishBuf = await readBytes(snapshot);
const translated = translatedBuf.toString('utf8');
const english = englishBuf.toString('utf8');
const marker = await readMarkerOf(file, translated);
const snapshotHash = blobHash(english);
const snapshotHash = blobHash(englishBuf);
if (!marker) {
const where = markerStyle(file) ? 'on the first line' : `in ${path.basename(sidecarOf(file))} next to it`;
this.err(label, `no source marker ${where} (expected: source: ${snapshotHash}; pnpm status ${lang} --mark ${rel})`);
Expand All @@ -149,6 +162,10 @@ class Checker {
if ((await exists(adevSrc(originDir))) && !(await exists(upstream))) {
this.warn(label, 'upstream no longer has this file: move or delete the pair');
}
if (!looksLikeText(englishBuf) || !looksLikeText(translatedBuf)) {
if (translatedBuf.equals(englishBuf)) this.warn(label, 'identical to the English snapshot: nothing translated yet');
return;
}

const ext = path.extname(file).toLowerCase();
const body = stripMarker(translated);
Expand Down
19 changes: 14 additions & 5 deletions tools/commands/status.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -16,10 +16,10 @@ import path from 'node:path';
import pc from 'picocolors';
import {command, langArg, resolveLang} from '../lib/cli.mjs';
import {applyMechanicalChange, formatDiff} from '../lib/diff.mjs';
import {exists, readText} from '../lib/fs.mjs';
import {exists, readBytes, readText} from '../lib/fs.mjs';
import {fail, log} from '../lib/log.mjs';
import {adevSrc, langSrcDir, originDir} from '../lib/paths.mjs';
import {blobHash, copySnapshot, normalizeNewlines, readMarkerOf, snapshotOf, stripMarker, writeMarked} from '../lib/snapshots.mjs';
import {blobHash, copySnapshot, looksLikeText, markerStyle, normalizeNewlines, readMarkerOf, snapshotOf, stripMarker, writeMarked} from '../lib/snapshots.mjs';
import {STATES, blobContent, computeStatus} from '../lib/status-core.mjs';

export default command({
Expand Down Expand Up @@ -68,6 +68,10 @@ async function applyMechanical({lang, src}, target) {
log.error(`${rel}: need both the translation and its snapshot`);
continue;
}
if (!looksLikeText(await readBytes(snapshot))) {
log.error(`${rel}: binary file, nothing to apply mechanically: update it by hand, then run --mark`);
continue;
}
const current = await readText(translated);
const marker = await readMarkerOf(translated, current);
const newEnglish = await readText(snapshot);
Expand Down Expand Up @@ -195,7 +199,8 @@ async function init({src, args}, rel) {
const translated = path.join(src, rel);
const snapshot = snapshotOf(translated);
if ((await exists(translated)) && !args.force) fail(`${path.relative(process.cwd(), translated)} exists; use --force to overwrite`);
const content = normalizeNewlines(await readText(upstream));
const content = normalizeNewlines(await readBytes(upstream));
if (markerStyle(translated) && !looksLikeText(content)) fail(`${rel} is binary and ${path.extname(rel)} has no sidecar marker: add the extension to MARKER_STYLES in tools/lib/snapshots.mjs`);
await copySnapshot(upstream, snapshot);
await writeMarked(translated, content, blobHash(content));
log.ok(`snapshot: ${path.relative(process.cwd(), snapshot)}`);
Expand All @@ -206,8 +211,8 @@ async function mark({lang, src}, rel) {
const translated = path.join(src, rel);
const snapshot = snapshotOf(translated);
if (!(await exists(translated)) || !(await exists(snapshot))) fail(`need both ${rel} and its .en snapshot under i18n/${lang}/src`);
const hash = blobHash(await readText(snapshot));
await writeMarked(translated, stripMarker(await readText(translated)), hash);
const hash = blobHash(await readBytes(snapshot));
await writeMarked(translated, markerStyle(translated) ? stripMarker(await readText(translated)) : await readBytes(translated), hash);
log.ok(`marker set to ${hash.slice(0, 10)} on ${rel}. Run: pnpm check ${lang} ${path.relative(process.cwd(), translated)}`);
}

Expand All @@ -216,6 +221,10 @@ async function showDiff({src}, rel) {
const snapshot = snapshotOf(translated);
const upstream = path.join(adevSrc(originDir), rel);
if (!(await exists(snapshot))) fail(`no snapshot for ${rel}; nothing translated yet (use --init)`);
if (!looksLikeText(await readBytes(snapshot))) {
log.warn(`${rel} is binary: no text diff. Compare it with origin/adev/src/${rel}, update the translation, then run --mark`);
return;
}
const snapshotText = await readText(snapshot);
const upstreamText = (await exists(upstream)) ? await readText(upstream) : null;
if (upstreamText !== null && blobHash(upstreamText) !== blobHash(snapshotText)) {
Expand Down
10 changes: 7 additions & 3 deletions tools/lib/snapshots.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,9 @@ const MARKER_STYLES = {
'.js': {open: '// source: ', close: ''},
'.mjs': {open: '// source: ', close: ''},
'.json': null, // JSON cannot carry a comment; the marker lives in <name>.source next to it
...Object.fromEntries(
['.png', '.jpg', '.jpeg', '.gif', '.webp', '.avif', '.ico', '.svg', '.woff', '.woff2', '.ttf', '.otf', '.pdf', '.mp4', '.webm'].map((ext) => [ext, null]),
),
};

/** Sidecar file holding the marker of a file that cannot carry one (JSON): `versions.json.source`. */
Expand Down Expand Up @@ -112,12 +115,13 @@ export async function readMarkerOf(file, content) {
return m ? m[1] : null;
}

/** Write a translated file with its marker set (in the content, or in the sidecar for JSON). */
/** Write a translated file with its marker set (in the content, or in the sidecar for JSON and binary assets). */
export async function writeMarked(file, content, hash) {
if (markerStyle(file)) {
await writeText(file, setMarker(file, content, hash));
await writeText(file, setMarker(file, content.toString('utf8'), hash));
} else {
await writeText(file, content);
if (Buffer.isBuffer(content)) await writeBytes(file, content);
else await writeText(file, content);
await writeText(sidecarOf(file), `${hash}\n`);
}
}
Expand Down
8 changes: 5 additions & 3 deletions tools/lib/status-core.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ import {
} from './adev.mjs';
import {loadDictionaries} from './languages.mjs';
import {INDEX_HTML, NAVIGATION_FILE, adevSrc, langSrcDir, originDir, rootDir} from './paths.mjs';
import {blobHash, listSnapshots, readMarkerOf, stripMarker} from './snapshots.mjs';
import {blobHash, listSnapshots, looksLikeText, readMarkerOf, stripMarker} from './snapshots.mjs';

export const STATES = {
upToDate: 'up-to-date',
Expand Down Expand Up @@ -90,10 +90,12 @@ export async function computeStatus(lang, {withClasses = true} = {}) {
}

async function describePair(rel, snap, originSrc, {withClasses}) {
const snapshotText = await readText(snap.snapshot);
const snapshotHash = blobHash(snapshotText);
const snapshotBuf = await readFile(snap.snapshot);
const snapshotText = snapshotBuf.toString('utf8');
const snapshotHash = blobHash(snapshotBuf);
const upstreamBuf = await readFile(path.join(originSrc, rel));
const upstreamHash = blobHash(upstreamBuf);
if (!looksLikeText(snapshotBuf)) withClasses = false;
const hasTranslation = await exists(snap.translated);
if (!hasTranslation) {
return {rel, state: STATES.orphanSnapshot, snapshotHash};
Expand Down
42 changes: 42 additions & 0 deletions tools/test/cli.test.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -241,6 +241,14 @@ describe('translating a page', () => {
let r = await run(['check', 'fr', lone]);
assert.equal(r.code, 1);
assert.match(r.stdout, /no snapshot NG0100.en.md next to it/);
r = await run(['check', 'fr']);
assert.equal(r.code, 1, 'the whole-language check (what CI runs) finds it too');
assert.match(r.stdout, /no snapshot NG0100.en.md next to it/);
const binary = path.join(root, 'i18n/fr/src/content/reference/license.md');
await write(binary, Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x00, 0x01]));
r = await run(['check', 'fr']);
assert.doesNotMatch(r.stdout, /license\.md/, 'binary overlays have no snapshot workflow yet: not selected');
await (await import('node:fs/promises')).rm(binary);
await write(lone.replace('.md', '.en.md'), await read(path.join(upstreamSrc, 'content/reference/errors/NG0100.md')));
r = await run(['check', 'fr', lone]);
assert.equal(r.code, 1);
Expand Down Expand Up @@ -317,6 +325,40 @@ describe('translating a page', () => {
assert.equal(entry.state, 'up-to-date');
assert.equal(entry.overlay, true);
});
test('binary overlays carry their marker in a sidecar and keep their bytes', async () => {
const rel = 'assets/images/logo.png';
const upstreamPng = Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a, 0x00, 0x00, 0x00, 0x0d, 0xff, 0x0d, 0x0a]);
const localizedPng = Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a, 0x00, 0x00, 0x00, 0x0d, 0xfe]);
const upstreamFile = path.join(upstreamSrc, rel);
const file = path.join(root, 'i18n/fr/src', rel);
const {readFile, rm} = await import('node:fs/promises');
await write(upstreamFile, upstreamPng);
try {
assert.equal((await run(['status', 'fr', '--init', rel])).code, 0);
assert.ok((await readFile(file)).equals(upstreamPng), 'bytes untouched, no marker line');
assert.ok((await readFile(file.replace('.png', '.en.png'))).equals(upstreamPng));
assert.ok(await exists(`${file}.source`));
await write(file, localizedPng);
let r = await run(['check', 'fr']);
assert.equal(r.code, 0, r.out);
let entry = JSON.parse((await run(['status', 'fr', '--json'])).stdout).files.find((f) => f.rel === rel);
assert.equal(entry.state, 'up-to-date');
await rm(`${file}.source`);
r = await run(['check', 'fr', rel]);
assert.equal(r.code, 1);
assert.match(r.stdout, /no source marker in logo.png.source next to it/);
assert.equal((await run(['status', 'fr', '--mark', rel])).code, 0);
assert.ok((await readFile(file)).equals(localizedPng), '--mark leaves the bytes alone');
assert.equal((await run(['check', 'fr', rel])).code, 0);
await write(file.replace('.png', '.en.png'), localizedPng);
entry = JSON.parse((await run(['status', 'fr', '--json'])).stdout).files.find((f) => f.rel === rel);
assert.equal(entry.state, 'stale-snapshot');
assert.equal(entry.class, undefined, 'no diff class for binaries');
assert.match((await run(['status', 'fr', '--diff', rel])).stdout, /is binary: no text diff/);
} finally {
for (const f of [upstreamFile, file, file.replace('.png', '.en.png'), `${file}.source`]) await rm(f, {force: true});
}
});
});

describe('sync: upstream changes', () => {
Expand Down