Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
107 changes: 107 additions & 0 deletions scripts/tests/translation-provider.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ import {
createLiteralBacktickProvider,
createLiteralMarkdownDelimiterProvider,
createInvalidIdRecoveryProvider,
createInvalidJsonRecoveryProvider,
createPreserveTermsProvider,
createSourceLineBreakNormalizationProvider,
createTerminologyProvider,
Expand Down Expand Up @@ -306,6 +307,112 @@ test("invalid-id provider preserves a terminal single-item failure", async () =>
);
});

test("invalid-JSON provider recursively splits a failed multi-item batch", async () => {
const calls: Array<{ instructions: string; size: number }> = [];
const provider = defineProvider({
async translateBatch(request) {
calls.push({
instructions: request.instructions ?? "",
size: request.items.length,
});
if (request.items.length > 1) {
throw new TranslationResponseError(
TranslationErrorCode.ResponseInvalidContainer,
"Chat-completions provider did not return valid JSON.",
);
}
return request.items.map((item) => ({
id: item.id,
text: `译文:${item.text}`,
}));
},
});
const recoveryProvider = createInvalidJsonRecoveryProvider(provider);
const output = await recoveryProvider.translateBatch({
items: [
{ context: {}, id: "one", text: "one" },
{ context: {}, id: "two", text: "two" },
{ context: {}, id: "three", text: "three" },
],
targetLanguage: "zh-CN",
});

assert.deepEqual(output, [
{ id: "one", text: "译文:one" },
{ id: "two", text: "译文:two" },
{ id: "three", text: "译文:three" },
]);
assert.deepEqual(
calls.map((call) => call.size),
[3, 2, 1, 1, 1],
);
for (const call of calls.slice(1)) {
assert.match(call.instructions, /Return exactly one JSON object/u);
}
});

test("invalid-JSON provider adds recovery guidance to a terminal failure", async () => {
const expected = new TranslationResponseError(
TranslationErrorCode.ResponseInvalidContainer,
"Chat-completions provider did not return valid JSON.",
{ retryInstruction: "Keep the original instruction." },
);
const provider = defineProvider({
async translateBatch() {
throw expected;
},
});
const recoveryProvider = createInvalidJsonRecoveryProvider(provider);

await assert.rejects(
recoveryProvider.translateBatch({
items: [{ context: {}, id: "one", text: "one" }],
targetLanguage: "zh-CN",
}),
(error: unknown) => {
assert.ok(error instanceof TranslationResponseError);
assert.equal(error.code, TranslationErrorCode.ResponseInvalidContainer);
assert.equal(error.cause, expected);
assert.match(
error.retryInstruction ?? "",
/Keep the original instruction/u,
);
assert.match(
error.retryInstruction ?? "",
/Return exactly one JSON object/u,
);
return true;
},
);
});

test("invalid-JSON provider does not split unrelated container failures", async () => {
const expected = new TranslationResponseError(
TranslationErrorCode.ResponseInvalidContainer,
"Chat-completions provider response has no choices.",
);
let calls = 0;
const provider = defineProvider({
async translateBatch() {
calls += 1;
throw expected;
},
});
const recoveryProvider = createInvalidJsonRecoveryProvider(provider);

await assert.rejects(
recoveryProvider.translateBatch({
items: [
{ context: {}, id: "one", text: "one" },
{ context: {}, id: "two", text: "two" },
],
targetLanguage: "zh-CN",
}),
(error: unknown) => error === expected,
);
assert.equal(calls, 1);
});

test("source line-break provider flattens matching translated layout breaks", async () => {
const provider = defineProvider({
async translateBatch(request) {
Expand Down
65 changes: 63 additions & 2 deletions scripts/translation/provider.ts
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,13 @@ const LITERAL_BACKTICK_INSTRUCTION =
const INVALID_ID_RECOVERY_INSTRUCTION =
"INVALID ID RECOVERY: This request is a smaller recovery sub-batch. " +
"Return every requested id exactly once, even when adjacent text fragments form one sentence.";
const INVALID_JSON_RECOVERY_INSTRUCTION =
"INVALID JSON RECOVERY: This request is a smaller recovery sub-batch. " +
"Return exactly one JSON object with no prose, reasoning, or Markdown code fences. " +
'Use exactly this shape: {"translations":[{"id":"...","text":"..."}]}. ' +
"Return every requested id exactly once.";
const CHAT_COMPLETIONS_INVALID_JSON_MESSAGE =
"Chat-completions provider did not return valid JSON.";
const LITERAL_BACKTICK_PATTERN = /`+/gu;
const LITERAL_MARKDOWN_DELIMITER_INSTRUCTION =
"{{ET_MD_*}} tokens represent protected literal Markdown emphasis or strikethrough delimiter runs. " +
Expand Down Expand Up @@ -615,6 +622,60 @@ export function createInvalidIdRecoveryProvider<TContext>(
return { name: provider.name, translateBatch };
}

export function createInvalidJsonRecoveryProvider<TContext>(
provider: TranslationProvider<TContext>,
): TranslationProvider<TContext> {
async function translateBatch(
request: TranslationBatchRequest<TContext>,
signal?: AbortSignal,
onActivity?: Parameters<TranslationProvider<TContext>["translateBatch"]>[2],
): Promise<TranslationOutputItem[]> {
try {
return await provider.translateBatch(request, signal, onActivity);
} catch (error) {
const invalidJson =
error instanceof TranslationResponseError &&
error.code === TranslationErrorCode.ResponseInvalidContainer &&
error.message === CHAT_COMPLETIONS_INVALID_JSON_MESSAGE;
if (!invalidJson) throw error;

const instructions = [
request.instructions,
INVALID_JSON_RECOVERY_INSTRUCTION,
]
.filter(Boolean)
.join("\n");
if (request.items.length < 2) {
throw new TranslationResponseError(error.code, error.message, {
cause: error,
details: error.details,
retryInstruction: [
error.retryInstruction,
INVALID_JSON_RECOVERY_INSTRUCTION,
]
.filter(Boolean)
.join("\n"),
});
}

const midpoint = Math.ceil(request.items.length / 2);
const left = await translateBatch(
{ ...request, instructions, items: request.items.slice(0, midpoint) },
signal,
onActivity,
);
const right = await translateBatch(
{ ...request, instructions, items: request.items.slice(midpoint) },
signal,
onActivity,
);
return [...left, ...right];
}
}

return { name: provider.name, translateBatch };
}

export function createTerminologyProvider<TContext>(
provider: TranslationProvider<TContext>,
preserveTerms: readonly string[],
Expand Down Expand Up @@ -951,8 +1012,8 @@ export function createConfiguredProvider(
createLiteralBacktickProvider(
createPreserveTermsProvider(
createTerminologyProvider(
createInvalidIdRecoveryProvider(
provider,
createInvalidJsonRecoveryProvider(
createInvalidIdRecoveryProvider(provider),
),
preserveTerms,
terminology,
Expand Down