diff --git a/lib/messages/utils.ts b/lib/messages/utils.ts index 2df246ff..65b3d591 100644 --- a/lib/messages/utils.ts +++ b/lib/messages/utils.ts @@ -7,7 +7,8 @@ const SUMMARY_ID_HASH_LENGTH = 16 const DCP_BLOCK_ID_TAG_REGEX = /(])[^>]*>)b\d+(<\/dcp-message-id>)/g const DCP_PAIRED_TAG_REGEX = /]*>[\s\S]*?<\/dcp[^>]*>/gi const DCP_UNPAIRED_TAG_REGEX = /<\/?dcp[^>]*>/gi -const HALLUCINATED_PARAMETER_SUFFIX_REGEX = /\nm\d+<\/parameter>\s*$/ +const INJECTED_MESSAGE_ID_SUFFIX_REGEX = /(?<=\n)]*>m\d+<\/dcp-message-id>\s*$/ +const HALLUCINATED_PARAMETER_SUFFIX_REGEX = /(?<=\n)m\d+<\/parameter>\s*$/ const generateStableId = (prefix: string, seed: string): string => { const hash = createHash("sha256").update(seed).digest("hex").slice(0, SUMMARY_ID_HASH_LENGTH) @@ -164,8 +165,12 @@ export const replaceBlockIdsWithBlocked = (text: string): string => { } export const stripHallucinationsFromString = (text: string): string => { - const withoutHallucinatedParameter = text.replace(HALLUCINATED_PARAMETER_SUFFIX_REGEX, "") - return withoutHallucinatedParameter.replace(DCP_PAIRED_TAG_REGEX, "").replace(DCP_UNPAIRED_TAG_REGEX, "") + const withoutKnownSuffixes = text + .replace(INJECTED_MESSAGE_ID_SUFFIX_REGEX, "") + .replace(HALLUCINATED_PARAMETER_SUFFIX_REGEX, "") + return withoutKnownSuffixes + .replace(DCP_PAIRED_TAG_REGEX, "") + .replace(DCP_UNPAIRED_TAG_REGEX, "") } export const stripHallucinations = (messages: WithParts[]): void => { diff --git a/package-lock.json b/package-lock.json index 2983b1ca..d90568eb 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1641,9 +1641,9 @@ } }, "node_modules/brace-expansion": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.1.1.tgz", - "integrity": "sha512-WR1cURNjuvBLMZBMbqM0UoE+WAfdUcEV1ccD8PVBVOI+Z3ND4+SZbN8RsfT2bMuG1qwz5RFvPukSZm5fF2D5eA==", + "version": "2.1.4", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.1.4.tgz", + "integrity": "sha512-hGfVzPxthbf3+2yjg/RBs60cB0FhqBS/zvdV/4wn4/BmN0bNMMHPc4V/BbFieqf1TKAGGAHnY4eSjajCl0f2Xg==", "license": "MIT", "dependencies": { "balanced-match": "^1.0.0" diff --git a/tests/message-priority.test.ts b/tests/message-priority.test.ts index 4d5b2375..d6105eae 100644 --- a/tests/message-priority.test.ts +++ b/tests/message-priority.test.ts @@ -814,6 +814,30 @@ test("hallucination stripping does not affect non-dcp tags", async () => { ) }) +test("hallucination stripping preserves content when dcp-message-id is mentioned in text (issue #556)", () => { + const input = + "The tag called `` is used to track messages. " + + "This text should survive.\n\n" + + "m0369" + + assert.equal( + stripHallucinationsFromString(input), + "The tag called `` is used to track messages. This text should survive.\n\n", + ) +}) + +test("hallucination stripping handles priority on injected message-id suffixes", () => { + const input = + "The tag called `` is used to track messages. " + + "This text should survive.\n\n" + + 'm0370' + + assert.equal( + stripHallucinationsFromString(input), + "The tag called `` is used to track messages. This text should survive.\n\n", + ) +}) + test("hallucination stripping removes trailing mXXXX artifact (issue #555)", () => { assert.equal( stripHallucinationsFromString("Total: maybe 20 lines changed.\n\nm0340\n\n"),