From e0f31a9870c4bb15d48ee489cc2e3246d54b13b1 Mon Sep 17 00:00:00 2001 From: sid597 Date: Wed, 19 Aug 2026 23:12:31 +0530 Subject: [PATCH] ENG-2153 Undecorate Roam node titles into core_title on publish Extract {content} from the page title with the node type's format and write it as literal_content.core_title in all three concept producers (publish, periodic sync, full-content). coreTitle is required on CrossAppNode so a producer omitting it fails to compile: upsert_concepts replaces literal_content wholesale, so an omitted key would erase core_title on the next re-upsert. Falls back to the full title when the type has no format or the title does not match it. --- .../utils/__tests__/conceptConversion.test.ts | 38 ++++++++++ .../__tests__/extractContentFromTitle.test.ts | 68 +++++++++++++++++ .../__tests__/publishNodesToGroups.test.ts | 12 ++- .../roamToCrossAppConverters.test.ts | 75 ++++++++++++++++++- apps/roam/src/utils/conceptConversion.ts | 5 ++ .../src/utils/roamToCrossAppConverters.ts | 10 +++ apps/roam/src/utils/syncDgNodesToSupabase.ts | 7 ++ packages/database/src/crossAppContracts.ts | 4 + .../src/crossAppNodeContract.example.ts | 2 + .../database/src/lib/crossAppConverters.ts | 3 + 10 files changed, 221 insertions(+), 3 deletions(-) create mode 100644 apps/roam/src/utils/__tests__/conceptConversion.test.ts create mode 100644 apps/roam/src/utils/__tests__/extractContentFromTitle.test.ts diff --git a/apps/roam/src/utils/__tests__/conceptConversion.test.ts b/apps/roam/src/utils/__tests__/conceptConversion.test.ts new file mode 100644 index 000000000..9f06bfc98 --- /dev/null +++ b/apps/roam/src/utils/__tests__/conceptConversion.test.ts @@ -0,0 +1,38 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; +import type { SupabaseContext } from "~/utils/supabaseContext"; + +vi.mock("~/utils/getBlockProps", () => ({ default: () => ({}) })); +vi.mock("~/utils/getDiscourseNodes", () => ({ default: () => [] })); +vi.mock("~/utils/getDiscourseRelations", () => ({ default: () => [] })); +vi.mock("~/utils/createReifiedBlock", () => ({ + DISCOURSE_GRAPH_PROP_NAME: "discourse-graph", +})); +vi.mock("roamjs-components/queries/getPageTitleByPageUid", () => ({ + default: () => "", +})); + +import { discourseNodeBlockToLocalConcept } from "~/utils/conceptConversion"; + +const context = { spaceId: 42 } as SupabaseContext; + +describe("discourseNodeBlockToLocalConcept", () => { + beforeEach(() => { + (globalThis as { window: unknown }).window = { + roamAlphaAPI: { + q: () => [["author-1", "page-1", 1000, 2000]], + }, + }; + }); + + it("writes the core title into literal_content", () => { + const concept = discourseNodeBlockToLocalConcept(context, { + nodeUid: "node-1", + schemaUid: "schema-1", + text: "CLM - my claim", + coreTitle: "my claim", + }); + expect(concept.literal_content).toEqual({ core_title: "my claim" }); + expect(concept.name).toBe("CLM - my claim"); + expect(concept.source_local_id).toBe("node-1"); + }); +}); diff --git a/apps/roam/src/utils/__tests__/extractContentFromTitle.test.ts b/apps/roam/src/utils/__tests__/extractContentFromTitle.test.ts new file mode 100644 index 000000000..78816c286 --- /dev/null +++ b/apps/roam/src/utils/__tests__/extractContentFromTitle.test.ts @@ -0,0 +1,68 @@ +import { describe, expect, it } from "vitest"; +import extractContentFromTitle from "~/utils/extractContentFromTitle"; + +describe("extractContentFromTitle", () => { + it("extracts the content from a title matching the format", () => { + expect( + extractContentFromTitle("[[CLM]] - my claim", { + format: "[[CLM]] - {content}", + }), + ).toBe("my claim"); + }); + + it("returns the title when the type has no format", () => { + expect(extractContentFromTitle("my claim", { format: "" })).toBe( + "my claim", + ); + }); + + it("returns the title when it does not match the format", () => { + expect( + extractContentFromTitle("random page", { + format: "[[CLM]] - {content}", + }), + ).toBe("random page"); + }); + + it("extracts the content from a format with a {Source} placeholder", () => { + expect( + extractContentFromTitle("[[EVD]] - finding - @smith2020", { + format: "[[EVD]] - {content} - {Source}", + }), + ).toBe("finding"); + }); + + it('keeps a trailing content containing " - " whole', () => { + expect( + extractContentFromTitle("[[CLM]] - a - b", { + format: "[[CLM]] - {content}", + }), + ).toBe("a - b"); + }); + + it('extracts the shortest match when the content contains " - " before another placeholder (accepted for v0)', () => { + expect( + extractContentFromTitle("[[EVD]] - a - b - @smith2020", { + format: "[[EVD]] - {content} - {Source}", + }), + ).toBe("a"); + }); + + it("round trips a title built from the core title", () => { + const coreTitle = "sleep improves memory"; + const simpleFormat = "[[CLM]] - {content}"; + expect( + extractContentFromTitle(simpleFormat.replace("{content}", coreTitle), { + format: simpleFormat, + }), + ).toBe(coreTitle); + + const sourceFormat = "[[EVD]] - {content} - {Source}"; + const title = sourceFormat + .replace("{content}", coreTitle) + .replace("{Source}", "@smith2020"); + expect(extractContentFromTitle(title, { format: sourceFormat })).toBe( + coreTitle, + ); + }); +}); diff --git a/apps/roam/src/utils/__tests__/publishNodesToGroups.test.ts b/apps/roam/src/utils/__tests__/publishNodesToGroups.test.ts index 46dae6534..d869fd4bb 100644 --- a/apps/roam/src/utils/__tests__/publishNodesToGroups.test.ts +++ b/apps/roam/src/utils/__tests__/publishNodesToGroups.test.ts @@ -72,12 +72,15 @@ const claimSchema: DiscourseNode = { const makeCrossAppNode = ({ uid, title, + coreTitle = title, }: { uid: string; title: string; + coreTitle?: string; }): CrossAppNode => ({ localId: uid, nodeType: SCHEMA_UID, + coreTitle, authorId: "user-1", createdAt: new Date("2026-01-02T00:00:00.000Z"), modifiedAt: new Date("2026-01-03T00:00:00.000Z"), @@ -160,7 +163,13 @@ describe("publishNodesToGroups", () => { client, spaceId: SPACE_ID, groupIds: [GROUP_ID], - nodes: [makeCrossAppNode({ uid: "node-1", title: "CLM - new claim" })], + nodes: [ + makeCrossAppNode({ + uid: "node-1", + title: "CLM - new claim", + coreTitle: "new claim", + }), + ], }); expect(rpcCalls).toHaveLength(1); @@ -177,6 +186,7 @@ describe("publishNodesToGroups", () => { source_local_id: "node-1", name: "CLM - new claim", schema_represented_by_local_id: SCHEMA_UID, + literal_content: { core_title: "new claim" }, }); expect(data[1].contents_inline).toEqual([ expect.objectContaining({ diff --git a/apps/roam/src/utils/__tests__/roamToCrossAppConverters.test.ts b/apps/roam/src/utils/__tests__/roamToCrossAppConverters.test.ts index 5596eb602..3afcfda32 100644 --- a/apps/roam/src/utils/__tests__/roamToCrossAppConverters.test.ts +++ b/apps/roam/src/utils/__tests__/roamToCrossAppConverters.test.ts @@ -1,5 +1,10 @@ -import { describe, expect, it, vi } from "vitest"; +import { beforeEach, describe, expect, it, vi } from "vitest"; import type { Json } from "@repo/database/dbTypes"; +import type { DiscourseNode } from "~/utils/getDiscourseNodes"; + +const mocks = vi.hoisted(() => ({ + getDiscourseNodes: vi.fn(), +})); vi.mock("roamjs-components/queries/getFullTreeByParentUid", () => ({ default: () => ({ children: [] }), @@ -8,8 +13,28 @@ vi.mock("roamjs-components/queries/getPageViewType", () => ({ default: () => "bullet", })); vi.mock("~/utils/pageToMarkdown", () => ({ toMarkdown: () => "" })); +vi.mock("~/utils/getDiscourseNodes", () => ({ + default: mocks.getDiscourseNodes, +})); + +import { + fullContentNodeToCrossApp, + nodeUidsWithTypeToCrossApp, +} from "~/utils/roamToCrossAppConverters"; -import { nodeUidsWithTypeToCrossApp } from "~/utils/roamToCrossAppConverters"; +const claimSchema: DiscourseNode = { + type: "schema-1", + text: "Claim", + shortcut: "C", + specification: [], + backedBy: "user", + canvasSettings: {}, + format: "CLM - {content}", +}; + +beforeEach(() => { + mocks.getDiscourseNodes.mockReturnValue([claimSchema]); +}); const USER_ROW = { ":db/id": 5, ":user/uid": "user-1" }; @@ -60,3 +85,49 @@ describe("nodeUidsWithTypeToCrossApp timestamps", () => { expect(node.modifiedAt).toEqual(new Date(1000)); }); }); + +describe("nodeUidsWithTypeToCrossApp coreTitle", () => { + it("extracts the content from a title matching the node type's format", async () => { + const node = await convertRow(baseRow); + expect(node.coreTitle).toBe("claim"); + }); + + it("keeps the whole title when the node type is unknown", async () => { + mocks.getDiscourseNodes.mockReturnValue([]); + const node = await convertRow(baseRow); + expect(node.coreTitle).toBe("CLM - claim"); + }); +}); + +describe("fullContentNodeToCrossApp coreTitle", () => { + const baseNode = { + author_local_id: "user-1", + source_local_id: "node-1", + created: 1000, + last_modified: 2000, + node_type_id: "schema-1", + text: "CLM - claim", + }; + + it("extracts the content from the title", () => { + const node = fullContentNodeToCrossApp(baseNode); + expect(node.coreTitle).toBe("claim"); + }); + + it("extracts from the page title when node_title is present", () => { + const node = fullContentNodeToCrossApp({ + ...baseNode, + text: "some block text", + node_title: "CLM - claim", + }); + expect(node.coreTitle).toBe("claim"); + }); + + it("keeps the whole title when it does not match the format", () => { + const node = fullContentNodeToCrossApp({ + ...baseNode, + text: "unrelated title", + }); + expect(node.coreTitle).toBe("unrelated title"); + }); +}); diff --git a/apps/roam/src/utils/conceptConversion.ts b/apps/roam/src/utils/conceptConversion.ts index 83abd1a14..35c8f76a5 100644 --- a/apps/roam/src/utils/conceptConversion.ts +++ b/apps/roam/src/utils/conceptConversion.ts @@ -104,10 +104,12 @@ export const discourseNodeBlockToLocalConcept = ( nodeUid, schemaUid, text, + coreTitle, }: { nodeUid: string; schemaUid: string; text: string; + coreTitle: string; }, ): LocalConceptDataInput => { return { @@ -116,6 +118,9 @@ export const discourseNodeBlockToLocalConcept = ( source_local_id: nodeUid, schema_represented_by_local_id: schemaUid, is_schema: false, + literal_content: { + core_title: coreTitle, + }, /* eslint-enable @typescript-eslint/naming-convention */ ...getNodeExtraData(nodeUid), }; diff --git a/apps/roam/src/utils/roamToCrossAppConverters.ts b/apps/roam/src/utils/roamToCrossAppConverters.ts index a3c33f399..2cd7c00cf 100644 --- a/apps/roam/src/utils/roamToCrossAppConverters.ts +++ b/apps/roam/src/utils/roamToCrossAppConverters.ts @@ -15,6 +15,14 @@ import { toMarkdown } from "./pageToMarkdown"; import getFullTreeByParentUid from "roamjs-components/queries/getFullTreeByParentUid"; import getPageViewType from "roamjs-components/queries/getPageViewType"; import { contentTypes } from "@repo/content-model"; +import getDiscourseNodes from "./getDiscourseNodes"; +import extractContentFromTitle from "./extractContentFromTitle"; + +const getCoreTitle = (title: string, nodeTypeUid: string): string => { + const format = + getDiscourseNodes().find((node) => node.type === nodeTypeUid)?.format ?? ""; + return extractContentFromTitle(title, { format }); +}; const FULL_MARKDOWN_OPTS = { refs: true, @@ -73,6 +81,7 @@ export const fullContentNodeToCrossApp = ( createdAt: new Date(node.created || Date.now()), modifiedAt: new Date(node.last_modified || Date.now()), nodeType: node.node_type_id, + coreTitle: getCoreTitle(title, node.node_type_id), content: { direct: { localId: node.source_local_id, @@ -122,6 +131,7 @@ export const nodeUidsWithTypeToCrossApp = async ( authorId: userUid, createdAt: new Date(createdTime), modifiedAt: new Date(Math.max(editTime, pageEditTime)), + coreTitle: getCoreTitle(title, typesByUid[uid]), content: { direct: { localId: uid, diff --git a/apps/roam/src/utils/syncDgNodesToSupabase.ts b/apps/roam/src/utils/syncDgNodesToSupabase.ts index 5427398e9..d0c90168d 100644 --- a/apps/roam/src/utils/syncDgNodesToSupabase.ts +++ b/apps/roam/src/utils/syncDgNodesToSupabase.ts @@ -4,6 +4,7 @@ import { nodeTypeSince, } from "./getAllDiscourseNodesSince"; import getDiscourseNodeFormatExpression from "./getDiscourseNodeFormatExpression"; +import extractContentFromTitle from "./extractContentFromTitle"; import { cleanupOrphanedNodes } from "./cleanupOrphanedNodes"; import { getLoggedInClient, @@ -667,11 +668,17 @@ export const convertDgToSupabaseConcepts = async ({ return discourseNodeSchemaToLocalConcept(context, node); }); + const formatByNodeTypeUid = new Map( + allNodeTypes.map((nodeType) => [nodeType.type, nodeType.format]), + ); const nodeBlockToLocalConcepts = nodesSince.map((node) => { const localConcept = discourseNodeBlockToLocalConcept(context, { nodeUid: node.source_local_id, schemaUid: node.type, text: node.node_title ? `${node.node_title} ${node.text}` : node.text, + coreTitle: extractContentFromTitle(node.node_title ?? node.text, { + format: formatByNodeTypeUid.get(node.type) ?? "", + }), }); return localConcept; }); diff --git a/packages/database/src/crossAppContracts.ts b/packages/database/src/crossAppContracts.ts index c1bf4268a..d5f1ea950 100644 --- a/packages/database/src/crossAppContracts.ts +++ b/packages/database/src/crossAppContracts.ts @@ -74,6 +74,10 @@ type InlineCrossAppTypedContent = InlineCrossAppContent & { // A node instance export type CrossAppNode = CrossAppBase & { nodeType: LocalId; + // The title stripped of the node type's title format ("[[CLM]] - {content}" + // -> the {content} part). Equals the title when the type has no format or + // the title does not match it. + coreTitle: string; content: { direct: InlineCrossAppContent; full?: InlineCrossAppTypedContent; diff --git a/packages/database/src/crossAppNodeContract.example.ts b/packages/database/src/crossAppNodeContract.example.ts index 33bdce9c3..e32ec31d2 100644 --- a/packages/database/src/crossAppNodeContract.example.ts +++ b/packages/database/src/crossAppNodeContract.example.ts @@ -14,6 +14,7 @@ Multiple studies show that sleep after learning strengthens memory traces. export const roamOriginNodeExample: CrossAppNode = { localId: ROAM_SOURCE_NODE_ID, nodeType: "rCLM0schema", + coreTitle: "Sleep improves memory consolidation", content: { direct: { value: "Sleep improves memory consolidation", @@ -47,6 +48,7 @@ Participants with more REM sleep showed better next-day recall. export const obsidianOriginNodeExample: CrossAppNode = { localId: OBSIDIAN_SOURCE_NODE_ID, nodeType: OBSIDIAN_SOURCE_NODE_TYPE_ID, + coreTitle: "REM sleep and recall", content: { direct: { value: "EVD - REM sleep and recall", diff --git a/packages/database/src/lib/crossAppConverters.ts b/packages/database/src/lib/crossAppConverters.ts index 4427c8f3b..e0f7941db 100644 --- a/packages/database/src/lib/crossAppConverters.ts +++ b/packages/database/src/lib/crossAppConverters.ts @@ -75,6 +75,9 @@ export const crossAppNodeToDbConcept = ( name: node.content.direct.value, author_local_id: node.authorId, schema_represented_by_local_id: node.nodeType, + literal_content: { + core_title: node.coreTitle, + }, contents_inline: filterUndefinedArray([ crossAppNodeToDbContent(node, "direct"), crossAppNodeToDbContent(node, "full"),