diff --git a/App/backend/local-api-contracts/src/memory-l3-world-model.ts b/App/backend/local-api-contracts/src/memory-l3-world-model.ts index 269d5f023..4d6d85f07 100644 --- a/App/backend/local-api-contracts/src/memory-l3-world-model.ts +++ b/App/backend/local-api-contracts/src/memory-l3-world-model.ts @@ -28,7 +28,7 @@ const L3WorldModelRuntimeNamespaceShape = { workspaceId: OptionalNonEmptyStringSchema, workspacePath: OptionalNonEmptyStringSchema, sessionKey: OptionalNonEmptyStringSchema, - userId: OptionalNonEmptyStringSchema, + userId: OptionalNonEmptyStringSchema.nullable(), tenantId: OptionalNonEmptyStringSchema } as const; diff --git a/App/backend/src/adapters/outbound/skill-writer/codex/tests/target.test.ts b/App/backend/src/adapters/outbound/skill-writer/codex/tests/target.test.ts index 3db0efef8..e161b7f5f 100644 --- a/App/backend/src/adapters/outbound/skill-writer/codex/tests/target.test.ts +++ b/App/backend/src/adapters/outbound/skill-writer/codex/tests/target.test.ts @@ -230,8 +230,10 @@ describe("codex skill target", () => { it("installs a UserPromptSubmit hook that blocks resume commands with top L1 candidates", async () => { const { rootDirectory, memmyConfigPath } = createFixture(); let requestBody: Record | undefined; + let requestCount = 0; let authorization = ""; const server = createServer(async (request: IncomingMessage, response: ServerResponse) => { + requestCount += 1; authorization = String(request.headers.authorization || ""); const url = new URL(request.url || "/", "http://127.0.0.1"); if (request.method === "POST" && url.pathname === "/api/v1/memory/search") { @@ -336,6 +338,11 @@ describe("codex skill target", () => { }; expect(hooksAfter.hooks?.UserPromptSubmit).toBeUndefined(); expect(hooksAfter.hooks?.Stop).toBeUndefined(); + const requestsBeforeRestart = requestCount; + const afterUninstall = await runNodeHook(hookScriptPath, JSON.stringify({hook_event_name:"Stop",session_id:"fresh-process"})); + expect(afterUninstall.status).not.toBe(0); + expect(afterUninstall.stderr).toContain("MODULE_NOT_FOUND"); + expect(requestCount).toBe(requestsBeforeRestart); expect(readFileSync(join(rootDirectory, "AGENTS.md"), "utf8")).toBe(existingTargetFile); expect(existsSync(join(rootDirectory, "skills", "memmy-memory"))).toBe(false); expect(existsSync(join(rootDirectory, "skills", "memmy-resume"))).toBe(false); diff --git a/App/backend/src/adapters/outbound/skill-writer/hermes/target.ts b/App/backend/src/adapters/outbound/skill-writer/hermes/target.ts index 72ce0aafb..eb9cb9124 100644 --- a/App/backend/src/adapters/outbound/skill-writer/hermes/target.ts +++ b/App/backend/src/adapters/outbound/skill-writer/hermes/target.ts @@ -1338,7 +1338,7 @@ def _load_runtime() -> Dict[str, str]: return { "baseUrl": base_url, "token": token, - "userId": _clean_text(app.get("userId")) or _clean_text(memory.get("userId")) or _clean_text(plugin_config.get("userId")) or "local-user", + "userId": "local-user", "workspaceHostId": _clean_text(plugin_config.get("workspaceHostId")), } diff --git a/App/backend/src/adapters/outbound/skill-writer/memmy-runtime-config.ts b/App/backend/src/adapters/outbound/skill-writer/memmy-runtime-config.ts index 58ac07d55..45e728890 100644 --- a/App/backend/src/adapters/outbound/skill-writer/memmy-runtime-config.ts +++ b/App/backend/src/adapters/outbound/skill-writer/memmy-runtime-config.ts @@ -28,7 +28,7 @@ export async function readMemmyMemoryServiceConfig(configPath: string): Promise< token: normalizeString(storage.token) || normalizeString(memmyMemory.token) || normalizeString(legacyStorage.token), - userId: normalizeString(app.userId) || normalizeString(memmyMemory.userId) || "local-user", + userId: "local-user", workspaceHostId: deriveWorkspaceHostId(getOrCreateInstallationId()) }; } diff --git a/App/backend/src/adapters/outbound/skill-writer/templates/tests/memmy-resume-hook.test.ts b/App/backend/src/adapters/outbound/skill-writer/templates/tests/memmy-resume-hook.test.ts index 82278434d..7ef35d6d3 100644 --- a/App/backend/src/adapters/outbound/skill-writer/templates/tests/memmy-resume-hook.test.ts +++ b/App/backend/src/adapters/outbound/skill-writer/templates/tests/memmy-resume-hook.test.ts @@ -294,7 +294,7 @@ describe("memmy resume hook stop capture", () => { workspaceHostId: "a".repeat(64), namespace: { source, - userId: "installed-owner", + userId: "local-user", sessionKey: `${source}-memory-host-session`, }, }); diff --git a/App/backend/src/adapters/outbound/skill-writer/workspace-bridge/runtime.test.ts b/App/backend/src/adapters/outbound/skill-writer/workspace-bridge/runtime.test.ts index 604c72e99..1e2bb66aa 100644 --- a/App/backend/src/adapters/outbound/skill-writer/workspace-bridge/runtime.test.ts +++ b/App/backend/src/adapters/outbound/skill-writer/workspace-bridge/runtime.test.ts @@ -23,7 +23,7 @@ afterEach(() => { }); describe("Memory lifecycle runtime", () => { - it("reads Memory connection and owner settings without a workspace scanning flag", async () => { + it("reads Memory connection with account-independent identity without a workspace scanning flag", async () => { const fixture = createFixture(); const configUrl = pathToFileURL(join(fixture, "memmy-memory-config.json")); const configPath = join(fixture, "config.yaml"); @@ -45,12 +45,12 @@ describe("Memory lifecycle runtime", () => { await expect(readRuntimeConfig(configUrl, true)).resolves.toEqual({ endpoint: "http://127.0.0.1:18888", token: "test-token", - userId: "installed-owner", + userId: "local-user", workspaceHostId: "a".repeat(64), }); }); - it.each([undefined, "pending-session"])("submits the pinned owner with pending Session %s across account changes", async (sessionId) => { + it.each([undefined, "pending-session"])("submits device identity with pending Session %s across account changes", async (sessionId) => { const fixture = createFixture(); const requests: Array<{ path: string; body: Record }> = []; const server = createServer(async (request, response) => { @@ -77,7 +77,7 @@ describe("Memory lifecycle runtime", () => { for (const request of requests) { expect(request.path).toBe("/api/v1/source-turns/complete"); expect(request.body.namespace).toEqual({ - source: "codex", profileId: "work", userId: "installed-owner", sessionKey: "native-conversation", + source: "codex", profileId: "work", userId: "local-user", sessionKey: "native-conversation", }); expect(request.body).toMatchObject({ sourceTurn: { source: "codex", profileId: "work", conversationId: "native-conversation" }, diff --git a/App/backend/src/adapters/outbound/skill-writer/workspace-bridge/runtime.ts b/App/backend/src/adapters/outbound/skill-writer/workspace-bridge/runtime.ts index db3305bc8..346a51b9c 100644 --- a/App/backend/src/adapters/outbound/skill-writer/workspace-bridge/runtime.ts +++ b/App/backend/src/adapters/outbound/skill-writer/workspace-bridge/runtime.ts @@ -91,9 +91,7 @@ export async function readRuntimeConfig(configUrl: URL, pinnedOwner = false): Pr return { endpoint: text(storage.endpoint) || text(memory.endpoint) || text(legacyStorage.endpoint) || text(snapshot.endpoint) || DEFAULT_ENDPOINT, token: text(storage.token) || text(memory.token) || text(legacyStorage.token) || text(snapshot.token), - userId: pinnedOwner - ? text(snapshot.userId) || "local-user" - : text(app.userId) || text(memory.userId) || text(snapshot.userId) || "local-user", + userId: "local-user", workspaceHostId: text(snapshot.workspaceHostId), }; } diff --git a/App/backend/src/infrastructure/memmy-config/index.ts b/App/backend/src/infrastructure/memmy-config/index.ts index 0da175c45..d39b23e46 100644 --- a/App/backend/src/infrastructure/memmy-config/index.ts +++ b/App/backend/src/infrastructure/memmy-config/index.ts @@ -681,7 +681,7 @@ export async function writeAccountModelProjectionToMemmyConfig( // inherit agent_chat or retain a stale BYOK memory endpoint. roleRouting.summary = "fixed"; roleRouting.evolution = "fixed"; - memory.userId = ownerAccountId; + delete memory.userId; memory.roleRouting = roleRouting; memory.summary = { ...withoutMemoryConnection(previousMemoryRecord(existingMemory.summary)), @@ -702,7 +702,7 @@ export async function writeAccountModelProjectionToMemmyConfig( mode: "cloud" }; memoryConfigAffected = memoryConfigAffected - || existingString(existingMemory.userId) !== ownerAccountId + || existingMemory.userId !== undefined || !accountMemoryConnectionMatches( existingMemory.summary, ACCOUNT_MODELS.memory_summary, diff --git a/App/backend/src/infrastructure/memmy-config/tests/account-model-projection.test.ts b/App/backend/src/infrastructure/memmy-config/tests/account-model-projection.test.ts index 321669001..1e964ee4c 100644 --- a/App/backend/src/infrastructure/memmy-config/tests/account-model-projection.test.ts +++ b/App/backend/src/infrastructure/memmy-config/tests/account-model-projection.test.ts @@ -143,8 +143,8 @@ describe("account model projection current catalog", () => { }); } expect(saved.modelAssignments.byok).toEqual(beforeByok); + expect(saved.memmyMemory).not.toHaveProperty("userId"); expect(saved.memmyMemory).toMatchObject({ - userId: "owner-a", roleRouting: { summary: "fixed", evolution: "fixed" }, summary: { provider: "openai_compatible", diff --git a/App/backend/src/services/index.ts b/App/backend/src/services/index.ts index b260406ba..d87fc7169 100644 --- a/App/backend/src/services/index.ts +++ b/App/backend/src/services/index.ts @@ -140,10 +140,7 @@ export function createBackendServices(options: CreateBackendServicesOptions): Ba const mode = options.appStateStore.repositories.bootstrap.getAppSettings().userMode; return mode === "account" || mode === "byok" ? mode : null; }; - const resolveMemoryUserId = () => { - const session = accountSessionRepository.get(); - return session.authenticated ? session.profile.userId : "local-user"; - }; + const resolveMemoryUserId = () => "local-user"; const memoryAddAnalytics = createMemoryDesktopAddAnalytics({ getUserId: resolveAnalyticsUserId, getUserMode: resolveAnalyticsUserMode, diff --git a/App/backend/src/services/tests/panel-task-identity.integration.test.ts b/App/backend/src/services/tests/panel-task-identity.integration.test.ts index b8f7c4bf1..98581db65 100644 --- a/App/backend/src/services/tests/panel-task-identity.integration.test.ts +++ b/App/backend/src/services/tests/panel-task-identity.integration.test.ts @@ -95,21 +95,21 @@ describe("desktop task identity / issue 409", () => { expect(await panel.tasks({ q: "no-matching-task" }, ctx)).toMatchObject({ total: 0, tasks: [] }); } - // Displaying local data must not migrate the original task ownership. - for (const [userId, total] of [["local-user", 21], ["account-user", 1], ["previous-account", 1]] as const) { - expect(f.service.panelTasks({ namespace: { userId } }).total).toBe(total); + // A legacy user namespace no longer partitions this device library. + for (const userId of ["local-user", "account-user", "previous-account"]) { + expect(f.service.panelTasks({ namespace: { userId } }).total).toBe(23); } }); - it("preserves Memory service authentication and token-bound user scopes", async () => { + it("preserves authentication and operation permissions for the shared device library", async () => { const f = await fixture(); const localId = await f.addTask("local-user", "local-scope-task"); const accountId = await f.addTask("account-user", "account-scope-task"); for (const token of ["scoped-panel-token", "cloud-panel-token"]) { const panel = f.panel(token, () => "account-user"); const result = await panel.tasks({}, ctx); - expect(result.total).toBe(1); - expect(result.tasks.map(task => task.id)).toEqual([accountId]); + expect(result.total).toBe(2); + expect(result.tasks.map(task => task.id).sort()).toEqual([localId,accountId].sort()); if (token === "scoped-panel-token") { await expect(panel.deleteTask(localId, ctx)).rejects.toMatchObject({ code: "forbidden" }); } diff --git a/App/memmy-agent/src/memmy-memory/config.ts b/App/memmy-agent/src/memmy-memory/config.ts index 4122661a8..caac5b557 100644 --- a/App/memmy-agent/src/memmy-memory/config.ts +++ b/App/memmy-agent/src/memmy-memory/config.ts @@ -7,15 +7,11 @@ export function resolveMemmyMemoryConfig(config: Config | Record | const raw = (config as any)?.memmyMemory ?? {}; return { enabled: Boolean(raw?.enabled ?? raw?.enable ?? true), - userId: stringOrUndefined(raw?.userId) ?? "local-user", + userId: "local-user", retrievalLayers: memoryLayersOrUndefined(raw?.retrievalLayers), }; } -function stringOrUndefined(value: any): string | undefined { - return typeof value === "string" && value.trim() ? value.trim() : undefined; -} - function memoryLayersOrUndefined(value: any): MemmyMemoryLayer[] | undefined { if (!Array.isArray(value)) return undefined; return [...new Set(value.filter((layer): layer is MemmyMemoryLayer => MEMORY_LAYERS.has(layer)))]; diff --git a/App/memmy-agent/src/memmy-memory/types.ts b/App/memmy-agent/src/memmy-memory/types.ts index d52044899..c8aff8604 100644 --- a/App/memmy-agent/src/memmy-memory/types.ts +++ b/App/memmy-agent/src/memmy-memory/types.ts @@ -30,7 +30,7 @@ export type MemmyMemoryRuntimeNamespace = { workspaceId?: string; workspacePath?: string; sessionKey?: string; - userId?: string; + userId?: string | null; tenantId?: string; }; diff --git a/App/memmy-agent/tests/memmy-memory/agent-loop-integration.test.ts b/App/memmy-agent/tests/memmy-memory/agent-loop-integration.test.ts index c3031501b..5430de6be 100644 --- a/App/memmy-agent/tests/memmy-memory/agent-loop-integration.test.ts +++ b/App/memmy-agent/tests/memmy-memory/agent-loop-integration.test.ts @@ -198,7 +198,7 @@ describe("AgentLoop memmy memory integration", () => { l3WorldModelProtocolVersion: 2, workspaceUri: expect.stringMatching(/^file:\/\//u), workspaceHostId: expect.stringMatching(/^[a-f0-9]{64}$/u), - namespace: { sessionKey: "websocket:memory-project", userId: "loop-user" }, + namespace: { sessionKey: "websocket:memory-project", userId: "local-user" }, }); expect(JSON.stringify(opened.body)).not.toContain(project.id); expect(requests.filter((request) => request.path === "/api/v1/sessions/open")).toHaveLength(1); diff --git a/App/memmy-agent/tests/memmy-memory/discovery.test.ts b/App/memmy-agent/tests/memmy-memory/discovery.test.ts index 14007326b..33b2d6632 100644 --- a/App/memmy-agent/tests/memmy-memory/discovery.test.ts +++ b/App/memmy-agent/tests/memmy-memory/discovery.test.ts @@ -124,7 +124,7 @@ describe("memmy memory discovery", () => { expect(defaultConfig.memmyMemory.enabled).toBe(true); expect(resolveMemmyMemoryConfig(enabled).enabled).toBe(true); expect(resolveMemmyMemoryConfig(defaultConfig).enabled).toBe(true); - expect(resolveMemmyMemoryConfig(enabled).userId).toBe("user_config_1"); + expect(resolveMemmyMemoryConfig(enabled).userId).toBe("local-user"); expect(resolveMemmyMemoryConfig(enabled).retrievalLayers).toEqual(["L1", "L3"]); expect(resolveMemmyMemoryConfig(disabled).enabled).toBe(false); expect(resolveMemmyMemoryConfig(disabled).userId).toBe("local-user"); diff --git a/Memory/src/agent-source/integration/hermes/target.ts b/Memory/src/agent-source/integration/hermes/target.ts index 12e80ff2c..d58904b8f 100644 --- a/Memory/src/agent-source/integration/hermes/target.ts +++ b/Memory/src/agent-source/integration/hermes/target.ts @@ -1338,7 +1338,7 @@ def _load_runtime() -> Dict[str, str]: return { "baseUrl": base_url, "token": token, - "userId": _clean_text(app.get("userId")) or _clean_text(memory.get("userId")) or _clean_text(plugin_config.get("userId")) or "local-user", + "userId": "local-user", "workspaceHostId": _clean_text(plugin_config.get("workspaceHostId")), } diff --git a/Memory/src/agent-source/integration/memmy-runtime-config.ts b/Memory/src/agent-source/integration/memmy-runtime-config.ts index f5a15c88d..06ac2a895 100644 --- a/Memory/src/agent-source/integration/memmy-runtime-config.ts +++ b/Memory/src/agent-source/integration/memmy-runtime-config.ts @@ -28,7 +28,7 @@ export async function readMemmyMemoryServiceConfig(configPath: string): Promise< token: normalizeString(storage.token) || normalizeString(memmyMemory.token) || normalizeString(legacyStorage.token), - userId: normalizeString(app.userId) || normalizeString(memmyMemory.userId) || "local-user", + userId: "local-user", workspaceHostId: deriveWorkspaceHostId(getOrCreateInstallationId()) }; } diff --git a/Memory/src/agent-source/integration/workspace-bridge/runtime.ts b/Memory/src/agent-source/integration/workspace-bridge/runtime.ts index c0868bf3c..dc4f6c348 100644 --- a/Memory/src/agent-source/integration/workspace-bridge/runtime.ts +++ b/Memory/src/agent-source/integration/workspace-bridge/runtime.ts @@ -91,9 +91,7 @@ export async function readRuntimeConfig(configUrl: URL, pinnedOwner = false): Pr return { endpoint: text(storage.endpoint) || text(memory.endpoint) || text(legacyStorage.endpoint) || text(snapshot.endpoint) || DEFAULT_ENDPOINT, token: text(storage.token) || text(memory.token) || text(legacyStorage.token) || text(snapshot.token), - userId: pinnedOwner - ? text(snapshot.userId) || "local-user" - : text(app.userId) || text(memory.userId) || text(snapshot.userId) || "local-user", + userId: "local-user", workspaceHostId: text(snapshot.workspaceHostId), }; } diff --git a/Memory/src/algorithm/plugin-algorithms.ts b/Memory/src/algorithm/plugin-algorithms.ts index 1b7f2390c..f94e988ac 100644 --- a/Memory/src/algorithm/plugin-algorithms.ts +++ b/Memory/src/algorithm/plugin-algorithms.ts @@ -55,7 +55,7 @@ export interface TraceMemoryMeta { rawTurnId?: string; episodeId?: string; sessionId?: string; - userId: string; + userId: string | null; summary: string; userText: string; agentText: string; @@ -4378,7 +4378,7 @@ export function retrievePluginMemories(input: { traceVectorTags, traceVectorTagsRequired, seededChannelScores: input.channelScoresByMemory?.get(memory.id), - useSeededChannelScores: input.channelScoresByMemory !== undefined, + useSeededChannelScores: input.channelScoresByMemory?.has(memory.id) ?? false, config } )) @@ -5202,7 +5202,7 @@ function hasTaggedTraceVectorMatch( if (shouldExcludeTraceFromTurnContext(trace, options)) return false; return vectorChannelsForMemory(memory, queryVec, options.config, { seededChannelScores: options.channelScoresByMemory?.get(memory.id), - useSeededChannelScores: options.channelScoresByMemory !== undefined + useSeededChannelScores: options.channelScoresByMemory?.has(memory.id) ?? false }).channels.some((channel) => channel.channel === "vec_summary" || channel.channel === "vec_action" ); diff --git a/Memory/src/algorithm/trace-direct-skill.ts b/Memory/src/algorithm/trace-direct-skill.ts index 62d01861e..3bfad77b3 100644 --- a/Memory/src/algorithm/trace-direct-skill.ts +++ b/Memory/src/algorithm/trace-direct-skill.ts @@ -91,7 +91,7 @@ export interface OutcomeThresholds { export interface EpisodeSkillFeatures { episodeId: string; - userId: string; + userId: string | null; projectId?: string; tools: string[]; artifacts: string[]; @@ -104,7 +104,7 @@ export interface EpisodeSkillFeatures { export interface SkillClusterFeatures { id: string; - userId: string; + userId: string | null; projectId?: string; tools: string[]; artifacts: string[]; @@ -282,7 +282,7 @@ export function firstUserQuery(turns: RawTurnLike[]): string { export function extractEpisodeSkillFeatures(input: { episodeId: string; - userId: string; + userId: string | null; projectId?: string; rTask?: number; turns: RawTurnLike[]; @@ -421,7 +421,7 @@ export function decideClusterAssignment( stage: "coarse" | "fine"; } | undefined; for (const cluster of clusters) { - if (cluster.userId !== episode.userId) continue; + if (episode.projectId && cluster.projectId && episode.projectId !== cluster.projectId) continue; const scored = scoreClusterJoin(episode, cluster, config); if (scored.score === null) continue; diff --git a/Memory/src/cli/config.ts b/Memory/src/cli/config.ts index 4156cb5ed..adad3b4d5 100644 --- a/Memory/src/cli/config.ts +++ b/Memory/src/cli/config.ts @@ -35,11 +35,7 @@ export function loadCliMemoryConfig(configPath?: string): { optionalString(process.env.MEMORY_SERVICE_TOKEN) ?? optionalString(storage.token) ?? optionalString(memmyMemory.token), - userId: optionalString(process.env.MEMMY_MEMORY_USER_ID) ?? - optionalString(process.env.MEMMY_USER_ID) ?? - optionalString(process.env.MEMORY_SERVICE_USER_ID) ?? - optionalString(memmyMemory.userId) ?? - optionalString(app.userId) + userId: undefined }, path: selectedPath }; diff --git a/Memory/src/cli/setup.ts b/Memory/src/cli/setup.ts index df8298243..008744bff 100644 --- a/Memory/src/cli/setup.ts +++ b/Memory/src/cli/setup.ts @@ -334,7 +334,7 @@ function setupMemmyMemoryConfig( const memmyMemory: Record = { ...existing, version: 1, - userId: optionalString(existing.userId) ?? options.appUserId ?? "local-user", + userId: undefined, roleRouting: { ...roleRouting, // Account mode has platform-owned models for both memory roles. Never diff --git a/Memory/src/config/index.ts b/Memory/src/config/index.ts index 4a6e722eb..1499ad85f 100644 --- a/Memory/src/config/index.ts +++ b/Memory/src/config/index.ts @@ -600,7 +600,7 @@ function parseConfigFile(path: string): Record { function configFromEnv(): Record { return compactRecord({ domain: process.env.MEMMY_MEMORY_DOMAIN ?? process.env.MEMMY_DOMAIN, - userId: process.env.MEMMY_MEMORY_USER_ID ?? process.env.MEMMY_USER_ID ?? process.env.MEMORY_SERVICE_USER_ID, + userId: undefined, storage: compactRecord({ mode: process.env.MEMMY_MEMORY_MODE, backend: process.env.MEMMY_MEMORY_BACKEND, @@ -701,7 +701,7 @@ function normalizeConfig(input: Record): MemmyConfig { version: 1, domain: memoryDomainName(input.domain, DEFAULT_MEMMY_CONFIG.domain), roleRouting: normalizeRoleRouting(asRecord(input.roleRouting)), - userId: optionalString(input.userId), + userId: undefined, ...(language ? { language } : {}), storage, summary, diff --git a/Memory/src/contracts/memory-l3-world-model.ts b/Memory/src/contracts/memory-l3-world-model.ts index 269d5f023..4d6d85f07 100644 --- a/Memory/src/contracts/memory-l3-world-model.ts +++ b/Memory/src/contracts/memory-l3-world-model.ts @@ -28,7 +28,7 @@ const L3WorldModelRuntimeNamespaceShape = { workspaceId: OptionalNonEmptyStringSchema, workspacePath: OptionalNonEmptyStringSchema, sessionKey: OptionalNonEmptyStringSchema, - userId: OptionalNonEmptyStringSchema, + userId: OptionalNonEmptyStringSchema.nullable(), tenantId: OptionalNonEmptyStringSchema } as const; diff --git a/Memory/src/server/http.ts b/Memory/src/server/http.ts index 9c84e0148..00bed46c4 100644 --- a/Memory/src/server/http.ts +++ b/Memory/src/server/http.ts @@ -1565,6 +1565,7 @@ function strictEnvelopeWithPrincipal( "source" ]; for (const field of fields) { + if (field === "userId" && (principal.kind === "local" || principal.kind === "anonymous")) continue; const requested = requestNamespace?.[field]; const scoped = principalNamespace?.[field]; if (typeof requested === "string" && requested && typeof scoped === "string" && scoped && requested !== scoped) { diff --git a/Memory/src/service/embedding/embedding-job-processor.ts b/Memory/src/service/embedding/embedding-job-processor.ts index 16ffa2e76..6367b4c6a 100644 --- a/Memory/src/service/embedding/embedding-job-processor.ts +++ b/Memory/src/service/embedding/embedding-job-processor.ts @@ -106,7 +106,7 @@ export interface PreparedEmbeddingJob { export interface EnqueueWorkerJobInput { jobType: JobType; - userId: string; + userId: string | null; sessionId?: string; episodeId?: string; targetMemoryId?: string; @@ -758,7 +758,7 @@ export class EmbeddingJobProcessor { : undefined; if ( decision.userMemoryAction === "correct_existing" && - (!correctionTarget || correctionTarget.userId !== sourceMemory.userId || correctionTarget.status !== "active") + (!correctionTarget || correctionTarget.status !== "active") ) return undefined; const candidate = buildUserMemory({ id: `user_memory_${stableHash(`${sourceTurnId}:${content}`).slice(0, 20)}`, diff --git a/Memory/src/service/embedding/embedding-maintenance.ts b/Memory/src/service/embedding/embedding-maintenance.ts index cb5ddd300..e0848ab67 100644 --- a/Memory/src/service/embedding/embedding-maintenance.ts +++ b/Memory/src/service/embedding/embedding-maintenance.ts @@ -64,7 +64,7 @@ export interface EmbeddingMaintenanceRunResult { interface Dependencies { repos: Repositories; embedder: Embedder; - userId: string; + userId: string | null; currentWork?(dimension: number): { retryQueue: { pending: number; active: number; stale: number; failed: number }; outdatedVectors: Set }; enqueueMemory(memory: MemoryRow, text: string, at: string, vectorField?: EmbeddingRetryVectorField): string; enqueueUserMemory(memory: UserMemoryRecord, at: string): string; @@ -74,7 +74,7 @@ interface Dependencies { export class EmbeddingMaintenance { constructor(private readonly deps: Dependencies) {} - private get key(): string { return `embedding-maintenance:${this.deps.userId}`; } + private get key(): string { return `embedding-maintenance:${this.deps.userId ?? "device"}`; } private entries(): EmbeddingMaintenanceEntry[] { return [...this.deps.repos.vectors.maintenanceEntries(), diff --git a/Memory/src/service/evolution/evolution-job-processor.ts b/Memory/src/service/evolution/evolution-job-processor.ts index a0fbc9517..0cc2c3d6e 100644 --- a/Memory/src/service/evolution/evolution-job-processor.ts +++ b/Memory/src/service/evolution/evolution-job-processor.ts @@ -69,7 +69,7 @@ export interface EvolutionJobProcessorDeps { trigger: "episode_rewarded" ): EvolutionJobRecord[]; resolvePendingSkillTrialsForReward(input: { - userId: string; + userId: string | null; episodeId: string; rHuman: number; feedbackId?: string; diff --git a/Memory/src/service/evolution/l3-world-model-pipeline.ts b/Memory/src/service/evolution/l3-world-model-pipeline.ts index 92b8b8de5..c8c299bac 100644 --- a/Memory/src/service/evolution/l3-world-model-pipeline.ts +++ b/Memory/src/service/evolution/l3-world-model-pipeline.ts @@ -63,7 +63,6 @@ export class L3WorldModelTraceFieldPipeline { } if ( job.jobType !== "l3_world_model_update" || - job.userId !== batch.userId || job.sessionId !== batch.sessionId || job.scopeKey !== target.fieldScopeKey || job.scopeSeq !== target.scopeSeq || @@ -71,7 +70,7 @@ export class L3WorldModelTraceFieldPipeline { ) { throw new L3WorldModelTerminalEvidenceError("L3 World Model job ownership mismatch"); } - if (sha256Hex(canonicalJson(batchPayloadForHash(batch))) !== batch.payloadHash) { + if (sha256Hex(canonicalJson(batchPayloadForHash(this.deps.repos.l3WorldModels.getStoredBatch(batch.id)!))) !== batch.payloadHash) { throw new L3WorldModelTerminalEvidenceError("L3 World Model batch payload hash mismatch"); } @@ -132,8 +131,10 @@ export class L3WorldModelTraceFieldPipeline { private loadEvidence(batchId: string): TraceEvidence { const batch = this.deps.repos.l3WorldModels.getBatch(batchId); if (!batch) throw new L3WorldModelTerminalEvidenceError(`missing batch: ${batchId}`); + const deletedAt = this.deps.repos.l3WorldModels.latestDeletion(batch.projectId); + if (deletedAt && batch.createdAt <= deletedAt) return {rawTurns:[],eligibleL1MemoryIds:[]}; const session = this.deps.repos.runtime.getSession(batch.sessionId); - if (!session || session.userId !== batch.userId || (session.projectId ?? null) !== (batch.projectId ?? null)) { + if (!session || (session.projectId ?? null) !== (batch.projectId ?? null)) { throw new L3WorldModelTerminalEvidenceError("L3 World Model batch session scope mismatch"); } const traces = this.deps.repos.l3WorldModels.listBatchTraces(batchId); @@ -158,7 +159,6 @@ export class L3WorldModelTraceFieldPipeline { : undefined; if ( !trace || - feedback.userId !== batch.userId || feedback.sessionId !== batch.sessionId || (feedback.projectId ?? null) !== (batch.projectId ?? null) ) { @@ -184,7 +184,7 @@ export class L3WorldModelTraceFieldPipeline { for (const trace of traces) { const memory = this.deps.repos.memories.get(trace.l1MemoryId); if (!memory) continue; - if (memory.userId !== batch.userId || memory.sessionId !== batch.sessionId) { + if (memory.sessionId !== batch.sessionId) { throw new L3WorldModelTerminalEvidenceError(`L1 scope mismatch: ${memory.id}`); } if (!memory.deletedAt && memory.status !== "deleted" && usableRawTurnIds.has(trace.rawTurnId)) { @@ -266,8 +266,8 @@ function assertJsonArray(value: unknown[], label: string): JsonValue[] { } } -function assertRawTurnScope(rawTurn: RawTurnRecord, userId: string, sessionId: string): void { - if (rawTurn.userId !== userId || rawTurn.sessionId !== sessionId) { +function assertRawTurnScope(rawTurn: RawTurnRecord, userId: string | null, sessionId: string): void { + if (rawTurn.sessionId !== sessionId) { throw new L3WorldModelTerminalEvidenceError(`RawTurn scope mismatch: ${rawTurn.id}`); } } diff --git a/Memory/src/service/evolution/negative-experience-pipeline.ts b/Memory/src/service/evolution/negative-experience-pipeline.ts index 0a557770a..893414345 100644 --- a/Memory/src/service/evolution/negative-experience-pipeline.ts +++ b/Memory/src/service/evolution/negative-experience-pipeline.ts @@ -263,7 +263,7 @@ export class NegativeExperiencePipeline { const episode = job.episodeId ? this.deps.repos.runtime.getEpisode(job.episodeId) : undefined; - if (!source || !episode || episode.userId !== job.userId) return undefined; + if (!source || !episode) return undefined; const feedbackId = text(job.payload.feedbackId); const repairId = text(job.payload.repairId); diff --git a/Memory/src/service/evolution/policy-induction.ts b/Memory/src/service/evolution/policy-induction.ts index 70a3aa6c1..fcdcdc5e5 100644 --- a/Memory/src/service/evolution/policy-induction.ts +++ b/Memory/src/service/evolution/policy-induction.ts @@ -69,13 +69,13 @@ type ReposPort = { runtime: { pruneCandidatePool(now: string): number; listPendingCandidatePool(input: { - userId?: string; + userId?: string | null; now?: string; limit?: number; }): CandidatePoolRecord[]; upsertCandidatePoolTrace(input: { id: string; - userId: string; + userId: string | null; sessionId?: string; sourceMemoryId: string; candidateKey: string; @@ -87,20 +87,20 @@ type ReposPort = { expiresAt?: string | null; }): void; markCandidatePoolPromoted(input: { - userId?: string; + userId?: string | null; candidateKey: string; sourceMemoryIds: string[]; policyId: string; at: string; }): void; listTracePolicyLinks(input: { - userId?: string; + userId?: string | null; l1MemoryId?: string; l2MemoryId?: string; limit?: number; }): PolicyTraceLink[]; insertTracePolicyLink(input: { - userId: string; + userId: string | null; l1MemoryId: string; l2MemoryId: string; relation?: string; @@ -137,7 +137,7 @@ export interface PolicyInductionDeps { kind: string; op: JobChangeKind; entityId: string; - userId: string; + userId: string | null; changeType: string; before?: unknown; after: unknown; @@ -828,7 +828,7 @@ export class PolicyInductionEngine { } private markCandidatePoolPromoted( - userId: string, + userId: string | null, signature: string, sourceMemoryIds: string[], policyId: string, diff --git a/Memory/src/service/evolution/reward-pipeline.ts b/Memory/src/service/evolution/reward-pipeline.ts index 4a8207a73..8ad850e24 100644 --- a/Memory/src/service/evolution/reward-pipeline.ts +++ b/Memory/src/service/evolution/reward-pipeline.ts @@ -55,7 +55,7 @@ export interface RewardPipelineDeps { enqueueJob(input: EnqueueJobInput): EvolutionJobRecord; finalizeClosedEpisode(episode: EpisodeRecord, at: string, trigger: "episode_rewarded"): EvolutionJobRecord[]; resolvePendingSkillTrialsForReward(input: { - userId: string; + userId: string | null; episodeId: string; rHuman: number; feedbackId?: string; diff --git a/Memory/src/service/evolution/skill-pipeline.ts b/Memory/src/service/evolution/skill-pipeline.ts index 640aa9c3b..f4d44b539 100644 --- a/Memory/src/service/evolution/skill-pipeline.ts +++ b/Memory/src/service/evolution/skill-pipeline.ts @@ -470,7 +470,6 @@ export class SkillPipeline { .map(skillMetaFromMemory) .filter((skill): skill is SkillMeta => Boolean( skill && - skill.memory.userId === policy.memory.userId && skill.status !== "archived" && !isReadOnlySkillMemory(skill.memory) )); @@ -1349,7 +1348,7 @@ function skillPolicyCompatibility( skill: NonNullable>, policy: PolicyMeta ): number { - if (skill.memory.userId !== policy.memory.userId) return 0; + const skillProject = projectIdFromMemory(skill.memory); const policyProject = projectIdFromMemory(policy.memory); if (skillProject && policyProject && skillProject !== policyProject) return 0; @@ -1371,7 +1370,7 @@ function skillPolicyCompatibility( } function skillsAreCompatibleForMerge(left: SkillMeta, right: SkillMeta): boolean { - if (left.memory.userId !== right.memory.userId) return false; + const leftProject = projectIdFromMemory(left.memory); const rightProject = projectIdFromMemory(right.memory); if (leftProject && rightProject && leftProject !== rightProject) return false; diff --git a/Memory/src/service/evolution/span-pipeline.ts b/Memory/src/service/evolution/span-pipeline.ts index 3152646e7..a45227d80 100644 --- a/Memory/src/service/evolution/span-pipeline.ts +++ b/Memory/src/service/evolution/span-pipeline.ts @@ -966,7 +966,7 @@ private reflectionDownstreamPreview(job: EvolutionJobRecord, memory: MemoryRow): ? recall.userMemoryCandidateIds : injectedIds; return this.deps.repos.userMemories.getMany(recalledUserMemoryIds) - .filter((memory) => memory.userId === trace.userId && memory.status === "active") + .filter((memory) => memory.status === "active") .map((memory) => ({ id: memory.id, memoryTypes: memory.memoryTypes, diff --git a/Memory/src/service/feedback/feedback-experience.ts b/Memory/src/service/feedback/feedback-experience.ts index b005fda0e..f5b95b700 100644 --- a/Memory/src/service/feedback/feedback-experience.ts +++ b/Memory/src/service/feedback/feedback-experience.ts @@ -65,7 +65,7 @@ import { export interface ResolvedContext { - userId: string; + userId: string | null; conversationId?: string; namespace: RuntimeNamespace; } @@ -628,7 +628,7 @@ maybeCreateDecisionRepair( } enqueueDeferredDecisionRepair(input: { - userId: string; + userId: string | null; sessionId?: string; episodeId?: string; repairId: string; @@ -656,7 +656,7 @@ maybeCreateDecisionRepair( } enqueueDeferredFeedbackExperience(input: { - userId: string; + userId: string | null; sessionId?: string; episodeId?: string; feedbackId: string; diff --git a/Memory/src/service/import/import-job-processor.ts b/Memory/src/service/import/import-job-processor.ts index ef000809c..01b5306f2 100644 --- a/Memory/src/service/import/import-job-processor.ts +++ b/Memory/src/service/import/import-job-processor.ts @@ -1,5 +1,6 @@ import { traceMetaFromMemory } from "../../algorithm/plugin-algorithms.js"; -import type { EvolutionJobRecord,SessionRecord } from "../../storage/repositories.js"; +import { captureNativeOrigin } from "../../storage/native-memory-origin.js"; +import type { EvolutionJobRecord, SessionRecord, MemoryCaptureClaimRepository } from "../../storage/repositories.js"; import type { JobRef, JobType, @@ -44,7 +45,7 @@ export const IMPORT_DEFAULT_PRIORITY = 0.5; type EnqueueInput = { jobType: JobType; - userId: string; + userId: string | null; sessionId?: string; episodeId?: string; targetMemoryId?: string; @@ -63,7 +64,7 @@ export interface ImportJobProcessorDeps { assertMemoryAddEnabled(): void; assertMemoryInScope(memory: MemoryRow, namespace: unknown): void; sanitizeMemoryAddRequest(request: MemoryAddRequest): MemoryAddRequest; - resolveContext(request: MemoryAddRequest): { userId: string; namespace: { source?: string; projectId?: string; profileId?: string } }; + resolveContext(request: MemoryAddRequest): { userId: string | null; namespace: { source?: string; projectId?: string; profileId?: string } }; requireSession(id: string): SessionRecord; assertSessionInScope(session: ReturnType, namespace: unknown): void; normalizeMemoryAddCreatedAt(value: string | undefined, timeZone?: string): string | undefined; @@ -98,7 +99,8 @@ export interface ImportJobProcessorDeps { get(id: string): MemoryRow | undefined; getIncludingDeleted(id: string): MemoryRow | undefined; getByKeyIncludingDeleted(layer: MemoryLayer, key: string): MemoryRow | undefined; - upsertByKey(memory: MemoryRow): { memory: MemoryRow; created: boolean; previous?: MemoryRow }; + listUserMemoriesByKeyIncludingDeleted(userId: string | null, key: string): MemoryRow[]; + upsertByKey(memory: MemoryRow, existingMemoryId?: string): { memory: MemoryRow; created: boolean; previous?: MemoryRow }; archivePriorReadOnlySkillVersions(input: { sourceAgentId: string; sourceSkillIdentity: string; @@ -112,19 +114,7 @@ export interface ImportJobProcessorDeps { toListItem(memory: MemoryRow): { id: string; kind: MemoryKind; memoryLayer: MemoryLayer; status: MemoryStatus; title: string; summary: string; tags: string[] }; update(memory: MemoryRow): MemoryRow; }; - captureClaims: { - claim(input: { - userId: string; - source: string; - qaHash: string; - primaryMemoryId: string; - capturedBy: "agent_source_scan"; - createdAt: string; - }): { - claimed: boolean; - claim: { primaryMemoryId: string; capturedBy: "turn_complete" | "agent_source_scan" }; - }; - }; + captureClaims: Pick; processing: { get(memoryId: string): MemoryProcessingRecord | undefined; getMany(memoryIds: string[]): MemoryProcessingRecord[]; @@ -175,7 +165,17 @@ export class ImportJobProcessor { : request.title?.trim() || firstLine(request.content).slice(0, 120) || "Untitled memory"); const importSummary = importTrace ? stringFromRecord(importTrace, "summary") || IMPORT_SUMMARY_QUEUED_TAG : undefined; const tags = d.memoryAddTags(request, importTrace !== null, importTrace ? stringArray(importTrace.tags) : []); - const memoryKey = d.memoryAddKey(request, layer, title); + const captureNamespace = { ...request.namespace, ...context.namespace }; + const captureIdentity = agentSourceImport + ? d.captureClaims.identity(captureNamespace, request.turnId, session?.id) + : undefined; + const importKey = d.memoryAddKey(request, layer, title); + // Even incomplete origins retain every known namespace boundary. UID and QA + // are deliberately excluded so account switches and edits keep one identity. + const importIdentity = agentSourceImport + ? captureIdentity ?? { ...d.captureClaims.scope(captureNamespace, session?.id), turnId: request.turnId ?? null } + : undefined; + const memoryKey = importIdentity ? `${importKey}:scope:${stableHash(importIdentity)}` : importKey; const deletedReadOnlySkill = readOnlySkill ? d.memories.getByKeyIncludingDeleted(layer, memoryKey) : undefined; @@ -198,7 +198,7 @@ export class ImportJobProcessor { conversationId: session?.conversationId, sessionId: session?.id ?? request.sessionId, agentId: session?.source ?? request.source?.trim() ?? context.namespace.source, - appId: session?.workspaceId, + appId: session?.workspaceId ?? captureNamespace.workspaceId, projectId: session?.projectId ?? context.namespace.projectId, profileId: session?.profileId ?? context.namespace.profileId, layer, kind, memoryType: layer === "Skill" ? "SkillMemory" : "LongTermMemory", @@ -211,6 +211,16 @@ export class ImportJobProcessor { }) : request.content, tags, info: { + ...(agentSourceImport ? { native_origin_v1: captureNativeOrigin({ + source: session?.source ?? request.source?.trim() ?? captureNamespace.source ?? "memmy", + profileId: session?.profileId ?? captureNamespace.profileId ?? "default", + projectId: session ? session.projectId ?? null : captureNamespace.projectId, + workspaceId: session ? session.workspaceId ?? null : captureNamespace.workspaceId, + tenantId: captureNamespace.tenantId, meta: session?.meta, + // The claim compatibility fallback is not verified native identity. + conversationId: session?.conversationId ?? null, + hostSessionKey: session?.hostSessionKey ?? captureNamespace.sessionKey ?? null + }) } : {}), title, summary: importSummary ?? firstLine(request.content), source: request.source ?? "manual", @@ -273,33 +283,49 @@ export class ImportJobProcessor { ? d.normalizeMemoryCaptureSource(memory.agentId ?? request.source ?? context.namespace.source ?? "") : ""; const persisted = d.transaction(() => { + let verifiedTarget: MemoryRow | undefined; + if (captureIdentity && captureSource && !d.memories.getByKeyIncludingDeleted(layer, memoryKey)) { + const legacy = d.memories.listUserMemoriesByKeyIncludingDeleted(null, importKey) + .filter(candidate => d.captureClaims.matchesOrigin(candidate.id, captureSource, captureIdentity)) + .sort((a, b) => Number(Boolean(b.deletedAt || b.status === "deleted")) - Number(Boolean(a.deletedAt || a.status === "deleted")))[0]; + // Content revisions cannot be matched by QA hash. Reuse a legacy key + // only with complete, equal native origin evidence; never rewrite it. + if (legacy) { + if (legacy.status === "deleted" || legacy.deletedAt) return { duplicateMemory: legacy } as const; + memory.memoryKey = legacy.memoryKey; + verifiedTarget = legacy; + } + } if (qaPair && captureSource) { - const priorCaptureMemory = d.memories.getByKeyIncludingDeleted(layer, memoryKey); - const capturePrimaryMemoryId = priorCaptureMemory && - priorCaptureMemory.status !== "deleted" && - !priorCaptureMemory.deletedAt - ? priorCaptureMemory.id - : memory.id; + const priorCaptureMemory = verifiedTarget ?? d.memories.getByKeyIncludingDeleted(layer, memory.memoryKey!); + if (priorCaptureMemory?.status === "deleted" || priorCaptureMemory?.deletedAt) return { duplicateMemory: priorCaptureMemory } as const; + const capturePrimaryMemoryId = priorCaptureMemory?.id ?? memory.id; const capture = d.captureClaims.claim({ userId: memory.userId, source: captureSource, qaHash: d.memoryCaptureQaHash(qaPair.query, qaPair.answer), + identity: captureIdentity, primaryMemoryId: capturePrimaryMemoryId, capturedBy: "agent_source_scan", createdAt: at }); - if (!capture.claimed && capture.claim.capturedBy !== "agent_source_scan") { + if (!capture.claimed) { const existing = d.memories.getIncludingDeleted(capture.claim.primaryMemoryId); if (!existing) { throw d.createError("conflict", "memory capture claim points to a missing memory"); } d.assertMemoryInScope(existing, request.namespace); - return { duplicateMemory: existing } as const; + if (capture.claim.capturedBy !== "agent_source_scan") return { duplicateMemory: existing } as const; + if (existing.status === "deleted" || existing.deletedAt) return { duplicateMemory: existing } as const; + // A proven legacy claim may point at the old, unscoped import key. + // Keep its identity when the same scanner revises that native turn. + memory.memoryKey = existing.memoryKey; + verifiedTarget = existing; } } - const existingByKey = memory.memoryKey + const existingByKey = verifiedTarget ?? (memory.memoryKey ? d.memories.getByKeyIncludingDeleted(layer, memory.memoryKey) - : undefined; + : undefined); if ( importTrace && existingByKey && @@ -310,7 +336,7 @@ export class ImportJobProcessor { repairPlaceholderImportSummary(d, existingByKey, receivedAt); return { upsert: { memory: existingByKey, created: false, previous: existingByKey }, changeSeq: 0, duplicateMemory: existingByKey } as const; } - const upsert = d.memories.upsertByKey(memory); + const upsert = d.memories.upsertByKey(memory, verifiedTarget?.id); const inserted = upsert.memory; const changeSeq = d.runtime.appendChange({ memoryId: inserted.id, namespaceId: d.namespaceIdFromMemory(inserted), kind: d.kindFromMemory(inserted), diff --git a/Memory/src/service/memory-service.ts b/Memory/src/service/memory-service.ts index fc4d85c79..ac1b37b26 100644 --- a/Memory/src/service/memory-service.ts +++ b/Memory/src/service/memory-service.ts @@ -1,5 +1,6 @@ import { EmbeddingMaintenance } from "./embedding/embedding-maintenance.js"; import { embeddingRetryTargetKindForMemory, embeddingRetryVectorFieldForMemory } from "./embedding/embedding-pipeline.js"; +import { canonicalProjectId, deviceNamespace, memoryFingerprint, matchesLegacyMemoryFingerprint, reloadDeviceScopeAliases } from "../storage/device-scope.js"; import { assertJsonValue, canonicalJson, @@ -281,6 +282,10 @@ function isAbortError(error: unknown): boolean { } export class MemoryService { + private normalizeDeviceNamespace(namespace?: RuntimeNamespace) { + return normalizeNamespace(deviceNamespace(this.repos.db, normalizeNamespace(namespace))); + } + private readonly embeddingJobs: EmbeddingJobProcessor; private readonly evolutionJobs: EvolutionJobProcessor; private readonly feedbackExperience: FeedbackExperienceService; @@ -597,6 +602,9 @@ export class MemoryService { }); this.episodeReadModel = new EpisodeReadModel({ repos: this.repos, + projectMemory: (memory) => isStrictL3WorldModelV2Memory(memory) + ? this.repos.l3WorldModels.getMemory(null, typeof memory.info.project_id === "string" ? memory.info.project_id : null) ?? memory + : memory, assertMemorySearchEnabled: this.assertMemorySearchEnabled.bind(this), resolveContext: this.resolveContext.bind(this), requireSession: this.requireSession.bind(this), @@ -1159,10 +1167,11 @@ export class MemoryService { if (!idempotencyKey) { return scopedRun(); } - const requestHash = stableHash({ operation, fingerprint }); + const hashFingerprint = (value: unknown) => stableHash({ operation, fingerprint: value }); + const requestHash = hashFingerprint(memoryFingerprint(this.repos.db, fingerprint)); const existing = this.repos.runtime.getIdempotency(idempotencyKey); if (existing) { - if (existing.requestHash !== requestHash) { + if (existing.requestHash !== requestHash && !matchesLegacyMemoryFingerprint(this.repos.db, fingerprint, existing.requestHash, hashFingerprint)) { throw new MemoryServiceError("conflict", "idempotency key reused with different request body"); } return withDuplicateFlag(existing.response) as T; @@ -1184,10 +1193,11 @@ export class MemoryService { ? `${operation}:${request.adapterId}:${request.requestId}` : undefined; if (!idempotencyKey) return scopedRun(); - const requestHash = sha256Hex(canonicalJson(assertJsonValue({ operation, fingerprint }))); + const hashFingerprint = (value: unknown) => sha256Hex(canonicalJson(assertJsonValue({ operation, fingerprint: value }))); + const requestHash = hashFingerprint(memoryFingerprint(this.repos.db, fingerprint)); const existing = this.repos.runtime.getIdempotency(idempotencyKey); if (existing) { - if (existing.requestHash !== requestHash) { + if (existing.requestHash !== requestHash && !matchesLegacyMemoryFingerprint(this.repos.db, fingerprint, existing.requestHash, hashFingerprint)) { throw new MemoryServiceError("conflict", "idempotency key reused with different request body"); } return existing.response as T; @@ -1214,7 +1224,7 @@ export class MemoryService { panel: boolean; }; effectiveNamespace: { - userId: string; + userId: string | null; projectId?: string; workspaceId?: string; profileId?: string; @@ -1222,7 +1232,7 @@ export class MemoryService { expiresAt?: string; serverTime: string; } { - const namespace = normalizeNamespace(request.namespace); + const namespace = this.normalizeDeviceNamespace(request.namespace); return { adapterId: request.adapterId ?? "anonymous", serviceVersion: PROJECT_VERSION, @@ -1244,7 +1254,7 @@ export class MemoryService { openSession(request: SessionOpenRequest): { sessionId: string; - userId: string; + userId: string | null; source: string; profileId: string; projectId?: string | null; @@ -1406,8 +1416,7 @@ export class MemoryService { } async completeSourceTurn(request: SourceTurnCompleteRequest): Promise { - // Native scans have no Hook envelope. Use the configured owner only when - // the request (including authenticated scope) did not provide one. + // Native scans use device identity regardless of legacy account UIDs. const response = await this.acceptCapture(() => this.withModelTaskContext(() => this.sessionTurns.withEpisodeSelection( { userText: request.query, turnId: request.sourceTurn?.turnId }, () => this.sessionTurns.completeSourceTurn(this.withTimeZone({ @@ -1417,7 +1426,7 @@ export class MemoryService { profileId: request.sourceTurn?.profileId, sessionKey: request.sourceTurn?.conversationId, ...request.namespace, - userId: request.namespace?.userId ?? this.config.userId + userId: null } })) ))); @@ -2182,7 +2191,7 @@ export class MemoryService { } timeline(input: RequestEnvelope & { - userId?: string; + userId?: string | null; sessionId?: string; episodeId?: string; layers?: MemoryLayer[]; @@ -2224,7 +2233,7 @@ export class MemoryService { } listSkills(input: RequestEnvelope & { - userId?: string; + userId?: string | null; q?: string; tags?: string[]; limit?: number; @@ -2416,51 +2425,59 @@ export class MemoryService { if (!request.bundle || !request.bundle.tables || typeof request.bundle.tables !== "object") { throw new MemoryServiceError("invalid_argument", "import bundle must contain tables"); } - const context = this.resolveContext(request); - const importedAt = nowIso(); - const result = this.repos.runtime.importBundleTables(request.bundle.tables, { - conflictStrategy: request.conflictStrategy ?? "skip" - }); - this.sessionTurns.migrateLegacyCaptures(); - const importedVectors = importedMemoryVectors(request.bundle.tables.memory_vectors); - this.repos.vectors.importRows(importedVectors); - result.inserted.memory_vectors = importedVectors.length; - const reembedMemoryIds = importedReembedMemoryIds( - request.bundle.tables, - this.embedder.config.model ?? this.embedder.config.provider - ); - const audit = this.repos.runtime.insertAudit({ - userId: context.userId, - actor: request.namespace ? { ...request.namespace } : {}, - action: "import", - targetKind: "bundle", - targetId: `import_${stableHash(importedAt).slice(0, 16)}`, - meta: { - sourceSchemaVersion: request.bundle.schemaVersion, - sourceExportedAt: request.bundle.exportedAt, - conflictStrategy: request.conflictStrategy ?? "skip", - inserted: result.inserted, - skipped: result.skipped, - replaced: result.replaced, - migrationMap: result.migrationMap, - conflicts: result.conflicts, - reembedMemoryIds - }, - createdAt: importedAt - }); - return { - ok: true, - importedAt, - conflictStrategy: request.conflictStrategy ?? "skip", - inserted: result.inserted, - skipped: result.skipped, - replaced: result.replaced, - migrationMap: result.migrationMap, - conflicts: result.conflicts, - reembedMemoryIds, - auditId: audit.id, - serverTime: nowIso() - }; + try { + return this.repos.transaction(() => { + const context = this.resolveContext(request); + const importedAt = nowIso(); + const result = this.repos.runtime.importBundleTables(request.bundle.tables, { + conflictStrategy: request.conflictStrategy ?? "skip" + }); + this.sessionTurns.migrateLegacyCaptures(); + const importedVectors = importedMemoryVectors(request.bundle.tables.memory_vectors) + .filter(row => !result.skippedMemoryIds.has(row.memory_id)); + this.repos.vectors.importRows(importedVectors); + result.inserted.memory_vectors = importedVectors.length; + const reembedMemoryIds = importedReembedMemoryIds( + { ...request.bundle.tables, memory_vectors: importedVectors }, + this.embedder.config.model ?? this.embedder.config.provider + ); + const audit = this.repos.runtime.insertAudit({ + userId: context.userId, + actor: request.namespace ? { ...request.namespace } : {}, + action: "import", + targetKind: "bundle", + targetId: `import_${stableHash(importedAt).slice(0, 16)}`, + meta: { + sourceSchemaVersion: request.bundle.schemaVersion, + sourceExportedAt: request.bundle.exportedAt, + conflictStrategy: request.conflictStrategy ?? "skip", + inserted: result.inserted, + skipped: result.skipped, + replaced: result.replaced, + migrationMap: result.migrationMap, + conflicts: result.conflicts, + reembedMemoryIds + }, + createdAt: importedAt + }); + return { + ok: true, + importedAt, + conflictStrategy: request.conflictStrategy ?? "skip", + inserted: result.inserted, + skipped: result.skipped, + replaced: result.replaced, + migrationMap: result.migrationMap, + conflicts: result.conflicts, + reembedMemoryIds, + auditId: audit.id, + serverTime: nowIso() + }; + }); + } catch (error) { + reloadDeviceScopeAliases(this.repos.db); + throw error; + } } archiveMemory(id: string, request: MemoryGovernanceRequest = {}): { @@ -2478,47 +2495,52 @@ export class MemoryService { this.assertMemoryInScope(memory, request.namespace); const kind = kindFromMemory(memory); const at = nowIso(); - const archived = this.repos.memories.archive(memory.id, at); - if (!archived) { - throw new MemoryServiceError("not_found", `memory not found: ${id}`); - } - const changeSeq = this.repos.runtime.appendChange({ - memoryId: archived.id, - namespaceId: namespaceIdFromMemory(archived), - kind: kindFromMemory(archived), - op: "archived", - entityId: archived.id, - userId: archived.userId, - changeType: "archive", - version: archived.version, - before: memory, - after: archived, - source: "panel.archive", - createdAt: archived.updatedAt - }); - const audit = this.repos.runtime.insertAudit({ - userId: archived.userId, - sessionId: archived.sessionId, - actor: request.namespace ? { ...request.namespace } : {}, - action: "archive", - targetKind: kindFromMemory(archived), - targetId: archived.id, - before: memory, - after: archived, - meta: { reason: request.reason }, - createdAt: archived.updatedAt - }); - this.evolutionJobs.invalidateMemoryDependencies(memory, at); - return { - ok: true, - id: archived.id, - kind: kindFromMemory(archived), - status: "archived", - changeSeq, - syncCursor: this.encodeChangeCursor(changeSeq, request.namespace ?? namespaceForMemory(archived)), - auditId: audit.id, - serverTime: nowIso() + const archive = () => { + const archived = isStrictL3WorldModelV2Memory(memory) + ? this.repos.l3WorldModels.archiveScopeMemory(memory.id, at) + : this.repos.memories.archive(memory.id, at); + if (!archived) { + throw new MemoryServiceError("not_found", `memory not found: ${id}`); + } + const changeSeq = this.repos.runtime.appendChange({ + memoryId: archived.id, + namespaceId: namespaceIdFromMemory(archived), + kind: kindFromMemory(archived), + op: "archived", + entityId: archived.id, + userId: archived.userId, + changeType: "archive", + version: archived.version, + before: memory, + after: archived, + source: "panel.archive", + createdAt: archived.updatedAt + }); + const audit = this.repos.runtime.insertAudit({ + userId: archived.userId, + sessionId: archived.sessionId, + actor: request.namespace ? { ...request.namespace } : {}, + action: "archive", + targetKind: kindFromMemory(archived), + targetId: archived.id, + before: memory, + after: archived, + meta: { reason: request.reason }, + createdAt: archived.updatedAt + }); + this.evolutionJobs.invalidateMemoryDependencies(memory, at); + return { + ok: true as const, + id: archived.id, + kind: kindFromMemory(archived), + status: "archived" as const, + changeSeq, + syncCursor: this.encodeChangeCursor(changeSeq, request.namespace ?? namespaceForMemory(archived)), + auditId: audit.id, + serverTime: nowIso() + }; }; + return isStrictL3WorldModelV2Memory(memory) ? this.repos.transaction(archive) : archive(); } deleteMemory(id: string, request: MemoryGovernanceRequest = {}): { @@ -2534,10 +2556,6 @@ export class MemoryService { this.assertMemoryAddEnabled(); const userMemory = this.repos.userMemories.get(id); if (userMemory) { - const namespaceUserId = request.namespace?.userId; - if (namespaceUserId && namespaceUserId !== userMemory.userId) { - throw new MemoryServiceError("forbidden", "user memory belongs to a different user"); - } const deleted = this.repos.userMemories.softDelete(userMemory.id, nowIso()); if (!deleted) throw new MemoryServiceError("not_found", `user memory not found: ${id}`); const changeSeq = this.repos.runtime.appendChange({ @@ -2590,12 +2608,8 @@ export class MemoryService { throw new MemoryServiceError("conflict", "invalid L3 World Model v2 record"); } if (strictV2WorldModel) { - const effectiveUserId = normalizeNamespace(request.namespace).userId; const projectId = typeof memory.info.project_id === "string" ? memory.info.project_id : null; - if (effectiveUserId !== memory.userId) { - throw new MemoryServiceError("forbidden", "L3 World Model belongs to a different user"); - } - if (request.namespace?.projectId && request.namespace.projectId !== projectId) { + if (request.namespace?.projectId && this.normalizeDeviceNamespace(request.namespace).projectId !== projectId) { throw new MemoryServiceError("forbidden", "L3 World Model belongs to a different project"); } } else { @@ -2672,9 +2686,6 @@ export class MemoryService { this.assertMemorySearchEnabled(); const event = this.repos.runtime.getRecallEventByQueryId(queryId); if (!event) throw new MemoryServiceError("not_found", `recall event not found: ${queryId}`); - if (request.namespace?.userId && request.namespace.userId !== event.userId) { - throw new MemoryServiceError("forbidden", "recall event belongs to a different user"); - } const eventRequest = isRecord(event.request) ? event.request : {}; const evidence = isRecord(eventRequest.recallEvidence) ? eventRequest.recallEvidence : {}; const storedHits = Array.isArray(evidence.hits) @@ -2734,7 +2745,7 @@ export class MemoryService { this.repos.transaction(() => { const spanIds = new Set(this.repos.memories.listTaskSpanIds( - episode.userId, episode.id, episode.l1MemoryIds + episode.id, episode.l1MemoryIds )); for (const spanId of spanIds) { this.deleteMemory(spanId, request); @@ -2835,7 +2846,7 @@ export class MemoryService { } auditLogs(input: RequestEnvelope & { - userId?: string; + userId?: string | null; targetKind?: string; targetId?: string; limit?: number; @@ -2847,7 +2858,7 @@ export class MemoryService { } serviceLogs(input: RequestEnvelope & { - userId?: string; + userId?: string | null; limit?: number; cursor?: string; } = {}): { @@ -2856,7 +2867,7 @@ export class MemoryService { type: "change" | "audit" | "job"; id: string; at: string; - userId?: string; + userId?: string | null; action: string; targetKind?: string; targetId?: string; @@ -2888,7 +2899,7 @@ export class MemoryService { return this.panelReadModel.apiLogs(input); } - serviceMetrics(input: RequestEnvelope & { userId?: string } = {}): { + serviceMetrics(input: RequestEnvelope & { userId?: string | null } = {}): { storage: StorageBackendCapabilities; schema: { version: number; lastMigrationId?: string }; memory: ReturnType["stats"]; @@ -2904,7 +2915,7 @@ export class MemoryService { return this.panelReadModel.serviceMetrics(input); } - adminStatus(input: RequestEnvelope & { userId?: string } = {}, routes: string[] = []): { + adminStatus(input: RequestEnvelope & { userId?: string | null } = {}, routes: string[] = []): { health: HealthResponse; overview: ReturnType; failedJobs: EvolutionJobRecord[]; @@ -2923,7 +2934,7 @@ export class MemoryService { return this.panelReadModel.configStatus(_input); } - panelOverview(input: RequestEnvelope & { userId?: string } = {}): { + panelOverview(input: RequestEnvelope & { userId?: string | null } = {}): { stats: { byLayer: Record; byStatus: Record<"activated" | "resolving" | "archived" | "deleted", number>; @@ -2942,7 +2953,7 @@ export class MemoryService { return this.panelReadModel.panelOverview(this.withTimeZone(input)); } - panelOverviewSummary(input: RequestEnvelope & { userId?: string } = {}): { + panelOverviewSummary(input: RequestEnvelope & { userId?: string | null } = {}): { counts: { memories: number; userMemories: number; @@ -2960,7 +2971,7 @@ export class MemoryService { return this.panelReadModel.panelOverviewSummary(this.withTimeZone(input)); } - panelAnalysis(input: RequestEnvelope & { userId?: string } = {}): { + panelAnalysis(input: RequestEnvelope & { userId?: string | null } = {}): { metrics: { avgRecallScore: number; recallEvents: number; @@ -2980,7 +2991,7 @@ export class MemoryService { } panelItems(input: RequestEnvelope & { - userId?: string; + userId?: string | null; layer?: RecallMemoryLayer; status?: "activated" | "resolving" | "archived" | "deleted"; q?: string; @@ -3025,7 +3036,7 @@ export class MemoryService { } panelChanges(input: RequestEnvelope & { - userId?: string; + userId?: string | null; limit?: number; cursor?: string; } = {}): { @@ -3047,7 +3058,7 @@ export class MemoryService { } panelJobs(input: RequestEnvelope & { - userId?: string; + userId?: string | null; status?: "queued" | "leased" | "succeeded" | "failed" | "dead_letter"; limit?: number; } = {}): { @@ -3091,8 +3102,8 @@ export class MemoryService { return new EmbeddingMaintenance({ repos: this.repos, embedder: this.embedder, - userId: this.config.userId?.trim() || "local-user", - currentWork: dimension => this.currentEmbeddingMaintenanceState(this.config.userId?.trim() || "local-user", dimension), + userId: null, + currentWork: dimension => this.currentEmbeddingMaintenanceState(null, dimension), enqueueMemory: (memory, text, at, vectorField = embeddingRetryVectorFieldForMemory(memory)) => { const existing = this.repos.runtime.getEmbeddingRetryByTarget( embeddingRetryTargetKindForMemory(memory), memory.id, vectorField @@ -3131,7 +3142,7 @@ export class MemoryService { }); } - private currentEmbeddingMaintenanceState(userId: string, dimension: number) { + private currentEmbeddingMaintenanceState(userId: string | null, dimension: number) { // Key by target and vector field so upstream action-vector maintenance also // uses the same current-content checks as the panel and repair enqueueing. const outdatedVectors = new Set(); @@ -3310,7 +3321,7 @@ export class MemoryService { private buildMemory(input: { id?: string; - userId: string; + userId: string | null; conversationId?: string; sessionId?: string; agentId?: string; @@ -3387,7 +3398,7 @@ export class MemoryService { } private readOnlyCursor(namespace?: RuntimeNamespace): { changeSeq: number; syncCursor: string } { - const scoped = namespace ? normalizeNamespace(namespace) : undefined; + const scoped = namespace ? this.normalizeDeviceNamespace(namespace) : undefined; const changeSeq = this.repos.runtime.latestChangeSeq( scoped?.userId, scoped ? namespaceIdFromContext(scoped) : undefined @@ -3399,12 +3410,12 @@ export class MemoryService { } private openSessionNoWrite(request: SessionOpenRequest): ReturnType { - const namespace = normalizeNamespace(request.namespace); + const namespace = this.normalizeDeviceNamespace(request.namespace); const existing = request.sessionId ? this.repos.runtime.getSession(request.sessionId) : namespace.sessionKey ? this.repos.runtime.findOpenSessionByHostKey({ - userId: namespace.userId, + tenantId: namespace.tenantId ?? null, source: request.source ?? namespace.source, profileId: request.profileId ?? namespace.profileId, hostSessionKey: namespace.sessionKey @@ -3427,10 +3438,13 @@ export class MemoryService { }; } const sessionId = request.sessionId ?? `session_${stableHash({ - userId: namespace.userId, source: request.source ?? namespace.source, profileId: request.profileId ?? namespace.profileId, - sessionKey: namespace.sessionKey ?? "readonly" + sessionKey: namespace.sessionKey ?? "readonly", + projectId: canonicalProjectId(this.repos.db, request.projectId) ?? namespace.projectId ?? namespace.workspaceId ?? null, + workspaceId: canonicalProjectId(this.repos.db, request.workspaceId) ?? namespace.workspaceId ?? null, + conversationId: stringFromMeta(request.meta, "conversationId") ?? null, + tenantId: namespace.tenantId ?? null }).slice(0, 20)}`; return { sessionId, @@ -3634,21 +3648,30 @@ export class MemoryService { } private assertSessionInScope(session: SessionRecord, namespace?: RuntimeNamespace): void { - void session; - void namespace; + const snakeTenant = session.meta.tenant_id; + const camelTenant = session.meta.tenantId; + const snakeId = typeof snakeTenant === "string" && snakeTenant.length > 0 ? snakeTenant : null; + const camelId = typeof camelTenant === "string" && camelTenant.length > 0 ? camelTenant : null; + if (snakeId !== null && camelId !== null && snakeId !== camelId) { + throw new MemoryServiceError("forbidden", "session has conflicting tenant metadata"); + } + if (!namespace) return; + const requestedTenant = namespace.tenantId ?? null; + if ((snakeId ?? camelId) !== requestedTenant) { + throw new MemoryServiceError("forbidden", "session belongs to a different tenant"); + } } private assertL3WorldModelSessionScope(session: SessionRecord, namespace: RuntimeNamespace): void { if (session.meta.l3_world_model_protocol_version !== 2) { throw new MemoryServiceError("conflict", "l3_world_model_protocol_v2_required"); } - const normalized = normalizeNamespace(namespace); + const normalized = this.normalizeDeviceNamespace(namespace); const conflicts = [ - normalized.userId !== session.userId, normalized.source !== session.source, normalized.profileId !== session.profileId, (normalized.projectId ?? null) !== (session.projectId ?? null), - Boolean(namespace.workspaceId && namespace.workspaceId !== session.workspaceId), + Boolean(normalized.workspaceId && normalized.workspaceId !== session.workspaceId), Boolean(namespace.sessionKey && namespace.sessionKey !== session.hostSessionKey) ]; if (conflicts.some(Boolean)) { @@ -3684,7 +3707,7 @@ export class MemoryService { v: 1, backendId: capabilities.backendId, schemaVersion: capabilities.schemaVersion, - namespaceId: namespace ? namespaceIdFromContext(normalizeNamespace(namespace)) : "", + namespaceId: namespace ? namespaceIdFromContext(this.normalizeDeviceNamespace(namespace)) : "", seq }; return `cur_${Buffer.from(JSON.stringify(payload)).toString("base64url")}`; @@ -3720,8 +3743,8 @@ export class MemoryService { return this.sessionTurns.ensureEpisode(session, episodeId); } - private resolveContext(request: RequestEnvelope & { sessionId?: string; userId?: string }): { - userId: string; + private resolveContext(request: RequestEnvelope & { sessionId?: string; userId?: string | null }): { + userId: string | null; conversationId?: string; namespace: RuntimeNamespace; } { @@ -3736,8 +3759,8 @@ export class MemoryService { }; } } - const namespace = normalizeNamespace(request.namespace); - const userId = request.userId ?? namespace.userId; + const namespace = this.normalizeDeviceNamespace(request.namespace); + const userId = null; return { userId, namespace: { diff --git a/Memory/src/service/namespace/namespace-scope.ts b/Memory/src/service/namespace/namespace-scope.ts index 126b727d4..30a93d63c 100644 --- a/Memory/src/service/namespace/namespace-scope.ts +++ b/Memory/src/service/namespace/namespace-scope.ts @@ -6,7 +6,7 @@ import { type ResolvedWorkspaceIdentity } from "./workspace-identity.js"; -export function normalizeNamespace(namespace?: RuntimeNamespace): RuntimeNamespace & { userId: string; source: string; profileId: string } { +export function normalizeNamespace(namespace?: RuntimeNamespace): RuntimeNamespace & { userId: string | null; source: string; profileId: string } { return { source: namespace?.source ?? DEFAULT_NAMESPACE_SOURCE, profileId: namespace?.profileId ?? "default", @@ -15,7 +15,7 @@ export function normalizeNamespace(namespace?: RuntimeNamespace): RuntimeNamespa workspaceId: namespace?.workspaceId, workspacePath: namespace?.workspacePath, sessionKey: namespace?.sessionKey, - userId: namespace?.userId ?? "local-user", + userId: null, tenantId: namespace?.tenantId }; } @@ -32,7 +32,7 @@ export function sessionScopeForOpenRequest(request: SessionOpenRequest, namespac export function resolveV2WorkspaceIdentityForOpenRequest( request: SessionOpenRequest, - namespace: RuntimeNamespace & { userId: string } + namespace: RuntimeNamespace & { userId: string | null } ): ResolvedWorkspaceIdentity { return resolveWorkspaceIdentity(namespace.userId, { workspaceUri: request.workspaceUri, @@ -41,7 +41,15 @@ export function resolveV2WorkspaceIdentityForOpenRequest( } export function namespaceForSession(session: SessionRecord): RuntimeNamespace { - return { source: session.source, profileId: session.profileId, profileLabel: session.profileLabel, projectId: session.projectId, workspaceId: session.workspaceId, workspacePath: session.workspacePath, sessionKey: session.hostSessionKey, userId: session.userId }; + const snakeTenant = session.meta.tenant_id; + const camelTenant = session.meta.tenantId; + const snakeId = typeof snakeTenant === "string" && snakeTenant ? snakeTenant : undefined; + const camelId = typeof camelTenant === "string" && camelTenant ? camelTenant : undefined; + if (snakeId && camelId && snakeId !== camelId) { + throw new Error("session has conflicting tenant metadata"); + } + const tenantId = snakeId ?? camelId; + return { source: session.source, profileId: session.profileId, profileLabel: session.profileLabel, projectId: session.projectId, workspaceId: session.workspaceId, workspacePath: session.workspacePath, sessionKey: session.hostSessionKey, userId: session.userId, tenantId }; } export function namespaceForMemory(memory: MemoryRow): RuntimeNamespace { diff --git a/Memory/src/service/namespace/workspace-identity.ts b/Memory/src/service/namespace/workspace-identity.ts index 02c5b1315..85ef7ecc9 100644 --- a/Memory/src/service/namespace/workspace-identity.ts +++ b/Memory/src/service/namespace/workspace-identity.ts @@ -20,11 +20,9 @@ export interface ResolvedWorkspaceIdentity { * the Agent Adapter that owns the workspace. */ export function resolveWorkspaceIdentity( - effectiveUserId: string, + effectiveUserId: string | null, input: WorkspaceIdentityFields ): ResolvedWorkspaceIdentity { - const userId = effectiveUserId.trim(); - if (!userId) throw new TypeError("effectiveUserId must be non-empty"); const parsed = WorkspaceIdentityFieldsSchema.parse(input); if (!parsed.workspaceUri) { return { @@ -38,7 +36,7 @@ export function resolveWorkspaceIdentity( const workspaceIdentity = isLocalWorkspaceUri(parsed.workspaceUri) ? `local\0${parsed.workspaceHostId}\0${parsed.workspaceUri}` : `remote\0${parsed.workspaceUri}`; - const workspaceId = sha256Hex(`${userId}\0${workspaceIdentity}`); + const workspaceId = sha256Hex(workspaceIdentity); return { workspaceUri: parsed.workspaceUri, workspaceHostId: parsed.workspaceHostId ?? null, diff --git a/Memory/src/service/project-environment/profile-pipeline.ts b/Memory/src/service/project-environment/profile-pipeline.ts index 17941c4ea..f8ed1085d 100644 --- a/Memory/src/service/project-environment/profile-pipeline.ts +++ b/Memory/src/service/project-environment/profile-pipeline.ts @@ -72,7 +72,6 @@ export class ProjectEnvironmentProfilePipeline { async process(job: EvolutionJobRecord, derived: ProjectEnvironmentDerivedEvidence): Promise { const payload = projectEnvironmentProfileJobPayload(job.payload); - if (job.userId !== payload.userId) throw new Error("project_environment_job_owner_mismatch"); const state = this.deps.repos.projectEnvironments.getState(payload.userId, payload.projectId); if (!state || state.currentScanId !== payload.scanId) return; @@ -132,16 +131,16 @@ export class ProjectEnvironmentProfilePipeline { } export function projectEnvironmentProfileJobPayload(value: Record): { - userId: string; + userId: string | null; projectId: string; scanId: string; trigger: "session_start" | "token_compaction"; } { - const userId = stringValue(value.userId); + const userId = null; const projectId = stringValue(value.projectId); const scanId = stringValue(value.scanId); const trigger = value.trigger; - if (!userId || !projectId || !scanId || (trigger !== "session_start" && trigger !== "token_compaction")) { + if (!projectId || !scanId || (trigger !== "session_start" && trigger !== "token_compaction")) { throw new TypeError(`invalid project environment job payload: ${canonicalJson(value as never)}`); } return { userId, projectId, scanId, trigger }; diff --git a/Memory/src/service/project-environment/project-environment-service.ts b/Memory/src/service/project-environment/project-environment-service.ts index b4ff673a5..c26d56456 100644 --- a/Memory/src/service/project-environment/project-environment-service.ts +++ b/Memory/src/service/project-environment/project-environment-service.ts @@ -63,10 +63,9 @@ export class ProjectEnvironmentService { async processProfileJob(job: EvolutionJobRecord): Promise { const payload = projectEnvironmentProfileJobPayload(job.payload); - if (job.userId !== payload.userId) throw new Error("project_environment_job_owner_mismatch"); if (!this.deps.repos.projectEnvironments.beginScan(payload.userId, payload.projectId, payload.scanId)) return; try { - const scope = this.deps.repos.l3WorldModels.getScope(payload.userId, payload.projectId); + const scope = this.deps.repos.l3WorldModels.ensureScope(payload.userId, payload.projectId); if (!scope?.workspaceUri) throw new Error("project_environment_workspace_uri_missing"); const scan = await scanLocalProject(scope.workspaceUri); const classification = classifyProjectInventory(scan.entries); diff --git a/Memory/src/service/project-environment/types.ts b/Memory/src/service/project-environment/types.ts index 244b21eae..12555862d 100644 --- a/Memory/src/service/project-environment/types.ts +++ b/Memory/src/service/project-environment/types.ts @@ -58,7 +58,7 @@ export interface ProjectEnvironmentDerivedEvidence { } export interface ProjectEnvironmentStateRecord { - userId: string; + userId: string | null; projectId: string; projectKind: ProjectEnvironmentKind; status: ProjectEnvironmentScanStatus; diff --git a/Memory/src/service/read-model/episode.ts b/Memory/src/service/read-model/episode.ts index 9f66de78f..1806f3b17 100644 --- a/Memory/src/service/read-model/episode.ts +++ b/Memory/src/service/read-model/episode.ts @@ -70,7 +70,7 @@ export type MemoryGetResponse = MemoryDetailResponse | (EpisodeDetailItem & { }); export interface TimelineRequest extends RequestEnvelope { - userId?: string; + userId?: string | null; sessionId?: string; episodeId?: string; layers?: MemoryLayer[]; @@ -99,13 +99,13 @@ export interface EpisodeReadModelRepositories { getRawTurn(id: string): RawTurnRecord | undefined; listRawTurnsByEpisode(episodeId: string, limit?: number): RawTurnRecord[]; listTracePolicyLinks(input: { - userId?: string; + userId?: string | null; l1MemoryId?: string; l2MemoryId?: string; limit?: number; }): TracePolicyLinkRecord[]; listSkillTrials(input: { - userId?: string; + userId?: string | null; skillMemoryId?: string; sessionId?: string; episodeId?: string; @@ -125,6 +125,7 @@ export interface EpisodeReadModelRepositories { */ export interface EpisodeReadModelDependencies { repos: EpisodeReadModelRepositories; + projectMemory?(memory: MemoryRow): MemoryRow; assertMemorySearchEnabled(): void; resolveContext(request: RequestEnvelope): unknown; requireSession(id: string): SessionRecord; @@ -243,7 +244,8 @@ export class EpisodeReadModel { throw this.deps.invalidArgument("raw turn ids are internal; use the containing trace_ or episode_ id"); } - const memory = this.deps.repos.memories.get(id); + const storedMemory = this.deps.repos.memories.get(id); + const memory = storedMemory && (this.deps.projectMemory?.(storedMemory) ?? storedMemory); if (!memory) throw this.deps.notFound(`memory not found: ${id}`); this.deps.assertMemoryInScope(memory, request.namespace); const detail = this.deps.detailFromMemory(memory, this.deps.repos.processing.get(memory.id)); diff --git a/Memory/src/service/read-model/l3-world-model-context.ts b/Memory/src/service/read-model/l3-world-model-context.ts index 0fcce5af5..5b2c69ff6 100644 --- a/Memory/src/service/read-model/l3-world-model-context.ts +++ b/Memory/src/service/read-model/l3-world-model-context.ts @@ -1,5 +1,7 @@ import { + canonicalJson, renderL3WorldModelFields, + sha256Hex, type SessionL3WorldModelContextResponse } from "../../contracts/index.js"; import type { Repositories, SessionRecord } from "../../storage/repositories.js"; @@ -27,7 +29,9 @@ export class L3WorldModelContextReadModel { schemaVersion: 2, projectId, memoryId: memory.id, - memoryVersion: memory.version, + // Clients cache the projected context, not the stored row. Scan validity + // and inherited provenance can change without changing the stored text. + memoryVersion: Number.parseInt(sha256Hex(canonicalJson({ fields: { ...fields }, sourceMemoryIds })).slice(0, 12), 16) || 1, renderedContext: renderL3WorldModelFields(fields), sourceMemoryIds, generalRulesAndSafetyConstraints: fields.generalRulesAndSafetyConstraints, @@ -39,7 +43,7 @@ export class L3WorldModelContextReadModel { } private environmentProfileIsApplied( - userId: string, + userId: string | null, projectId: string, memoryScanId: unknown ): boolean { @@ -47,8 +51,8 @@ export class L3WorldModelContextReadModel { const row = this.repos.db.prepare( `SELECT applied_scan_id FROM l3_world_model_project_environment_state - WHERE user_id = ? AND project_id = ?` - ).get(userId, projectId) as { applied_scan_id: string | null } | undefined; + WHERE memory_project_id(project_id) = ? AND applied_scan_id = ? LIMIT 1` + ).get(projectId, memoryScanId) as { applied_scan_id: string | null } | undefined; return row?.applied_scan_id === memoryScanId; } } diff --git a/Memory/src/service/read-model/panel-read.ts b/Memory/src/service/read-model/panel-read.ts index 0e34ac1c6..c48d8396f 100644 --- a/Memory/src/service/read-model/panel-read.ts +++ b/Memory/src/service/read-model/panel-read.ts @@ -75,8 +75,8 @@ export interface PanelReadModelDependencies { schemaVersion: () => { version: number; lastMigrationId?: string }; health: (routes?: string[]) => HealthResponse; models: () => HealthResponse["models"]; - resolveContext: (request: RequestEnvelope & { sessionId?: string; userId?: string }) => { - userId: string; + resolveContext: (request: RequestEnvelope & { sessionId?: string; userId?: string | null }) => { + userId: string | null; conversationId?: string; namespace: RuntimeNamespace; }; @@ -95,7 +95,7 @@ export class PanelReadModel { } auditLogs(input: RequestEnvelope & { - userId?: string; + userId?: string | null; targetKind?: string; targetId?: string; limit?: number; @@ -115,7 +115,7 @@ export class PanelReadModel { } serviceLogs(input: RequestEnvelope & { - userId?: string; + userId?: string | null; limit?: number; cursor?: string; } = {}): { @@ -124,7 +124,7 @@ export class PanelReadModel { type: "change" | "audit" | "job"; id: string; at: string; - userId?: string; + userId?: string | null; action: string; targetKind?: string; targetId?: string; @@ -263,7 +263,7 @@ export class PanelReadModel { return typeof goal === "string" && goal.trim() ? goal.trim() : undefined; } - serviceMetrics(input: RequestEnvelope & { userId?: string } = {}): { + serviceMetrics(input: RequestEnvelope & { userId?: string | null } = {}): { storage: StorageBackendCapabilities; schema: { version: number; lastMigrationId?: string }; memory: ReturnType["stats"]; @@ -291,7 +291,7 @@ export class PanelReadModel { }; } - adminStatus(input: RequestEnvelope & { userId?: string } = {}, routes: string[] = []): { + adminStatus(input: RequestEnvelope & { userId?: string | null } = {}, routes: string[] = []): { health: HealthResponse; overview: ReturnType; failedJobs: EvolutionJobRecord[]; @@ -322,7 +322,7 @@ export class PanelReadModel { }; } - panelOverview(_input: RequestEnvelope & { userId?: string } = {}): { + panelOverview(_input: RequestEnvelope & { userId?: string | null } = {}): { stats: { byLayer: Record; byStatus: Record<"activated" | "resolving" | "archived" | "deleted", number>; @@ -361,19 +361,19 @@ export class PanelReadModel { }; } - panelOverviewSummary(input: RequestEnvelope & { userId?: string } = {}): { + panelOverviewSummary(input: RequestEnvelope & { userId?: string | null } = {}): { counts: { memories: number; userMemories: number; skills: number; experiences: number; worldModels: number }; sourceDistribution: Array<{ source: string; count: number; percentage: number }>; dailyActivity: Array<{ date: string; count: number }>; } { const memories = this.deps.repos.memories.listStats(); - const userId = input.userId ?? input.namespace?.userId; + const userId = null; const timeZone = resolveTimeZone(input.timeZone); const dates = panelDateKeys(this.now(), PANEL_DAILY_ACTIVITY_DAYS, timeZone); return { counts: { memories: memories.filter((memory) => memory.memoryLayer === "L1").length, - userMemories: userId ? this.deps.repos.userMemories.countForPanel({ userId }) : 0, + userMemories: this.deps.repos.userMemories.countForPanel({ userId }), skills: memories.filter((memory) => memory.memoryLayer === "Skill").length, experiences: memories.filter((memory) => memory.memoryLayer === "L2").length, worldModels: memories.filter((memory) => memory.memoryLayer === "L3").length @@ -383,7 +383,7 @@ export class PanelReadModel { }; } - panelAnalysis(input: RequestEnvelope & { userId?: string } = {}): { + panelAnalysis(input: RequestEnvelope & { userId?: string | null } = {}): { metrics: { avgRecallScore: number; recallEvents: number; @@ -426,7 +426,7 @@ export class PanelReadModel { } panelItems(input: RequestEnvelope & { - userId?: string; + userId?: string | null; layer?: RecallMemoryLayer; status?: "activated" | "resolving" | "archived" | "deleted"; q?: string; @@ -450,9 +450,9 @@ export class PanelReadModel { } { const pageSize = normalizePanelItemsLimit(input.limit); if (input.layer === "UserMemory") { - const userId = input.userId ?? input.namespace?.userId; + const userId = null; const requestedPage = normalizePageNumber(input.page); - if (!userId || input.status === "resolving") { + if (input.status === "resolving") { return emptyPanelItems(requestedPage, pageSize, this.now()); } const status = input.status === "activated" @@ -497,14 +497,28 @@ export class PanelReadModel { agentId: input.sourceAgent, excludedAgentIds: input.excludedSourceAgents }; - const total = input.q?.trim() + const projectedWorlds = input.layer === "L3" ? (() => { + const seen = new Set(); + const projectionFilter = { ...filter, status: undefined }; + return this.deps.repos.memories.list(projectionFilter, 100000, 0).flatMap(memory => { + if (memory.properties.internal_info.schema_version !== 2) return [memory]; + const project = typeof memory.info.project_id === "string" ? memory.info.project_id : null; + if (seen.has(project ?? "")) return []; + seen.add(project ?? ""); + const shared = this.deps.repos.l3WorldModels.getMemory(null, project); + if (!shared) return [memory]; + return [shared]; + }).filter(memory => (!input.status || memory.status === input.status) && + (!input.q?.trim() || memory.memoryValue.toLowerCase().includes(input.q.trim().toLowerCase()))); + })() : undefined; + const total = projectedWorlds ? projectedWorlds.length : input.q?.trim() ? this.deps.repos.memories.searchCount(input.q, { ...filter, status: filter.status ?? ["activated", "resolving"] }) : this.deps.repos.memories.count(filter); const totalPages = Math.max(1, Math.ceil(total / pageSize)); const requestedPage = normalizePageNumber(input.page); const page = Math.min(requestedPage, totalPages); const offset = normalizeOffsetCursor(input.cursor) ?? ((page - 1) * pageSize); - const memories = input.q?.trim() + const memories = projectedWorlds ? projectedWorlds.slice(offset, offset + pageSize) : input.q?.trim() ? this.deps.repos.memories.getMany(this.deps.repos.memories.searchPanelIds( input.q, { ...filter, status: filter.status ?? ["activated", "resolving"] }, @@ -541,7 +555,7 @@ export class PanelReadModel { const item = sourceText ? { ...listed, sourceText } : listed; const scope = scopesByMemoryId.get(memory.id); const worldModelScope = memory.memoryLayer === "L3" && scope && - scope.memoryId === memory.id && scope.userId === memory.userId + scope.memoryId === memory.id && (scope.projectId ?? null) === (memory.info.project_id ?? null) ? panelWorldModelScope(scope) : undefined; return worldModelScope ? { ...item, worldModelScope } : item; @@ -609,7 +623,7 @@ export class PanelReadModel { } panelChanges(input: RequestEnvelope & { - userId?: string; + userId?: string | null; limit?: number; cursor?: string; } = {}): { @@ -633,7 +647,7 @@ export class PanelReadModel { } panelJobs(input: RequestEnvelope & { - userId?: string; + userId?: string | null; status?: "queued" | "leased" | "succeeded" | "failed" | "dead_letter"; limit?: number; } = {}): { diff --git a/Memory/src/service/read-model/skill.ts b/Memory/src/service/read-model/skill.ts index 03c51622c..c373a0ff5 100644 --- a/Memory/src/service/read-model/skill.ts +++ b/Memory/src/service/read-model/skill.ts @@ -75,7 +75,7 @@ export interface SkillRuntimeRepository { } | undefined; saveIdempotency(key: string, requestHash: string, response: unknown): void; listSkillTrials(input?: { - userId?: string; + userId?: string | null; skillMemoryId?: string; sessionId?: string; episodeId?: string; @@ -85,7 +85,7 @@ export interface SkillRuntimeRepository { outcome?: SkillTrialRecord["outcome"]; limit?: number; }): SkillTrialRecord[]; - latestChangeSeq(userId?: string, namespaceId?: string): number; + latestChangeSeq(userId?: string | null, namespaceId?: string): number; insertSkillTrial(trial: SkillTrialRecord): SkillTrialRecord; appendEpisodeDerivedMemory( episodeId: string, @@ -99,7 +99,7 @@ export interface SkillRuntimeRepository { kind: "skill_trial"; op: "created"; entityId: string; - userId: string; + userId: string | null; changeType: "skill_trial"; after: SkillTrialRecord; source: "skill.use"; @@ -146,7 +146,7 @@ export interface SkillReadModelDeps { } export interface ListSkillsRequest extends RequestEnvelope { - userId?: string; + userId?: string | null; q?: string; tags?: string[]; limit?: number; diff --git a/Memory/src/service/retrieval/cross-account-candidate-dedup.ts b/Memory/src/service/retrieval/cross-account-candidate-dedup.ts deleted file mode 100644 index aab41318a..000000000 --- a/Memory/src/service/retrieval/cross-account-candidate-dedup.ts +++ /dev/null @@ -1,151 +0,0 @@ -import type { SeededChannelScores } from "../../algorithm/plugin-algorithms.js"; -import type { Repositories } from "../../storage/repositories.js"; -import { attachMemoryVector, memoryVector, memoryVectorEntries } from "../../storage/memory-vector-state.js"; -import type { MemoryRow } from "../../types.js"; -import { stableHash } from "../../utils/id.js"; -import { isRecord } from "../../utils/json.js"; - -export interface CrossAccountCandidateDedupResult { - memories: MemoryRow[]; - channelScoresByMemory: ReadonlyMap; - memoryIdByMemoryId: ReadonlyMap; -} - -/** - * Collapse copies of the same locally sourced memory before ranking. Account IDs - * identify the write owner, but the same native source turn can have been written - * under more than one account before cross-account capture deduplication existed. - */ -export function deduplicateCrossAccountMemoryCandidates(input: { - memories: readonly MemoryRow[]; - repos: Repositories; - preferredUserId: string; - channelScoresByMemory?: ReadonlyMap; -}): CrossAccountCandidateDedupResult { - const groups = new Map(); - const nativeConversationBySessionId = new Map(); - input.memories.forEach((memory, index) => { - const key = duplicateKey(memory, input.repos, nativeConversationBySessionId); - if (!key) { - groups.set(`unique:${index}:${memory.id}`, { firstIndex: index, memories: [memory] }); - return; - } - const group = groups.get(key); - if (group) group.memories.push(memory); - else groups.set(key, { firstIndex: index, memories: [memory] }); - }); - - const selected = [...groups.values()] - .sort((left, right) => left.firstIndex - right.firstIndex) - .map((group) => { - const representative = [...group.memories].sort((left, right) => - Number(right.userId === input.preferredUserId) - Number(left.userId === input.preferredUserId) || - candidateScore(right, input.channelScoresByMemory) - candidateScore(left, input.channelScoresByMemory) || - left.createdAt.localeCompare(right.createdAt) || - left.id.localeCompare(right.id) - )[0]!; - return { representative, members: group.memories }; - }); - - const channelScoresByMemory = new Map(); - const memoryIdByMemoryId = new Map(); - for (const group of selected) { - if (group.members.length === 1) { - const memory = group.representative; - memoryIdByMemoryId.set(memory.id, memory.id); - const scores = input.channelScoresByMemory?.get(memory.id); - if (scores) channelScoresByMemory.set(memory.id, scores); - continue; - } - - const merged: SeededChannelScores = {}; - for (const memory of group.members) { - memoryIdByMemoryId.set(memory.id, group.representative.id); - for (const vector of memoryVectorEntries(memory)) { - if (!memoryVector(group.representative, vector.vectorField)) { - attachMemoryVector(group.representative, vector); - } - } - for (const [channel, score] of Object.entries(input.channelScoresByMemory?.get(memory.id) ?? {})) { - const key = channel as keyof SeededChannelScores; - const current = merged[key]; - if (typeof score === "number" && (current === undefined || score > current)) { - merged[key] = score; - } - } - } - if (Object.keys(merged).length > 0) channelScoresByMemory.set(group.representative.id, merged); - } - - return { - memories: selected.map((group) => group.representative), - channelScoresByMemory, - memoryIdByMemoryId - }; -} - -function duplicateKey( - memory: MemoryRow, - repos: Repositories, - nativeConversationBySessionId: Map -): string | undefined { - if (memory.memoryLayer !== "L1" || memory.properties.internal_info.memory_kind === "work_memory") { - return undefined; - } - const nativeConversation = nativeConversationKey(memory, repos, nativeConversationBySessionId); - if (!nativeConversation) return undefined; - const internal = memory.properties.internal_info; - const trace = isRecord(internal.trace) - ? internal.trace - : isRecord(memory.info.trace) - ? memory.info.trace - : undefined; - const turnId = text(memory.info.turn_id) ?? text(trace?.turn_id); - if (!turnId) return undefined; - const stepIndex = typeof trace?.step_index === "number" && Number.isFinite(trace.step_index) - ? trace.step_index - : memory.contentHash ?? stableHash(memory.memoryValue); - return `native-turn:${stableHash({ nativeConversation, turnId, stepIndex })}`; -} - -function nativeConversationKey( - memory: MemoryRow, - repos: Repositories, - cache: Map -): string | undefined { - if (!memory.sessionId) return undefined; - if (cache.has(memory.sessionId)) return cache.get(memory.sessionId); - const session = repos.runtime.getSession(memory.sessionId); - if (!session) { - cache.set(memory.sessionId, undefined); - return undefined; - } - const conversationId = session.conversationId?.trim() || memory.conversationId?.trim(); - const source = session.source.trim(); - const profileId = session.profileId.trim(); - if (!conversationId || !source || !profileId) { - cache.set(memory.sessionId, undefined); - return undefined; - } - - const storedNamespaceKey = text(session.meta.source_namespace_key); - const tenantId = text(session.meta.tenant_id) ?? text(session.meta.tenantId) ?? null; - const namespaceKey = storedNamespaceKey ?? stableHash({ - tenantId, - projectId: session.projectId ?? null, - workspaceId: session.workspaceId ?? null - }); - const key = stableHash({ source, profileId, namespaceKey, conversationId }); - cache.set(memory.sessionId, key); - return key; -} - -function candidateScore(memory: MemoryRow, scores?: ReadonlyMap): number { - return Math.max(0, ...Object.values(scores?.get(memory.id) ?? {}).filter((value): value is number => - typeof value === "number" && Number.isFinite(value) - )); -} - -function text(value: unknown): string | undefined { - return typeof value === "string" && value.trim() ? value.trim() : undefined; -} diff --git a/Memory/src/service/retrieval/indexed-candidate-pool.ts b/Memory/src/service/retrieval/indexed-candidate-pool.ts index e177b97ec..5fc2fc9df 100644 --- a/Memory/src/service/retrieval/indexed-candidate-pool.ts +++ b/Memory/src/service/retrieval/indexed-candidate-pool.ts @@ -5,6 +5,10 @@ import type { import { isMemoryReadyForRetrieval } from "../../algorithm/plugin-algorithms.js"; +import { + deduplicateSameSourceL1Candidates, + memoryMatchesRequestedTags +} from "./same-source-l1-dedup.js"; import { Repositories, type MemorySearchIdHit @@ -14,7 +18,6 @@ import type { MemoryLayer, MemoryRow } from "../../types.js"; -import { deduplicateCrossAccountMemoryCandidates } from "./cross-account-candidate-dedup.js"; export function dedupeStrings(values: readonly string[]): string[] { const out: string[] = []; @@ -36,7 +39,7 @@ export class IndexedCandidatePool { constructor(private readonly deps: IndexedCandidatePoolDependencies) {} retrievalCandidateCount(input: { - userId: string; + userId: string | null; layers: MemoryLayer[]; tags?: string[]; projectId?: string | null; @@ -52,7 +55,7 @@ export class IndexedCandidatePool { } hasRetrievalVectorCandidates(input: { - userId: string; + userId: string | null; layers: MemoryLayer[]; tags?: string[]; projectId?: string | null; @@ -69,7 +72,7 @@ export class IndexedCandidatePool { } async indexedRetrievalCandidatePool(input: { - userId: string; + userId: string | null; compiledQuery: CompiledRetrievalQuery; queryVector?: number[]; layers: MemoryLayer[]; @@ -77,6 +80,7 @@ export class IndexedCandidatePool { projectId?: string | null; targetSkillId?: string; currentAgentId?: string; + excludeTraceRawTurnIds?: ReadonlySet; config: { tier1TopK: number; tier2TopK: number; @@ -110,24 +114,36 @@ export class IndexedCandidatePool { if (queryVector) { if (layer === "L1") { - addRoute(() => this.searchTraceVectorRoutes(queryVector, filter, vectorPool, input.compiledQuery, input.config)); + addRoute(() => this.expandL1Route(filter, vectorPool, (excludedFilter, limit) => + this.searchTraceVectorRoutes(queryVector, excludedFilter, limit, input.compiledQuery, input.config), + input.excludeTraceRawTurnIds)); } else { addRoute(() => this.deps.repos.memories.searchVectorIds(queryVector, "vec", filter, vectorPool)); } } if (input.compiledQuery.ftsMatch) { - addRoute(() => this.deps.repos.memories.searchFtsIds(input.compiledQuery.ftsMatch, filter, keywordPool)); + addRoute(() => layer === "L1" + ? this.expandL1Route(filter, keywordPool, (excludedFilter, limit) => + this.deps.repos.memories.searchFtsIds(input.compiledQuery.ftsMatch, excludedFilter, limit), + input.excludeTraceRawTurnIds) + : this.deps.repos.memories.searchFtsIds(input.compiledQuery.ftsMatch, filter, keywordPool)); } if (input.compiledQuery.patternTerms.length > 0) { - addRoute(() => this.deps.repos.memories.searchPatternIds(input.compiledQuery.patternTerms, filter, keywordPool)); + addRoute(() => layer === "L1" + ? this.expandL1Route(filter, keywordPool, (excludedFilter, limit) => + this.deps.repos.memories.searchPatternIds(input.compiledQuery.patternTerms, excludedFilter, limit), + input.excludeTraceRawTurnIds) + : this.deps.repos.memories.searchPatternIds(input.compiledQuery.patternTerms, filter, keywordPool)); } if (layer === "L1" && input.compiledQuery.structuralFragments.length > 0) { - addRoute(() => this.deps.repos.memories.searchStructuralIds( - input.compiledQuery.structuralFragments, - filter, - Math.max(input.config.tier2TopK, 10) - )); + const structuralPool = Math.max(input.config.tier2TopK, 10); + addRoute(() => this.expandL1Route(filter, structuralPool, (excludedFilter, limit) => + this.deps.repos.memories.searchStructuralIds( + input.compiledQuery.structuralFragments, + excludedFilter, + limit + ), input.excludeTraceRawTurnIds)); } } @@ -152,12 +168,10 @@ export class IndexedCandidatePool { this.isMemoryReadyForRetrieval(memory) && this.isSkillVisibleToAgent(memory, input.currentAgentId) ); - return deduplicateCrossAccountMemoryCandidates({ + return { memories: readyMemories, - repos: this.deps.repos, - preferredUserId: input.userId, channelScoresByMemory - }); + }; } private isSkillVisibleToAgent(memory: MemoryRow, currentAgentId?: string): boolean { @@ -196,6 +210,57 @@ export class IndexedCandidatePool { return this.deps.repos.memories.searchVectorIds(queryVector, "vec_summary", filter, vectorPool); } + private expandL1Route( + filter: MemoryFilter, + limit: number, + search: (filter: MemoryFilter, limit: number) => MemorySearchIdHit[], + excludeTraceRawTurnIds?: ReadonlySet + ): MemorySearchIdHit[] { + const hits: MemorySearchIdHit[] = []; + const candidateIds = new Set(); + const excludedIds = new Set(); + let sourceCount = 0; + + while (true) { + if (sourceCount >= limit) break; + + const batch = search({ ...filter, excludedIds: [...excludedIds] }, Math.max(1, limit - sourceCount)); + const batchIds = new Set(); + const freshHits = batch.filter(hit => { + if (candidateIds.has(hit.id) || batchIds.has(hit.id)) return false; + batchIds.add(hit.id); + return true; + }); + if (freshHits.length === 0) break; + const freshIds = freshHits.map(hit => hit.id); + const candidateMemories = this.deps.repos.memories.getMany(freshIds); + const deduplicated = deduplicateSameSourceL1Candidates({ + memories: candidateMemories, + repos: this.deps.repos, + excludeTraceRawTurnIds, + isMemoryEligible: memory => (memory.status === "activated" || memory.status === "resolving") && + this.isMemoryReadyForRetrieval(memory), + isMemoryInScope: memory => memoryMatchesRequestedTags(memory, filter.tags) + }); + sourceCount += deduplicated.memories.length; + const memoriesById = new Map(candidateMemories.map(memory => [memory.id, memory])); + for (const hit of freshHits) { + hits.push(hit); + candidateIds.add(hit.id); + const memory = memoriesById.get(hit.id); + if (!memory) { + excludedIds.add(hit.id); + continue; + } + for (const id of deduplicated.sourceIdsByCandidateId.get(hit.id) ?? [memory.id]) { + excludedIds.add(id); + } + } + } + + return hits; + } + isMemoryReadyForRetrieval(memory: MemoryRow): boolean { const processing = this.deps.repos.processing.get(memory.id); if (!processing || !this.deps.memoryHasImportPipeline(memory)) return isMemoryReadyForRetrieval(memory); diff --git a/Memory/src/service/retrieval/retrieval-service.ts b/Memory/src/service/retrieval/retrieval-service.ts index 2a0e9624b..71e1d30bd 100644 --- a/Memory/src/service/retrieval/retrieval-service.ts +++ b/Memory/src/service/retrieval/retrieval-service.ts @@ -1,3 +1,4 @@ +import { deduplicateSameSourceL1Candidates, memoryMatchesRequestedTags } from "./same-source-l1-dedup.js"; import type { RetrievalQueryExtract, RetrievalResult @@ -70,7 +71,6 @@ import { normalizeRetrievalExtractKeywords } from "../turn/turn-normalization.js"; import { IndexedCandidatePool } from "./indexed-candidate-pool.js"; -import { deduplicateCrossAccountMemoryCandidates } from "./cross-account-candidate-dedup.js"; import { filterL1TraceSpanRecallHits } from "./l1-trace-span-filter.js"; type InternalMemorySearchRequest = MemorySearchRequest & { @@ -1827,8 +1827,8 @@ interface RetrievalDependencies { memorySearchEnabled(): boolean; queryRewriteEnabled(): boolean; requireEpisode(episodeId: string): EpisodeRecord; - resolveContext(request: RequestEnvelope & { sessionId?: string; userId?: string }): { - userId: string; + resolveContext(request: RequestEnvelope & { sessionId?: string; userId?: string | null }): { + userId: string | null; conversationId?: string; namespace: RuntimeNamespace; }; @@ -1975,8 +1975,7 @@ export class RetrievalService { ? this.retrieveTimeFilteredTraceMemories({ timeFilter, tags: request.tags, - limit: retrievalLimit, - preferredUserId: context.userId + limit: retrievalLimit }) : await this.retrieveSearchMemories({ userId: context.userId, @@ -2200,7 +2199,6 @@ export class RetrievalService { timeFilter: RetrievalTimeFilter; tags?: string[]; limit: number; - preferredUserId: string; }): { retrieval: RetrievalResult; memories: MemoryRow[] } { const filter: MemoryFilter = { memoryLayer: "L1", @@ -2210,15 +2208,20 @@ export class RetrievalService { ...(input.tags?.length ? { tags: input.tags } : {}) }; const candidateCount = this.deps.repos.memories.count(filter); - const loadedCandidates = this.deps.repos.memories + const rawCandidates = this.deps.repos.memories .list(filter, candidateCount) .filter((memory) => this.isMemoryReadyForRetrieval(memory)) -.filter((memory) => kindFromMemory(memory) === "observed_activity" || Boolean(traceMetaFromMemory(memory)?.summary.trim())); - const candidates = deduplicateCrossAccountMemoryCandidates({ - memories: loadedCandidates, + .filter((memory) => kindFromMemory(memory) === "observed_activity" || Boolean(traceMetaFromMemory(memory)?.summary.trim())); + const { memories: candidates } = deduplicateSameSourceL1Candidates({ + memories: rawCandidates, repos: this.deps.repos, - preferredUserId: input.preferredUserId - }).memories; + isMemoryEligible: memory => memory.memoryLayer === "L1" && + (memory.status === "activated" || memory.status === "resolving") && + this.isMemoryReadyForRetrieval(memory), + isMemoryInScope: memory => memory.createdAt >= input.timeFilter.startAt && + memory.createdAt < input.timeFilter.endAt && memoryMatchesRequestedTags(memory, input.tags) && + (kindFromMemory(memory) === "observed_activity" || Boolean(traceMetaFromMemory(memory)?.summary.trim())) + }); const selected = [...candidates] .sort(compareTimeFilteredTraceRecency) .slice(0, Math.max(0, input.limit)) @@ -2249,7 +2252,7 @@ export class RetrievalService { } private async retrieveUserMemories(input: { - userId: string; + userId: string | null; query: string; queryVectorText: string; queryExtract: RetrievalQueryExtract | null; @@ -2306,7 +2309,7 @@ export class RetrievalService { } private async retrieveSearchMemories(input: { - userId: string; + userId: string | null; projectId?: string | null; query: string; queryVectorText: string; @@ -2347,9 +2350,18 @@ export class RetrievalService { tags: input.tags, targetSkillId: input.targetSkillId, currentAgentId: input.currentAgentId, + excludeTraceRawTurnIds: input.excludeTraceRawTurnIds, config }); - const memories = candidatePool.memories; + const { memories, channelScoresByMemory } = deduplicateSameSourceL1Candidates({ + ...candidatePool, + repos: this.deps.repos, + excludeTraceRawTurnIds: input.excludeTraceRawTurnIds, + isMemoryEligible: memory => input.layers.includes(memory.memoryLayer) && + (memory.status === "activated" || memory.status === "resolving") && + this.isMemoryReadyForRetrieval(memory), + isMemoryInScope: memory => memoryMatchesRequestedTags(memory, input.tags) + }); if (memories.length === 0) { return { retrieval: emptyRetrievalResult(), memories }; } @@ -2365,7 +2377,7 @@ export class RetrievalService { mode: input.mode, excludeTraceRawTurnIds: input.excludeTraceRawTurnIds, targetSkillId: input.targetSkillId, - channelScoresByMemory: candidatePool.channelScoresByMemory, + channelScoresByMemory, config }) }; @@ -2392,9 +2404,29 @@ export class RetrievalService { query === input.query ? input.queryExtract : null ) )); + // Different rewrites may retrieve different copies of one native turn. + // Normalize their hit identities before RRF and its final limit as well. + const allMemories = uniqMemories(outputs.flatMap(output => output.memories)); + const hitsById = new Map(outputs.flatMap(output => output.retrieval.hits).map(hit => [hit.id, hit])); + const dedup = deduplicateSameSourceL1Candidates({ + memories: allMemories.filter(memory => hitsById.has(memory.id)), + repos: this.deps.repos, + includeUnreturnedCopies: false + }); + const retrievals = outputs.map(output => { + const seen = new Set(); + const hits = output.retrieval.hits.flatMap(hit => { + const id = dedup.memoryIdByMemoryId.get(hit.id) ?? hit.id; + if (seen.has(id)) return []; + seen.add(id); + return [{ ...hitsById.get(id)!, score: hit.score }]; + }); + return { ...output.retrieval, hits }; + }); return { - retrieval: mergeRetrievalResults(outputs.map((output) => output.retrieval), input.limit, QUERY_REWRITE_RRF_CONSTANT, QUERY_REWRITE_PER_QUERY_MIN_KEEP), - memories: uniqMemories(outputs.flatMap((output) => output.memories)) + retrieval: mergeRetrievalResults(retrievals, input.limit, QUERY_REWRITE_RRF_CONSTANT, QUERY_REWRITE_PER_QUERY_MIN_KEEP), + memories: allMemories.filter(memory => !dedup.memoryIdByMemoryId.has(memory.id) || + dedup.memoryIdByMemoryId.get(memory.id) === memory.id) }; } diff --git a/Memory/src/service/retrieval/same-source-l1-dedup.ts b/Memory/src/service/retrieval/same-source-l1-dedup.ts new file mode 100644 index 000000000..a77848a0e --- /dev/null +++ b/Memory/src/service/retrieval/same-source-l1-dedup.ts @@ -0,0 +1,174 @@ +import { traceMetaFromMemory, type SeededChannelScores } from "../../algorithm/plugin-algorithms.js"; +import type { Repositories } from "../../storage/repositories.js"; +import { attachMemoryVector, memoryVector, memoryVectorEntries } from "../../storage/memory-vector-state.js"; +import type { MemoryRow } from "../../types.js"; + +export interface SameSourceL1CandidateDedupResult { + memories: MemoryRow[]; + channelScoresByMemory: ReadonlyMap; + memoryIdByMemoryId: ReadonlyMap; + sourceIdsByCandidateId: ReadonlyMap; +} + +function timestampValue(value: string): number { + const parsed = Date.parse(value); + return Number.isFinite(parsed) ? parsed : Number.NEGATIVE_INFINITY; +} + +function newestMemoryFirst(left: MemoryRow, right: MemoryRow): number { + return timestampValue(right.updatedAt) - timestampValue(left.updatedAt) || + right.version - left.version || + timestampValue(right.createdAt) - timestampValue(left.createdAt) || + left.id.localeCompare(right.id); +} + +function sameContentRevision(left: MemoryRow, right: MemoryRow): boolean { + const leftTrace = traceMetaFromMemory(left); + const rightTrace = traceMetaFromMemory(right); + return (left.contentHash ?? null) === (right.contentHash ?? null) && + left.memoryValue === right.memoryValue && + JSON.stringify(left.tags) === JSON.stringify(right.tags) && + JSON.stringify([ + leftTrace?.summary, leftTrace?.userText, leftTrace?.agentText, + leftTrace?.toolCalls, leftTrace?.reflection, leftTrace?.errorSignatures + ]) === JSON.stringify([ + rightTrace?.summary, rightTrace?.userText, rightTrace?.agentText, + rightTrace?.toolCalls, rightTrace?.reflection, rightTrace?.errorSignatures + ]); +} + +export function memoryMatchesRequestedTags(memory: MemoryRow, tags: readonly string[] = []): boolean { + if (tags.length === 0) return true; + const infoTags = Array.isArray(memory.info.tags) ? memory.info.tags : []; + const propertyTags = Array.isArray(memory.properties.tags) ? memory.properties.tags : []; + const available = new Set([...memory.tags, ...infoTags, ...propertyTags] + .filter((tag): tag is string => typeof tag === "string") + .map(tag => tag.trim().toLowerCase()) + .filter(Boolean)); + return tags.every(tag => available.has(tag.trim().toLowerCase())); +} + +/** + * Collapse proven copies of the same native L1 source step before ranking. + * UID never participates in grouping or representative selection. Historical + * rows and their content are preserved; incomplete source identities stay separate. + */ +export function deduplicateSameSourceL1Candidates(input: { + memories: readonly MemoryRow[]; + repos: Repositories; + excludeTraceRawTurnIds?: ReadonlySet; + channelScoresByMemory?: ReadonlyMap; + isMemoryEligible?: (memory: MemoryRow) => boolean; + isMemoryInScope?: (memory: MemoryRow) => boolean; + /** Disable repository expansion when merging already-normalized query-rewrite results. */ + includeUnreturnedCopies?: boolean; +}): SameSourceL1CandidateDedupResult { + const groups = new Map; + memoriesById: Map; + }>(); + const candidateMemoriesById = new Map(); + const sourceIdsByCandidateId = new Map(); + input.memories.forEach((memory, index) => { + const groupState = input.repos.memories.nativeDuplicates(memory); + sourceIdsByCandidateId.set(memory.id, groupState.ids); + if (groupState.deleted) return; + const key = groupState.ids.length > 1 ? `native:${groupState.ids[0]}` : undefined; + const groupKey = key ?? `unique:${index}:${memory.id}`; + const group = groups.get(groupKey) ?? { + firstIndex: index, + nativeIds: groupState.ids.length > 0 ? [...groupState.ids] : [memory.id], + candidateIds: new Set(), + memoriesById: new Map() + }; + group.candidateIds.add(memory.id); + group.memoriesById.set(memory.id, memory); + groups.set(groupKey, group); + candidateMemoriesById.set(memory.id, memory); + }); + + const includeUnreturnedCopies = input.includeUnreturnedCopies !== false; + const sourceMemoriesById = new Map(candidateMemoriesById); + if (includeUnreturnedCopies) { + const sourceIds = [...new Set([...groups.values()] + .filter(group => group.nativeIds.length > 1) + .flatMap(group => group.nativeIds))]; + // Keep each IN query below SQLite's conservative parameter limit. + for (let offset = 0; offset < sourceIds.length; offset += 500) { + for (const memory of input.repos.memories.getMany(sourceIds.slice(offset, offset + 500))) { + if (!sourceMemoriesById.has(memory.id)) sourceMemoriesById.set(memory.id, memory); + } + } + } + + const selected = [...groups.values()] + .sort((left, right) => left.firstIndex - right.firstIndex) + .map((group) => { + const eligible = group.nativeIds.flatMap(id => { + const memory = sourceMemoriesById.get(id) ?? group.memoriesById.get(id); + if (!memory) return []; + const isEligible = input.isMemoryEligible?.(memory) ?? + (memory.status === "activated" || memory.status === "resolving"); + return isEligible ? [memory] : []; + }); + const excludedByRecentTurn = Boolean(input.excludeTraceRawTurnIds?.size && group.nativeIds.some(id => { + const member = sourceMemoriesById.get(id) ?? input.repos.memories.get(id); + const rawTurnId = member ? traceMetaFromMemory(member)?.rawTurnId : undefined; + return rawTurnId && input.excludeTraceRawTurnIds?.has(rawTurnId); + })); + const membersById = new Map(eligible.map(memory => [memory.id, memory])); + const representative = [...membersById.values()].sort(newestMemoryFirst)[0]; + const representativeInScope = representative && (input.isMemoryInScope?.(representative) ?? true); + return representative && representativeInScope && !excludedByRecentTurn + ? { representative, members: [...membersById.values()], nativeIds: group.nativeIds, candidateIds: group.candidateIds } + : undefined; + }) + .filter((group): group is NonNullable => Boolean(group)); + + const channelScoresByMemory = new Map(); + const memoryIdByMemoryId = new Map(); + for (const group of selected) { + if (group.members.length === 1) { + const memory = group.representative; + for (const id of group.nativeIds) memoryIdByMemoryId.set(id, memory.id); + const scores = group.candidateIds.has(memory.id) + ? input.channelScoresByMemory?.get(memory.id) + : undefined; + if (scores) channelScoresByMemory.set(memory.id, scores); + continue; + } + + const merged: SeededChannelScores = {}; + for (const memory of group.members) { + memoryIdByMemoryId.set(memory.id, group.representative.id); + if (sameContentRevision(memory, group.representative)) { + for (const vector of memoryVectorEntries(memory)) { + if (!memoryVector(group.representative, vector.vectorField)) { + attachMemoryVector(group.representative, vector); + } + } + const scores = group.candidateIds.has(memory.id) + ? input.channelScoresByMemory?.get(memory.id) + : undefined; + for (const [channel, score] of Object.entries(scores ?? {})) { + const key = channel as keyof SeededChannelScores; + const current = merged[key]; + if (typeof score === "number" && (current === undefined || score > current)) { + merged[key] = score; + } + } + } + } + for (const id of group.nativeIds) memoryIdByMemoryId.set(id, group.representative.id); + if (Object.keys(merged).length > 0) channelScoresByMemory.set(group.representative.id, merged); + } + + return { + memories: selected.map((group) => group.representative), + channelScoresByMemory, + memoryIdByMemoryId, + sourceIdsByCandidateId + }; +} diff --git a/Memory/src/service/session/session-turn-service.ts b/Memory/src/service/session/session-turn-service.ts index b3a3ca929..8fb614742 100644 --- a/Memory/src/service/session/session-turn-service.ts +++ b/Memory/src/service/session/session-turn-service.ts @@ -1,3 +1,6 @@ +import type { RuntimeNamespace } from "../../types.js"; +import { canonicalProjectId, deviceNamespace } from "../../storage/device-scope.js"; +import { captureNativeOrigin, nativeClaimIdentity, pendingOriginKey, readNativeOrigin } from "../../storage/native-memory-origin.js"; import { EpisodeSelectionRequired, selectionInput, selectionKey, selectEpisode, type EpisodeSelectionContext, type EpisodeSelectionResult } from "./episode-selection.js"; import { captureTurnSteps, @@ -73,7 +76,7 @@ import type { SynthesizeDecisionRepairDraft } from "../feedback/feedback-experience.js"; import { recordApiLog } from "../model-audit/model-call-audit.js"; -import { deduplicateCrossAccountMemoryCandidates } from "../retrieval/cross-account-candidate-dedup.js"; +import { deduplicateSameSourceL1Candidates } from "../retrieval/same-source-l1-dedup.js"; import { namespaceForRawTurn, namespaceForSession, @@ -499,6 +502,29 @@ export function failureBurstAntiPattern(burst: ToolFailureBurst, reason: string) } export class SessionTurnService { + private normalizeDeviceNamespace(namespace?: RuntimeNamespace) { + return normalizeNamespace(deviceNamespace(this.deps.repos.db, normalizeNamespace(namespace))); + } + + private idempotencyRequest(request: T): T { + return { ...request, namespace: this.normalizeDeviceNamespace(request.namespace) }; + } + + private legacyIdempotencyMatches( + savedHash: string, request: RequestEnvelope, sessionId: string | undefined, + hash: (request: RequestEnvelope) => string + ): boolean { + if (savedHash === hash(request)) return true; + if (!sessionId || !request.namespace) return false; + const original = this.deps.repos.db.prepare("SELECT user_id FROM sessions WHERE id = ?") + .get(sessionId) as { user_id: string | null } | undefined; + // Only substitute the historical owner, leaving every semantic request field + // intact. A reused request ID with different content must still conflict. + return original?.user_id != null && savedHash === hash({ + ...request, namespace: { ...request.namespace, userId: original.user_id } + }); + } + private readonly toolFailureStates = new Map(); private readonly toolSuccessSteps = new Map(); private readonly toolStepCounters = new Map(); @@ -767,7 +793,7 @@ export class SessionTurnService { openSession(request: SessionOpenRequest, options: { createNew?: boolean; at?: string } = {}): { sessionId: string; - userId: string; + userId: string | null; source: string; profileId: string; projectId?: string | null; @@ -787,17 +813,22 @@ export class SessionTurnService { const idempotencyKey = request.adapterId && request.requestId ? `session.open:${request.adapterId}:${request.requestId}` : undefined; - const requestHash = stableHash({ operation: "session.open", request }); + const hashRequest = (body: RequestEnvelope) => stableHash({ operation: "session.open", request: body }); + const requestHash = hashRequest(this.idempotencyRequest(request)); if (idempotencyKey) { const existing = this.deps.repos.runtime.getIdempotency(idempotencyKey); if (existing) { - if (existing.requestHash !== requestHash) { + const response = existing.response as ReturnType; + if (existing.requestHash !== requestHash && !this.legacyIdempotencyMatches(existing.requestHash, request, response.sessionId, hashRequest)) { throw new MemoryServiceError("conflict", "idempotency key reused with different session.open request body"); } - return this.deps.withDuplicateFlag(existing.response) as ReturnType; + return this.deps.withDuplicateFlag({ ...response, userId: null, + projectId: canonicalProjectId(this.deps.repos.db, response.projectId), + workspaceId: canonicalProjectId(this.deps.repos.db, response.workspaceId) ?? undefined + }) as ReturnType; } } - const namespace = normalizeNamespace(request.namespace); + const namespace = this.normalizeDeviceNamespace(request.namespace); const at = options.at ?? nowIso(); if (request.l3WorldModelProtocolVersion === undefined && ( request.l3WorldModelTransition !== undefined || @@ -847,7 +878,7 @@ export class SessionTurnService { const hostSessionKey = namespace.sessionKey; if (hostSessionKey && !options.createNew) { const existingSession = this.deps.repos.runtime.findOpenSessionByHostKey({ - userId: namespace.userId, + tenantId: namespace.tenantId ?? null, source: request.source ?? namespace.source, profileId: request.profileId ?? namespace.profileId, hostSessionKey @@ -892,6 +923,7 @@ export class SessionTurnService { status: "open" as const, meta: { ...sanitizedSessionMeta(request.meta), + ...(namespace.tenantId !== undefined ? { tenant_id: namespace.tenantId } : {}), ...(request.timeZone ? { time_zone: request.timeZone } : {}) }, openedAt: at, @@ -960,10 +992,11 @@ export class SessionTurnService { let existing = request.sessionId ? this.deps.repos.runtime.getSession(request.sessionId) : this.deps.repos.runtime.findOpenSessionByHostKey({ - userId: namespace.userId, + tenantId: namespace.tenantId ?? null, source, profileId, - hostSessionKey: namespace.sessionKey! + hostSessionKey: namespace.sessionKey!, + projectId: workspace.projectId ?? namespace.projectId }); if (request.sessionId && !existing) { @@ -1066,11 +1099,11 @@ export class SessionTurnService { ): void { const savedWorkspaceUri = optionalMetaString(session.meta, "workspace_uri"); const savedWorkspaceHostId = optionalMetaString(session.meta, "workspace_host_id"); - const requestProjectId = request.projectId ?? request.namespace?.projectId; - const requestWorkspaceId = request.workspaceId ?? request.namespace?.workspaceId; - const mismatch = session.userId !== namespace.userId || - session.source !== (request.source ?? namespace.source) || + const requestProjectId = request.projectId === undefined ? namespace.projectId : canonicalProjectId(this.deps.repos.db, request.projectId) ?? undefined; + const requestWorkspaceId = request.workspaceId === undefined ? namespace.workspaceId : canonicalProjectId(this.deps.repos.db, request.workspaceId) ?? undefined; + const mismatch = session.source !== (request.source ?? namespace.source) || session.profileId !== (request.profileId ?? namespace.profileId) || + !sessionTenantMatches(session, namespace.tenantId ?? null) || session.hostSessionKey !== namespace.sessionKey || (requestProjectId !== undefined && requestProjectId !== session.projectId) || (requestWorkspaceId !== undefined && requestWorkspaceId !== session.workspaceId) || @@ -1449,6 +1482,10 @@ export class SessionTurnService { const session = this.deps.requireOpenSession(request.sessionId); this.deps.assertSessionInScope(session, request.namespace); const turnId = request.turnId ?? newId("turn"); + const originKey = pendingOriginKey(session.id, turnId); + if (!this.deps.repos.runtime.getRawTurnBySessionTurn(session.id, turnId) && !this.deps.repos.runtime.getKv(originKey)) { + this.deps.repos.runtime.setKv(originKey, captureNativeOrigin({ ...session, tenantId: request.namespace?.tenantId }), nowIso()); + } const intentDecision = classifyIntent(request.query); const turnJudgeRouting = this.turnJudgeRouting(); const endTopicDecision = turnJudgeRouting ? undefined : explicitEndTopicDecision(request.query); @@ -1536,7 +1573,7 @@ export class SessionTurnService { if (request.channel !== "hook" && request.channel !== "agent_source_scan") { throw new MemoryServiceError("invalid_argument", "invalid source capture channel"); } - const namespace = normalizeNamespace(request.namespace ?? { + const namespace = this.normalizeDeviceNamespace(request.namespace ?? { source: identity.source, profileId: identity.profileId, sessionKey: identity.conversationId }); if (namespace.source !== identity.source || namespace.profileId !== identity.profileId || @@ -1625,6 +1662,9 @@ export class SessionTurnService { // A replayed capture was accepted before later turns were stored. Keep it as // detached history instead of dropping it as out of order. const legacyBackfill = allowsLegacyHistoryCapture(request); + if (legacyBackfill && this.completedLegacySourceTurn(namespace, scope, sourceTurn.turnId)) { + return { status: "pending", reason: "legacy_source_turn_already_completed" }; + } const detachLegacyHistory = (reason: string): SourceTurnCompleteResponse => { if (!legacyBackfill) return { status: "pending", reason }; return this.captureDetachedLegacyHistory(request, normalized, sourceTurn, scope, contentHash); @@ -1675,7 +1715,7 @@ export class SessionTurnService { const episode = requestedEpisodeId ? this.deps.repos.runtime.getEpisode(requestedEpisodeId) : this.deps.repos.runtime.latestEpisodeForSession(session.id); - if (request.episodeId && (!episode || episode.sessionId !== session.id || episode.userId !== session.userId)) { + if (request.episodeId && (!episode || episode.sessionId !== session.id)) { throw new MemoryServiceError("forbidden", "source_episode_scope_conflict"); } if (episode && episode.status !== "open") { @@ -1708,6 +1748,9 @@ export class SessionTurnService { }, identity.conversationId)) { throw new MemoryServiceError("forbidden", "source_raw_turn_scope_conflict"); } + if (!this.deps.repos.runtime.bindPendingTurnSourceConversation(session.id, identity.turnId, identity.conversationId)) { + throw new MemoryServiceError("forbidden", "source_pending_turn_scope_conflict"); + } const result = this.completeTurn(identity.turnId, { ...normalized, sessionId: session.id, ...(gapEpisode ? { episodeId: gapEpisode.id } : {}) }, sourceTurn); if (gapEpisode) this.deps.repos.runtime.orderEpisodeTurnsBySourceTime(gapEpisode.id); @@ -1734,13 +1777,20 @@ export class SessionTurnService { const legacyTurnId = request.legacyImportTurnId?.trim(); if (!legacyTurnId) return undefined; const source = request.sourceTurn.source; - const memories = this.deps.repos.memories.listUserMemoriesByKeyIncludingDeleted( - namespace.userId, - `memory.add:agent-source:${source}:turn:${legacyTurnId}` - ).filter((memory) => legacyImportMatchesScope(memory, namespace, source, legacyTurnId)); + const legacyKey = `memory.add:agent-source:${source}:turn:${legacyTurnId}`; + // Accept pre-scope keys and newer imports with/without native session evidence. + const keys = new Set([legacyKey, ...[null, request.sourceTurn.conversationId].map(conversationId => + `${legacyKey}:scope:${stableHash({ ...this.deps.repos.captureClaims.scope(namespace), conversationId, turnId: legacyTurnId })}`)]); + const memories = [...keys].flatMap(key => this.deps.repos.memories.listUserMemoriesByKeyIncludingDeleted(namespace.userId, key)) + .filter((memory) => legacyImportMatchesScope(memory, namespace, source, legacyTurnId)); if (memories.length === 0) return undefined; - if (memories.length > 1) return { status: "pending", reason: "legacy_identity_unresolved" }; - const memory = memories[0]!; + const distinctContent = new Set(memories.filter(memory => !memory.deletedAt && memory.status !== "deleted") + .map(memory => stableHash(legacyImportTexts(memory)))); + if (distinctContent.size > 1) return { status: "pending", reason: "legacy_identity_unresolved" }; + // An explicit deletion wins over another account's historical copy. + const memory = [...memories].sort((a, b) => + Number(Boolean(b.deletedAt || b.status === "deleted")) - Number(Boolean(a.deletedAt || a.status === "deleted")) || + a.createdAt.localeCompare(b.createdAt) || a.id.localeCompare(b.id))[0]!; const texts = legacyImportTexts(memory); const deleted = memory.status === "deleted" || Boolean(memory.deletedAt); if (!deleted && texts.user && texts.agent && (texts.user !== normalized.query || texts.agent !== normalized.answer)) { @@ -1769,7 +1819,7 @@ export class SessionTurnService { scope: SourceTurnCaptureScope, contentHash: string ): SourceTurnCompleteResponse { - const namespace = normalizeNamespace(request.namespace); + const namespace = this.normalizeDeviceNamespace(request.namespace); this.assertExplicitEpisodeInSourceScope(request, namespace, sourceTurn, scope); if (this.completedLegacySourceTurn(namespace, scope, sourceTurn.turnId)) { return { status: "pending", reason: "legacy_source_turn_already_completed" }; @@ -1822,8 +1872,6 @@ export class SessionTurnService { const episode = this.deps.repos.runtime.getEpisode(request.episodeId); const episodeSession = episode ? this.deps.repos.runtime.getSession(episode.sessionId) : undefined; const inScope = Boolean(episode && episodeSession && - episode.userId === namespace.userId && - episodeSession.userId === namespace.userId && episodeSession.source === sourceTurn.source && episodeSession.profileId === sourceTurn.profileId && (!episodeSession.conversationId || episodeSession.conversationId === sourceTurn.conversationId) && @@ -1902,7 +1950,7 @@ export class SessionTurnService { scope: SourceTurnCaptureScope, mappedSessionId?: string ): SessionRecord | { status: "pending"; reason: string } { - const namespace = normalizeNamespace(request.namespace); + const namespace = this.normalizeDeviceNamespace(request.namespace); const inScope = (candidate: SessionRecord): boolean => candidate.source === identity.source && candidate.profileId === identity.profileId && (!candidate.conversationId || candidate.conversationId === identity.conversationId) && @@ -1973,15 +2021,13 @@ export class SessionTurnService { const idempotencyKey = !sourceTurn && request.adapterId && request.requestId ? `turn.complete:${request.adapterId}:${request.requestId}` : undefined; - const requestHash = stableHash({ - turnId, - request - }); + const hashRequest = (body: RequestEnvelope) => stableHash({ turnId, request: body }); + const requestHash = hashRequest(this.idempotencyRequest(request)); if (idempotencyKey) { const existing = this.deps.repos.runtime.getIdempotency(idempotencyKey); if (existing) { - if (existing.requestHash !== requestHash) { + if (existing.requestHash !== requestHash && !this.legacyIdempotencyMatches(existing.requestHash, request, request.sessionId, hashRequest)) { throw new MemoryServiceError("conflict", "idempotency key reused with different request body"); } const raw = this.deps.repos.runtime.getRawTurn((existing.response as CompleteTurnResponse).rawTurnId); @@ -2001,6 +2047,18 @@ export class SessionTurnService { const session = this.deps.requireSession(request.sessionId); this.deps.assertSessionInScope(session, request.namespace); const existingRawTurn = this.deps.repos.runtime.getRawTurnBySessionTurn(session.id, turnId); + const capturedOrigin = readNativeOrigin(existingRawTurn?.messagePayload?.native_origin_v1) + ?? readNativeOrigin(this.deps.repos.runtime.getKv(pendingOriginKey(session.id, turnId))?.value); + if (capturedOrigin && request.namespace) { + const ns = request.namespace; + if ((ns.projectId !== undefined && canonicalProjectId(this.deps.repos.db, ns.projectId) !== canonicalProjectId(this.deps.repos.db, capturedOrigin.projectId)) + || (ns.workspaceId !== undefined && canonicalProjectId(this.deps.repos.db, ns.workspaceId) !== canonicalProjectId(this.deps.repos.db, capturedOrigin.workspaceId)) + || (ns.tenantId !== undefined && ns.tenantId !== capturedOrigin.tenantId) + || (ns.profileId !== undefined && ns.profileId !== capturedOrigin.profileId) + || (ns.source !== undefined && normalizeMemoryCaptureSource(ns.source) !== capturedOrigin.source)) { + throw new MemoryServiceError("conflict", "turn origin conflicts with the captured namespace"); + } + } if (existingRawTurn) { this.deps.assertRawTurnInScope(existingRawTurn, request.namespace); } @@ -2010,8 +2068,10 @@ export class SessionTurnService { const existingCaptureClaim = !sourceTurn && existingRawTurn.userText && existingRawTurn.assistantText ? this.deps.repos.captureClaims.get( session.userId, - normalizeMemoryCaptureSource(session.source), - memoryCaptureQaHash(existingRawTurn.userText, existingRawTurn.assistantText) + normalizeMemoryCaptureSource(capturedOrigin?.source ?? session.source), + memoryCaptureQaHash(existingRawTurn.userText, existingRawTurn.assistantText), + capturedOrigin ? nativeClaimIdentity(capturedOrigin, existingRawTurn.turnId) + : this.deps.repos.captureClaims.identity({}, existingRawTurn.turnId, session.id) ) : undefined; const l1MemoryIds = this.deps.repos.runtime.l1MemoryIdsForRawTurn(existingRawTurn.id).filter((memoryId: string) => { @@ -2224,6 +2284,7 @@ export class SessionTurnService { sourceMemoryIds, usage: isRecord(request.usage) ? request.usage : {}, messagePayload: { + native_origin_v1: captureNativeOrigin({ ...session, tenantId: request.namespace?.tenantId }), ...(sourceTurn ? { source_turn: sourceTurn } : {}), turn_start: turnStartPayload, turn_complete: { @@ -2337,6 +2398,7 @@ export class SessionTurnService { }), tags: step.tags, info: { + native_origin_v1: sourceRawTurn.messagePayload?.native_origin_v1, turn_id: step.turnId, raw_turn_id: stepRawTurnId, episode_id: episode?.id, @@ -2404,10 +2466,13 @@ export class SessionTurnService { const qaQuery = sourceRawTurn.userText ?? ""; const qaAnswer = sourceRawTurn.assistantText ?? ""; if (qaQuery.trim() && qaAnswer.trim()) { + const origin = readNativeOrigin(sourceRawTurn.messagePayload?.native_origin_v1); const capture = this.deps.repos.captureClaims.claim({ userId: session.userId, - source: normalizeMemoryCaptureSource(session.source), + source: normalizeMemoryCaptureSource(origin?.source ?? session.source), qaHash: memoryCaptureQaHash(qaQuery, qaAnswer), + identity: origin ? nativeClaimIdentity(origin, sourceRawTurn.turnId) + : this.deps.repos.captureClaims.identity({}, sourceRawTurn.turnId, session.id), primaryMemoryId: l1Memory.id, capturedBy: "turn_complete", createdAt: at @@ -3343,10 +3408,12 @@ export class SessionTurnService { }, 500 ).filter((memory) => this.deps.isMemoryReadyForRetrieval(memory)); - const candidates = deduplicateCrossAccountMemoryCandidates({ + const candidates = deduplicateSameSourceL1Candidates({ memories: loadedCandidates, repos: this.deps.repos, - preferredUserId: session.userId + isMemoryEligible: memory => repairLayers.includes(memory.memoryLayer) && + (memory.status === "activated" || memory.status === "resolving") && + this.deps.isMemoryReadyForRetrieval(memory) }).memories; const retrieval = retrievePluginMemories({ query, @@ -3489,7 +3556,7 @@ export class SessionTurnService { const correction = request.userMemoryCorrection; if (correction) { const target = this.deps.repos.userMemories.get(correction.targetMemoryId); - if (!target || target.userId !== rawTurn.userId || target.status !== "active") { + if (!target || target.status !== "active") { throw new MemoryServiceError("not_found", `active user memory not found: ${correction.targetMemoryId}`); } const content = correction.revisedContent.trim(); @@ -3552,7 +3619,7 @@ export class SessionTurnService { } private userMemoryEmbeddingJobs( - memory: { id: string; userId: string; content: string }, + memory: { id: string; userId: string | null; content: string }, rawTurn: RawTurnRecord, at: string ): EvolutionJobRecord[] { @@ -3570,7 +3637,7 @@ export class SessionTurnService { } private appendUserMemoryChange( - memory: { id: string; userId: string }, + memory: { id: string; userId: string | null }, before: unknown, op: "created" | "updated" | "archived", at: string @@ -4601,6 +4668,8 @@ function sanitizedSessionMeta(meta?: Record): Record { return { ...sanitizedSessionMeta(request.meta), + ...(request.namespace?.tenantId !== undefined ? { tenant_id: request.namespace.tenantId } : {}), l3_world_model_protocol_version: 2, ...(workspace.workspaceUri ? { workspace_uri: workspace.workspaceUri } : {}), ...(workspace.workspaceHostId ? { workspace_host_id: workspace.workspaceHostId } : {}), @@ -4618,6 +4688,15 @@ function v2SessionMeta( }; } +function sessionTenantMatches(session: SessionRecord, tenantId: string | null): boolean { + const snake = session.meta.tenant_id; + const camel = session.meta.tenantId; + const snakeId = typeof snake === "string" && snake.length > 0 ? snake : null; + const camelId = typeof camel === "string" && camel.length > 0 ? camel : null; + if (snakeId !== null && camelId !== null && snakeId !== camelId) return false; + return (snakeId ?? camelId) === tenantId; +} + function optionalMetaString(meta: Record, key: string): string | undefined { const value = meta[key]; return typeof value === "string" && value ? value : undefined; diff --git a/Memory/src/service/trials/skill-trial-resolver.ts b/Memory/src/service/trials/skill-trial-resolver.ts index d95dc6df9..6275eea66 100644 --- a/Memory/src/service/trials/skill-trial-resolver.ts +++ b/Memory/src/service/trials/skill-trial-resolver.ts @@ -211,7 +211,7 @@ updateSkillTrialStats(trial: SkillTrialRecord, at: string): void { } resolvePendingSkillTrialsForReward(input: { - userId: string; + userId: string | null; episodeId: string; rHuman: number; feedbackId?: string; diff --git a/Memory/src/service/user-memory/user-memory.ts b/Memory/src/service/user-memory/user-memory.ts index ddc9e42af..d17f58c53 100644 --- a/Memory/src/service/user-memory/user-memory.ts +++ b/Memory/src/service/user-memory/user-memory.ts @@ -52,7 +52,7 @@ export function userMemoryTextHash(text: string): string { export function buildUserMemory(input: { id: string; sourceTurnId: string; - userId: string; + userId: string | null; memoryTypes: UserMemoryType[]; content: string; createdAt: string; diff --git a/Memory/src/service/work-memory/work-memory-pipeline.ts b/Memory/src/service/work-memory/work-memory-pipeline.ts index da59a64aa..03e631d88 100644 --- a/Memory/src/service/work-memory/work-memory-pipeline.ts +++ b/Memory/src/service/work-memory/work-memory-pipeline.ts @@ -162,7 +162,7 @@ export class WorkMemoryPipeline { */ private takeWindow( chunk: readonly L3WorldModelInputTraceRecord[], - scope: { userId: string; sessionId: string } + scope: { userId: string | null; sessionId: string } ): { qa: WorkMemoryQaPair[]; throughTraceSeq: number; truncated: boolean } { const rawTurnIds = [...new Set(chunk.map((trace) => trace.rawTurnId))]; const lastTraceByRawTurnId = new Map(); @@ -244,17 +244,23 @@ export class WorkMemoryPipeline { throw new Error(`work memory trajectory hash mismatch: ${job.id}`); } const session = job.sessionId ? this.deps.repos.runtime.getSession(job.sessionId) : undefined; - if (!session || session.userId !== job.userId) { + if (!session) { throw new Error(`work memory job session scope mismatch: ${job.id}`); } const projectId = normalizeProjectId(session.projectId); const source = session.source.trim() || "unknown"; - const expectedScopeKey = stableHash(["work_memory", job.userId, projectId]); + const stored = this.deps.repos.db.prepare(`SELECT job.user_id, session.project_id + FROM evolution_jobs job JOIN sessions session ON session.id = job.session_id WHERE job.id = ?`) + .get(job.id) as {user_id: string | null; project_id: string | null} | undefined; + // Immutable legacy jobs retain their original hash; new jobs use device identity, + // including when their session was created before the schema upgrade. + const hashProjectId = stored?.user_id != null ? stored.project_id : projectId; + const expectedScopeKey = stableHash(["work_memory", stored?.user_id ?? null, hashProjectId]); const expectedDedupeKey = stableHash([ "work_memory_extract", - job.userId, + stored?.user_id ?? null, source, - projectId, + hashProjectId, payload.trajectoryHash ]); if (job.scopeKey !== expectedScopeKey || job.dedupeKey !== expectedDedupeKey) { @@ -293,7 +299,7 @@ export class WorkMemoryPipeline { const batch = this.deps.repos.l3WorldModels.getBatch(batchId); if (!batch) throw new Error(`work memory batch not found: ${batchId}`); const session = this.deps.repos.runtime.getSession(batch.sessionId); - if (!session || session.userId !== batch.userId || normalizeProjectId(session.projectId) !== normalizeProjectId(batch.projectId)) { + if (!session || normalizeProjectId(session.projectId) !== normalizeProjectId(batch.projectId)) { throw new Error(`work memory batch session scope mismatch: ${batchId}`); } return this.scheduleQaInTransaction({ @@ -315,11 +321,11 @@ export class WorkMemoryPipeline { */ private qaForRawTurn( rawTurnId: string, - scope: { userId: string; sessionId: string } + scope: { userId: string | null; sessionId: string } ): WorkMemoryQaPair | undefined { const rawTurn = this.deps.repos.runtime.getRawTurn(rawTurnId); if (!rawTurn || rawTurn.deletedAt || rawTurn.redactedAt) return undefined; - if (rawTurn.userId !== scope.userId || rawTurn.sessionId !== scope.sessionId) { + if (rawTurn.sessionId !== scope.sessionId) { throw new Error(`work memory RawTurn scope mismatch: ${rawTurnId}`); } const user = normalizeQaText(rawTurn.userText ?? ""); @@ -332,14 +338,14 @@ export class WorkMemoryPipeline { private scheduleQaInTransaction( input: { qa: readonly WorkMemoryQaPair[]; - userId: string; + userId: string | null; sessionId: string; projectId: string | null; }, at: string ): EvolutionJobRecord | undefined { const session = this.deps.repos.runtime.getSession(input.sessionId); - if (!session || session.userId !== input.userId) { + if (!session) { throw new Error(`work memory session scope mismatch: ${input.sessionId}`); } const qa = input.qa; @@ -353,7 +359,12 @@ export class WorkMemoryPipeline { projectId, trajectoryHash ]); - const existing = this.deps.repos.runtime.getJobByDedupeKey(dedupeKey); + const historical = this.deps.repos.db.prepare(`SELECT job.id FROM evolution_jobs job + JOIN sessions session ON session.id = job.session_id + WHERE job.job_type = 'work_memory_extract' AND session.source = ? + AND memory_project_id(session.project_id) IS ? AND json_extract(job.payload_json, '$.trajectoryHash') = ? + ORDER BY job.created_at, job.id LIMIT 1`).get(source, projectId, trajectoryHash) as {id: string} | undefined; + const existing = historical ? this.deps.repos.runtime.getJob(historical.id) : this.deps.repos.runtime.getJobByDedupeKey(dedupeKey); if (existing?.status === "queued" || existing?.status === "leased" || existing?.status === "succeeded" || existing?.status === "dead_letter") { return existing; } @@ -773,7 +784,6 @@ function validateDecisionOutput( function isWorkMemoryInScope(memory: MemoryRow, filter: MemoryFilter): boolean { if (memory.memoryLayer !== "L1" || memory.properties.internal_info.memory_kind !== "work_memory") return false; - if (filter.workMemoryUserId !== undefined && memory.userId !== filter.workMemoryUserId) return false; const project = normalizeProjectId(typeof memory.info.project_id === "string" ? memory.info.project_id : null); return filter.workMemoryProjectId === undefined || project === filter.workMemoryProjectId; } diff --git a/Memory/src/service/worker/job-handlers.ts b/Memory/src/service/worker/job-handlers.ts index 88a569554..4ceec9501 100644 --- a/Memory/src/service/worker/job-handlers.ts +++ b/Memory/src/service/worker/job-handlers.ts @@ -40,7 +40,7 @@ export type ClosedEpisodeTrigger = "topic_boundary" | "session_closed" | "episod export interface EnqueueJobInput { jobType: JobType; - userId: string; + userId: string | null; sessionId?: string; episodeId?: string; targetMemoryId?: string; diff --git a/Memory/src/storage/bundle-import-plan.ts b/Memory/src/storage/bundle-import-plan.ts new file mode 100644 index 000000000..8bca9d5fa --- /dev/null +++ b/Memory/src/storage/bundle-import-plan.ts @@ -0,0 +1,149 @@ +import type Database from "better-sqlite3"; +import { canonicalProjectId } from "./device-scope.js"; +import { resolveWorkspaceIdentity } from "../service/namespace/workspace-identity.js"; +import { parseJson } from "../utils/json.js"; + +type Row = Record; +type Strategy = "skip" | "replace" | "error"; +const str = (value: unknown): string | undefined => typeof value === "string" && value ? value : undefined; +const json = (value: unknown): Row => typeof value === "string" ? parseJson(value, {}) : {}; + +/** A stable, pre-write decision shared by tables and their post-import work. */ +export function planBundleImport(db: Database.Database, tables: Record, strategy: Strategy) { + const rows = (table: string) => tables[table] ?? []; + const aliases = new Map(); + for (const row of db.prepare("SELECT legacy_id,project_id FROM memory_project_aliases").all() as Row[]) { + aliases.set(String(row.legacy_id), String(row.project_id)); + } + const existingAliases = new Map(aliases); + for (const row of rows("memory_project_aliases")) { + const from = str(row.legacy_id), to = str(row.project_id); + if (!from || !to) throw new Error("invalid project alias in bundle"); + if (!aliases.has(from) || strategy === "replace") aliases.set(from, to); + } + const canonical = (value: unknown): string | null => { + let current = str(value); + if (!current) return null; + const seen = new Set(); + while (aliases.has(current) && aliases.get(current) !== current) { + if (seen.has(current)) throw new Error("cyclic project aliases in bundle"); + seen.add(current); current = aliases.get(current)!; + } + return current; + }; + for (const id of aliases.keys()) canonical(id); + // Include inferred identities now, before override rows are considered. Only + // rows that would be accepted can contribute new alias evidence. + for (const table of ["sessions", "l3_world_model_scopes"]) { + const key = table === "sessions" ? "id" : "scope_key"; + for (const row of rows(table)) { + if (strategy === "skip" && db.prepare(`SELECT 1 FROM ${table} WHERE ${key}=?`).get(row[key])) continue; + const meta = table === "sessions" ? json(row.meta_json) : row; + const project = str(row.project_id), uri = str(meta.workspace_uri); + if (!project || !uri) continue; + let identity: ReturnType; + try { identity = resolveWorkspaceIdentity(null, { workspaceUri: uri, workspaceHostId: str(meta.workspace_host_id) }); } + catch { continue; } // Incomplete evidence does not create an alias. + if (!identity.projectId) continue; + const prior = canonical(project); + if (prior !== project && prior !== identity.projectId) throw new Error("conflicting project identity in bundle"); + aliases.set(project, identity.projectId); + const workspace = str(row.workspace_id); + if (workspace && identity.workspaceId) aliases.set(workspace, identity.workspaceId); + } + } + + const targetMemories = db.prepare("SELECT * FROM memories WHERE memory_layer='L3'").all() as Row[]; + const targetScopes = db.prepare("SELECT * FROM l3_world_model_scopes").all() as Row[]; + const protectedProjects = new Set(); + const originalProjects = new Set(); + const rawTargetProjects = new Set(); + const memoryProjects = new Map>(); + const remember = (id: unknown, project: unknown) => { + if (!str(id)) return; + const projects = memoryProjects.get(String(id)) ?? new Set(); + projects.add(canonical(project)); memoryProjects.set(String(id), projects); + }; + const rememberTarget = (project: unknown) => { + originalProjects.add(canonicalProjectId(db, str(project))); + protectedProjects.add(canonical(project)); + if (str(project)) rawTargetProjects.add(String(project)); + }; + if (strategy === "skip") { + for (const row of targetMemories) { + rememberTarget(json(row.info_json).project_id); + remember(row.id, json(row.info_json).project_id); + } + for (const row of targetScopes) { + rememberTarget(row.project_id); remember(row.memory_id, row.project_id); + } + } + for (const row of rows("memories")) if (row.memory_layer === "L3") remember(row.id, json(row.info_json).project_id); + for (const row of rows("l3_world_model_scopes")) remember(row.memory_id, row.project_id); + const skippedMemoryIds = new Set(); + if (strategy === "skip") { + for (const row of targetMemories) if (typeof row.id === "string") skippedMemoryIds.add(row.id); + for (const row of rows("memories")) { + if (typeof row.id === "string" && (db.prepare("SELECT 1 FROM memories WHERE id=?").get(row.id) + || (row.memory_layer === "L3" && protectedProjects.has(canonical(json(row.info_json).project_id))))) { + skippedMemoryIds.add(row.id); + } + } + } + const touchesMemory = (id: unknown) => typeof id === "string" && + (skippedMemoryIds.has(id) || [...(memoryProjects.get(id) ?? [])].some(p => protectedProjects.has(p))); + const touchesProject = (project: unknown) => protectedProjects.has(canonical(project)); + // An incoming row can lose a physical-ID conflict to the retained target row. + // Keep both dependency scopes: rejected input must neither mask the target's + // protection nor authorize dependents that would resume under another scope. + const dependencyProjects = (key: string, records: Row[]) => { + const projects = new Map>(); + for (const row of records) { + const id = str(row[key]); + if (!id) continue; + const values = projects.get(id) ?? new Set(); + values.add(canonical(row.project_id)); projects.set(id, values); + } + return projects; + }; + const batches = dependencyProjects("id", [...db.prepare("SELECT id,project_id FROM l3_world_model_evidence_batches").all() as Row[], + ...rows("l3_world_model_evidence_batches")]); + const scopes = dependencyProjects("scope_key", [...targetScopes, ...rows("l3_world_model_scopes")]); + const sessions = dependencyProjects("id", [...db.prepare("SELECT id,project_id FROM sessions").all() as Row[], ...rows("sessions")]); + const hasProtectedProject = (projects: Set | undefined) => + [...(projects ?? [])].some(project => protectedProjects.has(project)); + const batchProtected = (id: unknown) => typeof id === "string" && hasProtectedProject(batches.get(id)); + const scopeProtected = (key: unknown) => typeof key === "string" && [...scopes].some(([scope, projects]) => + (key === scope || key.startsWith(scope + ":")) && hasProtectedProject(projects)); + const sessionProtected = (id: unknown) => typeof id === "string" && hasProtectedProject(sessions.get(id)); + + const shouldSkip = (table: string, row: Row): boolean => { + if (strategy !== "skip") return false; + if (table === "memories") return typeof row.id === "string" && skippedMemoryIds.has(row.id); + if (table === "memory_l3_overrides") return touchesProject(row.project_key) || touchesMemory(row.memory_id); + if (table === "l3_world_model_scopes") return touchesProject(row.project_id) || touchesMemory(row.memory_id); + if (table === "memory_l3_legacy_field_edits") return touchesMemory(row.memory_id); + if (table === "memory_change_log") return (row.change_type === "l3_world_model_create" + || row.change_type === "l3_world_model_update" || row.change_type === "l3_world_model_delete" + || row.change_type === "archive") && touchesMemory(row.memory_id); + if (table === "audit_logs") return row.action === "archive" && touchesMemory(row.target_id); + if (table === "memory_project_aliases") { + const from = str(row.legacy_id); + return Boolean(from && rawTargetProjects.has(from) && !existingAliases.has(from)); + } + if (table === "l3_world_model_project_environment_state" || table === "l3_world_model_evidence_batches") return touchesProject(row.project_id); + if (table === "l3_world_model_batch_targets") return batchProtected(row.batch_id) || scopeProtected(row.field_scope_key); + if (table === "l3_world_model_session_cursors" || table === "l3_world_model_input_traces") return sessionProtected(row.session_id); + if (table === "memory_processing_state") return touchesMemory(row.memory_id); + if (table === "embedding_retry_queue") return touchesMemory(row.target_id); + if (table === "evolution_jobs") { + if (touchesMemory(row.target_memory_id)) return true; + if (row.job_type !== "l3_world_model_update" && row.job_type !== "project_environment_profile") return false; + const payload = json(row.payload_json); + return batchProtected(payload.batchId) || scopeProtected(row.scope_key) || sessionProtected(row.session_id) + || (Object.hasOwn(payload, "projectId") && touchesProject(payload.projectId)); + } + return false; + }; + return { shouldSkip, skippedMemoryIds, originalProjects, canonical }; +} diff --git a/Memory/src/storage/db.ts b/Memory/src/storage/db.ts index 2fbba53fd..b614b90b2 100644 --- a/Memory/src/storage/db.ts +++ b/Memory/src/storage/db.ts @@ -1,3 +1,4 @@ +import { installDeviceScope } from "./device-scope.js"; import Database from "better-sqlite3"; import { copyFileSync, existsSync, mkdirSync } from "node:fs"; import { basename, dirname, join, resolve } from "node:path"; @@ -38,6 +39,7 @@ export class MemoryDb { this.createPreMigrationBackup(); migrate(this.db); } + installDeviceScope(this.db); } catch (error) { this.db.close(); throw error; @@ -57,8 +59,10 @@ export class MemoryDb { private configure(): void { this.db.pragma("foreign_keys = ON"); - this.db.pragma("journal_mode = WAL"); - this.db.pragma("synchronous = NORMAL"); + if (!this.db.readonly) { + this.db.pragma("journal_mode = WAL"); + this.db.pragma("synchronous = NORMAL"); + } this.db.pragma("busy_timeout = 5000"); } diff --git a/Memory/src/storage/device-scope.ts b/Memory/src/storage/device-scope.ts new file mode 100644 index 000000000..02049d0ba --- /dev/null +++ b/Memory/src/storage/device-scope.ts @@ -0,0 +1,186 @@ +import { stableHash } from "../utils/id.js"; +import type { RuntimeNamespace } from "../types.js"; +import type Database from "better-sqlite3"; +import { resolveWorkspaceIdentity } from "../service/namespace/workspace-identity.js"; + +const installed = new WeakSet(); +const namespaces = new WeakMap>(); +const aliases = new WeakMap>(); + +/** Account identities are never memory scopes. Only evidence-backed workspace aliases are persisted. */ +export function installDeviceScope(db: Database.Database): void { + if (installed.has(db)) return; + aliases.set(db, readProjectAliases(db)); + db.function("memory_project_id", (value: unknown) => canonicalProjectId(db, typeof value === "string" ? value : null)); + namespaces.set(db, new Map()); + db.function("memory_namespace_key", (value: unknown) => typeof value === "string" ? canonicalNamespaceKey(db, value) : null); + refreshProjectAliases(db); + installed.add(db); +} + +export function canonicalProjectId(db: Database.Database, id: string | null | undefined): string | null { + if (!id?.trim()) return null; + return resolveAlias(aliases.get(db), id); +} + +export function registerProjectAlias(db: Database.Database, legacyId: string, uri: string, host?: string | null): string { + const identity = resolveWorkspaceIdentity(null, { workspaceUri: uri, ...(host ? { workspaceHostId: host } : {}) }); + if (!identity.projectId) return legacyId; + const existing = canonicalProjectId(db, legacyId); + if (existing !== legacyId && existing !== identity.projectId) throw new Error("conflicting legacy workspace identity"); + if (!db.readonly) db.prepare(`INSERT INTO memory_project_aliases(legacy_id, project_id, workspace_uri, workspace_host_id, created_at) + VALUES (?, ?, ?, ?, ?) ON CONFLICT(legacy_id) DO NOTHING`) + .run(legacyId, identity.projectId, uri, host ?? null, new Date().toISOString()); + aliases.get(db)?.set(legacyId, identity.projectId); + return identity.projectId; +} + +export function refreshProjectAliases(db: Database.Database): void { + reloadDeviceScopeAliases(db); + refreshProjectAliasEvidence(db); +} + +/** Reload committed mappings after a transaction rolls back; do not infer or write. */ +export function reloadDeviceScopeAliases(db: Database.Database): void { + const hasNamespaceAliases = tableExists(db, "memory_namespace_aliases"); + const hasProjectAliases = tableExists(db, "memory_project_aliases"); + const storedNamespaces = hasNamespaceAliases + ? db.prepare("SELECT legacy_key, namespace_key FROM memory_namespace_aliases").all() as Array<{legacy_key:string;namespace_key:string}> + : []; + if (hasNamespaceAliases || !db.readonly || !namespaces.has(db)) { + namespaces.set(db, new Map(storedNamespaces.map(row=>[row.legacy_key,row.namespace_key]))); + } + if (hasProjectAliases || !db.readonly || !aliases.has(db)) { + aliases.set(db, readProjectAliases(db)); + } +} + +function refreshProjectAliasEvidence(db: Database.Database): void { + const rows = db.prepare(`SELECT project_id, workspace_id, meta_json FROM sessions WHERE project_id IS NOT NULL`).all() as Array<{ project_id: string; workspace_id: string | null; meta_json: string }>; + // Resolve all evidence before writing any mapping; conflicting identities remain isolated. + const candidates = new Map>(); + const scopeRows = db.prepare(`SELECT project_id, workspace_uri FROM l3_world_model_scopes + WHERE project_id IS NOT NULL AND workspace_uri IS NOT NULL`).all() as Array<{project_id:string;workspace_uri:string}>; + // Remote URIs are self-contained; local paths still require the originating host ID. + const evidence = [...rows, ...scopeRows.map(row => ({project_id:row.project_id, + meta_json:JSON.stringify({workspace_uri:row.workspace_uri})}))]; + for (const row of evidence) { + try { + const meta = JSON.parse(row.meta_json); + const uri = typeof meta.workspace_uri === "string" ? meta.workspace_uri : undefined; + const host = typeof meta.workspace_host_id === "string" ? meta.workspace_host_id : undefined; + if (!uri) continue; + const identity = resolveWorkspaceIdentity(null, { workspaceUri: uri, ...(host ? { workspaceHostId: host } : {}) }); + if (!identity.projectId) continue; + const group = candidates.get(row.project_id) ?? new Map(); + group.set(identity.projectId, { uri, host }); + candidates.set(row.project_id, group); + } catch { /* Incomplete legacy evidence must never merge unrelated projects. */ } + } + db.transaction(() => { + for (const [id, candidatesForId] of candidates) { + const existing = aliases.get(db)?.get(id); + if (candidatesForId.size !== 1 || (existing && !candidatesForId.has(existing))) { + // Later imports can reveal conflicting evidence. Never keep a stale merge. + if (!db.readonly) db.prepare("DELETE FROM memory_project_aliases WHERE legacy_id = ?").run(id); + aliases.get(db)?.delete(id); + continue; + } + const candidate = [...candidatesForId.values()][0]!; + registerProjectAlias(db, id, candidate.uri, candidate.host); + } + })(); + const workspaceTargets = new Map>(); + for (const row of rows) { + const projectId = canonicalProjectId(db, row.project_id); + if (row.workspace_id) { + const targets = workspaceTargets.get(row.workspace_id) ?? new Set(); + targets.add(projectId !== row.project_id && projectId?.startsWith("ws_") ? projectId.slice(3) : row.workspace_id); + workspaceTargets.set(row.workspace_id, targets); + } + } + for (const [id, targets] of workspaceTargets) { + const target = targets.size === 1 ? [...targets][0]! : id; + if (target === id) { + if (!db.readonly) db.prepare("DELETE FROM memory_project_aliases WHERE legacy_id = ? AND workspace_uri = ''").run(id); + aliases.get(db)?.delete(id); + continue; + } + if (!db.readonly) db.prepare(`INSERT INTO memory_project_aliases(legacy_id, project_id, workspace_uri, created_at) + VALUES (?, ?, '', ?) ON CONFLICT(legacy_id) DO UPDATE SET project_id = excluded.project_id`) + .run(id, target, new Date().toISOString()); + aliases.get(db)?.set(id, target); + } + for (const row of rows) { + const projectId = canonicalProjectId(db, row.project_id); + let meta: Record; + try { meta = JSON.parse(row.meta_json); } catch { continue; } + if (!meta || typeof meta !== "object") continue; + const tenantId = meta.tenant_id ?? meta.tenantId ?? null; + const current = stableHash({tenantId, projectId, workspaceId: canonicalProjectId(db, row.workspace_id)}); + const old = stableHash({tenantId, projectId: row.project_id, workspaceId: row.workspace_id}); + // These hashes cannot be reconstructed once clear removes their Sessions. + // Keep the alias with the native-capture tombstones, including across export/import. + if (!db.readonly) db.prepare(`INSERT INTO memory_namespace_aliases(legacy_key,namespace_key) VALUES (?,?) + ON CONFLICT(legacy_key) DO UPDATE SET namespace_key=excluded.namespace_key`).run(old,current); + namespaces.get(db)?.set(old, current); + if (meta.source_namespace_key === old) namespaces.get(db)?.set(meta.source_namespace_key, current); + } +} + +export function canonicalNamespaceKey(db: Database.Database, key: string): string { + return resolveAlias(namespaces.get(db), key); +} + +function resolveAlias(map: ReadonlyMap | undefined, key: string): string { + let current = key; + const seen = new Set(); + while (map?.has(current) && map.get(current) !== current) { + if (seen.has(current)) return key; // Invalid historical cycles do not merge identities. + seen.add(current); current = map.get(current)!; + } + return current; +} + +export function deviceNamespace(db: Database.Database, namespace: RuntimeNamespace): RuntimeNamespace { + return {...namespace, userId: null, + projectId: canonicalProjectId(db, namespace.projectId) ?? undefined, + workspaceId: canonicalProjectId(db, namespace.workspaceId) ?? undefined}; +} + +function readProjectAliases(db: Database.Database): Map { + if (!tableExists(db, "memory_project_aliases")) return new Map(); + const rows = db.prepare("SELECT legacy_id, project_id FROM memory_project_aliases").all() as Array<{ + legacy_id: string; + project_id: string; + }>; + return new Map(rows.map(row => [row.legacy_id, row.project_id])); +} + +function tableExists(db: Database.Database, name: string): boolean { + return Boolean(db.prepare("SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = ?").get(name)); +} + +/** Normalize only protocol envelopes, never user content or nested tool payloads. */ +export function memoryFingerprint(db: Database.Database, value: unknown, legacyUserId?: string): unknown { + if (!value || typeof value !== "object" || Array.isArray(value)) return value; + const record = value as Record; + const result = { ...record }; + if (record.namespace && typeof record.namespace === "object" && !Array.isArray(record.namespace)) { + result.namespace = legacyUserId === undefined + ? deviceNamespace(db, record.namespace as RuntimeNamespace) + : { ...record.namespace, userId: legacyUserId }; + } + for (const key of ["request", "skill"]) { + if (record[key] && typeof record[key] === "object") result[key] = memoryFingerprint(db, record[key], legacyUserId); + } + return result; +} + +export function matchesLegacyMemoryFingerprint(db: Database.Database, fingerprint: unknown, expected: string, + hash: (value: unknown) => string): boolean { + if (hash(fingerprint) === expected) return true; + const owners = db.prepare(`SELECT DISTINCT user_id FROM sessions WHERE user_id IS NOT NULL + UNION SELECT DISTINCT user_id FROM memories WHERE user_id IS NOT NULL`).pluck().all() as string[]; + return owners.some(owner => hash(memoryFingerprint(db, fingerprint, owner)) === expected); +} diff --git a/Memory/src/storage/legacy-l3-field-edits.ts b/Memory/src/storage/legacy-l3-field-edits.ts new file mode 100644 index 000000000..0a1d398e0 --- /dev/null +++ b/Memory/src/storage/legacy-l3-field-edits.ts @@ -0,0 +1,58 @@ +import type Database from "better-sqlite3"; +import { L3WorldModelFieldNameSchema, type L3WorldModelFieldName } from "../contracts/index.js"; +import { isRecord, parseJson } from "../utils/json.js"; + +export interface LegacyL3Change { + memory_id: string; + before_json: string | null; + after_json: string | null; + created_at: string; + seq: number; +} + +export interface LegacyL3FieldEdit { + memory_id: string; + field: L3WorldModelFieldName; + deleted: number; + edited_at: string; + change_seq: number; +} + +/** Persist evidence of an edit, never infer a deletion from an uninitialized null. */ +export function legacyL3FieldEditsFromChanges(rows: LegacyL3Change[]): LegacyL3FieldEdit[] { + const body = (json: string | null) => { + const value = parseJson(json, null); + if (!isRecord(value) || !isRecord(value.properties)) return undefined; + const internal = value.properties.internal_info; + return isRecord(internal) && internal.schema_version === 2 && isRecord(internal.world_model) + ? internal.world_model : undefined; + }; + return rows.flatMap(row => { + const before = body(row.before_json), after = body(row.after_json); + if (!after) return []; + return L3WorldModelFieldNameSchema.options.flatMap(field => { + const deleted = typeof before?.[field] === "string" && after[field] === null; + const written = typeof after[field] === "string" && after[field] !== before?.[field]; + return deleted || written ? [{ memory_id: row.memory_id, field, deleted: Number(deleted), + edited_at: row.created_at, change_seq: row.seq }] : []; + }); + }); +} + +/** Called inside the migration/retention transaction before any evidence is removed. */ +export function preserveLegacyL3FieldEdits(db: Database.Database, importedChanges?: LegacyL3Change[]): void { + const rows = importedChanges ?? db.prepare(`SELECT change.memory_id, change.before_json, change.after_json, change.created_at, change.seq + FROM memory_change_log AS change JOIN l3_world_model_scopes AS scope ON scope.memory_id = change.memory_id + WHERE scope.user_id IS NOT NULL AND change.change_type IN ('l3_world_model_create', 'l3_world_model_update')`) + .all() as LegacyL3Change[]; + const legacyMemoryIds = importedChanges ? new Set((db.prepare( + "SELECT memory_id FROM l3_world_model_scopes WHERE user_id IS NOT NULL AND memory_id IS NOT NULL" + ).all() as Array<{memory_id: string}>).map(row => row.memory_id)) : undefined; + const save = db.prepare(`INSERT INTO memory_l3_legacy_field_edits(memory_id, field, deleted, edited_at, change_seq) + VALUES (@memory_id, @field, @deleted, @edited_at, @change_seq) + ON CONFLICT(memory_id, field, deleted) DO UPDATE SET edited_at=excluded.edited_at, change_seq=excluded.change_seq + WHERE excluded.edited_at > edited_at OR (excluded.edited_at = edited_at AND excluded.change_seq > change_seq)`); + for (const edit of legacyL3FieldEditsFromChanges(rows)) { + if (!legacyMemoryIds || legacyMemoryIds.has(edit.memory_id)) save.run(edit); + } +} diff --git a/Memory/src/storage/native-memory-origin.ts b/Memory/src/storage/native-memory-origin.ts new file mode 100644 index 000000000..558967c06 --- /dev/null +++ b/Memory/src/storage/native-memory-origin.ts @@ -0,0 +1,144 @@ +import type Database from "better-sqlite3"; +import type { MemoryRow } from "../types.js"; +import { canonicalNamespaceKey, canonicalProjectId } from "./device-scope.js"; +import { isRecord, parseJson } from "../utils/json.js"; +import { stableHash } from "../utils/id.js"; +import { normalizeMemoryCaptureSource } from "../utils/memory-capture-claim.js"; + +/** Present null is a captured empty scope. A missing snapshot is unknown, not a wildcard. */ +export interface NativeOrigin { + version: 1; + source: string; + profileId: string; + conversationId: string | null; + projectId: string | null; + workspaceId: string | null; + tenantId: string | null; + namespaceKey: string | null; + hostSessionKey?: string | null; +} + +export const pendingOriginKey = (sessionId: string, turnId: string): string => + `native_turn_origin:${stableHash({ sessionId, turnId })}`; + +export function captureNativeOrigin(input: { + source: string; profileId: string; conversationId?: string | null; + projectId?: string | null; workspaceId?: string | null; tenantId?: string | null; + meta?: Record; + hostSessionKey?: string | null; +}): NativeOrigin { + const meta = input.meta ?? {}; + return { version: 1, source: normalizeMemoryCaptureSource(input.source), profileId: input.profileId, + conversationId: input.conversationId ?? null, projectId: input.projectId ?? null, + workspaceId: input.workspaceId ?? null, + tenantId: input.tenantId ?? text(meta.tenant_id) ?? text(meta.tenantId) ?? null, + namespaceKey: text(meta.source_namespace_key) ?? null, hostSessionKey: input.hostSessionKey ?? null }; +} + +export function readNativeOrigin(value: unknown): NativeOrigin | undefined { + if (!isRecord(value) || value.version !== 1 || !text(value.source) || !text(value.profileId)) return undefined; + for (const key of ["conversationId", "projectId", "workspaceId", "tenantId", "namespaceKey"]) { + if (value[key] !== null && typeof value[key] !== "string") return undefined; + } + return value as unknown as NativeOrigin; +} + +export interface NativeIdentity { + source: string; profileId: string; conversationId: string; turnId: string; namespaceKey: string; +} + +/** Claim compatibility may use a host key, without promoting it to native provenance. */ +export function nativeClaimIdentity(origin: NativeOrigin, turnId: string) { + const conversationId = origin.conversationId ?? origin.hostSessionKey; + return conversationId ? { projectId: origin.projectId, workspaceId: origin.workspaceId, + tenantId: origin.tenantId, profileId: origin.profileId, conversationId, turnId } : undefined; +} + +export function nativeIdentity(db: Database.Database, origin: NativeOrigin, turnId: string, allowHostConversation = false): NativeIdentity | undefined { + const conversationId = origin.conversationId ?? (allowHostConversation ? origin.hostSessionKey : null); + if (!conversationId || !turnId) return undefined; + const scope = { tenantId: origin.tenantId, projectId: canonicalProjectId(db, origin.projectId), + workspaceId: canonicalProjectId(db, origin.workspaceId) }; + const namespaceKey = stableHash(scope); + if (origin.namespaceKey && canonicalNamespaceKey(db, origin.namespaceKey) !== namespaceKey) return undefined; + return { source: normalizeMemoryCaptureSource(origin.source), profileId: origin.profileId, + conversationId, turnId, + namespaceKey }; +} + +/** Called only for a new row, never to backfill a persisted historical row. */ +export function freezeMemoryOrigin(db: Database.Database, memory: MemoryRow): MemoryRow { + if (memory.memoryLayer !== "L1" || memory.properties.internal_info.memory_kind === "work_memory") return memory; + if (Object.hasOwn(memory.info, "native_origin_v1")) { + const captured = readNativeOrigin(memory.info.native_origin_v1); + return captured ? withMemoryOrigin(memory, captured) : memory; + } + const trace = memory.properties.internal_info.trace; + const rawId = text(memory.info.raw_turn_id) ?? (isRecord(trace) ? text(trace.raw_turn_id) : undefined); + const raw = rawId ? db.prepare("SELECT message_payload_json FROM raw_turns WHERE id=?").get(rawId) as + { message_payload_json: string } | undefined : undefined; + const rawOrigin = readNativeOrigin(parseJson>(raw?.message_payload_json, {}).native_origin_v1); + const session = memory.sessionId ? db.prepare("SELECT * FROM sessions WHERE id=?").get(memory.sessionId) as + {source: string; profile_id: string; conversation_id: string | null; host_session_key: string | null; meta_json: string} | undefined : undefined; + const origin = rawOrigin ?? (session ? captureNativeOrigin({ + source: memory.agentId ?? session.source, profileId: text(memory.info.profile_id) ?? session.profile_id, + conversationId: memory.conversationId ?? session.conversation_id, + projectId: text(memory.info.project_id) ?? null, workspaceId: memory.appId ?? null, + meta: parseJson>(session.meta_json, {}), hostSessionKey: session.host_session_key + }) : undefined); + return origin ? withMemoryOrigin(memory, origin) : memory; +} + +export function withMemoryOrigin(memory: MemoryRow, origin: NativeOrigin): MemoryRow { + const info = { ...memory.info, native_origin_v1: origin, project_id: origin.projectId, profile_id: origin.profileId }; + return { ...memory, agentId: origin.source, appId: origin.workspaceId ?? undefined, + conversationId: origin.conversationId ?? undefined, info, + properties: { ...memory.properties, info } }; +} + +function text(value: unknown): string | undefined { + return typeof value === "string" && value.trim() ? value : undefined; +} + +export function nativeTurnId(memory: MemoryRow): string | undefined { + const trace = memory.properties.internal_info.trace; + return text(memory.info.turn_id) ?? (isRecord(trace) ? text(trace.turn_id) : undefined); +} + +/** No mutable Session fields are used to reconstruct a historical origin. */ +export function resolveNativeIdentity(db: Database.Database, memory: MemoryRow, allowHostConversation = false): NativeIdentity | undefined { + const turn = nativeTurnId(memory); + if (!turn) return undefined; + if (Object.hasOwn(memory.info, "native_origin_v1")) { + const origin = readNativeOrigin(memory.info.native_origin_v1); + return origin ? nativeIdentity(db, origin, turn, allowHostConversation) : undefined; + } + const trace = memory.properties.internal_info.trace; + const rawId = text(memory.info.raw_turn_id) ?? (isRecord(trace) ? text(trace.raw_turn_id) : undefined); + const raw = rawId ? db.prepare("SELECT message_payload_json FROM raw_turns WHERE id=?").get(rawId) as + { message_payload_json: string } | undefined : undefined; + const rawOrigin = readNativeOrigin(parseJson>(raw?.message_payload_json, {}).native_origin_v1); + if (rawOrigin) return nativeIdentity(db, rawOrigin, turn, allowHostConversation); + // A Session+turn match alone is insufficient after scope reuse. Require a + // durable reference to this memory or its raw turn, including old input traces. + const linkedRawIds = db.prepare("SELECT raw_turn_id FROM l3_world_model_input_traces WHERE l1_memory_id=?") + .pluck().all(memory.id) as string[]; + if (rawId) linkedRawIds.push(rawId); + const captures = db.prepare(`SELECT source,profile_id,conversation_id,namespace_key FROM source_turn_captures + WHERE turn_id=? AND (raw_turn_id IN (SELECT value FROM json_each(?)) + OR json_extract(response_json,'$.legacyImportMemoryId')=? + OR EXISTS (SELECT 1 FROM json_each(response_json,'$.result.l1MemoryIds') WHERE value=?))`) + .all(turn, JSON.stringify(linkedRawIds), memory.id, memory.id) as Array<{ + source: string; profile_id: string; conversation_id: string; namespace_key: string; + }>; + const identities = new Map(); + for (const row of captures) { + const identity = { source: normalizeMemoryCaptureSource(row.source), profileId: row.profile_id, + conversationId: row.conversation_id, namespaceKey: canonicalNamespaceKey(db, row.namespace_key), turnId: turn }; + if (memory.agentId && normalizeMemoryCaptureSource(memory.agentId) !== identity.source) return undefined; + if (memory.conversationId && memory.conversationId !== identity.conversationId) return undefined; + if (text(memory.info.profile_id) && memory.info.profile_id !== identity.profileId) return undefined; + identities.set(stableHash(identity), identity); + } + return identities.size === 1 ? [...identities.values()][0] : undefined; +} diff --git a/Memory/src/storage/repositories.ts b/Memory/src/storage/repositories.ts index 897936263..4c152785c 100644 --- a/Memory/src/storage/repositories.ts +++ b/Memory/src/storage/repositories.ts @@ -1,10 +1,14 @@ import type { EmbeddingMaintenanceEntry } from "./sqlite-vec-store.js"; import { sameEmbeddingIdentity, vectorEmbeddingIdentity, type EmbeddingIdentity } from "../model/embedding-identity.js"; +import { planBundleImport } from "./bundle-import-plan.js"; +import { canonicalNamespaceKey, installDeviceScope, canonicalProjectId, refreshProjectAliases, reloadDeviceScopeAliases } from "./device-scope.js"; +import { captureNativeOrigin, freezeMemoryOrigin, nativeIdentity, pendingOriginKey, readNativeOrigin, resolveNativeIdentity, withMemoryOrigin } from "./native-memory-origin.js"; import type Database from "better-sqlite3"; import { canonicalJson, renderL3WorldModelFields, sha256Hex, + L3WorldModelFieldNameSchema, type JsonValue, type L3WorldModelFieldName, type L3WorldModelFields, @@ -32,6 +36,7 @@ import type { MemoryStatsRow, MemoryStatus, RecallHit, + RuntimeNamespace, SourceTurnCompleteResponse, UserMemoryRecord, UserMemoryStatus, @@ -40,6 +45,7 @@ import type { import { DEFAULT_NAMESPACE_SOURCE } from "../types.js"; import { agentSourceFamilyRoots, normalizeAgentIdKey } from "../utils/agent-source-id.js"; import { newId, stableHash } from "../utils/id.js"; +import { normalizeMemoryCaptureSource } from "../utils/memory-capture-claim.js"; import { asStringArray, isRecord, parseJson, toJson } from "../utils/json.js"; import { firstSemanticUserLine, semanticUserText } from "../utils/text.js"; import { nowIso } from "../utils/time.js"; @@ -57,10 +63,47 @@ import { type VectorSearchCandidate } from "./sqlite-vec-store.js"; import { compactHistoryChangePayload, compactHistoryChangeRow } from "./history-change-snapshot.js"; +import { legacyL3FieldEditsFromChanges, preserveLegacyL3FieldEdits, type LegacyL3Change, type LegacyL3FieldEdit } from "./legacy-l3-field-edits.js"; +import { getSchemaVersion, SCHEMA_VERSION } from "./schema.js"; type SqlValue = string | number | Buffer | null; +const OPTIONAL_PREVIOUS_SCHEMA_TABLES = new Set([ + "memory_project_aliases", + "memory_namespace_aliases", + "memory_l3_overrides", + "memory_l3_legacy_field_edits" +]); + +function hasSqliteTable(db: Database.Database, table: string): boolean { + return Boolean(db.prepare("SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = ?").get(table)); +} + +function isMissingOptionalTableFromPreviousSchema(db: Database.Database, table: string): boolean { + return db.readonly && OPTIONAL_PREVIOUS_SCHEMA_TABLES.has(table) && !hasSqliteTable(db, table) && + getSchemaVersion(db).version < SCHEMA_VERSION; +} + +/** Materialize archive evidence before its bounded audit/history log can be pruned. */ +function preserveExplicitL3ArchiveMarkers(db: Database.Database): void { + db.prepare(`UPDATE memories + SET properties_json = json_set(properties_json, + '$.internal_info.l3_world_model_explicitly_archived', json('true')) + WHERE memory_layer = 'L3' AND status = 'archived' + AND json_type(properties_json, '$.internal_info') = 'object' + AND json_extract(properties_json, '$.internal_info.schema_version') = 2 + AND json_extract(properties_json, '$.internal_info.l3_world_model_explicitly_archived') IS NOT 1 + AND ( + EXISTS (SELECT 1 FROM audit_logs WHERE action = 'archive' AND target_id = memories.id) + OR EXISTS (SELECT 1 FROM memory_change_log WHERE change_type = 'archive' AND memory_id = memories.id) + )`).run(); +} + const BUNDLE_TABLES = [ + "memory_project_aliases", + "memory_namespace_aliases", + "memory_l3_overrides", "memories", + "memory_l3_legacy_field_edits", "memory_capture_claims", "source_turn_captures", "l3_world_model_scopes", @@ -90,7 +133,7 @@ const BUNDLE_TABLES = [ "audit_logs" ] as const; const CLEAR_MEMORY_TABLES = [ - ...BUNDLE_TABLES, + ...BUNDLE_TABLES.filter((table) => table !== "memory_project_aliases" && table !== "memory_namespace_aliases"), "memories_fts", "user_memories_fts", "memory_vector_entries", @@ -132,7 +175,7 @@ interface MemorySqlRow { export interface SessionRecord { id: string; - userId: string; + userId: string | null; source: string; profileId: string; profileLabel?: string; @@ -152,7 +195,7 @@ export interface SessionRecord { export interface EpisodeRecord { id: string; sessionId: string; - userId: string; + userId: string | null; projectId?: string; conversationId?: string; status: "open" | "processing" | "closed"; @@ -179,7 +222,7 @@ export interface EpisodeRecord { export interface SkillClusterRecord { id: string; - userId: string; + userId: string | null; projectId?: string; tools: string[]; artifacts: string[]; @@ -206,7 +249,7 @@ export interface RawTurnRecord { sessionId: string; episodeId?: string; turnId: string; - userId: string; + userId: string | null; conversationId?: string; userText?: string; assistantText?: string; @@ -225,17 +268,31 @@ export interface RawTurnRecord { export type MemoryCaptureClaimOrigin = "turn_complete" | "agent_source_scan"; export interface MemoryCaptureClaimRecord { - userId: string; + userId: string | null; source: string; qaHash: string; primaryMemoryId: string; capturedBy: MemoryCaptureClaimOrigin; createdAt: string; + identity?: MemoryCaptureIdentity; +} + +export interface MemoryCaptureScope { + projectId: string | null; + workspaceId: string | null; + tenantId: string | null; + profileId: string; + conversationId: string | null; +} + +export interface MemoryCaptureIdentity extends MemoryCaptureScope { + conversationId: string; + turnId: string; } export interface FeedbackRecord { id: string; - userId: string; + userId: string | null; projectId?: string; conversationId?: string; sessionId?: string; @@ -257,7 +314,7 @@ export interface RecallEventRecord { sessionId?: string; episodeId?: string; turnId?: string; - userId: string; + userId: string | null; query: string; queryHash?: string; layers: MemoryLayer[]; @@ -312,7 +369,7 @@ export interface EvolutionJobRecord { jobType: JobType; status: JobStatus; dedupeKey?: string; - userId: string; + userId: string | null; sessionId?: string; episodeId?: string; targetMemoryId?: string; @@ -348,7 +405,7 @@ export type L3WorldModelTargetField = Exclude; action: string; @@ -572,6 +629,7 @@ export class MemoryRepository { ) {} insert(memory: MemoryRow): MemoryRow { + memory = freezeMemoryOrigin(this.db, { ...memory, userId: null }); const prepared = prepareMemoryForStorage(memory); this.db .prepare( @@ -594,6 +652,7 @@ export class MemoryRepository { } update(memory: MemoryRow): MemoryRow { + memory = { ...memory, userId: null }; return this.updateRow(memory, true); } @@ -606,6 +665,8 @@ export class MemoryRepository { if (!existing) { throw new Error(`memory not found: ${memory.id}`); } + const origin = readNativeOrigin(existing.info.native_origin_v1); + if (origin) memory = withMemoryOrigin(memory, origin); const prepared = prepareMemoryForStorage(memory); const updated = { ...prepared.memory, @@ -615,7 +676,6 @@ export class MemoryRepository { .prepare( `UPDATE memories SET timeline = @timeline, - user_id = @userId, conversation_id = @conversationId, session_id = @sessionId, agent_id = @agentId, @@ -651,14 +711,17 @@ export class MemoryRepository { return attachMemoryVectors(updated, updated.deletedAt || updated.status === "deleted" ? [] : mergedVectors); } - upsertByKey(memory: MemoryRow): { + upsertByKey(memory: MemoryRow, existingMemoryId?: string): { memory: MemoryRow; created: boolean; previous?: MemoryRow; } { - const previous = memory.memoryKey + const previous = existingMemoryId ? this.get(existingMemoryId) : memory.memoryKey ? this.getByKey(memory.memoryLayer, memory.memoryKey) : undefined; + if (existingMemoryId && (!previous || previous.memoryLayer !== memory.memoryLayer || previous.memoryKey !== memory.memoryKey)) { + throw new Error('memory upsert target no longer matches its verified key'); + } if (!previous) { return { memory: this.insert(memory), created: true }; } @@ -698,29 +761,29 @@ export class MemoryRepository { const row = this.db .prepare(`SELECT * FROM memories WHERE id = ? AND deleted_at IS NULL`) .get(id) as MemorySqlRow | undefined; - return row ? this.hydrate(memoryFromSql(row)) : undefined; + return row ? this.hydrate(memoryFromSql(row, this.db)) : undefined; } getIncludingDeleted(id: string): MemoryRow | undefined { const row = this.db .prepare(`SELECT * FROM memories WHERE id = ?`) .get(id) as MemorySqlRow | undefined; - return row ? this.hydrate(memoryFromSql(row)) : undefined; + return row ? this.hydrate(memoryFromSql(row, this.db)) : undefined; } - listTaskSpanIds(userId: string, episodeId: string, sourceTraceIds: string[]): string[] { + listTaskSpanIds(episodeId: string, sourceTraceIds: string[]): string[] { // Parent traces may already be soft-deleted, and their span_ids only describe // the latest split. Use the spans' own ownership links for task deletion. const rows = this.db.prepare( `SELECT id FROM memories - WHERE user_id = ? AND memory_layer = 'L1' + WHERE memory_layer = 'L1' AND deleted_at IS NULL AND status != 'deleted' AND json_extract(properties_json, '$.internal_info.memory_kind') = 'span' AND json_extract(properties_json, '$.internal_info.span.episode_id') = ? AND json_extract(properties_json, '$.internal_info.span.source_trace_id') IN (SELECT value FROM json_each(?)) ORDER BY id` - ).all(userId, episodeId, JSON.stringify(sourceTraceIds)) as Array<{ id: string }>; + ).all(episodeId, JSON.stringify(sourceTraceIds)) as Array<{ id: string }>; return rows.map((row) => row.id); } @@ -736,7 +799,7 @@ export class MemoryRepository { LIMIT 1` ) .get(memoryLayer, key) as MemorySqlRow | undefined; - return row ? this.hydrate(memoryFromSql(row)) : undefined; + return row ? this.hydrate(memoryFromSql(row, this.db)) : undefined; } getByKeyIncludingDeleted(memoryLayer: MemoryLayer, key: string): MemoryRow | undefined { @@ -747,7 +810,7 @@ export class MemoryRepository { ORDER BY updated_at DESC, id DESC LIMIT 1` ) .get(memoryLayer, key) as MemorySqlRow | undefined; - return row ? this.hydrate(memoryFromSql(row)) : undefined; + return row ? this.hydrate(memoryFromSql(row, this.db)) : undefined; } listMemoryTags(ids: readonly string[]): string[] { @@ -769,13 +832,13 @@ export class MemoryRepository { })); } - listUserMemoriesByKeyIncludingDeleted(userId: string, memoryKey: string): MemoryRow[] { + listUserMemoriesByKeyIncludingDeleted(userId: string | null, memoryKey: string): MemoryRow[] { const rows = this.db.prepare(` SELECT * FROM memories - WHERE user_id = ? AND memory_layer = 'L1' AND memory_key = ? + WHERE memory_layer = 'L1' AND memory_key = ? ORDER BY updated_at DESC, id DESC - `).all(userId, memoryKey) as MemorySqlRow[]; - return rows.map((row) => this.hydrate(memoryFromSql(row))); + `).all(memoryKey) as MemorySqlRow[]; + return rows.map((row) => this.hydrate(memoryFromSql(row, this.db))); } archivePriorReadOnlySkillVersions(input: { @@ -802,7 +865,7 @@ export class MemoryRepository { input.sourceSkillIdentity ) as MemorySqlRow[]; return rows.map((row) => { - const memory = this.hydrate(memoryFromSql(row)); + const memory = this.hydrate(memoryFromSql(row, this.db)); const internalSkill = isRecordLike(memory.properties.internal_info.skill) ? memory.properties.internal_info.skill : {}; @@ -833,7 +896,7 @@ export class MemoryRepository { .all(...ids) as MemorySqlRow[]; const vectors = this.vectors.getMany(rows.map((row) => row.id)); const byId = new Map(rows.map((row) => { - const memory = memoryFromSql(row); + const memory = memoryFromSql(row, this.db); attachMemoryVectors(memory, vectors.get(row.id) ?? []); return [row.id, memory]; })); @@ -851,7 +914,7 @@ export class MemoryRepository { LIMIT ? OFFSET ?` ) .all(...built.params, limit, offset) as MemorySqlRow[]; - return this.hydrateMany(rows.map(memoryFromSql)); + return this.hydrateMany(rows.map((row) => memoryFromSql(row, this.db))); } listStats(): MemoryStatsRow[] { @@ -933,7 +996,7 @@ export class MemoryRepository { LIMIT ?` ) .all(...(targetMemoryIds ?? []), limit) as MemorySqlRow[]; - return this.hydrateMany(rows.map(memoryFromSql)); + return this.hydrateMany(rows.map((row) => memoryFromSql(row, this.db))); } listUnprocessedAgentSourceImports(limit = 10000): MemoryRow[] { @@ -962,7 +1025,7 @@ export class MemoryRepository { LIMIT ?` ) .all(limit) as MemorySqlRow[]; - return this.hydrateMany(rows.map(memoryFromSql)); + return this.hydrateMany(rows.map((row) => memoryFromSql(row, this.db))); } listImportMemoriesNeedingSummary(limit = 10000): MemoryRow[] { @@ -990,7 +1053,7 @@ export class MemoryRepository { LIMIT ?` ) .all(limit) as MemorySqlRow[]; - return this.hydrateMany(rows.map(memoryFromSql)); + return this.hydrateMany(rows.map((row) => memoryFromSql(row, this.db))); } /** Bounded, idempotent upgrade pass; deleted observations never re-enter processing. */ @@ -1001,7 +1064,7 @@ export class MemoryRepository { AND json_type(properties_json, '$.internal_info.external_evidence') = 'object' AND COALESCE(json_extract(properties_json, '$.internal_info.observation_memory.schema_version'), 0) < 1 ORDER BY created_at DESC, id DESC LIMIT ?`).all(Math.min(Math.max(limit, 0), 1000)) as MemorySqlRow[]; - return this.hydrateMany(rows.map(memoryFromSql)); + return this.hydrateMany(rows.map((row) => memoryFromSql(row, this.db))); } listUnindexedL1Imports(limit = 10000): MemoryRow[] { @@ -1034,7 +1097,7 @@ export class MemoryRepository { LIMIT ?` ) .all(limit) as MemorySqlRow[]; - return this.hydrateMany(rows.map(memoryFromSql)); + return this.hydrateMany(rows.map((row) => memoryFromSql(row, this.db))); } count(filter: MemoryFilter = {}): number { @@ -1067,7 +1130,7 @@ export class MemoryRepository { const tagFilter = filter.tags?.map((tag) => tag.toLowerCase()) ?? []; const scored = rows - .map(memoryFromSql) + .map((row) => memoryFromSql(row, this.db)) .filter((memory) => tagFilter.length === 0 ? true @@ -1259,6 +1322,33 @@ export class MemoryRepository { }); } + /** Only immutable capture evidence can propagate deletion or promotion. */ + nativeDuplicates(memory: MemoryRow): { ids: string[]; deleted: boolean } { + const fallback = { ids: [memory.id], deleted: Boolean(memory.deletedAt || memory.status === "deleted") }; + if (memory.memoryLayer !== "L1" || memory.properties.internal_info.memory_kind === "work_memory") return fallback; + const identity = resolveNativeIdentity(this.db, memory); + if (!identity) return fallback; + const trace = memory.properties.internal_info.trace as Record | undefined; + const step = typeof trace?.step_index === "number" ? trace.step_index : null; + // Use the native-turn index. Session scope is mutable and cannot restrict this set. + const rows = this.db.prepare(`SELECT * FROM memories + WHERE memory_layer='L1' + AND COALESCE(json_extract(info_json, '$.turn_id'), json_extract(properties_json, '$.internal_info.trace.turn_id')) = ? + AND COALESCE(json_extract(properties_json, '$.internal_info.memory_kind'), 'trace') = ? + AND (json_extract(properties_json, '$.internal_info.trace.step_index') = ? + OR (? IS NULL AND json_extract(properties_json, '$.internal_info.trace.step_index') IS NULL + AND content_hash IS ? AND memory_value = ?)) + ORDER BY created_at, id`).all(identity.turnId, memory.properties.internal_info.memory_kind ?? "trace", + step, step, memory.contentHash ?? null, memory.memoryValue) as MemorySqlRow[]; + const key = stableHash(identity); + const members = rows.filter(row => { + const candidate = resolveNativeIdentity(this.db, memoryFromSql(row, this.db)); + return candidate !== undefined && stableHash(candidate) === key; + }); + return members.length ? { ids: members.map(row => row.id), + deleted: members.some(row => row.deleted_at || row.status === "deleted") } : fallback; + } + softDelete(id: string, deletedAt = nowIso()): MemoryRow | undefined { const memory = this.get(id); if (!memory) { @@ -1276,7 +1366,7 @@ export class MemoryRepository { }); } - countByLayer(userId?: string): Record { + countByLayer(userId?: string | null): Record { void userId; const rows = this.db .prepare( @@ -1295,7 +1385,7 @@ export class MemoryRepository { }; } - countByStatus(userId?: string): Record { + countByStatus(userId?: string | null): Record { void userId; const rows = this.db .prepare( @@ -1392,44 +1482,80 @@ export class MemoryRepository { export class MemoryCaptureClaimRepository { constructor(private readonly db: Database.Database) {} + scope(namespace: Partial, sessionId?: string): MemoryCaptureScope { + const session = sessionId ? this.db.prepare(`SELECT * FROM sessions WHERE id = ?`).get(sessionId) as SqlSessionRow | undefined : undefined; + const meta = parseJson>(session?.meta_json, {}); + const tenantId = meta.tenant_id ?? meta.tenantId ?? namespace.tenantId; + const conversationId = session?.conversation_id?.trim() ? session.conversation_id : session?.host_session_key ?? namespace.sessionKey; + return { + projectId: canonicalProjectId(this.db, session?.project_id ?? namespace.projectId), + workspaceId: canonicalProjectId(this.db, session?.workspace_id ?? namespace.workspaceId), + tenantId: typeof tenantId === 'string' && tenantId ? tenantId : null, + profileId: session?.profile_id ?? namespace.profileId ?? 'default', + conversationId: conversationId?.trim() ? conversationId : null + }; + } + + identity(namespace: Partial, turnId?: string, sessionId?: string): MemoryCaptureIdentity | undefined { + const scope = this.scope(namespace, sessionId); + // Missing conversation evidence disables claim reuse, not project isolation. + if (!scope.conversationId || !turnId?.trim()) return undefined; + return { ...scope, conversationId: scope.conversationId, turnId }; + } + + matchesOrigin(memoryId: string, source: string, identity: MemoryCaptureIdentity): boolean { + const row = this.db.prepare("SELECT * FROM memories WHERE id=?").get(memoryId) as MemorySqlRow | undefined; + if (!row) return false; + const origin = resolveNativeIdentity(this.db, memoryFromSql(row, this.db), true); + const expected = nativeIdentity(this.db, captureNativeOrigin({ ...identity, source }), identity.turnId); + return origin !== undefined && expected !== undefined && stableHash(origin) === stableHash(expected); + } + + private scopedHash(qaHash: string, identity: MemoryCaptureIdentity): string { + return `v2:${stableHash({ qaHash, ...identity, + projectId: canonicalProjectId(this.db, identity.projectId), + workspaceId: canonicalProjectId(this.db, identity.workspaceId) })}`; + } + claim(input: MemoryCaptureClaimRecord): { claimed: boolean; claim: MemoryCaptureClaimRecord; } { - const result = this.db.prepare( - `INSERT OR IGNORE INTO memory_capture_claims ( - user_id, source, qa_hash, primary_memory_id, captured_by, created_at - ) VALUES (?, ?, ?, ?, ?, ?)` - ).run( - input.userId, - input.source, - input.qaHash, - input.primaryMemoryId, - input.capturedBy, - input.createdAt - ); - const claim = this.get(input.userId, input.source, input.qaHash); - if (!claim) { - throw new Error("memory capture claim was not persisted"); - } - return { claimed: result.changes === 1, claim }; + return this.db.transaction(() => { + if (!input.identity) return { claimed: true, claim: { ...input, userId: null } }; + const existing = this.get(null, input.source, input.qaHash, input.identity); + if (existing) return { claimed: false, claim: existing }; + this.db.prepare(`INSERT INTO memory_capture_claims ( + user_id, source, qa_hash, primary_memory_id, captured_by, created_at + ) VALUES (NULL, ?, ?, ?, ?, ?)`) + .run(input.source, this.scopedHash(input.qaHash, input.identity), input.primaryMemoryId, input.capturedBy, input.createdAt); + return { claimed: true, claim: { ...input, userId: null } }; + })(); } - get(userId: string, source: string, qaHash: string): MemoryCaptureClaimRecord | undefined { - const row = this.db.prepare( + get(userId: string | null, source: string, qaHash: string, identity?: MemoryCaptureIdentity): MemoryCaptureClaimRecord | undefined { + if (!identity) return undefined; + const scopedHash = this.scopedHash(qaHash, identity); + const rows = this.db.prepare( `SELECT user_id, source, qa_hash, primary_memory_id, captured_by, created_at FROM memory_capture_claims - WHERE user_id = ? AND source = ? AND qa_hash = ?` - ).get(userId, source, qaHash) as { + WHERE source = ? AND qa_hash IN (?, ?) ORDER BY created_at, primary_memory_id` + ).all(source, scopedHash, qaHash) as Array<{ user_id: string; source: string; qa_hash: string; primary_memory_id: string; captured_by: MemoryCaptureClaimOrigin; created_at: string; - } | undefined; + }>; + const row = rows.find(candidate => { + if (candidate.qa_hash === scopedHash) return true; + // Keep v1 claims on disk, but reuse one only when its referenced memory + // supplies the same complete origin. Legacy text-only claims stay inert. + return this.matchesOrigin(candidate.primary_memory_id, source, identity); + }); return row ? { - userId: row.user_id, + userId: null, source: row.source, qaHash: row.qa_hash, primaryMemoryId: row.primary_memory_id, @@ -1439,6 +1565,23 @@ export class MemoryCaptureClaimRepository { } } +// Historical equal-text UserMemory rows remain on disk. Corrections/deletions +// suppress older copies; a genuinely new expression after the tombstone may be stored. +const ACTIVE_USER_MEMORY_VIEW_SQL = ` + NOT EXISTS (SELECT 1 FROM user_memories tombstone + WHERE tombstone.normalized_user_text_hash = user_memories.normalized_user_text_hash + AND (tombstone.deleted_at IS NOT NULL OR tombstone.replaced_by_memory_id IS NOT NULL) + AND tombstone.updated_at >= user_memories.created_at) + AND NOT EXISTS (SELECT 1 FROM user_memories earlier + WHERE earlier.normalized_user_text_hash = user_memories.normalized_user_text_hash + AND earlier.status = 'active' AND earlier.deleted_at IS NULL + AND NOT EXISTS (SELECT 1 FROM user_memories tombstone + WHERE tombstone.normalized_user_text_hash = earlier.normalized_user_text_hash + AND (tombstone.deleted_at IS NOT NULL OR tombstone.replaced_by_memory_id IS NOT NULL) + AND tombstone.updated_at >= earlier.created_at) + AND (earlier.created_at < user_memories.created_at OR + (earlier.created_at = user_memories.created_at AND earlier.id < user_memories.id)))`; + export class UserMemoryRepository { constructor(private readonly db: Database.Database) {} @@ -1462,13 +1605,13 @@ export class UserMemoryRepository { confirmExisting(input: { id: string; - userId: string; + userId: string | null; sourceTurnId: string; memoryTypes: UserMemoryType[]; updatedAt: string; }): { memory: UserMemoryRecord; previous: UserMemoryRecord } | undefined { const previous = this.get(input.id); - if (!previous || previous.userId !== input.userId || previous.status !== "active") return undefined; + if (!previous || previous.status !== "active") return undefined; const memory = this.update({ ...previous, memoryTypes: uniq([...previous.memoryTypes, ...input.memoryTypes]), @@ -1481,6 +1624,7 @@ export class UserMemoryRepository { } insert(memory: UserMemoryRecord): UserMemoryRecord { + memory = { ...memory, userId: null }; this.db.prepare( `INSERT INTO user_memories ( id, source_turn_id, user_id, memory_types_json, content, @@ -1490,7 +1634,7 @@ export class UserMemoryRepository { created_at, updated_at, deleted_at ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)` ).run( - memory.id, memory.sourceTurnId, memory.userId, toJson(memory.memoryTypes), memory.content, + memory.id, memory.sourceTurnId, null, toJson(memory.memoryTypes), memory.content, memory.normalizedUserTextHash, toJson(memory.sourceTurnRefs), memory.status, memory.replacesMemoryId ?? null, memory.replacedByMemoryId ?? null, memory.archivedAt ?? null, memory.archiveReason ?? null, @@ -1503,16 +1647,17 @@ export class UserMemoryRepository { } update(memory: UserMemoryRecord): UserMemoryRecord { + memory = { ...memory, userId: null }; this.db.prepare( `UPDATE user_memories SET - source_turn_id = ?, user_id = ?, memory_types_json = ?, content = ?, + source_turn_id = ?, memory_types_json = ?, content = ?, normalized_user_text_hash = ?, source_turn_refs_json = ?, status = ?, replaces_memory_id = ?, replaced_by_memory_id = ?, archived_at = ?, archive_reason = ?, embedding_json = ?, embedding_model = ?, embedding_provider = ?, updated_at = ?, deleted_at = ? WHERE id = ?` ).run( - memory.sourceTurnId, memory.userId, toJson(memory.memoryTypes), memory.content, + memory.sourceTurnId, toJson(memory.memoryTypes), memory.content, memory.normalizedUserTextHash, toJson(memory.sourceTurnRefs), memory.status, memory.replacesMemoryId ?? null, memory.replacedByMemoryId ?? null, memory.archivedAt ?? null, memory.archiveReason ?? null, @@ -1547,16 +1692,16 @@ export class UserMemoryRepository { return ids.map((id) => byId.get(id)).filter((item): item is UserMemoryRecord => Boolean(item)); } - listActive(userId: string, limit = 2000): UserMemoryRecord[] { + listActive(userId: string | null, limit = 2000): UserMemoryRecord[] { return (this.db.prepare( `SELECT * FROM user_memories - WHERE user_id = ? AND status = 'active' AND deleted_at IS NULL + WHERE status = 'active' AND deleted_at IS NULL AND ${ACTIVE_USER_MEMORY_VIEW_SQL} ORDER BY updated_at DESC, id DESC LIMIT ?` - ).all(userId, limit) as UserMemorySqlRow[]).map(userMemoryFromSql); + ).all(limit) as UserMemorySqlRow[]).map(userMemoryFromSql); } listForPanel(input: { - userId: string; + userId: string | null; status?: UserMemoryStatus; query?: string; sourceAgent?: string; @@ -1573,7 +1718,7 @@ export class UserMemoryRepository { } countForPanel(input: { - userId: string; + userId: string | null; status?: UserMemoryStatus; query?: string; sourceAgent?: string; @@ -1584,40 +1729,41 @@ export class UserMemoryRepository { return row.count; } - embeddingMaintenanceEntries(userId: string): EmbeddingMaintenanceEntry[] { + embeddingMaintenanceEntries(_userId: string | null): EmbeddingMaintenanceEntry[] { return this.db.prepare(` SELECT id, 'user_memory' AS kind, COALESCE(json_array_length(embedding_json), 0) AS dimension, embedding_model AS embeddingModel, embedding_provider AS embeddingProvider - FROM user_memories WHERE user_id = ? AND status = 'active' AND deleted_at IS NULL + FROM user_memories + WHERE status = 'active' AND deleted_at IS NULL AND ${ACTIVE_USER_MEMORY_VIEW_SQL} ORDER BY id - `).all(userId) as EmbeddingMaintenanceEntry[]; + `).all() as EmbeddingMaintenanceEntry[]; } - embeddingDimensionCounts(userId: string): { + embeddingDimensionCounts(_userId: string | null): { totalSlots: number; dimensions: Array<{ dimension: number; count: number }>; } { - const where = "user_id = ? AND status = 'active' AND deleted_at IS NULL"; + const where = `status = 'active' AND deleted_at IS NULL AND ${ACTIVE_USER_MEMORY_VIEW_SQL}`; const totalSlots = Number(this.db.prepare( `SELECT COUNT(*) FROM user_memories WHERE ${where}` - ).pluck().get(userId) ?? 0); + ).pluck().get() ?? 0); const dimensions = this.db.prepare( `SELECT json_array_length(embedding_json) AS dimension, COUNT(*) AS count FROM user_memories WHERE ${where} AND embedding_json IS NOT NULL GROUP BY json_array_length(embedding_json) ORDER BY count DESC, dimension DESC` - ).all(userId) as Array<{ dimension: number; count: number }>; + ).all() as Array<{ dimension: number; count: number }>; return { totalSlots, dimensions }; } - getActiveByNormalizedText(userId: string, hash: string): UserMemoryRecord | undefined { + getActiveByNormalizedText(userId: string | null, hash: string): UserMemoryRecord | undefined { const row = this.db.prepare( `SELECT * FROM user_memories - WHERE user_id = ? AND normalized_user_text_hash = ? - AND status = 'active' AND deleted_at IS NULL + WHERE normalized_user_text_hash = ? + AND status = 'active' AND deleted_at IS NULL AND ${ACTIVE_USER_MEMORY_VIEW_SQL} ORDER BY updated_at DESC, id DESC LIMIT 1` - ).get(userId, hash) as UserMemorySqlRow | undefined; + ).get(hash) as UserMemorySqlRow | undefined; return row ? userMemoryFromSql(row) : undefined; } @@ -1668,39 +1814,38 @@ export class UserMemoryRepository { }) : undefined; } - searchFtsIds(userId: string, ftsMatch: string | undefined | null, limit: number): MemorySearchIdHit[] { + searchFtsIds(userId: string | null, ftsMatch: string | undefined | null, limit: number): MemorySearchIdHit[] { if (!ftsMatch || limit <= 0) return []; try { const rows = this.db.prepare( `SELECT user_memories.id AS id FROM user_memories_fts JOIN user_memories ON user_memories.id = user_memories_fts.id - WHERE user_memories.user_id = ? - AND user_memories.status = 'active' - AND user_memories.deleted_at IS NULL + WHERE user_memories.status = 'active' + AND user_memories.deleted_at IS NULL AND ${ACTIVE_USER_MEMORY_VIEW_SQL} AND user_memories_fts MATCH ? ORDER BY rank LIMIT ?` - ).all(userId, ftsMatch, limit) as Array<{ id: string }>; + ).all(ftsMatch, limit) as Array<{ id: string }>; return rows.map((row, index) => ({ id: row.id, score: 1 / (index + 1), channel: "fts" })); } catch { return []; } } - searchPatternIds(userId: string, terms: readonly string[], limit: number): MemorySearchIdHit[] { + searchPatternIds(userId: string | null, terms: readonly string[], limit: number): MemorySearchIdHit[] { const normalized = terms.map((term) => term.trim().toLowerCase()).filter(Boolean).slice(0, 16); if (normalized.length === 0 || limit <= 0) return []; const clauses = normalized.map(() => `lower(content) LIKE ? ESCAPE '\\'`); const rows = this.db.prepare( `SELECT id FROM user_memories - WHERE user_id = ? AND status = 'active' AND deleted_at IS NULL + WHERE status = 'active' AND deleted_at IS NULL AND ${ACTIVE_USER_MEMORY_VIEW_SQL} AND (${clauses.join(" OR ")}) ORDER BY updated_at DESC, id DESC LIMIT ?` - ).all(userId, ...normalized.map((term) => `%${escapeLikePattern(term)}%`), limit) as Array<{ id: string }>; + ).all(...normalized.map((term) => `%${escapeLikePattern(term)}%`), limit) as Array<{ id: string }>; return rows.map((row, index) => ({ id: row.id, score: 1 / (index + 1), channel: "pattern" })); } - searchVectorIds(userId: string, query: readonly number[], limit: number, identity = vectorEmbeddingIdentity(query)): MemorySearchIdHit[] { + searchVectorIds(userId: string | null, query: readonly number[], limit: number, identity = vectorEmbeddingIdentity(query)): MemorySearchIdHit[] { if (query.length === 0 || limit <= 0) return []; return this.listActive(userId) .flatMap((memory) => memory.embedding?.length === query.length && (!identity || sameEmbeddingIdentity(memory, identity)) @@ -1821,7 +1966,7 @@ export class MemoryProcessingRepository { } export interface SourceTurnCaptureScope { - userId: string; + userId: string | null; source: string; profileId: string; namespaceKey: string; @@ -1843,7 +1988,7 @@ export interface SourceTurnCaptureRecord extends SourceTurnCaptureScope { function sourceTurnCaptureFromSql(row: Record): SourceTurnCaptureRecord { return { - userId: String(row.user_id), source: String(row.source), profileId: String(row.profile_id), + userId: null, source: String(row.source), profileId: String(row.profile_id), namespaceKey: String(row.namespace_key), conversationId: String(row.conversation_id), turnId: String(row.turn_id), contentHash: String(row.content_hash), sessionId: typeof row.session_id === "string" ? row.session_id : undefined, episodeId: typeof row.episode_id === "string" ? row.episode_id : undefined, @@ -1934,8 +2079,8 @@ export class RuntimeRepository { getSourceTurnCapture(scope: SourceTurnCaptureScope, turnId: string): SourceTurnCaptureRecord | undefined { const row = this.db.prepare(`SELECT * FROM source_turn_captures WHERE source = @source AND profile_id = @profileId - AND namespace_key = @namespaceKey AND conversation_id = @conversationId AND turn_id = @turnId - ORDER BY created_at ASC, user_id ASC LIMIT 1` + AND memory_namespace_key(namespace_key) = memory_namespace_key(@namespaceKey) AND conversation_id = @conversationId AND turn_id = @turnId + ORDER BY created_at ASC, raw_turn_id ASC LIMIT 1` ).get({ ...scope, turnId }) as Record | undefined; return row ? sourceTurnCaptureFromSql(row) : undefined; } @@ -1943,7 +2088,7 @@ export class RuntimeRepository { latestSourceTurnCapture(scope: SourceTurnCaptureScope): SourceTurnCaptureRecord | undefined { const row = this.db.prepare(`SELECT * FROM source_turn_captures WHERE source = @source AND profile_id = @profileId - AND namespace_key = @namespaceKey AND conversation_id = @conversationId AND session_id IS NOT NULL + AND memory_namespace_key(namespace_key) = memory_namespace_key(@namespaceKey) AND conversation_id = @conversationId AND session_id IS NOT NULL ORDER BY completed_at DESC, source_sequence DESC LIMIT 1` ).get(scope) as Record | undefined; return row ? sourceTurnCaptureFromSql(row) : undefined; @@ -1955,7 +2100,7 @@ export class RuntimeRepository { } { const prefix = `SELECT * FROM source_turn_captures WHERE source = @source AND profile_id = @profileId - AND namespace_key = @namespaceKey AND conversation_id = @conversationId AND session_id IS NOT NULL`; + AND memory_namespace_key(namespace_key) = memory_namespace_key(@namespaceKey) AND conversation_id = @conversationId AND session_id IS NOT NULL`; const before = this.db.prepare(`${prefix} AND started_at < @startedAt ORDER BY started_at DESC LIMIT 1`) .get({ ...scope, startedAt }) as Record | undefined; const after = this.db.prepare(`${prefix} AND started_at > @startedAt ORDER BY started_at ASC LIMIT 1`) @@ -1970,7 +2115,7 @@ export class RuntimeRepository { session_id, episode_id, raw_turn_id, response_json, started_at, completed_at, source_sequence, created_at ) VALUES (@userId, @source, @profileId, @namespaceKey, @conversationId, @turnId, @contentHash, @sessionId, @episodeId, @rawTurnId, @responseJson, @startedAt, @completedAt, @sequence, @createdAt)` - ).run({ ...capture, sessionId: capture.sessionId ?? null, episodeId: capture.episodeId ?? null, + ).run({ ...capture, userId: null, sessionId: capture.sessionId ?? null, episodeId: capture.episodeId ?? null, rawTurnId: capture.rawTurnId ?? null, responseJson: toJson(capture.response), sequence: capture.sequence ?? null }); } @@ -1991,11 +2136,11 @@ export class RuntimeRepository { .run(toJson(rawTurnIds), toJson(l1MemoryIds), episodeId); } - sourceConversationSessions(input: { userId?: string; source: string; profileId: string; conversationId: string }): SessionRecord[] { + sourceConversationSessions(input: { userId?: string | null; source: string; profileId: string; conversationId: string }): SessionRecord[] { return (this.db.prepare(`SELECT * FROM sessions WHERE source = @source AND profile_id = @profileId AND (host_session_key = @conversationId OR conversation_id = @conversationId OR host_session_key = @source || '-memory-' || @conversationId) - ORDER BY opened_at DESC`).all(input) as SqlSessionRow[]).map(sessionFromSql); + ORDER BY opened_at DESC`).all(input) as SqlSessionRow[]).map((row) => sessionFromSql(row, this.db)); } bindSessionSourceConversation(id: string, conversationId: string): boolean { @@ -2009,14 +2154,31 @@ export class RuntimeRepository { scope: Pick, conversationId: string ): boolean { + const origin = readNativeOrigin(this.getRawTurn(scope.id)?.messagePayload?.native_origin_v1); + if (origin?.conversationId && origin.conversationId !== conversationId) return false; const result = this.db.prepare(`UPDATE raw_turns SET conversation_id = @conversationId - WHERE id = @id AND session_id = @sessionId AND user_id = @userId AND turn_id = @turnId + WHERE id = @id AND session_id = @sessionId AND turn_id = @turnId AND (conversation_id IS NULL OR conversation_id = '' OR conversation_id = @conversationId)`) .run({ ...scope, conversationId }); + if (result.changes === 1 && origin && !origin.conversationId) { + this.db.prepare("UPDATE raw_turns SET message_payload_json=json_set(message_payload_json,'$.native_origin_v1.conversationId',?) WHERE id=?") + .run(conversationId, scope.id); + } return result.changes === 1; } + bindPendingTurnSourceConversation(sessionId: string, turnId: string, conversationId: string): boolean { + const key = pendingOriginKey(sessionId, turnId); + const pending = this.getKv(key); + if (!pending) return true; + const origin = readNativeOrigin(pending.value); + if (!origin || (origin.conversationId && origin.conversationId !== conversationId)) return false; + if (!origin.conversationId) this.setKv(key, { ...origin, conversationId }, nowIso()); + return true; + } + createSession(session: SessionRecord): SessionRecord { + session = { ...session, userId: null }; this.db .prepare( `INSERT INTO sessions ( @@ -2031,6 +2193,7 @@ export class RuntimeRepository { ) .run({ ...session, + userId: null, profileLabel: session.profileLabel ?? null, projectId: session.projectId ?? null, workspaceId: session.workspaceId ?? null, @@ -2048,29 +2211,25 @@ export class RuntimeRepository { const row = this.db .prepare(`SELECT * FROM sessions WHERE id = ?`) .get(id) as SqlSessionRow | undefined; - return row ? sessionFromSql(row) : undefined; + return row ? sessionFromSql(row, this.db) : undefined; } findOpenSessionByHostKey(input: { - userId: string; + tenantId: string | null; source: string; profileId: string; hostSessionKey: string; + projectId?: string | null; }): SessionRecord | undefined { - const row = this.db - .prepare( - `SELECT * - FROM sessions - WHERE user_id = ? - AND source = ? - AND profile_id = ? - AND host_session_key = ? - AND status = 'open' - ORDER BY updated_at DESC - LIMIT 1` - ) - .get(input.userId, input.source, input.profileId, input.hostSessionKey) as SqlSessionRow | undefined; - return row ? sessionFromSql(row) : undefined; + const rows = this.db.prepare(`SELECT * FROM sessions + WHERE source = ? AND profile_id = ? AND host_session_key = ? AND status = 'open' + ORDER BY opened_at ASC, id ASC`) + .all(input.source,input.profileId,input.hostSessionKey) as SqlSessionRow[]; + const candidates = rows.map(row => sessionFromSql(row,this.db)).filter(session => + sessionMatchesTenant(session, input.tenantId) + && (input.projectId === undefined || (session.projectId ?? null) === canonicalProjectId(this.db,input.projectId))); + if (new Set(candidates.map(session=>session.projectId ?? "")).size > 1) return undefined; + return candidates[0]; } updateSessionScope( @@ -2161,7 +2320,7 @@ export class RuntimeRepository { ORDER BY updated_at DESC` ) .all(sessionId) as SqlEpisodeRow[]; - return rows.map(episodeFromSql); + return rows.map((row) => episodeFromSql(row, this.db)); } listIdleEpisodes( @@ -2203,7 +2362,7 @@ export class RuntimeRepository { ORDER BY episodes.opened_at ASC, episodes.id ASC` ) .all(...params) as SqlEpisodeRow[]; - return rows.map(episodeFromSql); + return rows.map((row) => episodeFromSql(row, this.db)); } closeOpenEpisodesForSession(sessionId: string, at = nowIso()): EpisodeRecord[] { @@ -2221,6 +2380,7 @@ export class RuntimeRepository { .run(at, at, sessionId); return rows.map((episode) => ({ ...episode, + userId: null, status: "closed" as const, closedAt: episode.closedAt ?? at, updatedAt: at @@ -2248,6 +2408,7 @@ export class RuntimeRepository { if (result.changes === 0) return undefined; return { ...episode, + userId: null, status: "closed", closedAt: episode.closedAt ?? at, meta, @@ -2277,6 +2438,7 @@ export class RuntimeRepository { .run(toJson(meta), at, episodeId); return { ...episode, + userId: null, status: "open", closedAt: null, rTask: undefined, @@ -2286,7 +2448,7 @@ export class RuntimeRepository { }; } - countEpisodesByStatus(userId?: string): Record<"open" | "processing" | "closed", number> { + countEpisodesByStatus(userId?: string | null): Record<"open" | "processing" | "closed", number> { void userId; const clauses = ["1=1"]; const params: SqlValue[] = []; @@ -2307,7 +2469,7 @@ export class RuntimeRepository { return counts; } - countEpisodes(userId?: string, query?: string, sourceAgent?: string): number { + countEpisodes(userId?: string | null, query?: string, sourceAgent?: string): number { const built = buildEpisodeWhere(userId, query, sourceAgent); const row = this.db .prepare( @@ -2319,7 +2481,7 @@ export class RuntimeRepository { return Number(row?.count ?? 0); } - listEpisodes(userId?: string, limit = 50, offset = 0, query?: string, sourceAgent?: string): EpisodeRecord[] { + listEpisodes(userId?: string | null, limit = 50, offset = 0, query?: string, sourceAgent?: string): EpisodeRecord[] { const built = buildEpisodeWhere(userId, query, sourceAgent); const rows = this.db .prepare( @@ -2330,7 +2492,7 @@ export class RuntimeRepository { LIMIT ? OFFSET ?` ) .all(...built.params, limit, offset) as SqlEpisodeRow[]; - return rows.map(episodeFromSql); + return rows.map((row) => episodeFromSql(row, this.db)); } deleteEpisode(id: string): boolean { @@ -2339,6 +2501,7 @@ export class RuntimeRepository { } createEpisode(episode: EpisodeRecord): EpisodeRecord { + episode = { ...episode, userId: null }; this.db .prepare( `INSERT INTO episodes ( @@ -2359,6 +2522,7 @@ export class RuntimeRepository { ) .run({ ...episode, + userId: null, projectId: episode.projectId ?? null, conversationId: episode.conversationId ?? null, title: episode.title ?? null, @@ -2386,8 +2550,8 @@ export class RuntimeRepository { if (!id) return undefined; const row = this.db .prepare(`SELECT * FROM episodes WHERE id = ?`) - .get(id ?? null) as SqlEpisodeRow | undefined; - return row ? episodeFromSql(row) : undefined; + .get(id) as SqlEpisodeRow | undefined; + return row ? episodeFromSql(row, this.db) : undefined; } updateEpisodeMeta(episodeId: string, patch: Record, at = nowIso()): EpisodeRecord | undefined { @@ -2407,6 +2571,7 @@ export class RuntimeRepository { .run(toJson(meta), at, episodeId); return { ...episode, + userId: null, meta, updatedAt: at }; @@ -2432,6 +2597,7 @@ export class RuntimeRepository { .run(input.title, input.summary, toJson(meta), at, episodeId); return { ...episode, + userId: null, title: input.title, summary: input.summary, meta, @@ -2457,7 +2623,7 @@ export class RuntimeRepository { LIMIT 1` ) .get(sessionId) as SqlEpisodeRow | undefined; - return row ? episodeFromSql(row) : undefined; + return row ? episodeFromSql(row, this.db) : undefined; } appendEpisodeTurn(episodeId: string, rawTurnId: string, l1MemoryId: string, at = nowIso()): EpisodeRecord { @@ -2479,6 +2645,7 @@ export class RuntimeRepository { .run(toJson(rawTurnIds), toJson(l1MemoryIds), rawTurnIds.length, at, episodeId); return { ...episode, + userId: null, rawTurnIds, l1MemoryIds, turnCount: rawTurnIds.length, @@ -2503,6 +2670,7 @@ export class RuntimeRepository { .run(toJson(rawTurnIds), rawTurnIds.length, at, episodeId); return { ...episode, + userId: null, rawTurnIds, turnCount: rawTurnIds.length, updatedAt: at @@ -2670,6 +2838,7 @@ export class RuntimeRepository { .run(input.rTask, toJson(input.rewardDetail), toJson(meta), at, episodeId); return { ...episode, + userId: null, rTask: input.rTask, rewardDetail: input.rewardDetail, meta, @@ -2700,6 +2869,7 @@ export class RuntimeRepository { .run(toJson(values), at, episodeId); return { ...episode, + userId: null, [field]: values, updatedAt: at }; @@ -2752,6 +2922,16 @@ export class RuntimeRepository { } insertRawTurn(rawTurn: RawTurnRecord): RawTurnRecord { + rawTurn = { ...rawTurn, userId: null }; + const originKey = pendingOriginKey(rawTurn.sessionId, rawTurn.turnId); + const pendingOrigin = readNativeOrigin(this.getKv(originKey)?.value); + if (pendingOrigin) { + rawTurn = { ...rawTurn, messagePayload: { ...rawTurn.messagePayload, native_origin_v1: pendingOrigin } }; + } else if (!Object.hasOwn(rawTurn.messagePayload ?? {}, "native_origin_v1")) { + const session = this.getSession(rawTurn.sessionId); + if (session) rawTurn = { ...rawTurn, messagePayload: { ...rawTurn.messagePayload, + native_origin_v1: captureNativeOrigin({ ...session, conversationId: rawTurn.conversationId }) } }; + } this.db .prepare( `INSERT INTO raw_turns ( @@ -2768,6 +2948,7 @@ export class RuntimeRepository { ) .run({ ...rawTurn, + userId: null, episodeId: rawTurn.episodeId ?? null, conversationId: rawTurn.conversationId ?? null, userText: rawTurn.userText ?? null, @@ -2781,10 +2962,14 @@ export class RuntimeRepository { redactedAt: rawTurn.redactedAt ?? null, deletedAt: rawTurn.deletedAt ?? null }); + this.db.prepare("DELETE FROM runtime_kv WHERE key=?").run(originKey); return rawTurn; } updateRawTurn(rawTurn: RawTurnRecord): RawTurnRecord { + rawTurn = { ...rawTurn, userId: null }; + const origin = this.getRawTurn(rawTurn.id)?.messagePayload?.native_origin_v1; + if (origin !== undefined) rawTurn = { ...rawTurn, messagePayload: { ...rawTurn.messagePayload, native_origin_v1: origin } }; this.db .prepare( `UPDATE raw_turns @@ -2937,7 +3122,7 @@ export class RuntimeRepository { } hasCompletedSourceTurnInScope(input: { - userId: string; + userId: string | null; source: string; profileId: string; conversationId: string; @@ -2948,17 +3133,32 @@ export class RuntimeRepository { storedProjectId: string | null; workspaceId: string | null; }): boolean { + // Captured provenance must be checked even if its Session has moved away + // from the requested scope. Never let the mutable fallback override it. + const captured = this.db.prepare(`SELECT message_payload_json FROM raw_turns + WHERE turn_id = ? AND json_type(message_payload_json, '$.turn_complete') = 'object' + AND json_type(message_payload_json, '$.native_origin_v1') IS NOT NULL`) + .all(input.turnId) as Array<{ message_payload_json: string }>; + for (const row of captured) { + const origin = readNativeOrigin(parseJson>(row.message_payload_json, {}).native_origin_v1); + const identity = origin ? nativeIdentity(this.db, origin, input.turnId) : undefined; + if (identity && identity.source === normalizeMemoryCaptureSource(input.source) + && identity.profileId === input.profileId && identity.conversationId === input.conversationId + && (identity.namespaceKey === canonicalNamespaceKey(this.db, input.namespaceKey) + || (!origin!.namespaceKey && identity.namespaceKey === stableHash({ tenantId: input.tenantId, + projectId: canonicalProjectId(this.db, input.storedProjectId), + workspaceId: canonicalProjectId(this.db, input.workspaceId) })))) return true; + } const row = this.db.prepare(` SELECT 1 AS found FROM raw_turns - WHERE raw_turns.user_id = @userId - AND raw_turns.turn_id = @turnId + WHERE raw_turns.turn_id = @turnId AND json_type(raw_turns.message_payload_json, '$.turn_complete') = 'object' + AND json_type(raw_turns.message_payload_json, '$.native_origin_v1') IS NULL AND EXISTS ( SELECT 1 FROM sessions WHERE sessions.id = raw_turns.session_id - AND sessions.user_id = @userId AND sessions.source = @source AND sessions.profile_id = @profileId AND ( @@ -2967,15 +3167,15 @@ export class RuntimeRepository { OR sessions.host_session_key = @source || '-memory-' || @conversationId ) AND ( - json_extract(sessions.meta_json, '$.source_namespace_key') = @namespaceKey + memory_namespace_key(json_extract(sessions.meta_json, '$.source_namespace_key')) = memory_namespace_key(@namespaceKey) OR ( json_extract(sessions.meta_json, '$.source_namespace_key') IS NULL AND @tenantId IS NULL AND ( @namespaceKey = @defaultNamespaceKey OR ( - sessions.project_id IS @storedProjectId - AND sessions.workspace_id IS @workspaceId + memory_project_id(sessions.project_id) IS memory_project_id(@storedProjectId) + AND memory_project_id(sessions.workspace_id) IS memory_project_id(@workspaceId) ) ) ) @@ -3058,7 +3258,7 @@ export class RuntimeRepository { ) .run({ id: cluster.id, - userId: cluster.userId, + userId: null, projectId: cluster.projectId ?? null, toolsJson: toJson(cluster.tools), artifactsJson: toJson(cluster.artifacts), @@ -3110,7 +3310,7 @@ export class RuntimeRepository { const row = this.db .prepare(`SELECT * FROM skill_clusters WHERE id = ?`) .get(id) as SqlSkillClusterRow | undefined; - return row ? skillClusterFromSql(row) : undefined; + return row ? skillClusterFromSql(row, this.db) : undefined; } getSkillClusterForEpisode(episodeId: string): SkillClusterRecord | undefined { @@ -3124,11 +3324,11 @@ export class RuntimeRepository { LIMIT 1` ) .get(episodeId) as SqlSkillClusterRow | undefined; - return row ? skillClusterFromSql(row) : undefined; + return row ? skillClusterFromSql(row, this.db) : undefined; } listSkillClustersByScope(input: { - userId: string; + userId: string | null; projectId?: string; limit?: number; }): SkillClusterRecord[] { @@ -3137,22 +3337,21 @@ export class RuntimeRepository { .prepare( `SELECT * FROM skill_clusters - WHERE user_id = ? - AND (project_id = ? OR project_id IS NULL) + WHERE (memory_project_id(project_id) = memory_project_id(?) OR project_id IS NULL) ORDER BY updated_at DESC LIMIT ?` ) - .all(input.userId, input.projectId, input.limit ?? 200) as SqlSkillClusterRow[] + .all(input.projectId, input.limit ?? 200) as SqlSkillClusterRow[] : this.db .prepare( `SELECT * FROM skill_clusters - WHERE user_id = ? + WHERE 1=1 ORDER BY updated_at DESC LIMIT ?` ) - .all(input.userId, input.limit ?? 200) as SqlSkillClusterRow[]; - return rows.map(skillClusterFromSql); + .all(input.limit ?? 200) as SqlSkillClusterRow[]; + return rows.map((row) => skillClusterFromSql(row, this.db)); } upsertSkillClusterMember(member: SkillClusterMemberRecord): SkillClusterMemberRecord { @@ -3222,11 +3421,11 @@ export class RuntimeRepository { const row = this.db .prepare(`SELECT * FROM feedback WHERE id = ?`) .get(id) as SqlFeedbackRow | undefined; - return row ? feedbackFromSql(row) : undefined; + return row ? feedbackFromSql(row, this.db) : undefined; } listFeedback(input: { - userId?: string; + userId?: string | null; sessionId?: string; episodeId?: string; rawTurnId?: string; @@ -3249,7 +3448,7 @@ export class RuntimeRepository { LIMIT ?` ) .all(...params, input.limit ?? 20) as SqlFeedbackRow[]; - return rows.map(feedbackFromSql); + return rows.map(row => feedbackFromSql(row, this.db)); function addOptional(column: string, value: string | undefined): void { if (value === undefined) return; @@ -3276,7 +3475,7 @@ export class RuntimeRepository { event.sessionId ?? null, event.episodeId ?? null, event.turnId ?? null, - event.userId, + null, event.query, event.queryHash ?? stableHash(event.query), toJson(event.layers), @@ -3376,7 +3575,7 @@ export class RuntimeRepository { return seq; } - latestChangeSeq(userId?: string, namespaceId?: string): number { + latestChangeSeq(userId?: string | null, namespaceId?: string): number { void userId; void namespaceId; const clauses = ["1=1"]; @@ -3389,7 +3588,7 @@ export class RuntimeRepository { return Number(row?.seq ?? 0); } - listChanges(userId?: string, limit = 50, cursor?: number, namespaceId?: string): ChangeLogRecord[] { + listChanges(userId?: string | null, limit = 50, cursor?: number, namespaceId?: string): ChangeLogRecord[] { void userId; void namespaceId; const clauses = ["1=1"]; @@ -3414,7 +3613,7 @@ export class RuntimeRepository { kind: row.kind ?? undefined, op: row.op ?? undefined, entityId: row.entity_id ?? undefined, - userId: row.user_id, + userId: null, changeType: row.change_type, version: row.version ?? undefined, before: row.before_json ? parseJson(row.before_json, undefined) : undefined, @@ -3454,6 +3653,7 @@ export class RuntimeRepository { } enqueueJobInTransaction(job: EvolutionJobRecord): EvolutionJobRecord { + job = { ...job, userId: null }; const existing = job.dedupeKey ? this.getActiveJobByDedupeKey(job.dedupeKey) : undefined; if (existing) { const updatedAt = job.updatedAt ?? nowIso(); @@ -3584,7 +3784,7 @@ export class RuntimeRepository { */ armWorkMemoryIdleFlush(input: { sessionId: string; - userId: string; + userId: string | null; lastActivityAt: string; runAfter: string; at?: string; @@ -3611,14 +3811,14 @@ export class RuntimeRepository { target_memory_id, scope_key, scope_seq, payload_json, attempts, max_attempts, leased_until, last_error, created_at, updated_at ) VALUES (?, 'work_memory_idle_flush', 'queued', ?, ?, ?, NULL, NULL, NULL, NULL, ?, 0, 3, NULL, NULL, ?, ?)` - ).run(newId("job"), dedupeKey, input.userId, input.sessionId, payload, at, at); + ).run(newId("job"), dedupeKey, null, input.sessionId, payload, at, at); } const job = this.getJobByDedupeKey(dedupeKey); if (!job) throw new Error(`failed to arm work memory idle flush: ${input.sessionId}`); return job; } - listJobs(status?: JobStatus, limit = 50, userId?: string): EvolutionJobRecord[] { + listJobs(status?: JobStatus, limit = 50, userId?: string | null): EvolutionJobRecord[] { void userId; const clauses: string[] = []; const params: SqlValue[] = []; @@ -3905,6 +4105,34 @@ export class RuntimeRepository { AND leased_l3_job.status = 'leased' AND leased_l3_job.id <> evolution_jobs.id ) + AND NOT EXISTS ( + SELECT 1 FROM evolution_jobs AS shared_job + LEFT JOIN sessions AS shared_session ON shared_session.id = shared_job.session_id + WHERE shared_job.job_type = evolution_jobs.job_type + AND shared_job.id <> evolution_jobs.id + AND shared_job.scope_key <> evolution_jobs.scope_key + AND memory_project_id(shared_session.project_id) IS + (SELECT memory_project_id(project_id) FROM sessions WHERE id = evolution_jobs.session_id) + AND json_extract(shared_job.payload_json, '$.targetField') IS json_extract(evolution_jobs.payload_json, '$.targetField') + AND (shared_job.status = 'leased' OR ( + shared_job.status = 'queued' + AND shared_job.scope_seq IS NOT NULL + AND shared_job.attempts < shared_job.max_attempts + AND (json_extract(shared_job.payload_json, '$.runAfter') IS NULL + OR CAST(json_extract(shared_job.payload_json, '$.runAfter') AS TEXT) <= ?) + -- Only runnable scope heads compete globally. Comparing a + -- later chunk by UUID against another scope creates cycles. + AND NOT EXISTS ( + SELECT 1 FROM evolution_jobs AS shared_predecessor + WHERE shared_predecessor.job_type = shared_job.job_type + AND shared_predecessor.scope_key = shared_job.scope_key + AND shared_predecessor.scope_seq < shared_job.scope_seq + AND shared_predecessor.status IN ('queued', 'leased', 'failed') + ) AND + (shared_job.created_at < evolution_jobs.created_at OR + (shared_job.created_at = evolution_jobs.created_at AND shared_job.id < evolution_jobs.id)) + )) + ) AND NOT EXISTS ( SELECT 1 FROM evolution_jobs AS earlier_l3_job @@ -3966,7 +4194,7 @@ export class RuntimeRepository { ORDER BY ${evolutionJobOrderSql()} LIMIT ?` ) - .all(at, at, ...(allowedJobTypes ?? []), ...(excludedJobTypes ?? []), ...(targetMemoryIds ?? []), limit) as Array; const queuePriority = candidates[0]?.queue_priority; @@ -4241,15 +4469,14 @@ export class RuntimeRepository { } if (status === "dead_letter" && row.job_type === "project_environment_profile") { const payload = parseJson>(row.payload_json, {}); - const userId = typeof payload.userId === "string" ? payload.userId : undefined; const projectId = typeof payload.projectId === "string" ? payload.projectId : undefined; const scanId = typeof payload.scanId === "string" ? payload.scanId : undefined; - if (userId && projectId && scanId) { + if (projectId && scanId) { this.db.prepare( `UPDATE l3_world_model_project_environment_state SET status = 'failed', last_error = ?, updated_at = ? - WHERE user_id = ? AND project_id = ? AND current_scan_id = ?` - ).run(error, at, userId, projectId, scanId); + WHERE memory_project_id(project_id) = memory_project_id(?) AND current_scan_id = ?` + ).run(error, at, projectId, scanId); } } return this.getJob(id); @@ -4349,7 +4576,7 @@ export class RuntimeRepository { listEmbeddingRetries( status?: EmbeddingRetryStatus, limit = 50, - userId?: string, + userId?: string | null, offset = 0 ): EmbeddingRetryRecord[] { void userId; @@ -4374,7 +4601,7 @@ export class RuntimeRepository { countEmbeddingRetriesByStatus( status: EmbeddingRetryStatus, - userId?: string + userId?: string | null ): number { void userId; const row = this.db @@ -4403,7 +4630,7 @@ export class RuntimeRepository { } embeddingMaintenanceQueueStats( - userId: string, + userId: string | null, now = Date.now(), isCurrentWork: (task: EmbeddingMaintenanceTask) => boolean = () => true ): { @@ -4423,8 +4650,8 @@ export class RuntimeRepository { m.memory_layer FROM evolution_jobs j LEFT JOIN memories m ON m.id = j.target_memory_id WHERE j.status != 'succeeded' AND ( - j.job_type = 'embedding' OR (j.job_type = 'user_memory_embedding' AND j.user_id = ?))` - ).all(userId) as Array & { memory_layer: string | null }>; const tasks: EmbeddingMaintenanceTask[] = [ ...retries.map(row => ({ @@ -4699,7 +4926,7 @@ export class RuntimeRepository { ) .run( trial.id, - trial.userId, + null, trial.projectId ?? null, trial.skillMemoryId, trial.sessionId ?? null, @@ -4725,7 +4952,7 @@ export class RuntimeRepository { } listSkillTrials(input: { - userId?: string; + userId?: string | null; skillMemoryId?: string; sessionId?: string; episodeId?: string; @@ -4788,7 +5015,7 @@ export class RuntimeRepository { sessionId?: string; episodeId?: string; rawTurnId?: string; - userId: string; + userId: string | null; projectId?: string; contextHash?: string; issue: string; @@ -4821,7 +5048,7 @@ export class RuntimeRepository { input.sessionId ?? null, input.episodeId ?? null, input.rawTurnId ?? null, - input.userId, + null, input.projectId ?? null, input.contextHash ?? null, input.issue, @@ -4842,7 +5069,7 @@ export class RuntimeRepository { sessionId: input.sessionId, episodeId: input.episodeId, rawTurnId: input.rawTurnId, - userId: input.userId, + userId: null, projectId: input.projectId, contextHash: input.contextHash, issue: input.issue, @@ -4861,17 +5088,13 @@ export class RuntimeRepository { } listDecisionRepairs(input: { - userId?: string; + userId?: string | null; contextHash?: string; since?: string; limit?: number; } = {}): DecisionRepairRecord[] { const clauses = ["1=1"]; const params: SqlValue[] = []; - if (input.userId) { - clauses.push("user_id = ?"); - params.push(input.userId); - } if (input.contextHash) { clauses.push("context_hash = ?"); params.push(input.contextHash); @@ -4933,7 +5156,7 @@ export class RuntimeRepository { upsertCandidatePoolTrace(input: { id: string; - userId: string; + userId: string | null; sessionId?: string; sourceMemoryId: string; candidateKey: string; @@ -4960,7 +5183,7 @@ export class RuntimeRepository { ) .run( input.id, - input.userId, + null, input.sessionId ?? null, input.sourceMemoryId, input.candidateKey, @@ -4974,7 +5197,7 @@ export class RuntimeRepository { } listPendingCandidatePool(input: { - userId?: string; + userId?: string | null; candidateKey?: string; now?: string; limit?: number; @@ -4994,7 +5217,17 @@ export class RuntimeRepository { LIMIT ?` ) .all(...params, input.limit ?? 1000) as SqlCandidatePoolRow[]; - return rows.map(candidatePoolFromSql); + const memories = new MemoryRepository(this.db, new SqliteVecStore(this.db)); + const seen = new Set(); + return rows.filter(row => { + const memory = memories.get(row.source_memory_id); + if (!memory) return false; + const group = memories.nativeDuplicates(memory); + const key = `${row.candidate_key}:${group.ids[0] ?? memory.id}`; + if (group.deleted || seen.has(key)) return false; + seen.add(key); + return true; + }).map(candidatePoolFromSql); } pruneCandidatePool(now = nowIso()): number { @@ -5005,13 +5238,17 @@ export class RuntimeRepository { } markCandidatePoolPromoted(input: { - userId?: string; + userId?: string | null; candidateKey: string; sourceMemoryIds: string[]; policyId: string; at: string; }): void { - const sourceMemoryIds = uniq(input.sourceMemoryIds); + const memories = new MemoryRepository(this.db, new SqliteVecStore(this.db)); + const sourceMemoryIds = uniq(input.sourceMemoryIds.flatMap(id => { + const memory = memories.get(id); + return memory ? memories.nativeDuplicates(memory).ids : [id]; + })); if (sourceMemoryIds.length === 0) return; const placeholders = sourceMemoryIds.map(() => "?").join(", "); this.db @@ -5028,7 +5265,7 @@ export class RuntimeRepository { insertTracePolicyLink(input: { id?: string; - userId: string; + userId: string | null; l1MemoryId: string; l2MemoryId: string; relation?: string; @@ -5044,7 +5281,7 @@ export class RuntimeRepository { ) .run( id, - input.userId, + null, input.l1MemoryId, input.l2MemoryId, input.relation ?? "supports", @@ -5055,7 +5292,7 @@ export class RuntimeRepository { } listTracePolicyLinks(input: { - userId?: string; + userId?: string | null; l1MemoryId?: string; l2MemoryId?: string; limit?: number; @@ -5089,7 +5326,7 @@ export class RuntimeRepository { }): AuditLogRecord { const row: AuditLogRecord = { id: input.id ?? newId("audit"), - userId: input.userId, + userId: null, sessionId: input.sessionId, actor: input.actor, action: input.action, @@ -5125,7 +5362,7 @@ export class RuntimeRepository { } listAudit(input: { - userId?: string; + userId?: string | null; targetKind?: string; targetId?: string; limit?: number; @@ -5215,10 +5452,24 @@ export class RuntimeRepository { exportBundleTables(includeRawText = false): Record>> { const tables: Record>> = {}; + const explicitArchiveEvidence = new Set(); + if (hasSqliteTable(this.db, "audit_logs")) { + for (const row of this.db.prepare("SELECT target_id FROM audit_logs WHERE action = 'archive'").all() as Array<{target_id: string | null}>) { + if (row.target_id) explicitArchiveEvidence.add(row.target_id); + } + } + if (hasSqliteTable(this.db, "memory_change_log")) { + for (const row of this.db.prepare("SELECT memory_id FROM memory_change_log WHERE change_type = 'archive'").all() as Array<{memory_id: string | null}>) { + if (row.memory_id) explicitArchiveEvidence.add(row.memory_id); + } + } for (const table of BUNDLE_TABLES) { + if (isMissingOptionalTableFromPreviousSchema(this.db, table)) continue; const rows = this.db.prepare(`SELECT * FROM ${table}`).all() as Array>; tables[table] = rows.map((row) => { - const projected = table === "memory_change_log" ? compactHistoryChangeRow(row) : row; + const projected = table === "memory_change_log" + ? compactHistoryChangeRow(row) + : table === "memories" ? bundleMemoryArchiveProjection(row, explicitArchiveEvidence) : row; return serializeBundleRow(table, includeRawText ? projected : redactBundleRow(table, projected)); }); } @@ -5231,6 +5482,7 @@ export class RuntimeRepository { conflictStrategy?: "skip" | "replace" | "error"; } = {} ): { + skippedMemoryIds: Set; inserted: Record; skipped: Record; replaced: Record; @@ -5245,6 +5497,7 @@ export class RuntimeRepository { } { const conflictStrategy = options.conflictStrategy ?? "skip"; const result = { + skippedMemoryIds: new Set(), inserted: {} as Record, skipped: {} as Record, replaced: {} as Record, @@ -5257,7 +5510,34 @@ export class RuntimeRepository { action: "skipped" | "replaced" | "error"; }> }; - this.db.transaction(() => { + try { this.db.transaction(() => { + const normalizedTables = Object.fromEntries(BUNDLE_TABLES.map(table => [table, + (Array.isArray(tables[table]) ? tables[table] as Array> : []).map(row => + applyBundleDefaults(table, deserializeBundleRow(row))) + ])); + for (const table of BUNDLE_TABLES) { + const seen = new Map(); + for (const row of normalizedTables[table]!) { + const identity = bundleIdentity(table, row); + if (!identity) continue; + const signature = stableHash(row); + const previous = seen.get(identity.sourceId); + if (previous && previous !== signature) throw new Error(`contradictory duplicate identity in bundle: ${table}.${identity.sourceId}`); + seen.set(identity.sourceId, signature); + } + } + const plan = planBundleImport(this.db, normalizedTables, conflictStrategy); + const world = new L3WorldModelRepository(this.db, new MemoryRepository(this.db, new SqliteVecStore(this.db))); + const projection = (project: string | null) => { + const memory = world.getMemory(null, project); + return stableHash(memory ? { id: memory.id, version: memory.version, status: memory.status, + fields: memory.properties.internal_info.world_model, + sources: memory.info.source_memory_ids ?? [], scan: memory.info.project_environment_applied_scan_id ?? null } : null); + }; + const before = new Map([...plan.originalProjects].map(project => [project, projection(project)])); + result.skippedMemoryIds = plan.skippedMemoryIds; + const importedMemoryIds = new Set(); + const importedLegacyChanges: LegacyL3Change[] = []; const hadRuntimeData = Boolean(this.db.prepare(`SELECT EXISTS( SELECT 1 FROM memories UNION ALL SELECT 1 FROM sessions UNION ALL SELECT 1 FROM raw_turns UNION ALL SELECT 1 FROM source_turn_captures @@ -5267,6 +5547,26 @@ export class RuntimeRepository { for (const row of rows) { const defaults = applyBundleDefaults(table, deserializeBundleRow(row)); const normalized = table === "memory_change_log" ? compactHistoryChangeRow(defaults) : defaults; + if (plan.shouldSkip(table, normalized)) { + const identity = bundleIdentity(table, normalized); + if (identity) { + recordMigrationMap(result.migrationMap, table, identity.sourceId, identity.sourceId); + result.conflicts.push({ table, primaryKey: identity.primaryKey, sourceId: identity.sourceId, + targetId: identity.sourceId, action: "skipped" }); + } + result.skipped[table] = (result.skipped[table] ?? 0) + 1; + continue; + } + // Log sequence numbers belong to the source database. A collision must + // not discard the deletion history of successfully imported memories. + if (table === "memory_change_log" + && (normalized.change_type === "l3_world_model_create" || normalized.change_type === "l3_world_model_update") + && typeof normalized.memory_id === "string" && typeof normalized.created_at === "string" + && typeof normalized.seq === "number") { + importedLegacyChanges.push({memory_id: normalized.memory_id, created_at: normalized.created_at, + seq: normalized.seq, before_json: typeof normalized.before_json === "string" ? normalized.before_json : null, + after_json: typeof normalized.after_json === "string" ? normalized.after_json : null}); + } const identity = bundleIdentity(table, normalized); if (identity) { recordMigrationMap(result.migrationMap, table, identity.sourceId, identity.sourceId); @@ -5310,6 +5610,7 @@ export class RuntimeRepository { this.db .prepare(`${verb} INTO ${table} (${columns.join(", ")}) VALUES (${placeholders})`) .run(...values); + if (table === "memories" && typeof normalized.id === "string") importedMemoryIds.add(normalized.id); if (existed) { result.conflicts.push({ table, @@ -5324,7 +5625,15 @@ export class RuntimeRepository { } } } - })(); + preserveLegacyL3FieldEdits(this.db, importedLegacyChanges.filter(change => importedMemoryIds.has(change.memory_id))); + refreshProjectAliases(this.db); + for (const [project, expected] of before) { + if (projection(project) !== expected) throw new Error(`skip import changes protected L3 scope: ${project ?? ""}`); + } + })(); } catch (error) { + reloadDeviceScopeAliases(this.db); + throw error; + } this.scheduleLogTablesPrune(); this.scheduleHistoryLogCompaction(true); return result; @@ -5354,12 +5663,12 @@ export class RuntimeRepository { return existing ? "replaced" : "inserted"; } - private rowExists(table: BundleTableName, columns: string[], values: Array): boolean { + private rowExists(table: BundleTableName, columns: string[], values: Array): boolean { const tableColumns = this.tableColumns(table); if (columns.length === 0 || columns.some((column) => !tableColumns.includes(column))) { return false; } - const where = columns.map((column) => `${column} = ?`).join(" AND "); + const where = columns.map((column) => `${column} IS ?`).join(" AND "); const row = this.db.prepare(`SELECT 1 AS ok FROM ${table} WHERE ${where} LIMIT 1`).get(...values) as | { ok: number } | undefined; @@ -5400,18 +5709,22 @@ export class RuntimeRepository { } private pruneLogTable(table: LogTableName): number { - const result = this.db - .prepare( - `DELETE FROM ${table} + return this.db.transaction(() => { + if (table === "memory_change_log") preserveLegacyL3FieldEdits(this.db); + if (table === "memory_change_log" || table === "audit_logs") preserveExplicitL3ArchiveMarkers(this.db); + const result = this.db + .prepare( + `DELETE FROM ${table} WHERE rowid IN ( SELECT rowid FROM ${table} ORDER BY ${LOG_TABLE_RETENTION_ORDER[table]} LIMIT -1 OFFSET ? )` - ) - .run(LOG_TABLE_RETENTION_LIMIT); - return result.changes; + ) + .run(LOG_TABLE_RETENTION_LIMIT); + return result.changes; + })(); } insertArtifact(input: { @@ -5419,7 +5732,7 @@ export class RuntimeRepository { sessionId?: string; episodeId?: string; rawTurnId?: string; - userId: string; + userId: string | null; kind: string; uri?: string; payload: unknown; @@ -5438,7 +5751,7 @@ export class RuntimeRepository { input.sessionId ?? null, input.episodeId ?? null, input.rawTurnId ?? null, - input.userId, + null, input.kind, input.uri ?? null, toJson(input.payload), @@ -5461,31 +5774,31 @@ export class L3WorldModelRepository { private readonly memories: MemoryRepository ) {} - getScope(userId: string, projectId?: string | null): L3WorldModelScopeRecord | undefined { + getScope(_userId: string | null, projectId?: string | null): L3WorldModelScopeRecord | undefined { const row = this.db.prepare( `SELECT * FROM l3_world_model_scopes - WHERE user_id = ? AND project_id IS ?` - ).get(userId, projectId ?? null) as SqlL3WorldModelScopeRow | undefined; - return row ? l3WorldModelScopeFromSql(row) : undefined; + WHERE user_id IS NULL AND project_id IS ?` + ).get(canonicalProjectId(this.db, projectId)) as SqlL3WorldModelScopeRow | undefined; + return row ? l3WorldModelScopeFromSql(row, this.db) : undefined; } - ensureScope(userId: string, projectId?: string | null, at = nowIso()): L3WorldModelScopeRecord { - const scopeKey = l3WorldModelScopeKey(userId, projectId); + ensureScope(_userId: string | null, projectId?: string | null, at = nowIso()): L3WorldModelScopeRecord { + projectId = canonicalProjectId(this.db, projectId); + const scopeKey = l3WorldModelScopeKey(null, projectId); + const uris = this.db.prepare(`SELECT DISTINCT workspace_uri FROM l3_world_model_scopes + WHERE memory_project_id(project_id) IS ? AND workspace_uri IS NOT NULL`).all(projectId) as Array<{workspace_uri: string}>; + const workspaceUri = uris.length === 1 ? uris[0]!.workspace_uri : null; this.db.prepare( `INSERT INTO l3_world_model_scopes ( - scope_key, user_id, project_id, memory_id, next_scope_seq, updated_at - ) VALUES (?, ?, ?, NULL, 1, ?) + scope_key, user_id, project_id, workspace_uri, memory_id, next_scope_seq, updated_at + ) VALUES (?, NULL, ?, ?, NULL, 1, ?) ON CONFLICT(scope_key) DO NOTHING` - ).run(scopeKey, userId, projectId ?? null, at); - const scope = this.getScope(userId, projectId); - if (!scope || scope.scopeKey !== scopeKey || scope.userId !== userId || (scope.projectId ?? null) !== (projectId ?? null)) { - throw new Error("corrupt L3 World Model scope ownership"); - } - return scope; + ).run(scopeKey, projectId, workspaceUri, at); + return this.getScope(null, projectId)!; } bindWorkspaceUri( - userId: string, + userId: string | null, projectId: string, workspaceUri: WorkspaceUri, at = nowIso() @@ -5514,7 +5827,7 @@ export class L3WorldModelRepository { const placeholders = ids.map(() => "?").join(", "); return (this.db.prepare( `SELECT * FROM l3_world_model_scopes WHERE memory_id IN (${placeholders})` - ).all(...ids) as SqlL3WorldModelScopeRow[]).map(l3WorldModelScopeFromSql); + ).all(...ids) as SqlL3WorldModelScopeRow[]).map((row) => l3WorldModelScopeFromSql(row, this.db)); } registerInputTrace(input: { @@ -5640,7 +5953,14 @@ export class L3WorldModelRepository { const row = this.db.prepare( `SELECT * FROM l3_world_model_evidence_batches WHERE id = ?` ).get(batchId) as SqlL3WorldModelEvidenceBatchRow | undefined; - return row ? l3WorldModelEvidenceBatchFromSql(row) : undefined; + return row ? l3WorldModelEvidenceBatchFromSql(row, this.db) : undefined; + } + + /** Integrity checks must hash the immutable stored identity, before alias projection. */ + getStoredBatch(batchId: string): L3WorldModelEvidenceBatchRecord | undefined { + const row = this.db.prepare(`SELECT * FROM l3_world_model_evidence_batches WHERE id = ?`) + .get(batchId) as SqlL3WorldModelEvidenceBatchRow | undefined; + return row ? { ...l3WorldModelEvidenceBatchFromSql(row), userId: row.user_id } : undefined; } getTarget(batchId: string, targetField: L3WorldModelTargetField): L3WorldModelBatchTargetRecord | undefined { @@ -5663,22 +5983,177 @@ export class L3WorldModelRepository { .map(l3WorldModelInputTraceFromSql); } - getMemory(userId: string, projectId?: string | null): MemoryRow | undefined { - const scope = this.getScope(userId, projectId); - if (!scope?.memoryId) return undefined; - const memory = this.memories.get(scope.memoryId); - if (!memory) throw new Error(`corrupt L3 World Model scope memory: ${scope.memoryId}`); - validateL3WorldModelMemory(memory, userId, projectId); - return memory; + latestDeletion(projectId?: string | null): string | undefined { + const row = this.db.prepare(`SELECT MAX(COALESCE(deleted_at, updated_at)) AS deleted_at + FROM memories WHERE memory_layer = 'L3' AND (deleted_at IS NOT NULL OR status = 'deleted') + AND json_extract(properties_json, '$.internal_info.schema_version') = 2 + AND memory_project_id(json_extract(info_json, '$.project_id')) IS ?`) + .get(canonicalProjectId(this.db, projectId)) as {deleted_at: string | null}; + return row.deleted_at ?? undefined; + } + + /** Read explicit legacy field edits without rewriting historical rows or + * treating an uninitialized null field as a deletion. */ + private legacyFieldEdits(projectId: string | null) { + type EditOrder = { at: string; seq: number }; + const deletions = new Map(); + const writes = new Map>(); + const rows = this.db.prepare(`SELECT change.memory_id, change.before_json, change.after_json, change.created_at, change.seq + FROM memory_change_log AS change + JOIN l3_world_model_scopes AS scope ON scope.memory_id = change.memory_id + WHERE scope.user_id IS NOT NULL AND memory_project_id(scope.project_id) IS ? + AND change.change_type IN ('l3_world_model_create', 'l3_world_model_update') + ORDER BY change.created_at, change.seq`).all(projectId) as LegacyL3Change[]; + // Read-only v10 databases still use their available log evidence until upgraded. + const hasDurableEdits = !this.db.readonly || Boolean(this.db.prepare( + "SELECT 1 FROM sqlite_master WHERE type='table' AND name='memory_l3_legacy_field_edits'").get()); + const persisted = hasDurableEdits ? this.db.prepare(`SELECT edits.* FROM memory_l3_legacy_field_edits AS edits + JOIN l3_world_model_scopes AS scope ON scope.memory_id = edits.memory_id + WHERE scope.user_id IS NOT NULL AND memory_project_id(scope.project_id) IS ?`).all(projectId) as LegacyL3FieldEdit[] : []; + const edits = [...persisted, ...legacyL3FieldEditsFromChanges(rows)] + .sort((a, b) => a.edited_at.localeCompare(b.edited_at) || a.change_seq - b.change_seq); + for (const row of edits) { + const property = l3WorldModelFieldProperty(row.field); + const order = { at: row.edited_at, seq: row.change_seq }; + if (row.deleted) deletions.set(property, order); + else { + const written = writes.get(row.memory_id) ?? new Map(); + written.set(property, order); + writes.set(row.memory_id, written); + } + } + return { deletions, writes }; + } + + getMemory(_userId: string | null, projectId?: string | null): MemoryRow | undefined { + projectId = canonicalProjectId(this.db, projectId); + const scopes = this.db.prepare(`SELECT * FROM l3_world_model_scopes + WHERE memory_project_id(project_id) IS ? AND memory_id IS NOT NULL + ORDER BY (user_id IS NULL) DESC, updated_at DESC, scope_key ASC`) + .all(projectId) as SqlL3WorldModelScopeRow[]; + const candidates = scopes.flatMap(scope => { + const memory = this.memories.get(scope.memory_id!); + if (!memory || memory.deletedAt) return []; + validateL3WorldModelMemory(memory, null, projectId); + return [{ memory, canonical: scope.user_id === null }]; + }); + if (!candidates.length) return undefined; + const overrides = isMissingOptionalTableFromPreviousSchema(this.db, "memory_l3_overrides") + ? [] + : this.db.prepare(`SELECT field, deleted FROM memory_l3_overrides WHERE project_key = ?`) + .all(projectId ?? '') as Array<{field: keyof L3WorldModelFields; deleted: number}>; + const fields = emptyL3WorldModelFields(); + const deletedAt = this.latestDeletion(projectId); + const legacyEdits = this.legacyFieldEdits(projectId); + const selectedSources: string[] = []; + let profileScanId: unknown; + for (const property of Object.keys(fields) as Array) { + const override = overrides.find(row => row.field === property); + if (override?.deleted) continue; + for (const candidate of candidates) { + if (override && !candidate.canonical) continue; + if (deletedAt && (candidate.memory.updatedAt < deletedAt || + (!candidate.canonical && candidate.memory.updatedAt === deletedAt))) continue; + const fieldDeletedAt = legacyEdits.deletions.get(property); + // A later edit to another field is not evidence that this field was + // recreated. A proven later write (or an explicit device override) is. + const fieldWrittenAt = legacyEdits.writes.get(candidate.memory.id)?.get(property) + ?? { at: candidate.memory.createdAt, seq: -1 }; + if (!override && fieldDeletedAt && (fieldWrittenAt.at < fieldDeletedAt.at || + (fieldWrittenAt.at === fieldDeletedAt.at && fieldWrittenAt.seq <= fieldDeletedAt.seq))) continue; + const value = fieldsFromL3WorldModelMemory(candidate.memory)[property]; + if (value !== null) { + fields[property] = value; + selectedSources.push(...l3WorldModelSourceMemoryIds(candidate.memory)); + if (property === "projectEnvironmentProfile") profileScanId = candidate.memory.info.project_environment_applied_scan_id; + break; + } + } + } + const base = candidates[0]!.memory; + const explicitlyArchived = candidates.some(({memory}) => { + const markerAt = isExplicitlyArchivedL3WorldModel(memory) ? memory.updatedAt : undefined; + const auditAt = this.latestExplicitArchiveAt(memory.id); + const archivedAt = [markerAt, auditAt].filter((value): value is string => Boolean(value)).sort().at(-1); + return Boolean(archivedAt && (!deletedAt || archivedAt > deletedAt)); + }); + const sourceMemoryIds = uniq(selectedSources).slice(-256); + const memoryValue = renderL3WorldModelFields(fields); + const projectedVersionState = explicitlyArchived ? { fields, explicitlyArchived: true } : fields; + const projectedInternalInfo = { + ...base.properties.internal_info, body: memoryValue, source_memory_ids: sourceMemoryIds + }; + if (explicitlyArchived) Object.assign(projectedInternalInfo, {l3_world_model_explicitly_archived: true}); + else delete (projectedInternalInfo as typeof projectedInternalInfo & { + l3_world_model_explicitly_archived?: boolean; + }).l3_world_model_explicitly_archived; + return { + ...base, memoryValue, contentHash: sha256Hex(memoryValue), + info: {...base.info, source_memory_ids: sourceMemoryIds, project_environment_applied_scan_id: profileScanId}, + // Explicit governance wins over legacy field merging; empty projections remain archived. + status: explicitlyArchived || !memoryValue ? "archived" : "activated", + version: candidates.length > 1 || deletedAt || explicitlyArchived + ? Number.parseInt(sha256Hex(canonicalJson(projectedVersionState)).slice(0, 12), 16) + : base.version, + properties: { ...base.properties, internal_info: { + ...projectedInternalInfo, + world_model: { + general_rules_and_safety_constraints: fields.generalRulesAndSafetyConstraints, + project_environment_profile: fields.projectEnvironmentProfile, + project_contract: fields.projectContract, + domain_knowledge: fields.domainKnowledge + } + }} + }; + } + + archiveScopeMemory(memoryId: string, at = nowIso()): MemoryRow | undefined { + return this.db.transaction(() => { + const memory = this.memories.get(memoryId); + if (!memory) return undefined; + const projectId = projectIdFromL3WorldModelMemory(memory); + validateL3WorldModelMemory(memory, null, projectId); + const archived = this.memories.update({ + ...memory, + status: "archived", + properties: { + ...memory.properties, + status: "archived", + internal_info: { + ...memory.properties.internal_info, + l3_world_model_explicitly_archived: true + } + }, + updatedAt: at + }); + this.db.prepare("UPDATE l3_world_model_scopes SET updated_at = ? WHERE memory_id = ?") + .run(at, memoryId); + return archived; + })(); + } + + private latestExplicitArchiveAt(memoryId: string): string | undefined { + const timestamps: string[] = []; + if (hasSqliteTable(this.db, "audit_logs")) { + const row = this.db.prepare(`SELECT MAX(created_at) AS created_at + FROM audit_logs WHERE action = 'archive' AND target_id = ?`).get(memoryId) as {created_at: string | null}; + if (row.created_at) timestamps.push(row.created_at); + } + if (hasSqliteTable(this.db, "memory_change_log")) { + const row = this.db.prepare(`SELECT MAX(created_at) AS created_at + FROM memory_change_log WHERE change_type = 'archive' AND memory_id = ?`).get(memoryId) as {created_at: string | null}; + if (row.created_at) timestamps.push(row.created_at); + } + return timestamps.sort().at(-1); } - fields(userId: string, projectId?: string | null): L3WorldModelFields { + fields(userId: string | null, projectId?: string | null): L3WorldModelFields { const memory = this.getMemory(userId, projectId); return memory ? fieldsFromL3WorldModelMemory(memory) : emptyL3WorldModelFields(); } upsertField(input: { - userId: string; + userId: string | null; projectId?: string | null; targetField: L3WorldModelFieldName; value: string | null; @@ -5717,7 +6192,7 @@ export class L3WorldModelRepository { throw new Error(`L3 World Model target is terminal: ${input.batchId}:${input.targetField}`); } const expectedScopeKey = l3WorldModelFieldScopeKey( - l3WorldModelScopeKey(batch.userId, batch.projectId), + batch.scopeKey, input.targetField ); if (target.fieldScopeKey !== expectedScopeKey || target.scopeSeq !== batch.scopeSeq) { @@ -5725,6 +6200,17 @@ export class L3WorldModelRepository { } assertL3WorldModelFieldOwnership(batch.projectId ?? null, input.targetField); + let memory = this.getMemory(batch.userId, batch.projectId); + if (isExplicitlyArchivedL3WorldModel(memory)) { + this.db.prepare( + `UPDATE l3_world_model_batch_targets + SET status = 'applied', no_change = 1, applied_at = ?, updated_at = ? + WHERE batch_id = ? AND target_field = ? AND status = 'queued'` + ).run(at, at, input.batchId, input.targetField); + updateL3WorldModelBatchTerminalOutcome(this.db, input.batchId, at); + return { alreadyApplied: false, noChange: true, memory }; + } + const fields = this.fields(batch.userId, batch.projectId); const currentField = fields[l3WorldModelFieldProperty(input.targetField)]; if (sha256Hex(currentField ?? "") !== input.expectedFieldHash) { @@ -5738,8 +6224,11 @@ export class L3WorldModelRepository { } let noChange = false; - let memory = this.getMemory(batch.userId, batch.projectId); - if (input.operation === "noop") { + const deletedAt = this.latestDeletion(batch.projectId); + if (deletedAt && batch.createdAt <= deletedAt) { + // An immutable pre-upgrade batch must not recreate a subsequently deleted scope. + noChange = true; + } else if (input.operation === "noop") { if (input.value !== "") throw new TypeError("noop L3 World Model output must be empty"); noChange = true; } else if (input.operation === "create") { @@ -5783,7 +6272,11 @@ export class L3WorldModelRepository { `SELECT * FROM l3_world_model_scopes WHERE memory_id = ?` ).get(memoryId) as SqlL3WorldModelScopeRow | undefined; if (!scopeRow) return undefined; - const scope = l3WorldModelScopeFromSql(scopeRow); + const scope = l3WorldModelScopeFromSql(scopeRow, this.db); + for (const field of Object.keys(emptyL3WorldModelFields())) { + this.db.prepare(`INSERT INTO memory_l3_overrides(project_key, field, deleted) VALUES (?, ?, 1) + ON CONFLICT(project_key, field) DO UPDATE SET deleted = 1`).run(scope.projectId ?? "", field); + } const before = this.memories.get(memoryId); if (!before) throw new Error(`corrupt L3 World Model scope memory: ${memoryId}`); validateL3WorldModelMemory(before, scope.userId, scope.projectId); @@ -5797,8 +6290,8 @@ export class L3WorldModelRepository { `SELECT target.batch_id, target.target_field FROM l3_world_model_batch_targets AS target JOIN l3_world_model_evidence_batches AS batch ON batch.id = target.batch_id - WHERE batch.scope_key = ? AND target.status = 'queued'` - ).all(scope.scopeKey) as Array<{ batch_id: string; target_field: L3WorldModelTargetField }>; + WHERE memory_project_id(batch.project_id) IS ? AND target.status = 'queued'` + ).all(scope.projectId ?? null) as Array<{ batch_id: string; target_field: L3WorldModelTargetField }>; const affectedBatchIds = new Set(); for (const target of pendingTargets) { const job = this.db.prepare( @@ -5829,22 +6322,22 @@ export class L3WorldModelRepository { `UPDATE evolution_jobs SET status = 'succeeded', leased_until = NULL, last_error = NULL, updated_at = ? WHERE job_type = 'project_environment_profile' - AND user_id = ? - AND json_extract(payload_json, '$.projectId') = ? + AND memory_project_id(json_extract(payload_json, '$.projectId')) = ? AND status IN ('queued', 'failed')` - ).run(at, scope.userId, scope.projectId); + ).run(at, scope.projectId); this.db.prepare( `UPDATE l3_world_model_project_environment_state SET project_kind = 'unknown', status = 'uninitialized', current_scan_id = NULL, applied_scan_id = NULL, fingerprint = NULL, last_error = NULL, updated_at = ? - WHERE user_id = ? AND project_id = ?` - ).run(at, scope.userId, scope.projectId); + WHERE memory_project_id(project_id) = ?` + ).run(at, scope.projectId); } return { before, deleted, scope }; })(); } insertImmutableJob(job: EvolutionJobRecord): EvolutionJobRecord { + job = { ...job, userId: null }; if (job.jobType !== "l3_world_model_update" && job.jobType !== "project_environment_profile") { throw new TypeError("immutable L3 job must use an L3 World Model job type"); } @@ -5965,7 +6458,7 @@ export class L3WorldModelRepository { const payload = { scopeKey: scope.scopeKey, scopeSeq: claimed.scope_seq, - userId: session.userId, + userId: null, projectId: session.projectId ?? null, sessionId: session.id, trigger: input.trigger, @@ -5987,7 +6480,7 @@ export class L3WorldModelRepository { batchId, scope.scopeKey, claimed.scope_seq, - session.userId, + null, session.projectId ?? null, session.id, input.trigger, @@ -6015,7 +6508,7 @@ export class L3WorldModelRepository { jobType: "l3_world_model_update", status: "queued", dedupeKey: `l3_world_model:${batchId}:${targetField}`, - userId: session.userId, + userId: null, sessionId: session.id, scopeKey: fieldScopeKey, scopeSeq: claimed.scope_seq, @@ -6043,7 +6536,7 @@ export class L3WorldModelRepository { } private upsertFieldInTransaction(input: { - userId: string; + userId: string | null; projectId?: string | null; targetField: L3WorldModelFieldName; value: string | null; @@ -6052,20 +6545,28 @@ export class L3WorldModelRepository { at?: string; source?: string; }): MemoryRow | undefined { - const projectId = input.projectId ?? null; + const projectId = canonicalProjectId(this.db, input.projectId); assertL3WorldModelFieldOwnership(projectId, input.targetField); const at = input.at ?? nowIso(); + // Inherit each field together with metadata from the same selected historical + // record. An explicit archive must stop automated writes before creating a + // replacement device scope or changing its sequence. + const inherited = this.getMemory(null, projectId); + if (isExplicitlyArchivedL3WorldModel(inherited)) return inherited; const scope = this.ensureScope(input.userId, projectId, at); const existing = scope.memoryId ? this.memories.get(scope.memoryId) : undefined; if (scope.memoryId && !existing) throw new Error(`corrupt L3 World Model scope memory: ${scope.memoryId}`); if (existing) validateL3WorldModelMemory(existing, input.userId, projectId); - const fields = existing ? fieldsFromL3WorldModelMemory(existing) : emptyL3WorldModelFields(); + const fields = inherited ? fieldsFromL3WorldModelMemory(inherited) : emptyL3WorldModelFields(); const property = l3WorldModelFieldProperty(input.targetField); fields[property] = normalizeL3WorldModelFieldValue(input.value); + this.db.prepare(`INSERT INTO memory_l3_overrides(project_key, field, deleted) VALUES (?, ?, ?) + ON CONFLICT(project_key, field) DO UPDATE SET deleted = excluded.deleted`) + .run(projectId ?? "", property, fields[property] === null ? 1 : 0); const memoryValue = renderL3WorldModelFields(fields); if (!existing && !memoryValue) return undefined; - const existingSourceMemoryIds = l3WorldModelSourceMemoryIds(existing); + const existingSourceMemoryIds = l3WorldModelSourceMemoryIds(inherited); const sourceMemoryIds = input.eligibleL1MemoryIds === undefined ? existingSourceMemoryIds : this.orderedRecentSourceMemoryIds( @@ -6081,7 +6582,7 @@ export class L3WorldModelRepository { ...(projectId ? { project_id: projectId } : {}), source_memory_ids: sourceMemoryIds }; - const existingScanId = existing?.info.project_environment_applied_scan_id; + const existingScanId = inherited?.info.project_environment_applied_scan_id; const scanId = input.projectEnvironmentAppliedScanId === undefined ? existingScanId : input.projectEnvironmentAppliedScanId; @@ -6092,7 +6593,7 @@ export class L3WorldModelRepository { const memory: MemoryRow = { id: existing?.id ?? newId("memory"), timeline: existing?.timeline ?? at, - userId: input.userId, + userId: null, memoryType: "LongTermMemory", status, visibility: "private", @@ -6144,7 +6645,7 @@ export class L3WorldModelRepository { scope.scopeKey, existing ? (status === "archived" ? "archived" : "updated") : "created", saved.id, - input.userId, + null, existing ? "l3_world_model_update" : "l3_world_model_create", saved.version, existing ? toJson(existing) : null, @@ -6199,7 +6700,7 @@ export class L3WorldModelRepository { private requireV2Session(sessionId: string): SessionRecord { const row = this.db.prepare(`SELECT * FROM sessions WHERE id = ?`).get(sessionId) as SqlSessionRow | undefined; if (!row) throw new Error(`session not found: ${sessionId}`); - const session = sessionFromSql(row); + const session = sessionFromSql(row, this.db); if (session.meta.l3_world_model_protocol_version !== 2) { throw new Error("l3_world_model_protocol_v2_required"); } @@ -6226,16 +6727,17 @@ export class ProjectEnvironmentRepository { private readonly runtime: RuntimeRepository ) {} - getState(userId: string, projectId: string): ProjectEnvironmentStateRecord | undefined { + getState(userId: string | null, projectId: string): ProjectEnvironmentStateRecord | undefined { const row = this.db.prepare( `SELECT * FROM l3_world_model_project_environment_state - WHERE user_id = ? AND project_id = ?` - ).get(userId, projectId) as SqlProjectEnvironmentStateRow | undefined; - return row ? projectEnvironmentStateFromSql(row) : undefined; + WHERE memory_project_id(project_id) = ? + ORDER BY (user_id IS NULL) DESC, updated_at DESC, project_id ASC LIMIT 1` + ).get(canonicalProjectId(this.db, projectId)) as SqlProjectEnvironmentStateRow | undefined; + return row ? projectEnvironmentStateFromSql(row, this.db) : undefined; } requestScan(input: { - userId: string; + userId: string | null; projectId: string; sessionId: string; trigger: "session_start" | "token_compaction"; @@ -6253,19 +6755,19 @@ export class ProjectEnvironmentRepository { user_id, project_id, project_kind, status, current_scan_id, applied_scan_id, fingerprint, last_error, updated_at ) VALUES (?, ?, 'unknown', 'queued', ?, NULL, NULL, NULL, ?) - ON CONFLICT(user_id, project_id) DO UPDATE SET + ON CONFLICT(project_id) WHERE user_id IS NULL DO UPDATE SET status = 'queued', current_scan_id = excluded.current_scan_id, last_error = NULL, updated_at = excluded.updated_at` - ).run(input.userId, input.projectId, scanId, at); + ).run(null, canonicalProjectId(this.db, input.projectId), scanId, at); const job: EvolutionJobRecord = { id: newId("job"), jobType: "project_environment_profile", status: "queued", dedupeKey: input.dedupeKey, - userId: input.userId, + userId: null, sessionId: input.sessionId, payload: { - userId: input.userId, + userId: null, projectId: input.projectId, scanId, trigger: input.trigger @@ -6280,26 +6782,26 @@ export class ProjectEnvironmentRepository { })(); } - beginScan(userId: string, projectId: string, scanId: string, at = nowIso()): boolean { + beginScan(userId: string | null, projectId: string, scanId: string, at = nowIso()): boolean { const result = this.db.prepare( `UPDATE l3_world_model_project_environment_state SET status = 'scanning', last_error = NULL, updated_at = ? - WHERE user_id = ? AND project_id = ? AND current_scan_id = ?` - ).run(at, userId, projectId, scanId); + WHERE memory_project_id(project_id) = memory_project_id(?) AND current_scan_id = ?` + ).run(at, projectId, scanId); return result.changes === 1; } - markSummarizing(userId: string, projectId: string, scanId: string, at = nowIso()): boolean { + markSummarizing(userId: string | null, projectId: string, scanId: string, at = nowIso()): boolean { const result = this.db.prepare( `UPDATE l3_world_model_project_environment_state SET status = 'summarizing', updated_at = ? - WHERE user_id = ? AND project_id = ? AND current_scan_id = ?` - ).run(at, userId, projectId, scanId); + WHERE memory_project_id(project_id) = memory_project_id(?) AND current_scan_id = ?` + ).run(at, projectId, scanId); return result.changes === 1; } markCleanWithoutModel(input: { - userId: string; + userId: string | null; projectId: string; scanId: string; projectKind: Exclude; @@ -6309,11 +6811,10 @@ export class ProjectEnvironmentRepository { const result = this.db.prepare( `UPDATE l3_world_model_project_environment_state SET project_kind = ?, status = 'clean', last_error = NULL, updated_at = ? - WHERE user_id = ? AND project_id = ? AND current_scan_id = ?` + WHERE memory_project_id(project_id) = memory_project_id(?) AND current_scan_id = ?` ).run( input.projectKind, at, - input.userId, input.projectId, input.scanId ); @@ -6321,7 +6822,7 @@ export class ProjectEnvironmentRepository { } applyProfile(input: { - userId: string; + userId: string | null; projectId: string; scanId: string; projectKind: Exclude; @@ -6363,7 +6864,7 @@ export class ProjectEnvironmentRepository { const existingMemory = this.l3WorldModels.getMemory(input.userId, input.projectId); if (!emptyNoop && (nextProfile !== null || existingMemory)) { this.l3WorldModels.upsertField({ - userId: input.userId, + userId: null, projectId: input.projectId, targetField: "project_environment_profile", value: nextProfile, @@ -6376,13 +6877,12 @@ export class ProjectEnvironmentRepository { `UPDATE l3_world_model_project_environment_state SET project_kind = ?, status = 'clean', applied_scan_id = ?, fingerprint = ?, last_error = NULL, updated_at = ? - WHERE user_id = ? AND project_id = ? AND current_scan_id = ?` + WHERE memory_project_id(project_id) = memory_project_id(?) AND current_scan_id = ?` ).run( input.projectKind, emptyNoop ? null : input.scanId, emptyNoop ? null : input.fingerprint, at, - input.userId, input.projectId, input.scanId ); @@ -6390,18 +6890,18 @@ export class ProjectEnvironmentRepository { })(); } - failCurrentScan(userId: string, projectId: string, scanId: string, error: string, at = nowIso()): void { + failCurrentScan(userId: string | null, projectId: string, scanId: string, error: string, at = nowIso()): void { this.db.prepare( `UPDATE l3_world_model_project_environment_state SET status = 'failed', last_error = ?, updated_at = ? - WHERE user_id = ? AND project_id = ? AND current_scan_id = ?` - ).run(error, at, userId, projectId, scanId); + WHERE memory_project_id(project_id) = memory_project_id(?) AND current_scan_id = ?` + ).run(error, at, projectId, scanId); } } -function projectEnvironmentStateFromSql(row: SqlProjectEnvironmentStateRow): ProjectEnvironmentStateRecord { +function projectEnvironmentStateFromSql(row: SqlProjectEnvironmentStateRow, db?: Database.Database): ProjectEnvironmentStateRecord { return { - userId: row.user_id, + userId: null, projectId: row.project_id, projectKind: row.project_kind, status: row.status, @@ -6424,6 +6924,7 @@ export class Repositories { readonly vectors: SqliteVecStore; constructor(readonly db: Database.Database) { + installDeviceScope(db); this.vectors = new SqliteVecStore(db); this.memories = new MemoryRepository(db, this.vectors); this.captureClaims = new MemoryCaptureClaimRepository(db); @@ -6524,8 +7025,8 @@ interface SqlL3WorldModelBatchTargetRow { updated_at: string; } -export function l3WorldModelScopeKey(userId: string, projectId?: string | null): string { - return `l3wm:${sha256Hex(canonicalJson({ userId, projectId: projectId ?? null }))}`; +export function l3WorldModelScopeKey(userId: string | null, projectId?: string | null): string { + return `l3wm:${sha256Hex(canonicalJson({ scope: "device", projectId: projectId ?? null }))}`; } export function l3WorldModelFieldScopeKey( @@ -6535,17 +7036,17 @@ export function l3WorldModelFieldScopeKey( return `${scopeKey}:${targetField}`; } -export function l3WorldModelMemoryKey(userId: string, projectId?: string | null): string { +export function l3WorldModelMemoryKey(userId: string | null, projectId?: string | null): string { return projectId - ? `world_model:project:${userId}:${projectId}` - : `world_model:general_rules:${userId}:no_project`; + ? `world_model:device:project:${projectId}` + : `world_model:device:general_rules`; } export function isL3WorldModelV2Memory(memory: MemoryRow): boolean { try { validateL3WorldModelMemory( memory, - memory.userId, + null, projectIdFromL3WorldModelMemory(memory) ); return true; @@ -6554,6 +7055,10 @@ export function isL3WorldModelV2Memory(memory: MemoryRow): boolean { } } +function isExplicitlyArchivedL3WorldModel(memory: MemoryRow | undefined): boolean { + return memory?.properties.internal_info.l3_world_model_explicitly_archived === true; +} + export function fieldsFromL3WorldModelMemory(memory: MemoryRow): L3WorldModelFields { const world = memory.properties.internal_info.world_model; if (!isRecordLike(world)) throw new Error(`invalid L3 World Model v2 fields: ${memory.id}`); @@ -6567,7 +7072,7 @@ export function fieldsFromL3WorldModelMemory(memory: MemoryRow): L3WorldModelFie function validateL3WorldModelMemory( memory: MemoryRow, - expectedUserId: string, + expectedUserId: string | null, expectedProjectId?: string | null ): void { const internal = memory.properties.internal_info; @@ -6590,10 +7095,11 @@ function validateL3WorldModelMemory( if (memory.memoryLayer !== "L3" || internal.memory_kind !== "world_model") { throw new Error(`invalid L3 World Model layer or kind: ${memory.id}`); } - if (memory.userId !== expectedUserId || projectIdFromL3WorldModelMemory(memory) !== projectId) { + if (projectIdFromL3WorldModelMemory(memory) !== projectId) { throw new Error(`invalid L3 World Model owner: ${memory.id}`); } - if (memory.memoryKey !== l3WorldModelMemoryKey(expectedUserId, projectId)) { + const legacyKey = projectId ? `world_model:project:${memory.userId}:${projectId}` : `world_model:general_rules:${memory.userId}:no_project`; + if (memory.memoryKey !== l3WorldModelMemoryKey(null, projectId) && memory.memoryKey !== legacyKey) { throw new Error(`invalid L3 World Model key: ${memory.id}`); } if (projectId && fields.generalRulesAndSafetyConstraints !== null) { @@ -6684,11 +7190,11 @@ export function splitL3TracesByRawTurn( return chunks; } -function l3WorldModelScopeFromSql(row: SqlL3WorldModelScopeRow): L3WorldModelScopeRecord { +function l3WorldModelScopeFromSql(row: SqlL3WorldModelScopeRow, db?: Database.Database): L3WorldModelScopeRecord { return { scopeKey: row.scope_key, - userId: row.user_id, - projectId: row.project_id ?? undefined, + userId: null, + projectId: (db ? canonicalProjectId(db, row.project_id) : row.project_id) ?? undefined, workspaceUri: row.workspace_uri ? row.workspace_uri as WorkspaceUri : undefined, memoryId: row.memory_id ?? undefined, nextScopeSeq: row.next_scope_seq, @@ -6709,13 +7215,13 @@ function l3WorldModelInputTraceFromSql(row: SqlL3WorldModelInputTraceRow): L3Wor function l3WorldModelEvidenceBatchFromSql( row: SqlL3WorldModelEvidenceBatchRow -): L3WorldModelEvidenceBatchRecord { +, db?: Database.Database): L3WorldModelEvidenceBatchRecord { return { id: row.id, scopeKey: row.scope_key, scopeSeq: row.scope_seq, - userId: row.user_id, - projectId: row.project_id ?? undefined, + userId: null, + projectId: (db ? canonicalProjectId(db, row.project_id) : row.project_id) ?? undefined, sessionId: row.session_id, trigger: row.trigger, startTraceSeq: row.start_trace_seq, @@ -6746,13 +7252,19 @@ function l3WorldModelBatchTargetFromSql( }; } -export function memoryFromSql(row: MemorySqlRow): MemoryRow { +export function memoryFromSql(row: MemorySqlRow, db?: Database.Database): MemoryRow { const info = parseJson>(row.info_json, {}); const properties = parseJson(row.properties_json, { internal_info: { memory_layer: row.memory_layer } }); + const storedProject = typeof info.project_id === "string" ? info.project_id : null; + const legacyKey = storedProject ? `world_model:project:${row.user_id}:${storedProject}` : `world_model:general_rules:${row.user_id}:no_project`; + const projectedKey = row.memory_layer === "L3" && properties.internal_info?.schema_version === 2 && row.memory_key === legacyKey + ? l3WorldModelMemoryKey(null, db ? canonicalProjectId(db, storedProject) : storedProject) : row.memory_key; + if (db && typeof info.project_id === "string") info.project_id = canonicalProjectId(db, info.project_id); + if (db && typeof properties.info?.project_id === "string") properties.info.project_id = canonicalProjectId(db, properties.info.project_id); const tags = uniq([ ...asStringArray(parseJson(row.tags_json, [])), ...asStringArray(info.tags), @@ -6766,15 +7278,15 @@ export function memoryFromSql(row: MemorySqlRow): MemoryRow { return { id: row.id, timeline: row.timeline, - userId: row.user_id, + userId: null, conversationId: row.conversation_id ?? undefined, sessionId: row.session_id ?? undefined, agentId: row.agent_id ?? undefined, - appId: row.app_id ?? undefined, + appId: (db ? canonicalProjectId(db, row.app_id) : row.app_id) ?? undefined, memoryType: row.memory_type, status: row.status, visibility: row.visibility, - memoryKey: row.memory_key ?? undefined, + memoryKey: projectedKey ?? undefined, memoryValue: row.memory_value, tags, info, @@ -6795,7 +7307,7 @@ function userMemoryFromSql(row: UserMemorySqlRow): UserMemoryRecord { return { id: row.id, sourceTurnId: row.source_turn_id, - userId: row.user_id, + userId: null, memoryTypes: asStringArray(parseJson(row.memory_types_json, [])) as UserMemoryType[], content: row.content, normalizedUserTextHash: row.normalized_user_text_hash, @@ -6815,13 +7327,13 @@ function userMemoryFromSql(row: UserMemorySqlRow): UserMemoryRecord { } function userMemoryPanelFilter(input: { - userId: string; + userId: string | null; status?: UserMemoryStatus; query?: string; sourceAgent?: string; }): { where: string; params: Array } { - const clauses = ["user_id = ?"]; - const params = [input.userId]; + const clauses = [`(status != 'active' OR (${ACTIVE_USER_MEMORY_VIEW_SQL}))`]; + const params: string[] = []; if (input.status) { clauses.push("status = ?"); params.push(input.status); @@ -6872,7 +7384,7 @@ export function memoryToSql(memory: MemoryRow): Record { return { id: memory.id, timeline: memory.timeline, - userId: memory.userId, + userId: null, conversationId: memory.conversationId ?? null, sessionId: memory.sessionId ?? null, agentId: memory.agentId ?? null, @@ -7322,7 +7834,7 @@ function buildMemoryWhere(filter: MemoryFilter): { where: string; params: SqlVal const clauses = ["deleted_at IS NULL"]; const params: SqlValue[] = []; - addValueClause("user_id", filter.userId); + // Legacy account provenance is not a memory filter. addValueClause("session_id", filter.sessionId); addValueClause("conversation_id", filter.conversationId); addAgentIdClause(filter.agentId, filter.excludedAgentIds); @@ -7332,6 +7844,10 @@ function buildMemoryWhere(filter: MemoryFilter): { where: string; params: SqlVal addArrayClause("memory_layer", filter.memoryLayer); addArrayClause("status", filter.status); addArrayClause("id", filter.ids); + if (filter.excludedIds?.length) { + clauses.push("memories.id NOT IN (SELECT CAST(value AS TEXT) FROM json_each(?))"); + params.push(JSON.stringify(filter.excludedIds)); + } addMemoryKindClause(filter.memoryKind); addWorkMemoryScopeClause(filter.workMemoryUserId, filter.workMemoryProjectId); addTagClauses(filter.tags); @@ -7402,14 +7918,13 @@ function buildMemoryWhere(filter: MemoryFilter): { where: string; params: SqlVal params.push(...values); } - function addWorkMemoryScopeClause(userId: string | undefined, projectId: string | null | undefined): void { - if (userId === undefined || projectId === undefined) return; - const normalizedProjectExpression = "NULLIF(TRIM(CAST(json_extract(info_json, '$.project_id') AS TEXT)), '')"; + function addWorkMemoryScopeClause(_userId: string | null | undefined, projectId: string | null | undefined): void { + if (projectId === undefined) return; + const normalizedProjectExpression = "memory_project_id(NULLIF(TRIM(CAST(json_extract(info_json, '$.project_id') AS TEXT)), ''))"; clauses.push(`( COALESCE(json_extract(properties_json, '$.internal_info.memory_kind'), 'trace') != 'work_memory' - OR (user_id = ? AND ${projectId === null ? `${normalizedProjectExpression} IS NULL` : `${normalizedProjectExpression} = ?`}) + OR (${projectId === null ? `${normalizedProjectExpression} IS NULL` : `${normalizedProjectExpression} = ?`}) )`); - params.push(userId); if (projectId !== null) params.push(projectId); } @@ -7438,13 +7953,9 @@ function buildMemoryWhere(filter: MemoryFilter): { where: string; params: SqlVal } } -function buildEpisodeWhere(userId?: string, query?: string, sourceAgent?: string): { where: string; params: SqlValue[] } { +function buildEpisodeWhere(userId?: string | null, query?: string, sourceAgent?: string): { where: string; params: SqlValue[] } { const clauses = ["1=1"]; const params: SqlValue[] = []; - if (userId) { - clauses.push("episodes.user_id = ?"); - params.push(userId); - } const normalizedSourceAgent = sourceAgent?.trim(); if (normalizedSourceAgent) { @@ -7627,20 +8138,22 @@ interface SqlSessionRow { updated_at: string; } -function sessionFromSql(row: SqlSessionRow): SessionRecord { +function sessionFromSql(row: SqlSessionRow, db?: Database.Database): SessionRecord { + const meta = parseJson>(row.meta_json, {}); + if (db && typeof meta.source_namespace_key === "string") meta.source_namespace_key = canonicalNamespaceKey(db, meta.source_namespace_key); return { id: row.id, - userId: row.user_id, + userId: null, source: row.source, profileId: row.profile_id, profileLabel: row.profile_label ?? undefined, - projectId: row.project_id ?? undefined, - workspaceId: row.workspace_id ?? undefined, + projectId: (db ? canonicalProjectId(db, row.project_id) : row.project_id) ?? undefined, + workspaceId: (db ? canonicalProjectId(db, row.workspace_id) : row.workspace_id) ?? undefined, workspacePath: row.workspace_path ?? undefined, hostSessionKey: row.host_session_key ?? undefined, conversationId: row.conversation_id ?? undefined, status: row.status, - meta: parseJson(row.meta_json, {}), + meta, openedAt: row.opened_at, lastSeenAt: row.last_seen_at ?? row.updated_at, closedAt: row.closed_at, @@ -7648,6 +8161,15 @@ function sessionFromSql(row: SqlSessionRow): SessionRecord { }; } +function sessionMatchesTenant(session: SessionRecord, tenantId: string | null): boolean { + const snake = session.meta.tenant_id; + const camel = session.meta.tenantId; + const snakeId = typeof snake === "string" && snake.length > 0 ? snake : null; + const camelId = typeof camel === "string" && camel.length > 0 ? camel : null; + if (snakeId !== null && camelId !== null && snakeId !== camelId) return false; + return (snakeId ?? camelId) === tenantId; +} + interface SqlEpisodeRow { id: string; session_id: string; @@ -7676,13 +8198,13 @@ interface SqlEpisodeRow { updated_at: string; } -function episodeFromSql(row: SqlEpisodeRow): EpisodeRecord { +function episodeFromSql(row: SqlEpisodeRow, db?: Database.Database): EpisodeRecord { const rawTurnIds = asStringArray(parseJson(row.raw_turn_ids_json, [])); return { id: row.id, sessionId: row.session_id, - userId: row.user_id, - projectId: row.project_id ?? undefined, + userId: null, + projectId: (db ? canonicalProjectId(db, row.project_id) : row.project_id) ?? undefined, conversationId: row.conversation_id ?? undefined, status: row.status, title: row.title ?? undefined, @@ -7723,12 +8245,12 @@ interface SqlSkillClusterRow { updated_at: string; } -function skillClusterFromSql(row: SqlSkillClusterRow): SkillClusterRecord { +function skillClusterFromSql(row: SqlSkillClusterRow, db?: Database.Database): SkillClusterRecord { const centroid = parseJson(row.centroid_json, null); return { id: row.id, - userId: row.user_id, - projectId: row.project_id ?? undefined, + userId: null, + projectId: (db ? canonicalProjectId(db, row.project_id) : row.project_id) ?? undefined, tools: asStringArray(parseJson(row.tools_json, [])), artifacts: asStringArray(parseJson(row.artifacts_json, [])), toolBigrams: asStringArray(parseJson(row.tool_bigrams_json, [])), @@ -7805,7 +8327,7 @@ function rawTurnFromSql(row: SqlRawTurnRow): RawTurnRecord { sessionId: row.session_id, episodeId: row.episode_id ?? undefined, turnId: row.turn_id, - userId: row.user_id, + userId: null, conversationId: row.conversation_id ?? undefined, userText: row.user_text ?? undefined, assistantText: row.assistant_text ?? undefined, @@ -7822,11 +8344,11 @@ function rawTurnFromSql(row: SqlRawTurnRow): RawTurnRecord { }; } -function feedbackFromSql(row: SqlFeedbackRow): FeedbackRecord { +function feedbackFromSql(row: SqlFeedbackRow, db: Database.Database): FeedbackRecord { return { id: row.id, - userId: row.user_id, - projectId: row.project_id ?? undefined, + userId: null, + projectId: canonicalProjectId(db, row.project_id) ?? undefined, conversationId: row.conversation_id ?? undefined, sessionId: row.session_id ?? undefined, episodeId: row.episode_id ?? undefined, @@ -7873,7 +8395,7 @@ function recallEventFromSql(row: SqlRecallEventRow): RecallEventRecord { sessionId: row.session_id ?? undefined, episodeId: row.episode_id ?? undefined, turnId: row.turn_id ?? undefined, - userId: row.user_id, + userId: null, query: row.query, queryHash: row.query_hash ?? undefined, layers: asStringArray(parseJson(row.layers_json, [])) as MemoryLayer[], @@ -8030,7 +8552,7 @@ function jobFromSql(row: SqlJobRow): EvolutionJobRecord { jobType: row.job_type, status: row.status, dedupeKey: row.dedupe_key ?? undefined, - userId: row.user_id, + userId: null, sessionId: row.session_id ?? undefined, episodeId: row.episode_id ?? undefined, targetMemoryId: row.target_memory_id ?? undefined, @@ -8099,7 +8621,7 @@ function embeddingRetryFromSql(row: SqlEmbeddingRetryRow): EmbeddingRetryRecord function skillTrialFromSql(row: SqlSkillTrialRow): SkillTrialRecord { return { id: row.id, - userId: row.user_id, + userId: null, projectId: row.project_id ?? undefined, skillMemoryId: row.skill_memory_id, sessionId: row.session_id ?? undefined, @@ -8125,7 +8647,7 @@ function statusFromOutcome(outcome: SkillTrialRecord["outcome"]): SkillTrialReco function tracePolicyLinkFromSql(row: SqlTracePolicyLinkRow): TracePolicyLinkRecord { return { id: row.id, - userId: row.user_id, + userId: null, l1MemoryId: row.l1_memory_id, l2MemoryId: row.l2_memory_id, relation: row.relation, @@ -8140,7 +8662,7 @@ function decisionRepairFromSql(row: SqlDecisionRepairRow): DecisionRepairRecord sessionId: row.session_id ?? undefined, episodeId: row.episode_id ?? undefined, rawTurnId: row.raw_turn_id ?? undefined, - userId: row.user_id, + userId: null, projectId: row.project_id ?? undefined, contextHash: row.context_hash ?? undefined, issue: row.issue, @@ -8161,7 +8683,7 @@ function decisionRepairFromSql(row: SqlDecisionRepairRow): DecisionRepairRecord function candidatePoolFromSql(row: SqlCandidatePoolRow): CandidatePoolRecord { return { id: row.id, - userId: row.user_id, + userId: null, sessionId: row.session_id ?? undefined, sourceMemoryId: row.source_memory_id, candidateKey: row.candidate_key, @@ -8178,7 +8700,7 @@ function candidatePoolFromSql(row: SqlCandidatePoolRow): CandidatePoolRecord { function auditLogFromSql(row: SqlAuditLogRow): AuditLogRecord { return { id: row.id, - userId: row.user_id, + userId: null, sessionId: row.session_id ?? undefined, actor: parseJson(row.actor_json, {}), action: row.action, @@ -8292,7 +8814,7 @@ interface BundleIdentity { primaryKey: string; sourceId: string; columns: string[]; - values: Array; + values: Array; } function bundleIdentity( @@ -8300,6 +8822,10 @@ function bundleIdentity( row: Record ): BundleIdentity | undefined { const newTableIdentityColumns: Partial> = { + memory_project_aliases: ["legacy_id"], + memory_namespace_aliases: ["legacy_key"], + memory_l3_overrides: ["project_key", "field"], + memory_l3_legacy_field_edits: ["memory_id", "field", "deleted"], memory_capture_claims: ["user_id", "source", "qa_hash"], source_turn_captures: ["user_id", "source", "profile_id", "namespace_key", "conversation_id", "turn_id"], l3_world_model_scopes: ["scope_key"], @@ -8313,18 +8839,18 @@ function bundleIdentity( const newColumns = newTableIdentityColumns[table]; if (newColumns) { const values = newColumns.map((column) => row[column]); - if (values.some((value) => typeof value !== "string" && typeof value !== "number")) { + if (values.some((value) => value !== null && typeof value !== "string" && typeof value !== "number")) { return undefined; } - const identity: Record = {}; + const identity: Record = {}; for (const [index, column] of newColumns.entries()) { - identity[column] = values[index] as string | number; + identity[column] = values[index] as string | number | null; } return { primaryKey: newColumns.join("+"), sourceId: canonicalJson(identity), columns: newColumns, - values: values as Array + values: values as Array }; } @@ -8369,6 +8895,25 @@ function nestedString(record: Record, ...path: string[]): strin return stringLike(value); } +function bundleMemoryArchiveProjection( + row: Record, + explicitArchiveEvidence: ReadonlySet +): Record { + if (row.memory_layer !== "L3" || row.status !== "archived" || typeof row.id !== "string" || + !explicitArchiveEvidence.has(row.id) || typeof row.properties_json !== "string") return row; + const properties = parseJson>(row.properties_json, {}); + const internal = properties.internal_info; + if (!isRecord(internal) || internal.schema_version !== 2 || + internal.l3_world_model_explicitly_archived === true) return row; + return { + ...row, + properties_json: toJson({ + ...properties, + internal_info: { ...internal, l3_world_model_explicitly_archived: true } + }) + }; +} + function redactBundleRow(table: BundleTableName, row: Record): Record { if (table === "evolution_jobs" && row.job_type === "capture_replay") { return { ...row, payload_json: JSON.stringify({ rawTextExcluded: true }) }; diff --git a/Memory/src/storage/schema.ts b/Memory/src/storage/schema.ts index 61f3510a7..d94160770 100644 --- a/Memory/src/storage/schema.ts +++ b/Memory/src/storage/schema.ts @@ -1,8 +1,9 @@ import type Database from "better-sqlite3"; import { memoryCaptureQaHash, normalizeMemoryCaptureSource } from "../utils/memory-capture-claim.js"; +import { preserveLegacyL3FieldEdits } from "./legacy-l3-field-edits.js"; -export const SCHEMA_VERSION = 10; -export const SCHEMA_MIGRATION_ID = "010_pending_episode_assignment"; +export const SCHEMA_VERSION = 11; +export const SCHEMA_MIGRATION_ID = "011_legacy_l3_field_edits"; const API_LOG_SOURCE_AGENT_MIGRATION_FROM_VERSION = 2; const PROCESSING_TAGS = new Set([ "摘要排队中", @@ -25,7 +26,7 @@ const statements = [ `CREATE TABLE IF NOT EXISTS memories ( id TEXT PRIMARY KEY, timeline TEXT NOT NULL, - user_id TEXT NOT NULL, + user_id TEXT, conversation_id TEXT, session_id TEXT, agent_id TEXT, @@ -64,7 +65,7 @@ const statements = [ `CREATE TABLE IF NOT EXISTS l3_world_model_scopes ( scope_key TEXT PRIMARY KEY, - user_id TEXT NOT NULL, + user_id TEXT, project_id TEXT, workspace_uri TEXT, memory_id TEXT UNIQUE REFERENCES memories(id) ON DELETE SET NULL, @@ -72,6 +73,14 @@ const statements = [ updated_at TEXT NOT NULL, CHECK (workspace_uri IS NULL OR (project_id IS NOT NULL AND length(workspace_uri) > 0)) )`, + `CREATE TABLE IF NOT EXISTS memory_l3_legacy_field_edits ( + memory_id TEXT NOT NULL REFERENCES memories(id) ON DELETE CASCADE, + field TEXT NOT NULL CHECK (field IN ('general_rules_and_safety_constraints', 'project_environment_profile', 'project_contract', 'domain_knowledge')), + deleted INTEGER NOT NULL CHECK (deleted IN (0, 1)), + edited_at TEXT NOT NULL, + change_seq INTEGER NOT NULL, + PRIMARY KEY(memory_id, field, deleted) + )`, `CREATE UNIQUE INDEX IF NOT EXISTS uq_l3_world_model_scopes_general ON l3_world_model_scopes (user_id) WHERE project_id IS NULL`, @@ -82,7 +91,7 @@ const statements = [ `CREATE TABLE IF NOT EXISTS user_memories ( id TEXT PRIMARY KEY, source_turn_id TEXT NOT NULL, - user_id TEXT NOT NULL, + user_id TEXT, memory_types_json TEXT NOT NULL DEFAULT '[]' CHECK (json_valid(memory_types_json)), content TEXT NOT NULL, normalized_user_text_hash TEXT NOT NULL, @@ -135,7 +144,7 @@ const statements = [ `CREATE TABLE IF NOT EXISTS sessions ( id TEXT PRIMARY KEY, - user_id TEXT NOT NULL, + user_id TEXT, project_id TEXT, source TEXT NOT NULL, profile_id TEXT NOT NULL, @@ -173,7 +182,7 @@ const statements = [ `CREATE TABLE IF NOT EXISTS episodes ( id TEXT PRIMARY KEY, session_id TEXT NOT NULL REFERENCES sessions(id) ON DELETE CASCADE, - user_id TEXT NOT NULL, + user_id TEXT, project_id TEXT, conversation_id TEXT, status TEXT NOT NULL DEFAULT 'open' CHECK (status IN ('open', 'processing', 'closed')), @@ -209,7 +218,7 @@ const statements = [ session_id TEXT NOT NULL REFERENCES sessions(id) ON DELETE CASCADE, episode_id TEXT REFERENCES episodes(id) ON DELETE CASCADE, turn_id TEXT NOT NULL, - user_id TEXT NOT NULL, + user_id TEXT, conversation_id TEXT, user_text TEXT, assistant_text TEXT, @@ -245,7 +254,7 @@ const statements = [ `CREATE TABLE IF NOT EXISTS feedback ( id TEXT PRIMARY KEY, - user_id TEXT NOT NULL, + user_id TEXT, project_id TEXT, conversation_id TEXT, session_id TEXT, @@ -277,7 +286,7 @@ const statements = [ id TEXT PRIMARY KEY, scope_key TEXT NOT NULL REFERENCES l3_world_model_scopes(scope_key) ON DELETE CASCADE, scope_seq INTEGER NOT NULL CHECK (scope_seq >= 1), - user_id TEXT NOT NULL, + user_id TEXT, project_id TEXT, session_id TEXT NOT NULL REFERENCES sessions(id) ON DELETE CASCADE, trigger TEXT NOT NULL CHECK (trigger IN ( @@ -323,7 +332,7 @@ const statements = [ session_id TEXT, episode_id TEXT, raw_turn_id TEXT, - user_id TEXT NOT NULL, + user_id TEXT, project_id TEXT, context_hash TEXT, issue TEXT NOT NULL, @@ -346,7 +355,7 @@ const statements = [ `CREATE TABLE IF NOT EXISTS l2_candidate_pool ( id TEXT PRIMARY KEY, - user_id TEXT NOT NULL, + user_id TEXT, session_id TEXT, source_memory_id TEXT NOT NULL, candidate_key TEXT NOT NULL, @@ -365,7 +374,7 @@ const statements = [ `CREATE TABLE IF NOT EXISTS trace_policy_links ( id TEXT PRIMARY KEY, - user_id TEXT NOT NULL, + user_id TEXT, l1_memory_id TEXT NOT NULL, l2_memory_id TEXT NOT NULL, relation TEXT NOT NULL DEFAULT 'supports', @@ -380,7 +389,7 @@ const statements = [ `CREATE TABLE IF NOT EXISTS skill_trials ( id TEXT PRIMARY KEY, - user_id TEXT NOT NULL, + user_id TEXT, project_id TEXT, skill_memory_id TEXT NOT NULL, session_id TEXT, @@ -410,7 +419,7 @@ const statements = [ `CREATE TABLE IF NOT EXISTS skill_clusters ( id TEXT PRIMARY KEY, - user_id TEXT NOT NULL, + user_id TEXT, project_id TEXT, tools_json TEXT NOT NULL DEFAULT '[]' CHECK (json_valid(tools_json)), artifacts_json TEXT NOT NULL DEFAULT '[]' CHECK (json_valid(artifacts_json)), @@ -446,7 +455,7 @@ const statements = [ session_id TEXT, episode_id TEXT, turn_id TEXT, - user_id TEXT NOT NULL, + user_id TEXT, query TEXT NOT NULL, query_hash TEXT, layers_json TEXT NOT NULL DEFAULT '[]' CHECK (json_valid(layers_json)), @@ -488,7 +497,7 @@ const statements = [ kind TEXT, op TEXT, entity_id TEXT, - user_id TEXT NOT NULL, + user_id TEXT, change_type TEXT NOT NULL, version INTEGER, before_json TEXT, @@ -510,7 +519,7 @@ const statements = [ )`, `CREATE TABLE IF NOT EXISTS source_turn_captures ( - user_id TEXT NOT NULL, + user_id TEXT, source TEXT NOT NULL, profile_id TEXT NOT NULL, namespace_key TEXT NOT NULL, @@ -529,6 +538,8 @@ const statements = [ )`, `CREATE INDEX IF NOT EXISTS idx_source_turn_captures_conversation ON source_turn_captures (user_id, source, profile_id, namespace_key, conversation_id, completed_at DESC)`, + `CREATE INDEX IF NOT EXISTS idx_source_turn_captures_turn + ON source_turn_captures (turn_id)`, `CREATE INDEX IF NOT EXISTS idx_source_turn_captures_native_turn ON source_turn_captures (source, profile_id, namespace_key, conversation_id, turn_id)`, `CREATE INDEX IF NOT EXISTS idx_source_turn_captures_native_conversation @@ -539,7 +550,7 @@ const statements = [ ON sessions (source, profile_id, host_session_key)`, `CREATE TABLE IF NOT EXISTS memory_capture_claims ( - user_id TEXT NOT NULL, + user_id TEXT, source TEXT NOT NULL, qa_hash TEXT NOT NULL, primary_memory_id TEXT NOT NULL, @@ -551,7 +562,7 @@ const statements = [ ON memory_capture_claims (primary_memory_id)`, `CREATE TABLE IF NOT EXISTS l3_world_model_project_environment_state ( - user_id TEXT NOT NULL, + user_id TEXT, project_id TEXT NOT NULL, project_kind TEXT NOT NULL DEFAULT 'unknown' CHECK (project_kind IN ('unknown', 'code', 'folder')), status TEXT NOT NULL DEFAULT 'uninitialized' CHECK (status IN ( @@ -573,7 +584,7 @@ const statements = [ status TEXT NOT NULL DEFAULT 'queued' CHECK (status IN ('queued', 'leased', 'succeeded', 'failed', 'dead_letter')), dedupe_key TEXT, - user_id TEXT NOT NULL, + user_id TEXT, session_id TEXT, episode_id TEXT, target_memory_id TEXT, @@ -650,7 +661,7 @@ const statements = [ session_id TEXT, episode_id TEXT, raw_turn_id TEXT, - user_id TEXT NOT NULL, + user_id TEXT, kind TEXT NOT NULL, uri TEXT, payload_json TEXT NOT NULL DEFAULT '{}' CHECK (json_valid(payload_json)), @@ -667,7 +678,7 @@ const statements = [ `CREATE TABLE IF NOT EXISTS audit_logs ( id TEXT PRIMARY KEY, - user_id TEXT NOT NULL, + user_id TEXT, session_id TEXT, actor_json TEXT NOT NULL DEFAULT '{}' CHECK (json_valid(actor_json)), action TEXT NOT NULL, @@ -695,7 +706,7 @@ export function migrate(db: Database.Database): void { const hasMemories = tableExists(db, "memories"); const version = currentSchemaVersion(db); - if (hasMemories && version !== SCHEMA_VERSION && version !== 2 && version !== 3 && version !== 4 && version !== 5 && version !== 6 && version !== 7 && version !== 8 && version !== 9) { + if (hasMemories && version !== SCHEMA_VERSION && version !== 2 && version !== 3 && version !== 4 && version !== 5 && version !== 6 && version !== 7 && version !== 8 && version !== 9 && version !== 10) { throw new Error( `Unsupported memory database schema version ${version}; the database was left unchanged` ); @@ -717,6 +728,7 @@ export function migrate(db: Database.Database): void { addColumnIfMissing(db, "evolution_jobs", "scope_key", "TEXT"); addColumnIfMissing(db, "evolution_jobs", "scope_seq", "INTEGER"); } + if (hasMemories && version < 11) relaxLegacyUserColumns(db); // Rebuild only the parent table, with foreign keys disabled outside this // transaction. Preserve rowid: it is the tie-breaker for turn ordering. if (tableExists(db, "raw_turns") && (db.prepare("PRAGMA table_info(raw_turns)").all() as Array<{ name: string; notnull: number }>).some(c => c.name === "episode_id" && c.notnull)) { @@ -750,6 +762,32 @@ export function migrate(db: Database.Database): void { WHERE dedupe_key IS NOT NULL AND status IN ('queued', 'leased', 'failed')` ).run(); + db.exec(` + CREATE TABLE IF NOT EXISTS memory_project_aliases ( + legacy_id TEXT PRIMARY KEY, project_id TEXT NOT NULL, workspace_uri TEXT NOT NULL, + workspace_host_id TEXT, created_at TEXT NOT NULL + ); + CREATE INDEX IF NOT EXISTS idx_memory_project_alias_target ON memory_project_aliases(project_id); + CREATE TABLE IF NOT EXISTS memory_namespace_aliases ( + legacy_key TEXT PRIMARY KEY, namespace_key TEXT NOT NULL + ); + CREATE TABLE IF NOT EXISTS memory_l3_overrides ( + project_key TEXT NOT NULL, field TEXT NOT NULL, memory_id TEXT, deleted INTEGER NOT NULL DEFAULT 0, + PRIMARY KEY(project_key, field) + ); + CREATE UNIQUE INDEX IF NOT EXISTS uq_device_l3_scope + ON l3_world_model_scopes(COALESCE(project_id, '')) WHERE user_id IS NULL; + CREATE UNIQUE INDEX IF NOT EXISTS uq_device_project_environment + ON l3_world_model_project_environment_state(project_id) WHERE user_id IS NULL; + CREATE UNIQUE INDEX IF NOT EXISTS uq_device_capture_claim ON memory_capture_claims(source, qa_hash) WHERE user_id IS NULL; + CREATE UNIQUE INDEX IF NOT EXISTS uq_device_source_turn ON source_turn_captures(source, profile_id, namespace_key, conversation_id, turn_id) WHERE user_id IS NULL; + CREATE INDEX IF NOT EXISTS idx_raw_turns_turn ON raw_turns(turn_id); + CREATE INDEX IF NOT EXISTS idx_memory_native_turn ON memories(memory_layer, + COALESCE(json_extract(info_json, '$.turn_id'), json_extract(properties_json, '$.internal_info.trace.turn_id'))); + CREATE INDEX IF NOT EXISTS idx_user_memories_device_text ON user_memories(normalized_user_text_hash, status, updated_at); + + `); + // This boundary belongs to the Memory database and is shared by Hook and scanners. db.prepare(`INSERT INTO runtime_kv (key, value_json, updated_at) VALUES ('source_turn_capture_activated_at', ?, ?) @@ -766,6 +804,7 @@ export function migrate(db: Database.Database): void { if (hasMemories && version > 0 && version < 7) { backfillMemoryCaptureClaims(db); } + if (version < 11) preserveLegacyL3FieldEdits(db); db.prepare( `INSERT INTO schema_migrations (id, version, applied_at, checksum) @@ -781,6 +820,34 @@ export function migrate(db: Database.Database): void { } } +/** SQLite cannot drop NOT NULL in place. Copy rows verbatim; never rewrite legacy identities or content. */ +function relaxLegacyUserColumns(db: Database.Database): void { + const sequences = db.prepare("SELECT name, seq FROM sqlite_sequence").all() as Array<{name:string;seq:number}>; + const tables = db.prepare("SELECT name, sql FROM sqlite_master WHERE type = 'table' AND sql IS NOT NULL") + .all() as Array<{ name: string; sql: string }>; + for (const { name, sql } of tables) { + if (!/\buser_id TEXT NOT NULL\b/i.test(sql)) continue; + const dependent = db.prepare("SELECT sql FROM sqlite_master WHERE tbl_name = ? AND type IN ('index', 'trigger') AND sql IS NOT NULL") + .all(name) as Array<{ sql: string }>; + const temporary = `${name}_device_upgrade`; + const definition = sql.replace(/^(CREATE TABLE(?: IF NOT EXISTS)?\s+)(?:"[^"]+"|`[^`]+`|\w+)/i, `$1"${temporary}"`) + .replace(/\buser_id TEXT NOT NULL\b/gi, "user_id TEXT"); + db.exec(definition); + const columns = (db.prepare(`PRAGMA table_info("${name}")`).all() as Array<{ name: string }>).map(column => `"${column.name}"`).join(", "); + db.exec(`INSERT INTO "${temporary}" (rowid, ${columns}) SELECT rowid, ${columns} FROM "${name}"`); + db.exec(`DROP TABLE "${name}"`); + db.exec(`ALTER TABLE "${temporary}" RENAME TO "${name}"`); + for (const item of dependent) db.exec(item.sql); + } + for (const row of sequences) { + const exists = db.prepare("SELECT 1 FROM sqlite_sequence WHERE name = ?").get(row.name); + if (exists) db.prepare("UPDATE sqlite_sequence SET seq = MAX(seq, ?) WHERE name = ?").run(row.seq,row.name); + else db.prepare("INSERT INTO sqlite_sequence(name,seq) VALUES (?,?)").run(row.name,row.seq); + } + const violations = db.pragma("foreign_key_check") as unknown[]; + if (violations.length) throw new Error("Memory scope schema upgrade failed foreign-key validation"); +} + function backfillMemoryCaptureClaims(db: Database.Database): void { const rows = db.prepare( `SELECT raw_turns.user_id, @@ -881,7 +948,6 @@ function backfillLegacyAdapterHostSessionKeys(db: Database.Database, now: string FROM sessions AS existing WHERE existing.id != candidate.id AND existing.status = 'open' - AND existing.user_id = candidate.user_id AND existing.source = candidate.source AND existing.profile_id = candidate.profile_id AND existing.host_session_key = candidate.id diff --git a/Memory/src/types.ts b/Memory/src/types.ts index 7d87fa03e..c69eabb11 100644 --- a/Memory/src/types.ts +++ b/Memory/src/types.ts @@ -101,7 +101,7 @@ export interface RuntimeNamespace { workspaceId?: string; workspacePath?: string; sessionKey?: string; - userId?: string; + userId?: string | null; tenantId?: string; } @@ -139,7 +139,7 @@ export interface JobRef { export interface MemoryRow { id: string; timeline: IsoTime; - userId: string; + userId: string | null; conversationId?: string; sessionId?: string; agentId?: string; @@ -186,7 +186,7 @@ export interface MemoryStatsRow { } export interface MemoryFilter { - userId?: string; + userId?: string | null; sessionId?: string; conversationId?: string; agentId?: string; @@ -198,8 +198,9 @@ export interface MemoryFilter { status?: MemoryStatus | MemoryStatus[]; tags?: string[]; ids?: string[]; + excludedIds?: string[]; memoryKind?: MemoryKind | MemoryKind[]; - workMemoryUserId?: string; + workMemoryUserId?: string | null; workMemoryProjectId?: string | null; } @@ -436,7 +437,7 @@ export type UserMemoryStatus = "active" | "archived" | "deleted"; export interface UserMemoryRecord { id: string; sourceTurnId: string; - userId: string; + userId: string | null; memoryTypes: UserMemoryType[]; content: string; normalizedUserTextHash: string; diff --git a/Memory/tests/cli-command-map.test.ts b/Memory/tests/cli-command-map.test.ts index 9f3222953..a023612be 100644 --- a/Memory/tests/cli-command-map.test.ts +++ b/Memory/tests/cli-command-map.test.ts @@ -436,7 +436,7 @@ describe("memmy CLI command map", () => { fetch: mockFetch(requests) }); - expect(requests[0]?.headers["x-memmy-user-id"]).toBe("user_from_memory"); + expect(requests[0]?.headers["x-memmy-user-id"]).toBeUndefined(); }); it("uses the flat memory userId even when obsolete profile fields remain", async () => { @@ -464,7 +464,7 @@ describe("memmy CLI command map", () => { fetch: mockFetch(requests) }); - expect(requests[0]?.headers["x-memmy-user-id"]).toBe("user_from_flat_memory"); + expect(requests[0]?.headers["x-memmy-user-id"]).toBeUndefined(); }); it("lets explicit user-id override memmyMemory.userId from config", async () => { diff --git a/Memory/tests/cli-setup.test.ts b/Memory/tests/cli-setup.test.ts index b69d777fd..ce58a7d00 100644 --- a/Memory/tests/cli-setup.test.ts +++ b/Memory/tests/cli-setup.test.ts @@ -54,7 +54,6 @@ describe("memmy-memory CLI setup commands", () => { expect(saved).toMatchObject({ memmyMemory: { version: 1, - userId: "local-user", roleRouting: { summary: "follow", evolution: "follow" @@ -410,7 +409,6 @@ describe("memmy-memory CLI setup commands", () => { expect(saved.memmyMemory).toMatchObject({ futureMemoryField: "keep-memory", version: 1, - userId: "user_123", roleRouting: { summary: "follow", evolution: "follow" diff --git a/Memory/tests/codex-native-hook.test.ts b/Memory/tests/codex-native-hook.test.ts index c1c599045..058b321ae 100644 --- a/Memory/tests/codex-native-hook.test.ts +++ b/Memory/tests/codex-native-hook.test.ts @@ -11,7 +11,7 @@ import { loadMemmyWorkspaceBridgeRuntimeAsset } from "../src/agent-source/integr const directories: string[] = []; afterEach(() => { for (const dir of directories.splice(0)) rmSync(dir, { recursive: true, force: true }); }); const createdAt = "2026-09-09T10:00:00.000Z"; -const expectedNamespace = { source: "codex", profileId: "default", userId: "fixture-owner", sessionKey: "native-session" }; +const expectedNamespace = { source: "codex", profileId: "default", userId: "local-user", sessionKey: "native-session" }; function record(type: string, payload: unknown) { return JSON.stringify({ type, timestamp: createdAt, payload }); } function transcript(complete = true) { return [record("session_meta", { id: "native-session", cwd: "/project" }), record("event_msg", { type: "task_started", turn_id: "native-turn" }), diff --git a/Memory/tests/config.test.ts b/Memory/tests/config.test.ts index 22f64bda0..d3106032b 100644 --- a/Memory/tests/config.test.ts +++ b/Memory/tests/config.test.ts @@ -200,7 +200,7 @@ describe("memmy memory config", () => { expect(config.storage.sqlitePath).toBe(join(homedir(), ".memmy", "memory-service", "memory.sqlite")); }); - it("reads user id from memmyMemory config and environment aliases", () => { + it("ignores memory user id from config and environment aliases", () => { const root = tempRoot(); const configPath = join(root, "config.yaml"); writeFileSync(configPath, YAML.stringify({ @@ -212,10 +212,10 @@ describe("memmy memory config", () => { } })); - expect(loadMemmyConfig(configPath).config.userId).toBe("user_from_memory"); + expect(loadMemmyConfig(configPath).config.userId).toBeUndefined(); setEnv("MEMMY_MEMORY_USER_ID", "user_from_env"); - expect(loadMemmyConfig(configPath).config.userId).toBe("user_from_env"); + expect(loadMemmyConfig(configPath).config.userId).toBeUndefined(); }); it("reads memory gates from memmyMemory algorithm config", () => { @@ -400,7 +400,7 @@ describe("memmy memory config", () => { const { config } = loadMemmyConfig(configPath); expect(config.roleRouting).toEqual({ summary: "fixed", evolution: "fixed" }); - expect(config.userId).toBe("user_account"); + expect(config.userId).toBeUndefined(); expect(config.summary).toMatchObject({ provider: "openai_compatible", sourceProvider: "memmy_account", diff --git a/Memory/tests/contract/external-evidence-http.test.ts b/Memory/tests/contract/external-evidence-http.test.ts index 719f3bd46..14b2edcf4 100644 --- a/Memory/tests/contract/external-evidence-http.test.ts +++ b/Memory/tests/contract/external-evidence-http.test.ts @@ -80,10 +80,10 @@ describe("external passive evidence", () => { const anotherNamespace = { ...namespace, userId: "another-user" }; expect(service.externalEvidenceStatus({ source: evidence.source, sourceRecordIds: [], skillIds: [candidate.id!], namespace: anotherNamespace }).skills).toEqual([ - { id: candidate.id, status: "missing" } + { id: candidate.id, status: "activated" } ]); expect(() => service.approveExternalEvidenceSkill(candidate.id!, { namespace: anotherNamespace })) - .toThrow(/not found/u); + .toThrow(/no longer awaiting review/u); expect(() => service.approveExternalEvidenceSkill(candidate.id!, { namespace })).toThrow(/no longer awaiting review/u); expect(isMemoryReadyForRetrieval(repos.memories.get(candidate.id!)!)).toBe(true); service.deleteMemory(first.id, { namespace }); diff --git a/Memory/tests/contract/external-evidence-source-http.test.ts b/Memory/tests/contract/external-evidence-source-http.test.ts index 79bb45ca0..5ff9a3e10 100644 --- a/Memory/tests/contract/external-evidence-source-http.test.ts +++ b/Memory/tests/contract/external-evidence-source-http.test.ts @@ -267,7 +267,7 @@ describe("external evidence producer identity over HTTP", () => { expect(repos.memories.get(sourceB.id)?.status).toBe("activated"); const tombstone = service.syncExternalEvidence({ action: "delete", source: "computer_history", namespace: scopeA, sourceRecordId: "2026-09-27T11-00-00Z-10min-summary", revision: record.revision }); - expect(externalEvidenceNamespaceForMemory(repos.memories.getIncludingDeleted(tombstone.id)!)).toMatchObject(scopeA); + expect(externalEvidenceNamespaceForMemory(repos.memories.getIncludingDeleted(tombstone.id)!)).toMatchObject({ ...scopeA, userId: null }); }); it("keeps Skill review and status in the source tenant", async () => { diff --git a/Memory/tests/contract/source-turn-http.test.ts b/Memory/tests/contract/source-turn-http.test.ts index 0864a21ea..fd7b51303 100644 --- a/Memory/tests/contract/source-turn-http.test.ts +++ b/Memory/tests/contract/source-turn-http.test.ts @@ -33,7 +33,7 @@ describe("source turn HTTP contract", () => { expect(bodies.map((body) => body.status).sort()).toEqual(["existing", "stored"]); expect(bodies[0]!.result.rawTurnId).toBe(bodies[1]!.result.rawTurnId); expect(bodies[0]!.result.l1MemoryId).toBe(bodies[1]!.result.l1MemoryId); - expect(new Repositories(db.db).runtime.getSession(bodies[0]!.result.sessionId)).toMatchObject({ userId: "http-user", source: "codex", profileId: "default" }); + expect(new Repositories(db.db).runtime.getSession(bodies[0]!.result.sessionId)).toMatchObject({ userId: null, source: "codex", profileId: "default" }); const forbidden = await submit({ ...payload, sourceTurn: { ...payload.sourceTurn, profileId: "foreign-profile" } }); expect(forbidden.status).toBe(403); const foreignUser = await submit({ ...payload, namespace: { source: "codex", profileId: "default", userId: "foreign-user" } }); @@ -73,7 +73,7 @@ describe("source turn HTTP contract", () => { expect(first.result.sessionId).toBe(opened.sessionId); expect(second.result.l1MemoryId).toBe(first.result.l1MemoryId); const repos = new Repositories(db.db); - expect(repos.memories.get(first.result.l1MemoryId)).toMatchObject({ userId: "installed-owner", sessionId: opened.sessionId, + expect(repos.memories.get(first.result.l1MemoryId)).toMatchObject({ userId: null, sessionId: opened.sessionId, info: { project_id: repos.runtime.getSession(opened.sessionId)!.projectId } }); for (const table of ["sessions", "episodes", "raw_turns", "source_turn_captures"]) { expect(db.db.prepare(`SELECT COUNT(*) AS n FROM ${table}`).get()).toEqual({ n: 1 }); diff --git a/Memory/tests/embedding-maintenance.test.ts b/Memory/tests/embedding-maintenance.test.ts index 69e6321d7..8dfdae49c 100644 --- a/Memory/tests/embedding-maintenance.test.ts +++ b/Memory/tests/embedding-maintenance.test.ts @@ -137,18 +137,18 @@ describe("embedding maintenance regressions (MEMMY-709–711)", () => { expect(repos.vectors.exportRows()).toHaveLength(0); }); - it("includes only this account's active user memories and filters their old-model vectors", async () => { + it("includes device-visible active user memories and filters vectors by model identity", async () => { const { repos, service } = setup(); for (const [id, userId, current] of [["healthy", "local-user", true], ["old", "local-user", false], ["foreign", "other-user", false]] as const) { repos.userMemories.insert({ ...buildUserMemory({ id, sourceTurnId: id, userId, memoryTypes: ["User Preference"], content: `I prefer ${id}`, createdAt: at }), embedding: vector(), embeddingModel: current ? identity.model : "old-model", embeddingProvider: identity.provider }); } - expect(await service.embeddingMaintenanceStats()).toMatchObject({ totalSlots: 2, ready: 1, modelMismatch: 1 }); + expect(await service.embeddingMaintenanceStats()).toMatchObject({ totalSlots: 3, ready: 1, modelMismatch: 2 }); expect(repos.userMemories.searchVectorIds("local-user", rememberEmbeddingIdentity(vector(), identity), 10).map(hit => hit.id)).toEqual(["healthy"]); const run = await service.rebuildEmbeddings("repair"); await service.runWorkerOnce(200); - expect(service.embeddingMaintenanceProgress(run.runId)).toMatchObject({ enqueued: 1, updated: 1, failed: 0, done: true }); - expect(repos.userMemories.get("foreign")!.embeddingModel).toBe("old-model"); + expect(service.embeddingMaintenanceProgress(run.runId)).toMatchObject({ enqueued: 2, updated: 2, failed: 0, done: true }); + expect(repos.userMemories.get("foreign")!.embeddingModel).toBe(identity.model); }); it("keeps in-flight vector provenance when settings change and stops claiming the run completed", async () => { diff --git a/Memory/tests/fixtures/schema-v9.sql b/Memory/tests/fixtures/schema-v9.sql new file mode 100644 index 000000000..d6c1d3f48 --- /dev/null +++ b/Memory/tests/fixtures/schema-v9.sql @@ -0,0 +1,641 @@ +-- Schema v9 from upstream 5326b303, used only to test preservation on upgrade. +CREATE TABLE schema_migrations ( + id TEXT PRIMARY KEY, + version INTEGER NOT NULL, + applied_at TEXT NOT NULL, + checksum TEXT NOT NULL + ); +CREATE TABLE memories ( + id TEXT PRIMARY KEY, + timeline TEXT NOT NULL, + user_id TEXT NOT NULL, + conversation_id TEXT, + session_id TEXT, + agent_id TEXT, + app_id TEXT, + memory_type TEXT NOT NULL DEFAULT 'LongTermMemory', + status TEXT NOT NULL DEFAULT 'activated' + CHECK (status IN ('activated', 'resolving', 'archived', 'deleted')), + visibility TEXT NOT NULL DEFAULT 'private', + memory_key TEXT, + memory_value TEXT NOT NULL, + tags_json TEXT NOT NULL DEFAULT '[]' CHECK (json_valid(tags_json)), + info_json TEXT NOT NULL DEFAULT '{}' CHECK (json_valid(info_json)), + properties_json TEXT NOT NULL DEFAULT '{}' CHECK (json_valid(properties_json)), + memory_layer TEXT NOT NULL CHECK (memory_layer IN ('L1', 'L2', 'L3', 'Skill')), + content_hash TEXT, + version INTEGER NOT NULL DEFAULT 1, + created_at TEXT NOT NULL, + updated_at TEXT NOT NULL, + deleted_at TEXT + ); +CREATE INDEX idx_memories_layer_status_updated + ON memories (memory_layer, status, updated_at DESC); +CREATE INDEX idx_memories_layer_status_created + ON memories (memory_layer, status, created_at DESC); +CREATE INDEX idx_memories_conversation_updated + ON memories (conversation_id, updated_at DESC); +CREATE INDEX idx_memories_session_layer + ON memories (session_id, memory_layer, updated_at DESC); +CREATE INDEX idx_memories_agent_app + ON memories (agent_id, app_id, updated_at DESC); +CREATE INDEX idx_memories_content_hash_layer + ON memories (content_hash, memory_layer); +CREATE INDEX idx_memories_key_layer + ON memories (memory_key, memory_layer); +CREATE TABLE l3_world_model_scopes ( + scope_key TEXT PRIMARY KEY, + user_id TEXT NOT NULL, + project_id TEXT, + workspace_uri TEXT, + memory_id TEXT UNIQUE REFERENCES memories(id) ON DELETE SET NULL, + next_scope_seq INTEGER NOT NULL DEFAULT 1 CHECK (next_scope_seq >= 1), + updated_at TEXT NOT NULL, + CHECK (workspace_uri IS NULL OR (project_id IS NOT NULL AND length(workspace_uri) > 0)) + ); +CREATE UNIQUE INDEX uq_l3_world_model_scopes_general + ON l3_world_model_scopes (user_id) + WHERE project_id IS NULL; +CREATE UNIQUE INDEX uq_l3_world_model_scopes_project + ON l3_world_model_scopes (user_id, project_id) + WHERE project_id IS NOT NULL; +CREATE TABLE user_memories ( + id TEXT PRIMARY KEY, + source_turn_id TEXT NOT NULL, + user_id TEXT NOT NULL, + memory_types_json TEXT NOT NULL DEFAULT '[]' CHECK (json_valid(memory_types_json)), + content TEXT NOT NULL, + normalized_user_text_hash TEXT NOT NULL, + source_turn_refs_json TEXT NOT NULL DEFAULT '[]' CHECK (json_valid(source_turn_refs_json)), + status TEXT NOT NULL DEFAULT 'active' CHECK (status IN ('active', 'archived', 'deleted')), + replaces_memory_id TEXT, + replaced_by_memory_id TEXT, + archived_at TEXT, + archive_reason TEXT, + embedding_json TEXT CHECK (embedding_json IS NULL OR json_valid(embedding_json)), + embedding_model TEXT, + embedding_provider TEXT, + created_at TEXT NOT NULL, + updated_at TEXT NOT NULL, + deleted_at TEXT + ); +CREATE INDEX idx_user_memories_user_status_updated + ON user_memories (user_id, status, updated_at DESC); +CREATE INDEX idx_user_memories_source_turn + ON user_memories (source_turn_id); +CREATE INDEX idx_user_memories_exact_text + ON user_memories (user_id, normalized_user_text_hash, status, updated_at DESC); +CREATE VIRTUAL TABLE user_memories_fts USING fts5 ( + id UNINDEXED, + content, + memory_types, + tokenize='unicode61' + ); +CREATE VIRTUAL TABLE memories_fts USING fts5 ( + id UNINDEXED, + identifier, + memory_value, + tags, + tokenize='unicode61' + ); +CREATE TABLE memory_vector_entries ( + id INTEGER PRIMARY KEY, + memory_id TEXT NOT NULL REFERENCES memories(id) ON DELETE CASCADE, + vector_field TEXT NOT NULL CHECK (vector_field IN ('vec_summary', 'vec_action', 'vec')), + embedding_model TEXT, + embedding_provider TEXT, + embedding_dim INTEGER NOT NULL, + updated_at TEXT NOT NULL, + UNIQUE (memory_id, vector_field) + ); +CREATE INDEX idx_memory_vector_entries_field_updated + ON memory_vector_entries (vector_field, updated_at DESC, id DESC); +CREATE TABLE sessions ( + id TEXT PRIMARY KEY, + user_id TEXT NOT NULL, + project_id TEXT, + source TEXT NOT NULL, + profile_id TEXT NOT NULL, + profile_label TEXT, + workspace_id TEXT, + workspace_path TEXT, + host_session_key TEXT, + conversation_id TEXT, + status TEXT NOT NULL DEFAULT 'open' CHECK (status IN ('open', 'processing', 'closed')), + meta_json TEXT NOT NULL DEFAULT '{}' CHECK (json_valid(meta_json)), + opened_at TEXT NOT NULL, + last_seen_at TEXT NOT NULL, + closed_at TEXT, + updated_at TEXT NOT NULL + ); +CREATE INDEX idx_sessions_user_updated + ON sessions (user_id, updated_at DESC); +CREATE INDEX idx_sessions_host_key + ON sessions (host_session_key); +CREATE INDEX idx_sessions_host_scope + ON sessions (user_id, source, profile_id, host_session_key, status); +CREATE TABLE l3_world_model_session_cursors ( + session_id TEXT PRIMARY KEY REFERENCES sessions(id) ON DELETE CASCADE, + last_scheduled_seq INTEGER NOT NULL DEFAULT 0 CHECK (last_scheduled_seq >= 0), + updated_at TEXT NOT NULL + ); +CREATE TABLE work_memory_session_cursors ( + session_id TEXT PRIMARY KEY REFERENCES sessions(id) ON DELETE CASCADE, + last_extracted_seq INTEGER NOT NULL DEFAULT 0 CHECK (last_extracted_seq >= 0), + updated_at TEXT NOT NULL + ); +CREATE TABLE episodes ( + id TEXT PRIMARY KEY, + session_id TEXT NOT NULL REFERENCES sessions(id) ON DELETE CASCADE, + user_id TEXT NOT NULL, + project_id TEXT, + conversation_id TEXT, + status TEXT NOT NULL DEFAULT 'open' CHECK (status IN ('open', 'processing', 'closed')), + title TEXT, + summary TEXT, + l1_memory_ids_json TEXT NOT NULL DEFAULT '[]' CHECK (json_valid(l1_memory_ids_json)), + raw_turn_ids_json TEXT NOT NULL DEFAULT '[]' CHECK (json_valid(raw_turn_ids_json)), + feedback_ids_json TEXT NOT NULL DEFAULT '[]' CHECK (json_valid(feedback_ids_json)), + decision_repair_ids_json TEXT NOT NULL DEFAULT '[]' CHECK (json_valid(decision_repair_ids_json)), + l2_policy_ids_json TEXT NOT NULL DEFAULT '[]' CHECK (json_valid(l2_policy_ids_json)), + l3_world_model_ids_json TEXT NOT NULL DEFAULT '[]' CHECK (json_valid(l3_world_model_ids_json)), + skill_memory_ids_json TEXT NOT NULL DEFAULT '[]' CHECK (json_valid(skill_memory_ids_json)), + turn_count INTEGER NOT NULL DEFAULT 0, + r_task REAL, + reward_detail_json TEXT NOT NULL DEFAULT '{}' CHECK (json_valid(reward_detail_json)), + pipeline_run_id TEXT, + pipeline_status TEXT NOT NULL DEFAULT 'idle' CHECK (pipeline_status IN ('idle', 'running', 'succeeded', 'failed')), + pipeline_error TEXT, + meta_json TEXT NOT NULL DEFAULT '{}' CHECK (json_valid(meta_json)), + opened_at TEXT NOT NULL, + closed_at TEXT, + updated_at TEXT NOT NULL + ); +CREATE INDEX idx_episodes_session_updated + ON episodes (session_id, updated_at DESC); +CREATE INDEX idx_episodes_project_updated + ON episodes (project_id, updated_at DESC); +CREATE INDEX idx_episodes_pipeline + ON episodes (pipeline_status, updated_at DESC); +CREATE TABLE raw_turns ( + id TEXT PRIMARY KEY, + session_id TEXT NOT NULL REFERENCES sessions(id) ON DELETE CASCADE, + episode_id TEXT NOT NULL REFERENCES episodes(id) ON DELETE CASCADE, + turn_id TEXT NOT NULL, + user_id TEXT NOT NULL, + conversation_id TEXT, + user_text TEXT, + assistant_text TEXT, + reasoning_summary TEXT, + tool_calls_json TEXT NOT NULL DEFAULT '[]' CHECK (json_valid(tool_calls_json)), + tool_results_json TEXT NOT NULL DEFAULT '[]' CHECK (json_valid(tool_results_json)), + source_memory_ids_json TEXT NOT NULL DEFAULT '[]' CHECK (json_valid(source_memory_ids_json)), + usage_json TEXT NOT NULL DEFAULT '{}' CHECK (json_valid(usage_json)), + message_payload_json TEXT NOT NULL DEFAULT '{}' CHECK (json_valid(message_payload_json)), + status TEXT NOT NULL DEFAULT 'succeeded', + redacted_at TEXT, + deleted_at TEXT, + created_at TEXT NOT NULL, + UNIQUE (session_id, turn_id) + ); +CREATE INDEX idx_raw_turns_episode_created + ON raw_turns (episode_id, created_at ASC); +CREATE INDEX idx_raw_turns_user_turn + ON raw_turns (user_id, turn_id); +CREATE TABLE l3_world_model_input_traces ( + session_id TEXT NOT NULL REFERENCES sessions(id) ON DELETE CASCADE, + trace_seq INTEGER NOT NULL CHECK (trace_seq >= 1), + l1_memory_id TEXT NOT NULL REFERENCES memories(id) ON DELETE CASCADE, + raw_turn_id TEXT NOT NULL REFERENCES raw_turns(id) ON DELETE CASCADE, + episode_id TEXT REFERENCES episodes(id) ON DELETE SET NULL, + created_at TEXT NOT NULL, + PRIMARY KEY (session_id, trace_seq), + UNIQUE (session_id, l1_memory_id) + ); +CREATE INDEX idx_l3_world_model_input_traces_raw_turn + ON l3_world_model_input_traces (raw_turn_id, session_id, trace_seq); +CREATE TABLE feedback ( + id TEXT PRIMARY KEY, + user_id TEXT NOT NULL, + project_id TEXT, + conversation_id TEXT, + session_id TEXT, + episode_id TEXT, + l1_memory_id TEXT, + raw_turn_id TEXT, + channel TEXT NOT NULL CHECK (channel IN ('explicit', 'implicit')), + polarity TEXT NOT NULL CHECK (polarity IN ('positive', 'negative', 'neutral')), + magnitude REAL NOT NULL DEFAULT 1, + rationale TEXT, + raw_payload_json TEXT NOT NULL DEFAULT '{}' CHECK (json_valid(raw_payload_json)), + context_hash TEXT, + created_at TEXT NOT NULL + ); +CREATE INDEX idx_feedback_session_created + ON feedback (session_id, created_at DESC); +CREATE INDEX idx_feedback_memory + ON feedback (l1_memory_id, created_at DESC); +CREATE INDEX idx_feedback_user_created + ON feedback (user_id, created_at DESC); +CREATE INDEX idx_feedback_episode_created + ON feedback (episode_id, created_at DESC); +CREATE INDEX idx_feedback_raw_turn_created + ON feedback (raw_turn_id, created_at DESC); +CREATE INDEX idx_feedback_context + ON feedback (user_id, project_id, context_hash, created_at DESC); +CREATE TABLE l3_world_model_evidence_batches ( + id TEXT PRIMARY KEY, + scope_key TEXT NOT NULL REFERENCES l3_world_model_scopes(scope_key) ON DELETE CASCADE, + scope_seq INTEGER NOT NULL CHECK (scope_seq >= 1), + user_id TEXT NOT NULL, + project_id TEXT, + session_id TEXT NOT NULL REFERENCES sessions(id) ON DELETE CASCADE, + trigger TEXT NOT NULL CHECK (trigger IN ( + 'new_task', 'token_compaction', 'token_compaction_attempt', 'session_close', 'episode_idle_close' + )), + start_trace_seq INTEGER NOT NULL CHECK (start_trace_seq >= 1), + end_trace_seq INTEGER NOT NULL CHECK (end_trace_seq >= start_trace_seq), + l1_memory_ids_json TEXT NOT NULL CHECK (json_valid(l1_memory_ids_json)), + raw_turn_ids_json TEXT NOT NULL CHECK (json_valid(raw_turn_ids_json)), + feedback_ids_json TEXT NOT NULL DEFAULT '[]' CHECK (json_valid(feedback_ids_json)), + payload_hash TEXT NOT NULL, + terminal_outcome TEXT CHECK (terminal_outcome IS NULL OR terminal_outcome IN ( + 'applied', 'partial_dead_letter', 'dead_letter' + )), + completed_at TEXT, + created_at TEXT NOT NULL, + updated_at TEXT NOT NULL, + UNIQUE (scope_key, scope_seq) + ); +CREATE INDEX idx_l3_world_model_batches_session_trace + ON l3_world_model_evidence_batches (session_id, end_trace_seq); +CREATE TABLE l3_world_model_batch_targets ( + batch_id TEXT NOT NULL REFERENCES l3_world_model_evidence_batches(id) ON DELETE CASCADE, + target_field TEXT NOT NULL CHECK (target_field IN ( + 'general_rules_and_safety_constraints', 'project_contract', 'domain_knowledge' + )), + field_scope_key TEXT NOT NULL, + scope_seq INTEGER NOT NULL CHECK (scope_seq >= 1), + status TEXT NOT NULL DEFAULT 'queued' CHECK (status IN ('queued', 'applied', 'dead_letter')), + no_change INTEGER NOT NULL DEFAULT 0 CHECK (no_change IN (0, 1)), + applied_at TEXT, + updated_at TEXT NOT NULL, + PRIMARY KEY (batch_id, target_field), + UNIQUE (field_scope_key, scope_seq), + CHECK (status = 'applied' OR no_change = 0) + ); +CREATE INDEX idx_l3_world_model_targets_field_status + ON l3_world_model_batch_targets (field_scope_key, status, scope_seq); +CREATE TABLE decision_repairs ( + id TEXT PRIMARY KEY, + session_id TEXT, + episode_id TEXT, + raw_turn_id TEXT, + user_id TEXT NOT NULL, + project_id TEXT, + context_hash TEXT, + issue TEXT NOT NULL, + suggestion TEXT NOT NULL, + preference TEXT, + anti_pattern TEXT, + high_value_memory_ids_json TEXT NOT NULL DEFAULT '[]' CHECK (json_valid(high_value_memory_ids_json)), + low_value_memory_ids_json TEXT NOT NULL DEFAULT '[]' CHECK (json_valid(low_value_memory_ids_json)), + attached_policy_memory_ids_json TEXT NOT NULL DEFAULT '[]' CHECK (json_valid(attached_policy_memory_ids_json)), + feedback_id TEXT, + validated INTEGER NOT NULL DEFAULT 0 CHECK (validated IN (0, 1)), + source_json TEXT NOT NULL DEFAULT '{}' CHECK (json_valid(source_json)), + meta_json TEXT NOT NULL DEFAULT '{}' CHECK (json_valid(meta_json)), + created_at TEXT NOT NULL + ); +CREATE INDEX idx_decision_repairs_context + ON decision_repairs (user_id, project_id, context_hash); +CREATE INDEX idx_decision_repairs_episode + ON decision_repairs (episode_id, created_at DESC); +CREATE TABLE l2_candidate_pool ( + id TEXT PRIMARY KEY, + user_id TEXT NOT NULL, + session_id TEXT, + source_memory_id TEXT NOT NULL, + candidate_key TEXT NOT NULL, + candidate_value TEXT NOT NULL, + score REAL NOT NULL DEFAULT 0, + status TEXT NOT NULL DEFAULT 'pending' CHECK (status IN ('pending', 'promoted', 'rejected')), + evidence_json TEXT NOT NULL DEFAULT '[]' CHECK (json_valid(evidence_json)), + created_at TEXT NOT NULL, + updated_at TEXT NOT NULL, + expires_at TEXT + ); +CREATE INDEX idx_l2_candidate_status + ON l2_candidate_pool (user_id, status, updated_at DESC); +CREATE INDEX idx_l2_candidate_pending_expiry + ON l2_candidate_pool (user_id, candidate_key, status, expires_at); +CREATE TABLE trace_policy_links ( + id TEXT PRIMARY KEY, + user_id TEXT NOT NULL, + l1_memory_id TEXT NOT NULL, + l2_memory_id TEXT NOT NULL, + relation TEXT NOT NULL DEFAULT 'supports', + strength REAL NOT NULL DEFAULT 1, + created_at TEXT NOT NULL, + UNIQUE (l1_memory_id, l2_memory_id, relation) + ); +CREATE INDEX idx_trace_policy_links_l1 + ON trace_policy_links (user_id, l1_memory_id, created_at DESC); +CREATE INDEX idx_trace_policy_links_l2 + ON trace_policy_links (user_id, l2_memory_id, created_at DESC); +CREATE TABLE skill_trials ( + id TEXT PRIMARY KEY, + user_id TEXT NOT NULL, + project_id TEXT, + skill_memory_id TEXT NOT NULL, + session_id TEXT, + episode_id TEXT NOT NULL, + l1_memory_id TEXT, + raw_turn_id TEXT, + turn_id TEXT, + tool_call_id TEXT, + status TEXT NOT NULL DEFAULT 'pending' + CHECK (status IN ('pending', 'pass', 'fail', 'unknown')), + outcome TEXT NOT NULL DEFAULT 'unknown' + CHECK (outcome IN ('unknown', 'success', 'failure', 'cancelled')), + feedback_id TEXT, + created_at TEXT NOT NULL, + resolved_at TEXT + ); +CREATE INDEX idx_skill_trials_skill_created + ON skill_trials (skill_memory_id, created_at DESC); +CREATE INDEX idx_skill_trials_user_status + ON skill_trials (user_id, status, created_at DESC); +CREATE INDEX idx_skill_trials_episode_status + ON skill_trials (episode_id, status, created_at DESC); +CREATE INDEX idx_skill_trials_l1_status + ON skill_trials (l1_memory_id, status, created_at DESC); +CREATE INDEX idx_skill_trials_raw_status + ON skill_trials (raw_turn_id, status, created_at DESC); +CREATE TABLE skill_clusters ( + id TEXT PRIMARY KEY, + user_id TEXT NOT NULL, + project_id TEXT, + tools_json TEXT NOT NULL DEFAULT '[]' CHECK (json_valid(tools_json)), + artifacts_json TEXT NOT NULL DEFAULT '[]' CHECK (json_valid(artifacts_json)), + tool_bigrams_json TEXT NOT NULL DEFAULT '[]' CHECK (json_valid(tool_bigrams_json)), + centroid_json TEXT, + skill_memory_id TEXT, + meta_skill_md TEXT NOT NULL DEFAULT '', + processed_episode_ids_json TEXT NOT NULL DEFAULT '[]' CHECK (json_valid(processed_episode_ids_json)), + member_count INTEGER NOT NULL DEFAULT 0, + created_at TEXT NOT NULL, + updated_at TEXT NOT NULL + ); +CREATE INDEX idx_skill_clusters_scope + ON skill_clusters (user_id, project_id, updated_at DESC); +CREATE INDEX idx_skill_clusters_skill + ON skill_clusters (skill_memory_id); +CREATE TABLE skill_cluster_members ( + cluster_id TEXT NOT NULL, + episode_id TEXT NOT NULL, + outcome TEXT NOT NULL DEFAULT 'unknown' + CHECK (outcome IN ('success', 'failure', 'unknown')), + r_task REAL, + assigned_at TEXT NOT NULL, + PRIMARY KEY (cluster_id, episode_id) + ); +CREATE INDEX idx_skill_cluster_members_episode + ON skill_cluster_members (episode_id, assigned_at DESC); +CREATE TABLE recall_events ( + id TEXT PRIMARY KEY, + namespace_id TEXT, + session_id TEXT, + episode_id TEXT, + turn_id TEXT, + user_id TEXT NOT NULL, + query TEXT NOT NULL, + query_hash TEXT, + layers_json TEXT NOT NULL DEFAULT '[]' CHECK (json_valid(layers_json)), + candidate_memory_ids_json TEXT NOT NULL DEFAULT '[]' CHECK (json_valid(candidate_memory_ids_json)), + injected_memory_ids_json TEXT NOT NULL DEFAULT '[]' CHECK (json_valid(injected_memory_ids_json)), + hit_memory_ids_json TEXT NOT NULL DEFAULT '[]' CHECK (json_valid(hit_memory_ids_json)), + dropped_json TEXT NOT NULL DEFAULT '[]' CHECK (json_valid(dropped_json)), + outcome TEXT NOT NULL DEFAULT 'pending' CHECK (outcome IN ('pending', 'positive', 'negative', 'ignored')), + request_json TEXT NOT NULL DEFAULT '{}' CHECK (json_valid(request_json)), + query_id TEXT, + user_memory_candidate_ids_json TEXT NOT NULL DEFAULT '[]' CHECK (json_valid(user_memory_candidate_ids_json)), + l1_candidate_ids_json TEXT NOT NULL DEFAULT '[]' CHECK (json_valid(l1_candidate_ids_json)), + merged_source_turn_ids_json TEXT NOT NULL DEFAULT '[]' CHECK (json_valid(merged_source_turn_ids_json)), + member_memory_ids_by_source_turn_id_json TEXT NOT NULL DEFAULT '{}' CHECK (json_valid(member_memory_ids_by_source_turn_id_json)), + created_at TEXT NOT NULL + ); +CREATE INDEX idx_recall_events_session_created + ON recall_events (session_id, created_at DESC); +CREATE TABLE api_logs ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + tool_name TEXT NOT NULL CHECK (tool_name IN ('memory_add', 'memory_search', 'skill_generate', 'skill_evolve')), + source_agent TEXT, + input_json TEXT NOT NULL DEFAULT '{}' CHECK (json_valid(input_json)), + output_json TEXT NOT NULL DEFAULT '{}' CHECK (json_valid(output_json)), + duration_ms INTEGER NOT NULL DEFAULT 0, + success INTEGER NOT NULL DEFAULT 1 CHECK (success IN (0, 1)), + called_at TEXT NOT NULL + ); +CREATE INDEX idx_api_logs_tool_time + ON api_logs (tool_name, called_at DESC); +CREATE INDEX idx_api_logs_tool_source_time + ON api_logs (tool_name, source_agent, called_at DESC); +CREATE TABLE memory_change_log ( + seq INTEGER PRIMARY KEY AUTOINCREMENT, + memory_id TEXT NOT NULL, + namespace_id TEXT, + kind TEXT, + op TEXT, + entity_id TEXT, + user_id TEXT NOT NULL, + change_type TEXT NOT NULL, + version INTEGER, + before_json TEXT, + after_json TEXT, + source TEXT NOT NULL, + created_at TEXT NOT NULL + ); +CREATE INDEX idx_memory_change_log_namespace_seq + ON memory_change_log (namespace_id, seq); +CREATE INDEX idx_memory_change_log_created + ON memory_change_log (created_at DESC); +CREATE TABLE idempotency_keys ( + key TEXT PRIMARY KEY, + request_hash TEXT NOT NULL, + response_json TEXT NOT NULL CHECK (json_valid(response_json)), + created_at TEXT NOT NULL, + expires_at TEXT + ); +CREATE TABLE source_turn_captures ( + user_id TEXT NOT NULL, + source TEXT NOT NULL, + profile_id TEXT NOT NULL, + namespace_key TEXT NOT NULL, + conversation_id TEXT NOT NULL, + turn_id TEXT NOT NULL, + content_hash TEXT NOT NULL, + session_id TEXT, + episode_id TEXT, + raw_turn_id TEXT, + response_json TEXT NOT NULL CHECK (json_valid(response_json)), + started_at TEXT NOT NULL, + completed_at TEXT NOT NULL, + source_sequence INTEGER, + created_at TEXT NOT NULL, + PRIMARY KEY (user_id, source, profile_id, namespace_key, conversation_id, turn_id) + ); +CREATE INDEX idx_source_turn_captures_conversation + ON source_turn_captures (user_id, source, profile_id, namespace_key, conversation_id, completed_at DESC); +CREATE INDEX idx_source_turn_captures_native_turn + ON source_turn_captures (source, profile_id, namespace_key, conversation_id, turn_id); +CREATE INDEX idx_source_turn_captures_native_conversation + ON source_turn_captures (source, profile_id, namespace_key, conversation_id, completed_at DESC); +CREATE INDEX idx_source_sessions_native_conversation + ON sessions (source, profile_id, conversation_id); +CREATE INDEX idx_source_sessions_native_host_key + ON sessions (source, profile_id, host_session_key); +CREATE TABLE memory_capture_claims ( + user_id TEXT NOT NULL, + source TEXT NOT NULL, + qa_hash TEXT NOT NULL, + primary_memory_id TEXT NOT NULL, + captured_by TEXT NOT NULL CHECK (captured_by IN ('turn_complete', 'agent_source_scan')), + created_at TEXT NOT NULL, + PRIMARY KEY (user_id, source, qa_hash) + ); +CREATE INDEX idx_memory_capture_claims_primary_memory + ON memory_capture_claims (primary_memory_id); +CREATE TABLE l3_world_model_project_environment_state ( + user_id TEXT NOT NULL, + project_id TEXT NOT NULL, + project_kind TEXT NOT NULL DEFAULT 'unknown' CHECK (project_kind IN ('unknown', 'code', 'folder')), + status TEXT NOT NULL DEFAULT 'uninitialized' CHECK (status IN ( + 'uninitialized', 'queued', 'scanning', 'summarizing', 'clean', 'failed' + )), + current_scan_id TEXT, + applied_scan_id TEXT, + fingerprint TEXT, + last_error TEXT, + updated_at TEXT NOT NULL, + PRIMARY KEY (user_id, project_id) + ); +CREATE INDEX idx_l3_world_model_project_environment_status + ON l3_world_model_project_environment_state (status, updated_at); +CREATE TABLE evolution_jobs ( + id TEXT PRIMARY KEY, + job_type TEXT NOT NULL, + status TEXT NOT NULL DEFAULT 'queued' + CHECK (status IN ('queued', 'leased', 'succeeded', 'failed', 'dead_letter')), + dedupe_key TEXT, + user_id TEXT NOT NULL, + session_id TEXT, + episode_id TEXT, + target_memory_id TEXT, + scope_key TEXT, + scope_seq INTEGER, + payload_json TEXT NOT NULL DEFAULT '{}' CHECK (json_valid(payload_json)), + attempts INTEGER NOT NULL DEFAULT 0, + max_attempts INTEGER NOT NULL DEFAULT 3, + leased_until TEXT, + last_error TEXT, + created_at TEXT NOT NULL, + updated_at TEXT NOT NULL + ); +CREATE INDEX idx_evolution_jobs_status_created + ON evolution_jobs (status, created_at ASC); +CREATE INDEX idx_evolution_jobs_target + ON evolution_jobs (target_memory_id, job_type); +CREATE UNIQUE INDEX uq_evolution_jobs_l3_immutable_dedupe + ON evolution_jobs (dedupe_key) + WHERE dedupe_key IS NOT NULL + AND job_type IN ('l3_world_model_update', 'project_environment_profile'); +CREATE UNIQUE INDEX uq_evolution_jobs_scope_seq + ON evolution_jobs (scope_key, scope_seq) + WHERE scope_key IS NOT NULL; +CREATE TABLE embedding_retry_queue ( + id TEXT PRIMARY KEY, + target_kind TEXT NOT NULL CHECK (target_kind IN ('trace', 'policy', 'world_model', 'skill')), + target_id TEXT NOT NULL, + vector_field TEXT NOT NULL CHECK (vector_field IN ('vec_summary', 'vec_action', 'vec')), + source_text TEXT NOT NULL, + embed_role TEXT NOT NULL DEFAULT 'document' CHECK (embed_role IN ('document', 'query')), + status TEXT NOT NULL DEFAULT 'pending' + CHECK (status IN ('pending', 'in_progress', 'failed', 'succeeded')), + attempts INTEGER NOT NULL DEFAULT 0, + max_attempts INTEGER NOT NULL DEFAULT 6, + next_attempt_at INTEGER NOT NULL, + claimed_by TEXT, + lease_until INTEGER, + last_error TEXT, + created_at INTEGER NOT NULL, + updated_at INTEGER NOT NULL, + UNIQUE (target_kind, target_id, vector_field) + ); +CREATE INDEX idx_embedding_retry_due + ON embedding_retry_queue (status, next_attempt_at); +CREATE INDEX idx_embedding_retry_target + ON embedding_retry_queue (target_kind, target_id); +CREATE TABLE memory_processing_state ( + memory_id TEXT PRIMARY KEY REFERENCES memories(id) ON DELETE CASCADE, + state TEXT NOT NULL CHECK (state IN ( + 'summary_pending', 'summarizing', 'embedding_pending', 'embedding', + 'ready', 'ready_text_only', 'failed' + )), + stage TEXT CHECK (stage IN ('summary', 'embedding')), + active_job_id TEXT REFERENCES evolution_jobs(id) ON DELETE SET NULL, + attempt_count INTEGER NOT NULL DEFAULT 0, + manual_retry_count INTEGER NOT NULL DEFAULT 0, + retry_action TEXT NOT NULL DEFAULT 'retry' + CHECK (retry_action IN ('retry', 'open_settings', 'none')), + error_code TEXT, + error_message TEXT, + failed_at TEXT, + updated_at TEXT NOT NULL + ); +CREATE INDEX idx_memory_processing_state_state_updated + ON memory_processing_state (state, updated_at ASC); +CREATE INDEX idx_memory_processing_state_active_job + ON memory_processing_state (active_job_id); +CREATE TABLE artifacts ( + id TEXT PRIMARY KEY, + session_id TEXT, + episode_id TEXT, + raw_turn_id TEXT, + user_id TEXT NOT NULL, + kind TEXT NOT NULL, + uri TEXT, + payload_json TEXT NOT NULL DEFAULT '{}' CHECK (json_valid(payload_json)), + created_at TEXT NOT NULL + ); +CREATE INDEX idx_artifacts_session_created + ON artifacts (session_id, created_at DESC); +CREATE TABLE runtime_kv ( + key TEXT PRIMARY KEY, + value_json TEXT NOT NULL DEFAULT '{}' CHECK (json_valid(value_json)), + updated_at TEXT NOT NULL + ); +CREATE TABLE audit_logs ( + id TEXT PRIMARY KEY, + user_id TEXT NOT NULL, + session_id TEXT, + actor_json TEXT NOT NULL DEFAULT '{}' CHECK (json_valid(actor_json)), + action TEXT NOT NULL, + target_kind TEXT NOT NULL, + target_id TEXT NOT NULL, + before_json TEXT, + after_json TEXT, + meta_json TEXT NOT NULL DEFAULT '{}' CHECK (json_valid(meta_json)), + created_at TEXT NOT NULL + ); +CREATE INDEX idx_audit_logs_user_created + ON audit_logs (user_id, created_at DESC); +CREATE TABLE token_usage_outbox ( + sequence INTEGER PRIMARY KEY, + event_id TEXT NOT NULL UNIQUE, + payload_json TEXT NOT NULL CHECK (json_valid(payload_json)) + ); +CREATE UNIQUE INDEX uq_evolution_jobs_active_dedupe + ON evolution_jobs (dedupe_key) + WHERE dedupe_key IS NOT NULL AND status IN ('queued', 'leased', 'failed'); +INSERT INTO schema_migrations VALUES ('009_token_usage_outbox', 9, '2026-10-01', 'fixture'); diff --git a/Memory/tests/native-scanner-lifecycle.test.ts b/Memory/tests/native-scanner-lifecycle.test.ts index 825eaecfd..31987b8fe 100644 --- a/Memory/tests/native-scanner-lifecycle.test.ts +++ b/Memory/tests/native-scanner-lifecycle.test.ts @@ -48,8 +48,8 @@ describe("Codex scan and Hook share the actual Memory lifecycle", () => { expect(db.db.prepare("SELECT COUNT(*) AS n FROM memories WHERE memory_layer = 'L1'").get()).toEqual({ n: 1 }); const captured = (await hook()).result!; const repos = new Repositories(db.db); - expect(repos.runtime.getSession(captured.sessionId)?.userId).toBe(first === "hook" ? "hook-owner" : "scan-owner"); - expect(repos.memories.get(captured.l1MemoryId)?.userId).toBe(first === "hook" ? "hook-owner" : "scan-owner"); + expect(repos.runtime.getSession(captured.sessionId)?.userId).toBe(null); + expect(repos.memories.get(captured.l1MemoryId)?.userId).toBe(null); expect(repos.runtime.getEpisode(captured.episodeId)?.l1MemoryIds).toEqual([captured.l1MemoryId]); expect(repos.runtime.getRawTurn(captured.rawTurnId)?.toolCalls[0]).toMatchObject({ id: "read", input: { path: "src/main.ts" }, output: expect.stringContaining("source contents") }); expect(db.db.prepare("SELECT COUNT(*) AS n FROM evolution_jobs WHERE job_type = 'import_summary'").get()).toEqual({ n: 0 }); diff --git a/Memory/tests/repository/device-memory-scope.test.ts b/Memory/tests/repository/device-memory-scope.test.ts new file mode 100644 index 000000000..986b54845 --- /dev/null +++ b/Memory/tests/repository/device-memory-scope.test.ts @@ -0,0 +1,845 @@ +import { L3WorldModelContextReadModel } from "../../src/service/read-model/l3-world-model-context.js"; +import { L3WorldModelTraceFieldPipeline } from "../../src/service/evolution/l3-world-model-pipeline.js"; +import type { LlmClient } from "../../src/model/types.js"; +import { readFileSync, existsSync, mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import Database from "better-sqlite3"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { MemoryDb } from "../../src/storage/db.js"; +import { SCHEMA_VERSION } from "../../src/storage/schema.js"; +import { Repositories } from "../../src/storage/repositories.js"; +import { canonicalJson, sha256Hex } from "../../src/contracts/index.js"; +import { canonicalNamespaceKey, canonicalProjectId, refreshProjectAliases, reloadDeviceScopeAliases } from "../../src/storage/device-scope.js"; +import { resolveWorkspaceIdentity } from "../../src/service/namespace/workspace-identity.js"; +import { stableHash } from "../../src/utils/id.js"; +import { memoryCaptureQaHash } from "../../src/utils/memory-capture-claim.js"; +import { createMemoryServiceFixture } from "../fixtures/memory-service-fixture.js"; +const fixture = createMemoryServiceFixture(); +afterEach(fixture.cleanup); +const at = "2026-10-01T00:00:00.000Z", uri = "file:///workspace/shared", host = "a".repeat(64); +const identity = resolveWorkspaceIdentity(null, {workspaceUri: uri, workspaceHostId: host}); +function legacyProject(user: string) { + const workspaceId = sha256Hex(`${user}\0local\0${host}\0${uri}`); + return {projectId: `ws_${workspaceId}`, workspaceId}; +} +function seedV9(path: string, pendingEpisodeAssignment = false) { + const db = new Database(path); + const schema = readFileSync(new URL("../fixtures/schema-v9.sql", import.meta.url), "utf8"); + db.exec(pendingEpisodeAssignment ? schema.replace(/episode_id TEXT NOT NULL/g, "episode_id TEXT") : schema); + if (pendingEpisodeAssignment) db.exec("UPDATE schema_migrations SET version=10, id='010_pending_episode_assignment'"); + for (const user of ["account-a", "account-b"]) { + const {projectId, workspaceId} = legacyProject(user); + const namespaceKey = stableHash({tenantId: null, projectId, workspaceId}); + db.prepare(`INSERT INTO sessions(id,user_id,project_id,source,profile_id,workspace_id,host_session_key,conversation_id,meta_json,opened_at,last_seen_at,updated_at) + VALUES (?, ?, ?, 'codex', 'default', ?, ?, 'native-conversation', ?, ?, ?, ?)`) + .run(`session-${user}`,user,projectId,workspaceId,`host-${user}`,JSON.stringify({workspace_uri:uri,workspace_host_id:host,l3_world_model_protocol_version:2,source_namespace_key:namespaceKey}),at,at,at); + for (const layer of ["L1","L3"] as const) { + const info = {project_id:projectId,...(layer === "L1" ? {turn_id:"native-turn"} : {})}; + const internal = layer === "L1" ? {memory_layer:layer,memory_kind:"trace",trace:{turn_id:"native-turn",step_index:0}} : { + memory_layer:layer,memory_kind:"world_model",schema_version:2,world_model:{ + general_rules_and_safety_constraints:null,project_environment_profile:null, + project_contract:user === "account-a" ? "Run tests" : null, + domain_knowledge:user === "account-b" ? "Parser accepts UTF-8" : null + }}; + db.prepare(`INSERT INTO memories(id,timeline,user_id,session_id,memory_key,memory_value,info_json,properties_json,memory_layer,created_at,updated_at) + VALUES (?,?,?,?,?,?,?,?,?,?,?)`).run(`${layer}-${user}`,at,user,`session-${user}`,`world_model:project:${user}:${projectId}`,"original", + JSON.stringify(info),JSON.stringify({internal_info:internal}),layer,at,at); + } + db.prepare(`INSERT INTO l3_world_model_scopes(scope_key,user_id,project_id,workspace_uri,memory_id,updated_at) VALUES (?,?,?,?,?,?)`) + .run(`l3wm:${sha256Hex(canonicalJson({userId:user,projectId}))}`,user,projectId,uri,`L3-${user}`,at); + db.prepare(`INSERT INTO user_memories(id,source_turn_id,user_id,memory_types_json,content,normalized_user_text_hash,created_at,updated_at) + VALUES (?, ?, ?, '["User Preference"]', 'Prefer Chinese', 'same-user-text', ?, ?)`) + .run(`user-memory-${user}`,`raw-${user}`,user,at,at); + const scopeKey = `l3wm:${sha256Hex(canonicalJson({userId:user,projectId}))}`; + const payload = {scopeKey,scopeSeq:1,userId:user,projectId,sessionId:`session-${user}`,trigger:"session_close",startTraceSeq:1,endTraceSeq:1,l1MemoryIds:[`L1-${user}`],rawTurnIds:[`raw-${user}`],feedbackIds:[]}; + db.prepare(`INSERT INTO episodes(id,session_id,user_id,project_id,opened_at,updated_at) VALUES (?,?,?,?,?,?)`).run(`episode-${user}`,`session-${user}`,user,projectId,at,at); + db.prepare(`INSERT INTO raw_turns(id,session_id,episode_id,turn_id,user_id,user_text,assistant_text,created_at) VALUES (?,?,?,?,?,?,?,?)`) + .run(`raw-${user}`,`session-${user}`,`episode-${user}`,"native-turn",user,"Please preserve the project rule.","The rule is already recorded.",at); + db.prepare(`INSERT INTO l3_world_model_input_traces(session_id,trace_seq,l1_memory_id,raw_turn_id,created_at) VALUES (?,1,?,?,?)`) + .run(`session-${user}`,`L1-${user}`,`raw-${user}`,at); + db.prepare(`INSERT INTO l3_world_model_evidence_batches(id,scope_key,scope_seq,user_id,project_id,session_id,trigger,start_trace_seq,end_trace_seq,l1_memory_ids_json,raw_turn_ids_json,feedback_ids_json,payload_hash,created_at,updated_at) + VALUES (?,?,1,?,?,?,'session_close',1,1,?,?,'[]',?,?,?)`) + .run(`batch-${user}`,scopeKey,user,projectId,`session-${user}`,JSON.stringify(payload.l1MemoryIds),JSON.stringify(payload.rawTurnIds),sha256Hex(canonicalJson(payload)),at,at); + db.prepare(`INSERT INTO l3_world_model_batch_targets(batch_id,target_field,field_scope_key,scope_seq,updated_at) VALUES (?,'project_contract',?,1,?)`) + .run(`batch-${user}`,`${scopeKey}:project_contract`,at); + db.prepare(`INSERT INTO evolution_jobs(id,job_type,user_id,session_id,scope_key,scope_seq,payload_json,created_at,updated_at) + VALUES (?,'l3_world_model_update',?,?,?,1,?,?,?)`) + .run(`job-${user}`,user,`session-${user}`,`${scopeKey}:project_contract`,JSON.stringify({batchId:`batch-${user}`,targetField:"project_contract"}),at,at); + // Historical provenance must link the captured turn, not only its mutable Session. + db.prepare(`INSERT INTO source_turn_captures(user_id,source,profile_id,namespace_key,conversation_id,turn_id,content_hash,session_id,raw_turn_id,response_json,started_at,completed_at,created_at) + VALUES (?,'codex','default',?,'native-conversation','native-turn','hash',?,?,'{"status":"rejected","reason":"fixture"}',?,?,?)`) + .run(user,namespaceKey,`session-${user}`,`raw-${user}`,at,at,at); + } + db.prepare(`INSERT INTO token_usage_outbox VALUES (1,'old-token-event',?)`).run(JSON.stringify({ownerAccountId:"account-a",totalTokens:1234})); + db.prepare(`INSERT INTO runtime_kv VALUES ('token-budget-fixture', ?, ?)`).run(JSON.stringify({accountId:"account-a",used:1234,limit:5678}),at); + db.prepare(`INSERT INTO runtime_kv VALUES ('source_turn_capture_activated_at', ?, ?)`).run(JSON.stringify(at),at); + db.prepare("INSERT INTO sqlite_sequence(name,seq) VALUES ('memory_change_log',9000)").run(); + return db; +} + +// A v9 field update has full before/after snapshots, including explicit nulls. +function legacyFieldEdit(db: Database.Database, id: string, field: string, value: string | null, when: string) { + const row = db.prepare("SELECT properties_json FROM memories WHERE id = ?").get(id) as { properties_json: string }; + const before = { id, properties: JSON.parse(row.properties_json) }; + const after = structuredClone(before); + after.properties.internal_info.world_model[field] = value; + db.prepare("UPDATE memories SET properties_json=?, updated_at=? WHERE id=?").run(JSON.stringify(after.properties),when,id); + db.prepare("UPDATE l3_world_model_scopes SET updated_at=? WHERE memory_id=?").run(when,id); + db.prepare(`INSERT INTO memory_change_log(memory_id,user_id,change_type,before_json,after_json,created_at,source) + VALUES (?,'account-a','l3_world_model_update',?,?,?,'worker.l3_world_model.v1')`).run(id,JSON.stringify(before),JSON.stringify(after),when); +} + +async function triggerLogRetention(db: MemoryDb, repos: Repositories) { + db.db.transaction(() => { + for (let i = 0; i < 10_000; i++) repos.runtime.appendChange({memoryId: "L1-account-a", userId: null, + changeType: "fixture_unrelated", source: "fixture", createdAt: "2026-10-04T00:00:00.000Z"}); + })(); + await new Promise(resolve => setImmediate(resolve)); +} + +describe("device memory / review regressions", () => { + it.each([false, true])("preserves rowids while combining device and pending-Episode upgrades (upstream v10=%s)", pending => { + const root = mkdtempSync(join(tmpdir(), "memmy-combined-scope-upgrade-")); + const path = join(root, "memory.sqlite"); + let upgraded: MemoryDb | undefined; + try { + const legacy = seedV9(path, pending); + legacy.exec("UPDATE raw_turns SET rowid=rowid+100"); + const before = legacy.prepare("SELECT rowid,* FROM raw_turns ORDER BY rowid").all(); + legacy.close(); + upgraded = new MemoryDb({ path }); + expect(upgraded.db.prepare("SELECT rowid,* FROM raw_turns ORDER BY rowid").all()).toEqual(before); + const columns = upgraded.db.prepare("PRAGMA table_info(raw_turns)").all() as Array<{name: string; notnull: number}>; + expect(columns.filter(column => ["user_id", "episode_id"].includes(column.name)).map(column => column.notnull)).toEqual([0, 0]); + upgraded.db.prepare("INSERT INTO raw_turns(id,session_id,turn_id,created_at) VALUES ('pending-turn','session-account-a','new-turn',?)").run(at); + expect(upgraded.db.prepare("SELECT episode_id,user_id FROM raw_turns WHERE id='pending-turn'").get()).toEqual({episode_id: null,user_id: null}); + expect(upgraded.db.pragma("foreign_key_check")).toEqual([]); + } finally { upgraded?.close(); rmSync(root, { recursive: true, force: true }); } + }); + + it("does not apply an unowned deletion override when its device memory was skipped", () => { + const {db} = fixture.createTestService(), repos = new Repositories(db.db); + repos.l3WorldModels.upsertField({userId: null, projectId: "project-p", targetField: "project_contract", value: "Keep target rule"}); + const bundle = repos.runtime.exportBundleTables(true); + bundle.memory_l3_overrides = [{project_key: "project-p", field: "projectContract", memory_id: null, deleted: 1}]; + // No target override conflict: the memory-row skip must govern this insert. + db.db.exec("DELETE FROM memory_l3_overrides"); + const history = db.db.prepare("SELECT * FROM memories ORDER BY id").all(); + repos.runtime.importBundleTables(bundle, {conflictStrategy: "skip"}); + expect(repos.l3WorldModels.fields("another-account", "project-p").projectContract).toBe("Keep target rule"); + expect(db.db.prepare("SELECT * FROM memories ORDER BY id").all()).toEqual(history); + expect(db.db.prepare("SELECT COUNT(*) FROM memory_l3_overrides").pluck().get()).toBe(0); + }); + + it("allows L3 model recreation after deleting a shared explicitly archived copy", () => { + const {db} = fixture.createTestService(), repos = new Repositories(db.db); + const archivedAt = "2026-01-01T00:00:00.000Z"; + const original = repos.l3WorldModels.upsertField({userId: "old-a", projectId: "p", + targetField: "project_contract", value: "old rule", at: archivedAt})!; + const stored = db.db.prepare("SELECT * FROM memories WHERE id=?").get(original.id) as Record; + const properties = JSON.parse(String(stored.properties_json)) as { + status: string; + internal_info: Record; + }; + properties.status = "archived"; + properties.internal_info.l3_world_model_explicitly_archived = true; + const originalScopeKey = `l3wm:${sha256Hex(canonicalJson({userId: "old-a", projectId: "p"}))}`; + db.db.prepare(`UPDATE memories SET user_id='old-a', memory_key='world_model:project:old-a:p', status='archived', + properties_json=?, updated_at=? WHERE id=?`).run(JSON.stringify(properties), archivedAt, original.id); + db.db.prepare("UPDATE l3_world_model_scopes SET user_id='old-a', scope_key=?, updated_at=? WHERE memory_id=?") + .run(originalScopeKey, archivedAt, original.id); + + const secondId = "legacy-b"; + const secondUpdatedAt = "2026-01-02T00:00:00.000Z"; + const second: Record = {...stored, id: secondId, user_id: "old-b", memory_key: "world_model:project:old-b:p", + updated_at: secondUpdatedAt}; + const columns = Object.keys(second); + db.db.prepare(`INSERT INTO memories (${columns.join(",")}) VALUES (${columns.map(() => "?").join(",")})`) + .run(...columns.map(column => second[column])); + const secondScopeKey = `l3wm:${sha256Hex(canonicalJson({userId: "old-b", projectId: "p"}))}`; + db.db.prepare(`INSERT INTO l3_world_model_scopes(scope_key,user_id,project_id,memory_id,updated_at) + VALUES (?,'old-b','p',?,?)`).run(secondScopeKey, secondId, secondUpdatedAt); + db.db.exec("DELETE FROM memory_l3_overrides"); + + expect(repos.l3WorldModels.getMemory("old-b", "p")?.status).toBe("archived"); + repos.l3WorldModels.deleteScopeMemory(secondId, "2026-01-03T00:00:00.000Z"); + const dbPath = db.path; + db.close(); + const reopenedDb = new MemoryDb({path: dbPath}), reopenedRepos = new Repositories(reopenedDb.db); + try { + const afterDelete = reopenedRepos.l3WorldModels.getMemory("old-b", "p"); + expect(afterDelete?.properties.internal_info.l3_world_model_explicitly_archived).toBeUndefined(); + const recreated = reopenedRepos.l3WorldModels.upsertField({userId: "old-b", projectId: "p", + targetField: "project_contract", value: "new rule", at: "2026-01-04T00:00:00.000Z"}); + expect(recreated?.status).toBe("activated"); + expect(reopenedRepos.l3WorldModels.fields("old-b", "p").projectContract).toBe("new rule"); + } finally { reopenedDb.close(); } + }); + + it("skips archive evidence for protected L3 rows while importing unrelated bundle data", () => { + const {db} = fixture.createTestService(), repos = new Repositories(db.db); + const original = repos.l3WorldModels.upsertField({userId: "legacy", projectId: "p", + targetField: "project_contract", value: "rule", at: "2026-01-01T00:00:00.000Z"})!; + repos.l3WorldModels.upsertField({userId: "legacy", projectId: "p", targetField: "project_contract", + value: null, at: "2026-01-02T00:00:00.000Z"}); + const bundle = repos.runtime.exportBundleTables(true); + const incomingMemory = bundle.memories!.find(row => row.id === original.id)!; + const incomingProperties = JSON.parse(incomingMemory.properties_json as string) as { + internal_info: Record; + }; + incomingProperties.internal_info.l3_world_model_explicitly_archived = true; + incomingMemory.properties_json = JSON.stringify(incomingProperties); + bundle.audit_logs = [{id: "incoming-archive-audit", user_id: null, actor_json: "{}", action: "archive", + target_kind: "world_model", target_id: original.id, created_at: "2026-01-03T00:00:00.000Z"}]; + bundle.memory_change_log = [{seq: 99_999, memory_id: original.id, user_id: null, change_type: "archive", + source: "fixture", created_at: "2026-01-03T00:00:00.000Z"}]; + bundle.artifacts = [{id: "unrelated-artifact", user_id: null, kind: "fixture", payload_json: "{}", + created_at: "2026-01-03T00:00:00.000Z"}]; + + const result = repos.runtime.importBundleTables(bundle, {conflictStrategy: "skip"}); + expect(result.skipped.audit_logs).toBe(1); + expect(result.skipped.memory_change_log).toBe(1); + const dbPath = db.path; + db.close(); + const reopenedDb = new MemoryDb({path: dbPath}), reopenedRepos = new Repositories(reopenedDb.db); + try { + expect(reopenedDb.db.prepare("SELECT COUNT(*) FROM artifacts WHERE id='unrelated-artifact'").pluck().get()).toBe(1); + const projected = reopenedRepos.l3WorldModels.getMemory("legacy", "p"); + expect(projected?.status).toBe("archived"); + expect(projected?.properties.internal_info.l3_world_model_explicitly_archived).toBeUndefined(); + } finally { reopenedDb.close(); } + }); + + it.each([false, true])("keeps skipped historical memories unchanged by imported edit logs or metadata (metadata=%s)", async withMetadata => { + const root = mkdtempSync(join(tmpdir(), "memmy-l3-skip-import-")), path = join(root, "target.sqlite"); + let db: MemoryDb | undefined; + try { + seedV9(path).close(); db = new MemoryDb({path}); + let repos = new Repositories(db.db); + const bundle = repos.runtime.exportBundleTables(true); + const history = db.db.prepare("SELECT * FROM memories ORDER BY id").all(); + const incoming = bundle.memories!.find(row => row.id === "L3-account-a")!; + const before = {properties: JSON.parse(incoming.properties_json as string)}; + const after = structuredClone(before); after.properties.internal_info.world_model.project_contract = null; + incoming.properties_json = JSON.stringify(after.properties); incoming.memory_value = ""; incoming.updated_at = "2026-10-02T00:00:00.000Z"; + bundle.memory_change_log = [{seq: 9999, memory_id: incoming.id, user_id: "account-a", change_type: "l3_world_model_update", + before_json: JSON.stringify(before), after_json: JSON.stringify(after), source: "fixture", created_at: incoming.updated_at}]; + if (withMetadata) bundle.memory_l3_legacy_field_edits = [{memory_id: incoming.id, field: "project_contract", + deleted: 1, edited_at: incoming.updated_at, change_seq: 9999}]; + else delete bundle.memory_l3_legacy_field_edits; + const result = repos.runtime.importBundleTables(bundle, {conflictStrategy: "skip"}); + expect(result.skipped.memories).toBe(history.length); + expect(result.skipped.memory_change_log).toBe(1); + expect(repos.l3WorldModels.fields(null, identity.projectId).projectContract).toBe("Run tests"); + await triggerLogRetention(db, repos); + db.close(); db = new MemoryDb({path}); repos = new Repositories(db.db); + expect(repos.l3WorldModels.fields("new-account", identity.projectId).projectContract).toBe("Run tests"); + expect(db.db.prepare("SELECT * FROM memories ORDER BY id").all()).toEqual(history); + expect(db.db.prepare("SELECT COUNT(*) FROM memory_l3_legacy_field_edits").pluck().get()).toBe(0); + } finally {db?.close(); rmSync(root, {recursive: true, force: true});} + }); + + it.each([false, true])("preserves old bundle deletion/recreation despite target-local log sequence collisions (recreated=%s)", async recreated => { + const root = mkdtempSync(join(tmpdir(), "memmy-l3-import-collision-")), path = join(root, "source.sqlite"); + let source: MemoryDb | undefined, target: MemoryDb | undefined; + try { + const old = seedV9(path); + old.prepare(`UPDATE memories SET properties_json=json_set(properties_json,'$.internal_info.world_model.project_contract','Run tests') WHERE id='L3-account-b'`).run(); + legacyFieldEdit(old, "L3-account-a", "project_contract", null, "2026-10-02T00:00:00.000Z"); + if (recreated) legacyFieldEdit(old, "L3-account-b", "project_contract", "Recreated rule", "2026-10-02T00:00:00.000Z"); + const history = old.prepare("SELECT * FROM memories ORDER BY id").all(); old.close(); + source = new MemoryDb({path}); + const bundle = new Repositories(source.db).runtime.exportBundleTables(true); + delete bundle.memory_l3_legacy_field_edits; // v9/v10 exports have only logs. + const targetPath = join(root, "target.sqlite"); + target = new MemoryDb({path: targetPath}); + for (const change of bundle.memory_change_log!) target.db.prepare( + "INSERT INTO memory_change_log(seq,memory_id,change_type,source,created_at) VALUES (?,'unrelated','fixture','fixture',?)" + ).run(change.seq, at); + let repos = new Repositories(target.db); + const logs = target.db.prepare("SELECT * FROM memory_change_log ORDER BY seq").all(); + // Conflict errors must roll back both data and inferred metadata. + expect(() => repos.runtime.importBundleTables(bundle, {conflictStrategy: "error"})).toThrow("import conflict"); + expect(target.db.prepare("SELECT COUNT(*) FROM memories").pluck().get()).toBe(0); + expect(target.db.prepare("SELECT COUNT(*) FROM memory_l3_legacy_field_edits").pluck().get()).toBe(0); + const result = repos.runtime.importBundleTables(bundle); + expect(result.skipped.memory_change_log).toBe(bundle.memory_change_log!.length); + expect(target.db.prepare("SELECT * FROM memory_change_log ORDER BY seq").all()).toEqual(logs); + expect(target.db.prepare("SELECT * FROM memories ORDER BY id").all()).toEqual(history); + const expected = recreated ? "Recreated rule" : null; + expect(repos.l3WorldModels.fields("new-account", identity.projectId).projectContract).toBe(expected); + await triggerLogRetention(target, repos); + target.close(); target = new MemoryDb({path: targetPath}); repos = new Repositories(target.db); + expect(repos.l3WorldModels.fields(null, identity.projectId).projectContract).toBe(expected); + } finally {source?.close(); target?.close(); rmSync(root, {recursive: true, force: true});} + }); + + it.each([null, "2026-10-02T00:00:00.000Z", "2026-10-03T00:00:00.000Z"])( + "retains historical field deletion/recreation (%s) after actual log pruning, restart and bundle restore", async recreatedAt => { + const root = mkdtempSync(join(tmpdir(), "memmy-durable-l3-")), path = join(root, "memory.sqlite"); + let db: MemoryDb | undefined, restored: MemoryDb | undefined; + try { + const old = seedV9(path); + old.prepare(`UPDATE memories SET properties_json=json_set(properties_json,'$.internal_info.world_model.project_contract','Run tests') WHERE id='L3-account-b'`).run(); + legacyFieldEdit(old, "L3-account-a", "project_contract", null, "2026-10-02T00:00:00.000Z"); + if (recreatedAt) legacyFieldEdit(old, "L3-account-b", "project_contract", "Recreated rule", recreatedAt); + legacyFieldEdit(old, "L3-account-b", "domain_knowledge", "Unrelated update", "2026-10-03T01:00:00.000Z"); + const history = old.prepare("SELECT * FROM memories ORDER BY id").all(); + old.close(); + db = new MemoryDb({path}); + let repos = new Repositories(db.db); + await triggerLogRetention(db, repos); + expect(db.db.prepare("SELECT COUNT(*) FROM memory_change_log").pluck().get()).toBe(10_000); + expect(db.db.prepare("SELECT COUNT(*) FROM memory_change_log WHERE change_type='l3_world_model_update'").pluck().get()).toBe(0); + const expected = {projectContract: recreatedAt ? "Recreated rule" : null, domainKnowledge: "Unrelated update"}; + for (const user of ["account-a", "account-b", "account-c", null]) expect(repos.l3WorldModels.fields(user, identity.projectId)).toMatchObject(expected); + db.close(); db = new MemoryDb({path}); repos = new Repositories(db.db); + expect(repos.l3WorldModels.fields("account-c", identity.projectId)).toMatchObject(expected); + expect(db.db.prepare("SELECT * FROM memories ORDER BY id").all()).toEqual(history); + const bundle = repos.runtime.exportBundleTables(true); + expect(bundle.memory_l3_legacy_field_edits).toEqual(expect.arrayContaining([ + expect.objectContaining({memory_id: "L3-account-a", field: "project_contract", deleted: 1}) + ])); + restored = new MemoryDb({path: join(root, "restored.sqlite")}); + const restoredRepos = new Repositories(restored.db); + restoredRepos.runtime.importBundleTables(bundle); + expect(restoredRepos.l3WorldModels.fields("account-d", identity.projectId)).toMatchObject(expected); + expect(restored.db.prepare("SELECT * FROM memories ORDER BY id").all()).toEqual(history); + restoredRepos.clearAllMemoryData(); + expect(restored.db.prepare("SELECT COUNT(*) FROM memory_l3_legacy_field_edits").pluck().get()).toBe(0); + restoredRepos.l3WorldModels.upsertField({userId: "account-d", projectId: identity.projectId, targetField: "project_contract", value: "Fresh rule"}); + expect(restoredRepos.l3WorldModels.fields(null, identity.projectId).projectContract).toBe("Fresh rule"); + db.close(); db = new MemoryDb({path, readonly: true}); + expect(new Repositories(db.db).l3WorldModels.fields("account-d", identity.projectId)).toMatchObject(expected); + } finally {db?.close(); restored?.close(); rmSync(root, {recursive: true, force: true});} + } + ); + + it("keeps source logs if persisting deletion evidence fails, then retries atomically", async () => { + const root = mkdtempSync(join(tmpdir(), "memmy-l3-prune-failure-")), path = join(root, "memory.sqlite"); + let db: MemoryDb | undefined; + try { + seedV9(path).close(); db = new MemoryDb({path}); + legacyFieldEdit(db.db, "L3-account-a", "project_contract", null, "2026-10-02T00:00:00.000Z"); + db.db.exec(`CREATE TRIGGER reject_field_edits BEFORE INSERT ON memory_l3_legacy_field_edits + BEGIN SELECT RAISE(ABORT, 'fixture persistence failure'); END`); + const repos = new Repositories(db.db); + await triggerLogRetention(db, repos); + expect(db.db.prepare("SELECT COUNT(*) FROM memory_change_log").pluck().get()).toBe(10_001); + expect(repos.l3WorldModels.fields("account-b", identity.projectId).projectContract).toBeNull(); + db.db.exec("DROP TRIGGER reject_field_edits"); + repos.runtime.appendChange({memoryId: "L1-account-a", userId: null, changeType: "fixture_retry", source: "fixture", createdAt: "2026-10-05T00:00:00.000Z"}); + await new Promise(resolve => setImmediate(resolve)); + expect(db.db.prepare("SELECT COUNT(*) FROM memory_change_log").pluck().get()).toBe(10_000); + expect(repos.l3WorldModels.fields("account-c", identity.projectId).projectContract).toBeNull(); + expect(db.db.prepare("SELECT deleted FROM memory_l3_legacy_field_edits WHERE field='project_contract'").pluck().get()).toBe(1); + } finally {db?.close(); rmSync(root, {recursive: true, force: true});} + }); + + it("upgrades v10 metadata without changing historical rows and reads pre-upgrade v10 logs read-only", () => { + const root = mkdtempSync(join(tmpdir(), "memmy-v10-l3-upgrade-")), path = join(root, "memory.sqlite"); + let db: MemoryDb | undefined; + try { + seedV9(path).close(); db = new MemoryDb({path}); + // v10 has the same data tables, but no durable field-edit table. + db.db.exec(`DROP TABLE memory_l3_legacy_field_edits; DELETE FROM schema_migrations; + INSERT INTO schema_migrations VALUES ('010_device_memory_scope',10,'2026-10-01','fixture-v10')`); + legacyFieldEdit(db.db, "L3-account-a", "project_contract", null, "2026-10-02T00:00:00.000Z"); + const history = db.db.prepare("SELECT * FROM memories ORDER BY id").all(); + const logs = db.db.prepare("SELECT * FROM memory_change_log ORDER BY seq").all(); + db.close(); rmSync(`${path}.pre-v${SCHEMA_VERSION}.bak`); + db = new MemoryDb({path, readonly: true}); + expect(new Repositories(db.db).l3WorldModels.fields("account-c", identity.projectId).projectContract).toBeNull(); + expect(new Repositories(db.db).runtime.exportBundleTables(true).memory_l3_legacy_field_edits).toBeUndefined(); + expect(db.schemaVersion().version).toBe(10); + db.close(); db = new MemoryDb({path}); + expect(db.schemaVersion().version).toBe(SCHEMA_VERSION); + expect(db.db.prepare("SELECT * FROM memories ORDER BY id").all()).toEqual(history); + expect(db.db.prepare("SELECT * FROM memory_change_log ORDER BY seq").all()).toEqual(logs); + expect(db.db.prepare("SELECT deleted FROM memory_l3_legacy_field_edits WHERE field='project_contract'").pluck().get()).toBe(1); + const backup = new Database(`${path}.pre-v${SCHEMA_VERSION}.bak`, {readonly: true}); + try {expect(backup.prepare("SELECT MAX(version) FROM schema_migrations").pluck().get()).toBe(10);} + finally {backup.close();} + } finally {db?.close(); rmSync(root, {recursive: true, force: true});} + }); + + it("opens an unupgraded v9 database read-only without writing missing device-scope tables", () => { + const root = mkdtempSync(join(tmpdir(), "memmy-v9-readonly-device-scope-")), path = join(root, "memory.sqlite"); + let db: MemoryDb | undefined; + try { + seedV9(path).close(); + db = new MemoryDb({ path, readonly: true }); + expect(db.schemaVersion().version).toBe(9); + expect(canonicalProjectId(db.db, legacyProject("account-a").projectId)).toBe(identity.projectId); + const oldNamespaceKey = stableHash({ + tenantId: null, + projectId: legacyProject("account-a").projectId, + workspaceId: legacyProject("account-a").workspaceId + }); + const canonicalNamespace = stableHash({ + tenantId: null, + projectId: identity.projectId, + workspaceId: identity.workspaceId + }); + expect(canonicalNamespaceKey(db.db, oldNamespaceKey)).toBe(canonicalNamespace); + const repos = new Repositories(db.db); + expect(repos.l3WorldModels.getMemory(null, identity.projectId)?.memoryValue).toContain("Run tests"); + const bundle = repos.runtime.exportBundleTables(true); + expect(bundle.memories).toHaveLength(4); + expect(bundle.memory_project_aliases).toBeUndefined(); + expect(bundle.memory_namespace_aliases).toBeUndefined(); + expect(bundle.memory_l3_overrides).toBeUndefined(); + refreshProjectAliases(db.db); + reloadDeviceScopeAliases(db.db); + expect(canonicalNamespaceKey(db.db, oldNamespaceKey)).toBe(canonicalNamespace); + expect(db.db.prepare("SELECT 1 FROM sqlite_master WHERE type='table' AND name='memory_project_aliases'").get()).toBeUndefined(); + expect(db.db.prepare("SELECT 1 FROM sqlite_master WHERE type='table' AND name='memory_namespace_aliases'").get()).toBeUndefined(); + db.close(); db = undefined; + + db = new MemoryDb({ path }); + expect(db.schemaVersion().version).toBe(SCHEMA_VERSION); + expect(db.db.prepare("SELECT COUNT(*) FROM memory_project_aliases").pluck().get()).toBeGreaterThan(0); + } finally { db?.close(); rmSync(root, { recursive: true, force: true }); } + }); + + it("reports missing metadata tables as corruption when a read-only database claims the current schema", () => { + const root = mkdtempSync(join(tmpdir(), "memmy-current-readonly-corruption-")), path = join(root, "memory.sqlite"); + let db: MemoryDb | undefined; + try { + db = new MemoryDb({ path }); + const repos = new Repositories(db.db); + repos.l3WorldModels.upsertField({ + userId: "current-account", + projectId: "current-project", + targetField: "project_contract", + value: "Current schema test content." + }); + db.db.exec("DROP TABLE memory_l3_overrides; DROP TABLE memory_project_aliases;"); + db.close(); + db = new MemoryDb({ path, readonly: true }); + expect(db.schemaVersion().version).toBe(SCHEMA_VERSION); + const readonlyRepos = new Repositories(db.db); + expect(() => readonlyRepos.l3WorldModels.getMemory(null, "current-project")) + .toThrow(/no such table: memory_l3_overrides/); + expect(() => readonlyRepos.runtime.exportBundleTables(true)) + .toThrow(/no such table: memory_project_aliases/); + } finally { db?.close(); rmSync(root, { recursive: true, force: true }); } + }); + + it("reads a proven v1 capture through project aliases without rewriting the claim", () => { + const root=mkdtempSync(join(tmpdir(),"memmy-legacy-claim-")),path=join(root,"memory.sqlite"); + try { + const old=seedV9(path),qaHash=memoryCaptureQaHash("query","answer"); + old.prepare(`INSERT INTO memory_capture_claims VALUES ('account-a','codex',?,'L1-account-a','turn_complete',?)`).run(qaHash,at); + const claims=old.prepare("SELECT * FROM memory_capture_claims").all();old.close(); + const db=new MemoryDb({path}),repos=new Repositories(db.db); + try { + const scope=repos.captureClaims.identity({projectId:identity.projectId!,workspaceId:identity.workspaceId!,profileId:"default",sessionKey:"native-conversation"},"native-turn")!; + expect(repos.captureClaims.get("account-c","codex",qaHash,scope)?.primaryMemoryId).toBe("L1-account-a"); + expect(repos.captureClaims.get("account-c","codex",qaHash,{...scope,projectId:"other-project"})).toBeUndefined(); + expect(repos.captureClaims.get("account-c","codex",qaHash)).toBeUndefined(); + expect(db.db.prepare("SELECT * FROM memory_capture_claims").all()).toEqual(claims); + } finally {db.close();} + } finally {rmSync(root,{recursive:true,force:true});} + }); + + it("preserves explicit legacy field deletions without mistaking missing fields for tombstones", () => { + const root = mkdtempSync(join(tmpdir(),"memmy-legacy-field-edit-")), path = join(root,"memory.sqlite"); + let db: MemoryDb | undefined; + try { + const old = seedV9(path); + old.prepare(`UPDATE memories SET properties_json=json_set(properties_json,'$.internal_info.world_model.project_contract','Run tests') WHERE id='L3-account-b'`).run(); + legacyFieldEdit(old,"L3-account-a","project_contract",null,"2026-10-02T00:00:00.000Z"); + // Updating an unrelated field later must not resurrect the stale contract. + legacyFieldEdit(old,"L3-account-b","domain_knowledge","New unrelated knowledge","2026-10-03T00:00:00.000Z"); + const history = old.prepare("SELECT * FROM memories ORDER BY id").all(); + const logs = old.prepare("SELECT * FROM memory_change_log ORDER BY seq").all(); + old.close(); + db = new MemoryDb({path}); + let repos = new Repositories(db.db); + expect(repos.l3WorldModels.fields("account-b",identity.projectId)).toMatchObject({projectContract:null,domainKnowledge:"New unrelated knowledge"}); + db.close(); db = new MemoryDb({path}); repos = new Repositories(db.db); + expect(repos.l3WorldModels.fields("account-c",identity.projectId).projectContract).toBeNull(); + expect(db.db.prepare("SELECT * FROM memories ORDER BY id").all()).toEqual(history); + expect(db.db.prepare("SELECT * FROM memory_change_log ORDER BY seq").all()).toEqual(logs); + repos.l3WorldModels.upsertField({userId:"account-c",projectId:identity.projectId,targetField:"project_contract",value:"New deliberate rule"}); + expect(repos.l3WorldModels.fields("account-d",identity.projectId).projectContract).toBe("New deliberate rule"); + } finally {db?.close();rmSync(root,{recursive:true,force:true});} + }); + + it.each(["2026-10-02T00:00:00.000Z", "2026-10-03T00:00:00.000Z"])("allows an explicitly recreated historical field at %s", (recreatedAt) => { + const root=mkdtempSync(join(tmpdir(),"memmy-legacy-recreate-")),path=join(root,"memory.sqlite"); + try { + const old=seedV9(path); + legacyFieldEdit(old,"L3-account-a","project_contract",null,"2026-10-02T00:00:00.000Z"); + legacyFieldEdit(old,"L3-account-b","project_contract","Later rule",recreatedAt); + old.close();const db=new MemoryDb({path}); + try {expect(new Repositories(db.db).l3WorldModels.fields(null,identity.projectId).projectContract).toBe("Later rule");} + finally {db.close();} + } finally {rmSync(root,{recursive:true,force:true});} + }); + + it("inherits the provenance of all selected historical L3 fields on the first device write", () => { + const root=mkdtempSync(join(tmpdir(),"memmy-l3-lineage-")),path=join(root,"memory.sqlite"); + let db: MemoryDb | undefined; + try { + const old=seedV9(path); + for (const user of ["account-a","account-b"]) old.prepare(`UPDATE memories + SET info_json=json_set(info_json,'$.source_memory_ids',json(?)), + properties_json=json_set(properties_json,'$.internal_info.source_memory_ids',json(?)) WHERE id=?`) + .run(JSON.stringify([`L1-${user}`]),JSON.stringify([`L1-${user}`]),`L3-${user}`); + const history=old.prepare("SELECT * FROM memories ORDER BY id").all();old.close(); + db=new MemoryDb({path});let repos=new Repositories(db.db); + expect(repos.l3WorldModels.getMemory(null,identity.projectId)?.info.source_memory_ids).toEqual(expect.arrayContaining(["L1-account-a","L1-account-b"])); + repos.l3WorldModels.upsertField({userId:"account-c",projectId:identity.projectId,targetField:"domain_knowledge",value:"New knowledge"}); + db.close();db=new MemoryDb({path});repos=new Repositories(db.db); + const context=new L3WorldModelContextReadModel(repos).load(repos.runtime.getSession("session-account-a")!); + expect(context.projectContract).toBe("Run tests"); + expect(context.sourceMemoryIds).toHaveLength(2); + expect(context.sourceMemoryIds).toEqual(expect.arrayContaining(["L1-account-a","L1-account-b"])); + expect(db.db.prepare("SELECT * FROM memories WHERE user_id IS NOT NULL ORDER BY id").all()).toEqual(history); + } finally {db?.close();rmSync(root,{recursive:true,force:true});} + }); + + it("changes the context revision when an inherited same-text profile becomes valid", () => { + const root=mkdtempSync(join(tmpdir(),"memmy-l3-revision-")),path=join(root,"memory.sqlite"); + try { + const old=seedV9(path); + old.prepare(`UPDATE memories SET properties_json=json_set(properties_json,'$.internal_info.world_model.project_environment_profile','Uses Node 22') WHERE id='L3-account-a'`).run();old.close(); + const db=new MemoryDb({path}),repos=new Repositories(db.db); + try { + repos.l3WorldModels.upsertField({userId:null,projectId:identity.projectId,targetField:"project_contract",value:"Updated rule"}); + const session=repos.runtime.getSession("session-account-a")!,reader=new L3WorldModelContextReadModel(repos); + const before=reader.load(session); + repos.projectEnvironments.requestScan({userId:null,projectId:identity.projectId!,sessionId:session.id,trigger:"session_start",dedupeKey:"revision-scan"}); + repos.projectEnvironments.applyProfile({userId:null,projectId:identity.projectId!,scanId:repos.projectEnvironments.getState(null,identity.projectId!)!.currentScanId!,projectKind:"code",fingerprint:"same-tree",expectedCurrentProfile:"Uses Node 22",operation:"noop",profile:""}); + const after=reader.load(session); + expect(before.projectEnvironmentProfile).toBeNull(); + expect(after.projectEnvironmentProfile).toBe("Uses Node 22"); + expect(after.memoryId).toBe(before.memoryId); + expect(after.memoryVersion).not.toBe(before.memoryVersion); + expect(reader.load({...session,userId:"another-account"}).memoryVersion).toBe(after.memoryVersion); + } finally {db.close();} + } finally {rmSync(root,{recursive:true,force:true});} + }); + + it("accepts old workspace aliases while rejecting another real workspace", () => { + const root=mkdtempSync(join(tmpdir(),"memmy-l3-namespace-")),path=join(root,"memory.sqlite"); + try { + seedV9(path).close();const db=new MemoryDb({path}); + try { + const service=fixture.createTestMemoryService({db}); + const namespace={source:"codex",profileId:"default",userId:"account-c",...legacyProject("account-a"),sessionKey:"host-account-a"}; + const envelope={adapterId:"fixture",requestId:"123e4567-e89b-42d3-a456-426614174000"}; + expect(service.l3WorldModelContext("session-account-a",{...envelope,namespace}).projectContract).toBe("Run tests"); + expect(()=>service.l3WorldModelContext("session-account-a",{...envelope,namespace:{...namespace,workspaceId:"different-workspace"}})).toThrow("scope_conflict"); + } finally {db.close();} + } finally {rmSync(root,{recursive:true,force:true});} + }); + + it("leases only runnable historical scope heads without a cross-scope cycle", () => { + const root=mkdtempSync(join(tmpdir(),"memmy-l3-queue-")),path=join(root,"memory.sqlite"); + try { + const old=seedV9(path); + old.prepare("UPDATE evolution_jobs SET id='job-z' WHERE id='job-account-a'").run(); + old.prepare("UPDATE evolution_jobs SET id='job-m' WHERE id='job-account-b'").run(); + old.prepare(`INSERT INTO evolution_jobs(id,job_type,status,user_id,session_id,scope_key,scope_seq,payload_json,attempts,max_attempts,created_at,updated_at) + SELECT 'job-a',job_type,status,user_id,session_id,scope_key,2,payload_json,attempts,max_attempts,created_at,updated_at FROM evolution_jobs WHERE id='job-z'`).run(); + old.close();const db=new MemoryDb({path}),repos=new Repositories(db.db); + try { + const lease=()=>repos.runtime.leaseQueuedJobs(10,60,undefined,false,["l3_world_model_update"]).map(job=>job.id); + expect(lease()).toEqual(["job-m"]); + expect(lease()).toEqual([]); + db.db.prepare("UPDATE evolution_jobs SET status='succeeded' WHERE id='job-m'").run(); + expect(lease()).toEqual(["job-z"]); + repos.runtime.requeueLeasedJobsAfterRestart(); + expect(lease()).toEqual(["job-z"]); + db.db.prepare("UPDATE evolution_jobs SET status='succeeded' WHERE id='job-z'").run(); + expect(lease()).toEqual(["job-a"]); + } finally {db.close();} + } finally {rmSync(root,{recursive:true,force:true});} + }); + + it.each(["failed", "backoff"])("does not let a %s scope block another runnable scope", (state) => { + const root=mkdtempSync(join(tmpdir(),"memmy-l3-blocked-head-")),path=join(root,"memory.sqlite"); + try { + seedV9(path).close();const db=new MemoryDb({path}),repos=new Repositories(db.db); + try { + db.db.prepare(`UPDATE evolution_jobs SET status=?,payload_json=json_set(payload_json,'$.runAfter',?) WHERE id='job-account-a'`) + .run(state==="failed"?"failed":"queued",state==="backoff"?"2099-01-01T00:00:00.000Z":null); + expect(repos.runtime.leaseQueuedJobs(10,60,undefined,false,["l3_world_model_update"]).map(job=>job.id)).toEqual(["job-account-b"]); + db.db.prepare("UPDATE evolution_jobs SET status='succeeded' WHERE id='job-account-b'").run(); + db.db.prepare("UPDATE evolution_jobs SET status='queued',payload_json=json_remove(payload_json,'$.runAfter') WHERE id='job-account-a'").run(); + expect(repos.runtime.leaseQueuedJobs(10,60,undefined,false,["l3_world_model_update"]).map(job=>job.id)).toEqual(["job-account-a"]); + } finally {db.close();} + } finally {rmSync(root,{recursive:true,force:true});} + }); +}); +describe("device memory / preserved schema-v9 history", () => { + it("upgrades NOT NULL tables verbatim, resolves aliases and never resurrects deleted legacy fields", () => { + const root = mkdtempSync(join(tmpdir(),"memmy-uid-upgrade-")), path = join(root,"memory.sqlite"); + try { + const old = seedV9(path); + const snapshots = ["memories","sessions","l3_world_model_scopes","source_turn_captures","user_memories","runtime_kv","token_usage_outbox","raw_turns","l3_world_model_evidence_batches","evolution_jobs"].map(table => [table,old.prepare(`SELECT * FROM ${table}`).all()] as const); + old.close(); + const db = new MemoryDb({path}); + expect(db.schemaVersion().version).toBe(SCHEMA_VERSION); + expect(existsSync(`${path}.pre-v${SCHEMA_VERSION}.bak`)).toBe(true); + for (const [table,rows] of snapshots) expect(db.db.prepare(`SELECT * FROM ${table}`).all()).toEqual(rows); + expect(db.db.pragma("foreign_key_check")).toEqual([]); + expect(db.db.pragma("table_info(memories)")).toEqual(expect.arrayContaining([expect.objectContaining({name:"user_id",notnull:0})])); + const repos = new Repositories(db.db), a = legacyProject("account-a"), b = legacyProject("account-b"); + expect(canonicalProjectId(db.db,a.projectId)).toBe(identity.projectId); + expect(canonicalProjectId(db.db,b.projectId)).toBe(identity.projectId); + expect(canonicalProjectId(db.db,"unknown-project")).toBe("unknown-project"); + const scope = {userId:"new-account",source:"codex",profileId:"default",namespaceKey:stableHash({tenantId:null,projectId:identity.projectId,workspaceId:identity.workspaceId}),conversationId:"native-conversation"}; + expect(repos.runtime.getSourceTurnCapture(scope,"native-turn")?.sessionId).toMatch(/^session-account-/); + expect(repos.memories.nativeDuplicates(repos.memories.get("L1-account-a")!)).toEqual({ids:["L1-account-a","L1-account-b"],deleted:false}); + repos.memories.softDelete("L1-account-a"); + expect(repos.memories.nativeDuplicates(repos.memories.get("L1-account-b")!).deleted).toBe(true); + expect(repos.userMemories.listActive("any-account").map(row=>row.id)).toEqual(["user-memory-account-a"]); + repos.userMemories.softDelete("user-memory-account-a"); + expect(repos.userMemories.listActive("account-b")).toEqual([]); + expect(db.db.prepare("SELECT COUNT(*) FROM user_memories").pluck().get()).toBe(2); + expect(db.db.prepare("SELECT seq FROM sqlite_sequence WHERE name='memory_change_log'").pluck().get()).toBe(9000); + const fields = repos.l3WorldModels.fields("new-account",identity.projectId); + expect(fields).toMatchObject({projectContract:"Run tests",domainKnowledge:"Parser accepts UTF-8"}); + expect(repos.l3WorldModels.fields("account-b",b.projectId)).toEqual(fields); + const saved = repos.l3WorldModels.upsertField({userId:"account-c",projectId:identity.projectId,targetField:"project_contract",value:null})!; + expect(repos.l3WorldModels.fields("account-a",a.projectId)).toMatchObject({projectContract:null,domainKnowledge:"Parser accepts UTF-8"}); + expect(db.db.prepare(`SELECT user_id FROM memories WHERE id=?`).get(saved.id)).toEqual({user_id:null}); + expect(db.db.prepare(`SELECT user_id FROM memories WHERE id='L3-account-a'`).get()).toEqual({user_id:"account-a"}); + repos.l3WorldModels.deleteScopeMemory(saved.id); + expect(repos.l3WorldModels.fields(null,identity.projectId)).toEqual({generalRulesAndSafetyConstraints:null,projectEnvironmentProfile:null,projectContract:null,domainKnowledge:null}); + db.close(); + const reopened = new MemoryDb({path}); + expect(new Repositories(reopened.db).l3WorldModels.fields("account-d",identity.projectId).projectContract).toBeNull(); + expect(reopened.db.prepare(`SELECT value_json FROM runtime_kv WHERE key='token-budget-fixture'`).pluck().get()).toBe(JSON.stringify({accountId:"account-a",used:1234,limit:5678})); + reopened.close(); + const readonly = new MemoryDb({path,readonly:true}); + expect(new Repositories(readonly.db).l3WorldModels.fields("account-e",a.projectId).projectContract).toBeNull(); + readonly.close(); + } finally { rmSync(root,{recursive:true,force:true}); } + }); + it("continues immutable legacy L3 batches once and serializes different account scopes", async () => { + const root = mkdtempSync(join(tmpdir(),"memmy-uid-batches-")), path = join(root,"memory.sqlite"); + try { + seedV9(path).close(); + const db = new MemoryDb({path}), repos = new Repositories(db.db); + const complete = vi.fn(async () => JSON.stringify({op:"noop",project_contract:"",reason:"already recorded"})); + const llm = {config:{},isConfigured:()=>true,complete,completeJson:vi.fn(),status:()=>({provider:"fixture",configured:true,remote:false})} as unknown as LlmClient; + const jobs = repos.runtime.leaseQueuedJobs(10,60,undefined,false,["l3_world_model_update"]); + expect(jobs.map(job=>job.id)).toEqual(["job-account-a"]); + expect(repos.runtime.leaseQueuedJobs(10,60,undefined,false,["l3_world_model_update"])).toEqual([]); + const pipeline = new L3WorldModelTraceFieldPipeline({repos,skillLlm:llm}); + await pipeline.updateField(jobs[0]!); + expect(complete).toHaveBeenCalledTimes(1); + await pipeline.updateField(jobs[0]!); + expect(complete).toHaveBeenCalledTimes(1); + expect(repos.l3WorldModels.getTarget("batch-account-a","project_contract")?.status).toBe("applied"); + db.db.prepare(`UPDATE evolution_jobs SET status='succeeded' WHERE id='job-account-a'`).run(); + expect(repos.runtime.leaseQueuedJobs(10,60,undefined,false,["l3_world_model_update"]).map(job=>job.id)).toEqual(["job-account-b"]); + expect(repos.l3WorldModels.getStoredBatch("batch-account-a")?.userId).toBe("account-a"); + expect(repos.l3WorldModels.getBatch("batch-account-a")?.userId).toBeNull(); + expect(repos.l3WorldModels.getBatch("batch-account-a")?.projectId).toBe(identity.projectId); + db.close(); + } finally {rmSync(root,{recursive:true,force:true});} + }); + + it.each(["queued", "failed"] as const)("resumes %s legacy L3 jobs with feedback without rewriting history", async (status) => { + const root = mkdtempSync(join(tmpdir(), "memmy-feedback-upgrade-")), path = join(root, "memory.sqlite"); + try { + const old = seedV9(path); + const projectId = legacyProject("account-a").projectId; + old.prepare(`INSERT INTO feedback(id,user_id,project_id,session_id,l1_memory_id,raw_turn_id,channel,polarity,rationale,created_at) + VALUES ('feedback-a','account-a',?,'session-account-a','L1-account-a','raw-account-a','explicit','positive','Keep this rule',?)`).run(projectId,at); + const payload = {scopeKey:`l3wm:${sha256Hex(canonicalJson({userId:"account-a",projectId}))}`,scopeSeq:1,userId:"account-a",projectId,sessionId:"session-account-a",trigger:"session_close",startTraceSeq:1,endTraceSeq:1,l1MemoryIds:["L1-account-a"],rawTurnIds:["raw-account-a"],feedbackIds:["feedback-a"]}; + old.prepare("UPDATE l3_world_model_evidence_batches SET feedback_ids_json=?,payload_hash=? WHERE id='batch-account-a'") + .run(JSON.stringify(payload.feedbackIds),sha256Hex(canonicalJson(payload))); + old.prepare("UPDATE evolution_jobs SET status=? WHERE id='job-account-a'").run(status); + const feedbackBefore = old.prepare("SELECT * FROM feedback").all(); + const batchBefore = old.prepare("SELECT id,user_id,project_id,scope_key,scope_seq,session_id,trigger,start_trace_seq,end_trace_seq,l1_memory_ids_json,raw_turn_ids_json,feedback_ids_json,payload_hash,created_at FROM l3_world_model_evidence_batches").all(); + const memoriesBefore = old.prepare("SELECT id,user_id,memory_value FROM memories ORDER BY id").all(); + old.close(); + const db = new MemoryDb({path}), repos = new Repositories(db.db); + try { + expect(repos.runtime.getFeedback("feedback-a")?.projectId).toBe(identity.projectId); + expect(repos.runtime.listFeedback({userId:"another-account",sessionId:"session-account-a"})[0]?.projectId).toBe(identity.projectId); + const complete = vi.fn(async () => JSON.stringify({op:"noop",project_contract:"",reason:"already recorded"})); + const pipeline = new L3WorldModelTraceFieldPipeline({repos,skillLlm:{config:{},isConfigured:()=>true,complete} as unknown as LlmClient}); + const job = repos.runtime.getJob("job-account-a")!; + // A different project remains invalid even though account ownership is ignored. + db.db.prepare("UPDATE feedback SET project_id='unrelated-project' WHERE id='feedback-a'").run(); + await expect(pipeline.updateField(job)).rejects.toThrow("feedback scope mismatch"); + expect(complete).not.toHaveBeenCalled(); + db.db.prepare("UPDATE feedback SET project_id=? WHERE id='feedback-a'").run(projectId); + db.db.prepare("UPDATE l3_world_model_evidence_batches SET payload_hash='tampered' WHERE id='batch-account-a'").run(); + await expect(pipeline.updateField(job)).rejects.toThrow("batch payload hash mismatch"); + expect(complete).not.toHaveBeenCalled(); + db.db.prepare("UPDATE l3_world_model_evidence_batches SET payload_hash=? WHERE id='batch-account-a'").run(sha256Hex(canonicalJson(payload))); + await pipeline.updateField(job); + expect(complete).toHaveBeenCalledTimes(1); + expect(JSON.stringify(complete.mock.calls)).toContain("Keep this rule"); + expect(repos.l3WorldModels.getTarget("batch-account-a","project_contract")?.status).toBe("applied"); + expect(db.db.prepare("SELECT * FROM feedback").all()).toEqual(feedbackBefore); + expect(db.db.prepare("SELECT id,user_id,project_id,scope_key,scope_seq,session_id,trigger,start_trace_seq,end_trace_seq,l1_memory_ids_json,raw_turn_ids_json,feedback_ids_json,payload_hash,created_at FROM l3_world_model_evidence_batches").all()).toEqual(batchBefore); + expect(db.db.prepare("SELECT id,user_id,memory_value FROM memories ORDER BY id").all()).toEqual(memoriesBefore); + } finally { db.close(); } + } finally { rmSync(root,{recursive:true,force:true}); } + }); + + it.each(["project_contract", "domain_knowledge"] as const)("inherits the applied profile when first updating %s after upgrade", (targetField) => { + const root = mkdtempSync(join(tmpdir(), "memmy-profile-upgrade-")), path = join(root,"memory.sqlite"); + try { + const old = seedV9(path), projectId = legacyProject("account-a").projectId; + old.prepare(`UPDATE memories SET properties_json=json_set(properties_json,'$.internal_info.world_model.project_environment_profile','Uses Node 22'), + info_json=json_set(info_json,'$.project_environment_applied_scan_id','scan-a') WHERE id='L3-account-a'`).run(); + old.prepare(`INSERT INTO l3_world_model_project_environment_state(user_id,project_id,status,applied_scan_id,updated_at) + VALUES ('account-a',?,'clean','scan-a',?)`).run(projectId,at); + const history = old.prepare("SELECT * FROM memories ORDER BY id").all(); old.close(); + let db = new MemoryDb({path}); + try { + let repos = new Repositories(db.db); + const read = () => new L3WorldModelContextReadModel(repos).load({...repos.runtime.getSession("session-account-a")!,userId:"switched-account"}); + expect(read().projectEnvironmentProfile).toBe("Uses Node 22"); + repos.l3WorldModels.upsertField({userId:"account-c",projectId:identity.projectId,targetField,value:"New project fact"}); + expect(read().projectEnvironmentProfile).toBe("Uses Node 22"); + expect(repos.l3WorldModels.getMemory(null,identity.projectId)?.info.project_environment_applied_scan_id).toBe("scan-a"); + expect(db.db.prepare("SELECT * FROM memories WHERE user_id IS NOT NULL ORDER BY id").all()).toEqual(history); + db.close(); db = new MemoryDb({path}); repos = new Repositories(db.db); + expect(read().projectEnvironmentProfile).toBe("Uses Node 22"); + repos.l3WorldModels.upsertField({userId:"account-d",projectId:identity.projectId,targetField:"project_environment_profile",value:null}); + repos.l3WorldModels.upsertField({userId:"account-e",projectId:identity.projectId,targetField,value:"Another fact"}); + expect(read().projectEnvironmentProfile).toBeNull(); + db.close(); db = new MemoryDb({path}); repos = new Repositories(db.db); + expect(read().projectEnvironmentProfile).toBeNull(); + } finally { db.close(); } + } finally { rmSync(root,{recursive:true,force:true}); } + }); + + it("does not borrow a scan ID from a different historical profile", () => { + const root = mkdtempSync(join(tmpdir(),"memmy-profile-provenance-")), path=join(root,"memory.sqlite"); + try { + const old=seedV9(path); + old.prepare(`UPDATE memories SET properties_json=json_set(properties_json,'$.internal_info.world_model.project_environment_profile','Unapplied newer profile'),updated_at='2026-10-02T00:00:00.000Z' WHERE id='L3-account-a'`).run(); + old.prepare(`UPDATE l3_world_model_scopes SET updated_at='2026-10-02T00:00:00.000Z' WHERE user_id='account-a'`).run(); + old.prepare(`UPDATE memories SET properties_json=json_set(properties_json,'$.internal_info.world_model.project_environment_profile','Applied older profile'),info_json=json_set(info_json,'$.project_environment_applied_scan_id','scan-b') WHERE id='L3-account-b'`).run(); + old.prepare(`INSERT INTO l3_world_model_project_environment_state(user_id,project_id,status,applied_scan_id,updated_at) VALUES ('account-b',?,'clean','scan-b',?)`).run(legacyProject("account-b").projectId,at); + old.close(); + const db=new MemoryDb({path}),repos=new Repositories(db.db); + try { + repos.l3WorldModels.upsertField({userId:null,projectId:identity.projectId,targetField:"project_contract",value:"New rule"}); + expect(repos.l3WorldModels.fields(null,identity.projectId).projectEnvironmentProfile).toBe("Unapplied newer profile"); + expect(repos.l3WorldModels.getMemory(null,identity.projectId)?.info.project_environment_applied_scan_id).toBeUndefined(); + expect(new L3WorldModelContextReadModel(repos).load(repos.runtime.getSession("session-account-a")!).projectEnvironmentProfile).toBeNull(); + } finally {db.close();} + } finally {rmSync(root,{recursive:true,force:true});} + }); + it("shares writes and project identity across accounts, retaining separate host identities", async () => { + const {db,service} = fixture.createTestService(); + const namespace = {source:"codex",profileId:"default",sessionKey:"shared-session",userId:"account-a"}; + const request = {l3WorldModelProtocolVersion:2 as const,l3WorldModelTransition:"resume_only" as const,workspaceUri:uri,workspaceHostId:host,namespace}; + const first = service.openSession(request), second = service.openSession({...request,namespace:{...namespace,userId:"account-b"}}); + expect(second.sessionId).toBe(first.sessionId); + expect(second.projectId).toBe(first.projectId); + expect(resolveWorkspaceIdentity("account-c",{workspaceUri:uri,workspaceHostId:host})).toEqual(identity); + expect(resolveWorkspaceIdentity(null,{workspaceUri:uri,workspaceHostId:"b".repeat(64)}).projectId).not.toBe(identity.projectId); + (await service.completeTurn("shared-turn",{sessionId:first.sessionId,namespace,query:"我最喜欢的水果是苹果",answer:"记住了。"})); + const a = service.panelItems({layer:"UserMemory",namespace}), b = service.panelItems({layer:"UserMemory",namespace:{...namespace,userId:"account-b"}}); + expect(a.items.length).toBeGreaterThan(0); + expect(b.items).toEqual(a.items); + for (const table of ["memories","user_memories","sessions","raw_turns","episodes","evolution_jobs"]) expect(db.db.prepare(`SELECT COUNT(*) FROM ${table} WHERE user_id IS NOT NULL`).pluck().get()).toBe(0); + service.deleteMemory(a.items[0]!.id,{namespace:{...namespace,userId:"account-b"}}); + expect(service.panelItems({layer:"UserMemory",namespace}).items).toEqual([]); + }); + it("resets legacy captures on full clear while retaining project aliases after restart", () => { + const root = mkdtempSync(join(tmpdir(),"memmy-uid-clear-")), path = join(root,"memory.sqlite"); + try { + seedV9(path).close(); + const db = new MemoryDb({path}), repos = new Repositories(db.db); + repos.clearAllMemoryData(); + expect(db.db.prepare("SELECT COUNT(*) FROM sessions").pluck().get()).toBe(0); + const bundle = repos.runtime.exportBundleTables(true); + expect(bundle.memory_namespace_aliases?.length ?? 0).toBeGreaterThan(0); + db.close(); + const reopened = new MemoryDb({path}); + const capture = new Repositories(reopened.db).runtime.getSourceTurnCapture({userId:"account-c",source:"codex",profileId:"default",namespaceKey:stableHash({tenantId:null,projectId:identity.projectId,workspaceId:identity.workspaceId}),conversationId:"native-conversation"},"native-turn"); + expect(capture).toBeUndefined(); + expect(canonicalProjectId(reopened.db,legacyProject("account-a").projectId)).toBe(identity.projectId); + expect(new Repositories(reopened.db).runtime.getKv("source_turn_capture_activated_at")?.value).toBe(at); + reopened.close(); + const imported = new MemoryDb({path:join(root,"imported.sqlite")}); + const importedRepos = new Repositories(imported.db); + importedRepos.runtime.importBundleTables(bundle); + expect(importedRepos.runtime.getSourceTurnCapture({userId:"account-d",source:"codex",profileId:"default",namespaceKey:stableHash({tenantId:null,projectId:identity.projectId,workspaceId:identity.workspaceId}),conversationId:"native-conversation"},"native-turn")).toBeUndefined(); + imported.close(); + } finally {rmSync(root,{recursive:true,force:true});} + }); + it("respects pre-upgrade L3 deletion and does not replay older batches into another account's copy", async () => { + const root = mkdtempSync(join(tmpdir(),"memmy-uid-tombstone-")), path = join(root,"memory.sqlite"); + try { + const old = seedV9(path); + old.prepare("UPDATE memories SET deleted_at='2026-10-02T00:00:00.000Z',status='deleted',updated_at='2026-10-02T00:00:00.000Z' WHERE id='L3-account-a'").run(); + old.prepare("UPDATE l3_world_model_scopes SET memory_id=NULL WHERE user_id='account-a'").run(); + old.close(); + const db = new MemoryDb({path}), repos = new Repositories(db.db); + expect(repos.l3WorldModels.fields("account-c",identity.projectId).domainKnowledge).toBeNull(); + const complete = vi.fn(); + await new L3WorldModelTraceFieldPipeline({repos,skillLlm:{complete} as unknown as LlmClient}).updateField(repos.runtime.getJob("job-account-b")!); + expect(complete).not.toHaveBeenCalled(); + expect(repos.l3WorldModels.getTarget("batch-account-b","project_contract")).toMatchObject({status:"applied",noChange:true}); + repos.l3WorldModels.upsertField({userId:"account-c",projectId:identity.projectId,targetField:"domain_knowledge",value:"New evidence",at:"2026-10-03T00:00:00.000Z"}); + expect(repos.l3WorldModels.fields("account-d",identity.projectId).domainKnowledge).toBe("New evidence"); + db.close(); + } finally {rmSync(root,{recursive:true,force:true});} + }); + it("retries new and historical idempotency entries across accounts without accepting changed content", async () => { + const {db,service} = fixture.createTestService(); + const repos = new Repositories(db.db); + const request = {adapterId:"fixture",requestId:"open",timeZone:"+00:00",namespace:{source:"codex",profileId:"default",userId:"account-a"}}; + const opened = service.openSession(request); + expect(service.openSession({...request,namespace:{...request.namespace,userId:"account-b"}})) + .toMatchObject({sessionId:opened.sessionId,userId:null,duplicate:true}); + // Reproduce the exact pre-upgrade cache and session owner, without rewriting them on read. + db.db.prepare("UPDATE sessions SET user_id='account-a' WHERE id=?").run(opened.sessionId); + repos.runtime.saveIdempotency("session.open:fixture:open",stableHash({operation:"session.open",request}),{...opened,userId:"account-a"}); + expect(service.openSession({...request,namespace:{...request.namespace,userId:"account-b"}})).toMatchObject({userId:null,duplicate:true}); + expect(()=>service.openSession({...request,source:"different"})).toThrow(/different session.open request/); + const turn = {adapterId:"fixture",requestId:"complete",timeZone:"+00:00",sessionId:opened.sessionId,namespace:request.namespace,query:"Remember UTF-8",answer:"Confirmed"}; + const completed = (await service.completeTurn("turn",turn)); + const switched = {...turn,namespace:{...turn.namespace,userId:"account-b"}}; + expect((await service.completeTurn("turn",switched))).toMatchObject({duplicate:true,rawTurnId:completed.rawTurnId,jobs:[]}); + repos.runtime.saveIdempotency("turn.complete:fixture:complete",stableHash({turnId:"turn",request:turn}),completed); + expect((await service.completeTurn("turn",switched))).toMatchObject({duplicate:true,rawTurnId:completed.rawTurnId}); + await expect(service.completeTurn("turn",{...switched,answer:"Different"})).rejects.toThrow(/different request body/); + }); + it("invalidates ambiguous imported aliases and keeps remote identity without session evidence", () => { + const {db,service} = fixture.createTestService(); + const opened = service.openSession({l3WorldModelProtocolVersion:2,l3WorldModelTransition:"resume_only",workspaceUri:uri,workspaceHostId:host,namespace:{source:"codex",profileId:"default",sessionKey:"alias-a"}}); + db.db.prepare("UPDATE sessions SET project_id='legacy',workspace_id='legacy-workspace' WHERE id=?").run(opened.sessionId); + refreshProjectAliases(db.db); + expect(canonicalProjectId(db.db,"legacy")).toBe(identity.projectId); + const other = service.openSession({l3WorldModelProtocolVersion:2,l3WorldModelTransition:"resume_only",workspaceUri:uri,workspaceHostId:"b".repeat(64),namespace:{source:"codex",profileId:"default",sessionKey:"alias-b"}}); + db.db.prepare("UPDATE sessions SET project_id='legacy',workspace_id='legacy-workspace' WHERE id=?").run(other.sessionId); + refreshProjectAliases(db.db); + expect(canonicalProjectId(db.db,"legacy")).toBe("legacy"); + expect(canonicalProjectId(db.db,"legacy-workspace")).toBe("legacy-workspace"); + const remoteUri = "git://github.com/example/project"; + db.db.prepare("INSERT INTO l3_world_model_scopes(scope_key,user_id,project_id,workspace_uri,updated_at) VALUES ('old-remote','old-owner','old-project',?,?)").run(remoteUri,at); + refreshProjectAliases(db.db); + expect(canonicalProjectId(db.db,"old-project")).toBe(resolveWorkspaceIdentity(null,{workspaceUri:remoteUri}).projectId); + }); + it("shares HTTP operation fingerprints across accounts and keeps content conflicts", async () => { + const {db,service} = fixture.createTestService(); + const request = {adapterId:"fixture",requestId:"generic",namespace:{source:"codex",profileId:"default",userId:"a"}}; + const fingerprint = {path:"/memory/add",request:{...request,content:"same content"}}; + const run = vi.fn(async()=>({id:"result"})); + await service.idempotent("memory.add",request,fingerprint,run); + const switched = {...request,namespace:{...request.namespace,userId:"b"}}; + expect(await service.idempotent("memory.add",switched,{...fingerprint,request:{...fingerprint.request,...switched}},run)).toMatchObject({id:"result",duplicate:true}); + expect(run).toHaveBeenCalledTimes(1); + await expect(service.idempotent("memory.add",switched,{...fingerprint,request:{...fingerprint.request,...switched,content:"different"}},run)).rejects.toThrow(/different request body/); + expect(db.db.prepare("SELECT COUNT(*) FROM idempotency_keys").pluck().get()).toBe(1); + }); +}); diff --git a/Memory/tests/repository/log-retention.test.ts b/Memory/tests/repository/log-retention.test.ts index 7c2b1d7c4..9e763184e 100644 --- a/Memory/tests/repository/log-retention.test.ts +++ b/Memory/tests/repository/log-retention.test.ts @@ -77,6 +77,89 @@ describe("runtime log retention", () => { rmSync(root, { recursive: true, force: true }); } }); + + it("persists legacy explicit L3 archives before their audit evidence is pruned", async () => { + const root = mkdtempSync(join(tmpdir(), "mindock-l3-archive-retention-")); + const path = join(root, "memory.sqlite"); + let db: MemoryDb | undefined; + let restored: MemoryDb | undefined; + try { + db = new MemoryDb({ path }); + const repos = new Repositories(db.db); + const memory = repos.l3WorldModels.upsertField({ + userId: "legacy-archive-account", + projectId: "legacy-archive-project", + targetField: "project_contract", + value: "This legacy project rule was manually archived.", + at: isoAt(0) + })!; + const row = db.db.prepare("SELECT properties_json FROM memories WHERE id = ?").get(memory.id) as { + properties_json: string; + }; + const properties = JSON.parse(row.properties_json) as { + status?: string; + internal_info: Record; + }; + properties.status = "archived"; + delete properties.internal_info.l3_world_model_explicitly_archived; + db.db.prepare("UPDATE memories SET status = 'archived', properties_json = ?, updated_at = ? WHERE id = ?") + .run(JSON.stringify(properties), isoAt(1), memory.id); + repos.runtime.insertAudit({ + id: "legacy-l3-archive-audit", + userId: "legacy-archive-account", + actor: {}, + action: "archive", + targetKind: "world_model", + targetId: memory.id, + meta: {}, + createdAt: isoAt(-1) + }); + expect(repos.l3WorldModels.getMemory(null, "legacy-archive-project")?.status).toBe("archived"); + + const exportWithoutAudit = repos.runtime.exportBundleTables(true); + delete exportWithoutAudit.audit_logs; + const exportedMemory = exportWithoutAudit.memories!.find((row) => row.id === memory.id)!; + expect(JSON.parse(String(exportedMemory.properties_json)).internal_info.l3_world_model_explicitly_archived) + .toBe(true); + expect(JSON.parse((db.db.prepare("SELECT properties_json FROM memories WHERE id = ?").get(memory.id) as { + properties_json: string; + }).properties_json).internal_info.l3_world_model_explicitly_archived).toBeUndefined(); + restored = new MemoryDb({ path: join(root, "restored.sqlite") }); + const restoredRepos = new Repositories(restored.db); + restoredRepos.runtime.importBundleTables(exportWithoutAudit); + expect(restoredRepos.l3WorldModels.getMemory(null, "legacy-archive-project")?.status).toBe("archived"); + restored.close(); + restored = undefined; + + seedAuditLogs(db, LOG_RETENTION_LIMIT); + repos.runtime.insertAudit({ + id: "audit-trigger-retention", + userId: "user", + actor: {}, + action: "test", + targetKind: "fixture", + targetId: "fixture", + meta: {}, + createdAt: isoAt(LOG_RETENTION_LIMIT + 1) + }); + await waitForRowCount(db, "audit_logs", LOG_RETENTION_LIMIT); + + expect(db.db.prepare("SELECT id FROM audit_logs WHERE id = 'legacy-l3-archive-audit'").get()).toBeUndefined(); + const archived = db.db.prepare("SELECT properties_json FROM memories WHERE id = ?").get(memory.id) as { + properties_json: string; + }; + expect(JSON.parse(archived.properties_json).internal_info.l3_world_model_explicitly_archived).toBe(true); + expect(repos.l3WorldModels.getMemory(null, "legacy-archive-project")?.status).toBe("archived"); + + db.close(); + db = new MemoryDb({ path, readonly: true }); + expect(new Repositories(db.db).l3WorldModels.getMemory(null, "legacy-archive-project")?.status).toBe("archived"); + } finally { + db?.close(); + restored?.close(); + rmSync(root, { recursive: true, force: true }); + } + }); }); function seedApiLogs(db: MemoryDb, count: number): void { diff --git a/Memory/tests/repository/memory-retrieval-index.test.ts b/Memory/tests/repository/memory-retrieval-index.test.ts index 950254528..bc1ccdd2f 100644 --- a/Memory/tests/repository/memory-retrieval-index.test.ts +++ b/Memory/tests/repository/memory-retrieval-index.test.ts @@ -42,7 +42,7 @@ describe("memory retrieval indexes", () => { created: false, memory: { id: first.id, - userId: "global-key-user-a", + userId: null, memoryLayer: layer, memoryKey: "shared-memory-key", memoryValue: `${layer} second value` @@ -86,6 +86,10 @@ describe("memory retrieval indexes", () => { }).map((hit) => hit.id)).not.toContain("trace_indexed"); expect(repos.memories.searchFtsIds("\"specialterm\"", { memoryLayer: "L1" }, 5) .map((hit) => hit.id)).toContain("trace_indexed"); + expect(repos.memories.searchFtsIds("\"specialterm\"", { + memoryLayer: "L1", + excludedIds: ["trace_indexed"] + }, 5)).toEqual([]); expect(repos.memories.searchFtsIds("\"trace_indexed\"", { memoryLayer: "L1" }, 5) .map((hit) => hit.id)).toContain("trace_indexed"); expect(repos.memories.search("trace_indexed", { memoryLayer: "L1" }, 5) diff --git a/Memory/tests/repository/uid-origin-import-regression.test.ts b/Memory/tests/repository/uid-origin-import-regression.test.ts new file mode 100644 index 000000000..ba720e6e2 --- /dev/null +++ b/Memory/tests/repository/uid-origin-import-regression.test.ts @@ -0,0 +1,419 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; +import { Repositories } from "../../src/storage/repositories.js"; +import { createCapturingEmbedder, createMemoryServiceFixture } from "../fixtures/memory-service-fixture.js"; +import { MemoryDb } from "../../src/storage/db.js"; +import { join } from "node:path"; +import { canonicalProjectId } from "../../src/storage/device-scope.js"; +import { SqliteVecStore } from "../../src/storage/sqlite-vec-store.js"; +import { embeddingTextForMemory } from "../../src/service/embedding/embedding-pipeline.js"; + +const fixture = createMemoryServiceFixture(); +afterEach(() => { vi.restoreAllMocks(); fixture.cleanup(); }); +const at = "2026-10-01T00:00:00.000Z"; +const turnInput = { + query: "Inspect package.json and report the test command.", answer: "Use npm test.", + toolCalls: [{ id: "read", name: "read_file", input: { path: "package.json" } }], + toolResults: [{ id: "read", output: "npm test" }] +}; + +function deletedBackup() { + const target = fixture.createTestService(), source = fixture.createTestService(); + const tr = new Repositories(target.db.db), sr = new Repositories(source.db.db); + const memory = tr.l3WorldModels.upsertField({ userId: null, projectId: "project-p", + targetField: "project_contract", value: "Keep target rule", at })!; + target.db.db.prepare("UPDATE memories SET user_id='old' WHERE id=?").run(memory.id); + target.db.db.prepare("UPDATE l3_world_model_scopes SET user_id='old' WHERE memory_id=?").run(memory.id); + target.db.db.exec("DELETE FROM memory_l3_overrides"); + sr.runtime.importBundleTables(tr.runtime.exportBundleTables(true)); + sr.l3WorldModels.deleteScopeMemory(memory.id, "2026-10-02T00:00:00.000Z"); + return { target, source, tr, sr, memory, bundle: sr.runtime.exportBundleTables(true) }; +} + +describe("UID decoupling origin and import boundaries", () => { + it.each(["project", "workspace", "both"])("keeps independent evidence after empty %s is filled", async boundary => { + const { db, service } = fixture.createTestService(); + const repos = new Repositories(db.db); + const common = { source: "codex", profileId: "default", userId: "account-a" }; + const first = { ...common, sessionKey: "a", + ...(boundary === "project" ? { workspaceId: "workspace-b" } : {}), + ...(boundary === "workspace" ? { projectId: "project-b" } : {}) }; + const second = { ...common, sessionKey: "b", projectId: "project-b", workspaceId: "workspace-b" }; + const a = service.openSession({ namespace: first, meta: { conversationId: "native-chat" } }); + const b = service.openSession({ namespace: second, meta: { conversationId: "native-chat" } }); + const capture = async (sessionId: string, namespace: typeof first) => (await service.completeTurn("native-turn", { + sessionId, namespace, query: "Inspect package.json and report the test command.", answer: "Use npm test.", + toolCalls: [{ id: "read", name: "read_file", input: { path: "package.json" } }], + toolResults: [{ id: "read", output: "npm test" }] + })); + const ta = (await capture(a.sessionId, first)), tb = (await capture(b.sessionId, second)); + expect(ta.l1MemoryId).not.toBe(tb.l1MemoryId); + for (const [key, turn, session] of [["a", ta, a], ["b", tb, b]] as const) { + repos.runtime.upsertCandidatePoolTrace({ id: key, userId: null, sessionId: session.sessionId, + sourceMemoryId: turn.l1MemoryId!, candidateKey: "policy", candidateValue: "same evidence", score: 1, + evidence: {}, createdAt: at, updatedAt: at }); + } + service.openSession({ sessionId: a.sessionId, namespace: second }); + repos.memories.softDelete(ta.l1MemoryId!); + expect(repos.memories.nativeDuplicates(repos.memories.get(tb.l1MemoryId!)!).ids).toEqual([tb.l1MemoryId]); + expect(repos.runtime.listPendingCandidatePool({}).map(row => row.sourceMemoryId)).toContain(tb.l1MemoryId); + repos.runtime.markCandidatePoolPromoted({ candidateKey: "policy", sourceMemoryIds: [tb.l1MemoryId!], policyId: "p", at }); + expect(db.db.prepare("SELECT id,status FROM l2_candidate_pool ORDER BY id").all()).toEqual([ + { id: "a", status: "pending" }, { id: "b", status: "promoted" } + ]); + }); + + it.each(["historical", null])("preserves skipped L3 and permits unrelated imports (owner=%s)", owner => { + const target = fixture.createTestService(), source = fixture.createTestService(); + const tr = new Repositories(target.db.db), sr = new Repositories(source.db.db); + const memory = tr.l3WorldModels.upsertField({ userId: owner, projectId: "project-p", + targetField: "project_contract", value: "Keep target rule", at })!; + target.db.db.prepare("UPDATE memories SET user_id=? WHERE id=?").run(owner, memory.id); + target.db.db.prepare("UPDATE l3_world_model_scopes SET user_id=? WHERE memory_id=?").run(owner, memory.id); + target.db.db.exec("DELETE FROM memory_l3_overrides"); + const before = target.db.db.prepare("SELECT * FROM memories WHERE id=?").get(memory.id); + sr.runtime.importBundleTables(tr.runtime.exportBundleTables(true)); + sr.l3WorldModels.deleteScopeMemory(memory.id, "2026-10-02T00:00:00.000Z"); + sr.l3WorldModels.upsertField({ userId: null, projectId: "unrelated", targetField: "project_contract", value: "New rule", at }); + const bundle = sr.runtime.exportBundleTables(true); + tr.runtime.importBundleTables(bundle, { conflictStrategy: "skip" }); + expect(target.db.db.prepare("SELECT * FROM memories WHERE id=?").get(memory.id)).toEqual(before); + expect(tr.l3WorldModels.fields(null, "project-p").projectContract).toBe("Keep target rule"); + expect(tr.l3WorldModels.fields(null, "unrelated").projectContract).toBe("New rule"); + tr.runtime.importBundleTables(bundle, { conflictStrategy: "skip" }); + expect(tr.l3WorldModels.fields(null, "project-p").projectContract).toBe("Keep target rule"); + }); + + it("keeps a turn's scope captured before an asynchronous start and later session move", async () => { + const { db, service } = fixture.createTestService(); + const repos = new Repositories(db.db); + const a = service.openSession({ namespace: { source: "codex", profileId: "default", sessionKey: "a" }, meta: { conversationId: "chat" } }); + const start = service.startTurn({ sessionId: a.sessionId, turnId: "turn", query: turnInput.query }); + service.openSession({ sessionId: a.sessionId, namespace: { source: "codex", profileId: "default", projectId: "later", workspaceId: "later-w" } }); + await start; + const completed = (await service.completeTurn("turn", { sessionId: a.sessionId, ...turnInput })); + const memory = repos.memories.get(completed.l1MemoryId!)!; + expect(memory.info.native_origin_v1).toMatchObject({ projectId: null, workspaceId: null }); + expect(memory.info.project_id).toBeNull(); + expect(memory.appId).toBeUndefined(); + expect(db.db.prepare("SELECT key FROM runtime_kv WHERE key LIKE 'native_turn_origin:%'").all()).toEqual([]); + }); + + it("keeps true copies together after a session moves, preserving group deletion and promotion", async () => { + const { db, service } = fixture.createTestService(); + const repos = new Repositories(db.db); + const ns = { source: "codex", projectId: "p", workspaceId: "w", profileId: "default" }; + const memories = (await Promise.all(["a", "b"].map(async key => { + const session = service.openSession({ namespace: { ...ns, sessionKey: key, userId: key }, meta: { conversationId: "chat" } }); + const turn = (await service.completeTurn("turn", { sessionId: session.sessionId, namespace: ns, ...turnInput })); + repos.runtime.upsertCandidatePoolTrace({ id: key, userId: key, sessionId: session.sessionId, + sourceMemoryId: turn.l1MemoryId!, candidateKey: "policy", candidateValue: "evidence", score: 1, evidence: {}, createdAt: at, updatedAt: at }); + return repos.memories.get(turn.l1MemoryId!)!; + }))); + service.openSession({ sessionId: memories[0]!.sessionId, namespace: { source: "codex", profileId: "default", projectId: "different" } }); + for (const memory of memories) expect(new Set(repos.memories.nativeDuplicates(memory).ids)).toEqual(new Set(memories.map(row => row.id))); + repos.runtime.markCandidatePoolPromoted({ candidateKey: "policy", sourceMemoryIds: [memories[1]!.id], policyId: "policy", at }); + expect(db.db.prepare("SELECT DISTINCT status FROM l2_candidate_pool").all()).toEqual([{ status: "promoted" }]); + repos.memories.softDelete(memories[0]!.id); + expect(repos.memories.nativeDuplicates(memories[1]!).deleted).toBe(true); + }); + + it("rejects a reused turn with an explicitly different scope without changing its stored evidence", async () => { + const { db, service } = fixture.createTestService(); + const session = service.openSession({ namespace: { source: "codex", profileId: "default", sessionKey: "a" }, meta: { conversationId: "chat" } }); + const turn = (await service.completeTurn("turn", { sessionId: session.sessionId, ...turnInput })); + const before = db.db.prepare("SELECT * FROM raw_turns WHERE id=?").get(turn.rawTurnId); + const namespace = { source: "codex", profileId: "default", projectId: "later" }; + service.openSession({ sessionId: session.sessionId, namespace }); + await expect(service.completeTurn("turn", { sessionId: session.sessionId, namespace, ...turnInput })).rejects.toThrow("turn origin conflicts"); + expect(db.db.prepare("SELECT * FROM raw_turns WHERE id=?").get(turn.rawTurnId)).toEqual(before); + }); + + it("retains the session tenant when completion omits its namespace", async () => { + const { db, service } = fixture.createTestService(); + const session = service.openSession({ namespace: { source: "codex", profileId: "default", sessionKey: "a", tenantId: "tenant-a" }, meta: { conversationId: "chat" } }); + const turn = (await service.completeTurn("turn", { sessionId: session.sessionId, ...turnInput })); + expect(new Repositories(db.db).memories.get(turn.l1MemoryId!)!.info.native_origin_v1).toMatchObject({ tenantId: "tenant-a" }); + }); + + it.each(["absent", "unverified", "verified"])("keeps host keys separate from native conversations during %s Session imports", mode => { + const { db, service } = fixture.createTestService(); + const repos = new Repositories(db.db); + const namespace = { source: "codex", profileId: "default", sessionKey: "codex-memory-unverified-host" }; + const session = mode === "absent" ? undefined : service.openSession({ namespace, + ...(mode === "verified" ? { meta: { conversationId: "native-conversation" } } : {}) }); + const result = service.addMemory({ namespace, sessionId: session?.sessionId, adapterId: "agent-source:codex", requestId: "scan", + layer: "L1", source: "codex", tags: ["agent-source", "codex"], turnId: "native-turn", + content: `## user\n\n${turnInput.query}\n\n## assistant\n\n${turnInput.answer}` }); + const memory = repos.memories.get(result.id)!; + expect(memory.conversationId).toBe(mode === "verified" ? "native-conversation" : undefined); + expect(memory.info.native_origin_v1).toMatchObject({ + conversationId: mode === "verified" ? "native-conversation" : null, hostSessionKey: namespace.sessionKey + }); + }); + + it("retains a verified source conversation when consuming a pending turn origin", async () => { + const { db, service } = fixture.createTestService(); + const repos = new Repositories(db.db); + const namespace = { source: "codex", profileId: "default" }; + const session = service.openSession({ namespace: { ...namespace, sessionKey: "codex-memory-native-conversation" } }); + await service.startTurn({ sessionId: session.sessionId, turnId: "native-turn", namespace, query: turnInput.query }); + expect(repos.runtime.getRawTurnBySessionTurn(session.sessionId, "native-turn")).toBeUndefined(); + const result = (await service.completeSourceTurn({ sessionId: session.sessionId, namespace, source: "codex", channel: "hook", + sourceTurn: { source: "codex", profileId: "default", conversationId: "native-conversation", turnId: "native-turn", + startedAt: "2099-01-01T10:00:00.000Z", completedAt: "2099-01-01T10:01:00.000Z", sequence: 1, completionEvidence: "task_complete:10" }, + ...turnInput })); + expect(result.status).toBe("stored"); + if (result.status !== "stored" || !result.result) throw new Error("source capture was not stored"); + const memory = repos.memories.get(result.result.l1MemoryId!)!; + const raw = repos.runtime.getRawTurn(result.result.rawTurnId!)!; + expect(memory.conversationId).toBe("native-conversation"); + expect(raw.messagePayload?.native_origin_v1).toMatchObject({ conversationId: "native-conversation", projectId: null }); + expect(memory.info.native_origin_v1).toMatchObject({ conversationId: "native-conversation", projectId: null }); + }); + + it("keeps claims on the captured source through delayed completion and replay", async () => { + const { db, service } = fixture.createTestService(); + const repos = new Repositories(db.db); + const common = { profileId: "default", projectId: "project-p", workspaceId: "workspace-w" }; + const session = service.openSession({ namespace: { ...common, source: "codex", sessionKey: "host-codex" }, meta: { conversationId: "shared-chat" } }); + const start = service.startTurn({ sessionId: session.sessionId, turnId: "shared-turn", query: turnInput.query }); + service.openSession({ sessionId: session.sessionId, namespace: { ...common, source: "claude" } }); + await start; + const completed = (await service.completeTurn("shared-turn", { sessionId: session.sessionId, ...turnInput })); + const memory = repos.memories.get(completed.l1MemoryId!)!; + expect(memory.agentId).toBe("codex"); + expect(db.db.prepare("SELECT source FROM memory_capture_claims WHERE primary_memory_id=?").get(memory.id)).toEqual({ source: "codex" }); + expect((await service.completeTurn("shared-turn", { sessionId: session.sessionId, ...turnInput })).l1MemoryId).toBe(memory.id); + const scan = service.addMemory({ namespace: { ...common, source: "claude", sessionKey: "shared-chat" }, + adapterId: "agent-source:claude", requestId: "independent-scan", layer: "L1", source: "claude", tags: ["agent-source", "claude"], + turnId: "shared-turn", content: `## user\n\n${turnInput.query}\n\n## assistant\n\n${turnInput.answer}` }); + expect(scan.id).not.toBe(memory.id); + expect(repos.memories.get(scan.id)!.agentId).toBe("claude"); + }); + + it.each(["original", "moved"])("checks completed historical turns against %s immutable scope", async scope => { + const { db, service } = fixture.createTestService(); + const repos = new Repositories(db.db); + const common = { source: "codex", profileId: "default" }; + const original = { ...common, projectId: "project-a", workspaceId: "workspace-a" }; + const moved = { ...common, projectId: "project-b", workspaceId: "workspace-b" }; + const session = service.openSession({ namespace: { ...original, sessionKey: "host-a" }, meta: { conversationId: "shared-chat" } }); + const first = (await service.completeTurn("shared-turn", { sessionId: session.sessionId, ...turnInput })); + service.openSession({ sessionId: session.sessionId, namespace: moved }); + repos.runtime.closeSession(session.sessionId, "2099-01-01T00:00:00.000Z"); + const scan = (await service.completeSourceTurn({ namespace: scope === "original" ? original : moved, source: "codex", channel: "agent_source_scan", + captureLegacyHistory: true, sourceTurn: { source: "codex", profileId: "default", conversationId: "shared-chat", turnId: "shared-turn", + startedAt: "2000-01-01T10:00:00.000Z", completedAt: "2000-01-01T10:01:00.000Z", completionEvidence: "task_complete:10" }, ...turnInput })); + expect(scan.status).toBe(scope === "original" ? "pending" : "stored"); + expect(repos.memories.get(first.l1MemoryId!)!.info.project_id).toBe("project-a"); + if (scope === "moved") expect(scan.result?.l1MemoryId).not.toBe(first.l1MemoryId); + }); + + it("does not rebuild an immutable completed turn when another capture already exists", async () => { + const { db, service } = fixture.createTestService(); + const repos = new Repositories(db.db); + const namespace = { source: "codex", profileId: "default", projectId: "project-a", workspaceId: "workspace-a" }; + const old = service.openSession({ namespace: { ...namespace, sessionKey: "host-old" }, meta: { conversationId: "shared-chat" } }); + (await service.completeTurn("shared-turn", { sessionId: old.sessionId, ...turnInput })); + service.openSession({ sessionId: old.sessionId, namespace: { ...namespace, projectId: "project-b", workspaceId: "workspace-b" } }); + repos.runtime.closeSession(old.sessionId, "2099-01-01T00:00:00.000Z"); + const scan = async (turnId: string, minute: string) => (await service.completeSourceTurn({ namespace, source: "codex", channel: "agent_source_scan", + captureLegacyHistory: true, sourceTurn: { source: "codex", profileId: "default", conversationId: "shared-chat", turnId, + startedAt: `2000-01-01T10:${minute}:00.000Z`, completedAt: `2000-01-01T10:${minute}:30.000Z`, completionEvidence: "task_complete:10" }, ...turnInput })); + expect((await scan("earlier-turn", "00")).status).toBe("stored"); + const before = db.db.prepare("SELECT COUNT(*) AS n FROM raw_turns WHERE turn_id='shared-turn'").get(); + expect((await scan("shared-turn", "02"))).toMatchObject({ status: "pending", reason: "legacy_source_turn_already_completed" }); + expect(db.db.prepare("SELECT COUNT(*) AS n FROM raw_turns WHERE turn_id='shared-turn'").get()).toEqual(before); + }); + + it("uses the retained Session scope when deciding whether imported L3 evidence can resume", async () => { + const target = fixture.createTestService(), source = fixture.createTestService(); + const tr = new Repositories(target.db.db), sr = new Repositories(source.db.db); + const opened = target.service.openSession({ namespace: { source: "codex", profileId: "default", projectId: "project-p", sessionKey: "host" } }); + target.db.db.prepare("UPDATE sessions SET meta_json=? WHERE id=?").run(JSON.stringify({ l3_world_model_protocol_version: 2 }), opened.sessionId); + tr.l3WorldModels.upsertField({ userId: null, projectId: "project-p", targetField: "project_contract", value: "Keep target rule", at }); + sr.runtime.importBundleTables(tr.runtime.exportBundleTables(true)); + source.db.db.prepare("UPDATE sessions SET project_id='project-q' WHERE id=?").run(opened.sessionId); + const captured = (await source.service.completeTurn("other-project-turn", { sessionId: opened.sessionId, ...turnInput })); + const incoming = sr.runtime.exportBundleTables(true); + tr.runtime.importBundleTables(incoming, { conflictStrategy: "skip" }); + expect(tr.runtime.getSession(opened.sessionId)!.projectId).toBe("project-p"); + expect(tr.memories.get(captured.l1MemoryId!)!.info.project_id).toBe("project-q"); + expect(tr.l3WorldModels.inputTraceByL1MemoryId(opened.sessionId, captured.l1MemoryId!)).toBeUndefined(); + tr.l3WorldModels.freezeBatches({ sessionId: opened.sessionId, trigger: "session_close" }); + expect(target.db.db.prepare("SELECT project_id FROM l3_world_model_evidence_batches").all()).toEqual([]); + expect(tr.l3WorldModels.fields(null, "project-p").projectContract).toBe("Keep target rule"); + }); + + it.each(["batch", "scope"])("does not import jobs authorized by a skipped %s identity", async dependency => { + const { db, service } = fixture.createTestService(); + const repos = new Repositories(db.db); + const session = service.openSession({ namespace: { source: "codex", profileId: "default", projectId: "project-p", sessionKey: "host" } }); + db.db.prepare("UPDATE sessions SET meta_json=? WHERE id=?").run(JSON.stringify({ l3_world_model_protocol_version: 2 }), session.sessionId); + repos.l3WorldModels.upsertField({ userId: null, projectId: "project-p", targetField: "project_contract", value: "Keep target rule", at }); + (await service.completeTurn("target-turn", { sessionId: session.sessionId, ...turnInput })); + repos.l3WorldModels.freezeBatches({ sessionId: session.sessionId, trigger: "session_close" }); + const before = repos.runtime.exportBundleTables(true); + const batch = before.l3_world_model_evidence_batches!.find(row => row.project_id === "project-p")!; + expect(batch).toBeDefined(); + const scope = before.l3_world_model_scopes!.find(row => row.scope_key === batch.scope_key)!; + const incoming = { + l3_world_model_evidence_batches: dependency === "batch" ? [{ ...batch, project_id: "project-q" }] : [], + l3_world_model_scopes: dependency === "scope" ? [{ ...scope, project_id: "project-q", memory_id: null }] : [], + evolution_jobs: [{ id: "injected-job", job_type: "l3_world_model_update", status: "queued", attempts: 0, + scope_key: dependency === "scope" ? `${String(scope.scope_key)}:project_contract` : null, + payload_json: JSON.stringify(dependency === "batch" ? { batchId: batch.id } : {}), created_at: at, updated_at: at }] + }; + repos.runtime.importBundleTables(incoming, { conflictStrategy: "skip" }); + expect(db.db.prepare("SELECT id FROM evolution_jobs WHERE id='injected-job'").get()).toBeUndefined(); + expect(repos.runtime.exportBundleTables(true).l3_world_model_evidence_batches).toEqual(before.l3_world_model_evidence_batches); + expect(repos.l3WorldModels.fields(null, "project-p").projectContract).toBe("Keep target rule"); + }); + + it("uses the native-turn index to bound legacy provenance lookups", () => { + const { db } = fixture.createTestService(); + const plan = db.db.prepare("EXPLAIN QUERY PLAN SELECT source FROM source_turn_captures WHERE turn_id=?").all("native-turn") as { detail: string }[]; + expect(plan.some(row => row.detail.includes("idx_source_turn_captures_turn"))).toBe(true); + }); + + it.each([null, "tenant-a"])("preserves tenant provenance through session reuse (tenant=%s)", async tenant => { + const { db, service } = fixture.createTestService(); + const repos = new Repositories(db.db); + const memoryIds = (await Promise.all([tenant, "tenant-b"].map(async (tenantId, i) => { + const namespace = { source: "codex", profileId: "default", sessionKey: `host-${i}`, tenantId: tenantId ?? undefined, projectId: "p", workspaceId: "w" }; + const session = service.openSession({ namespace, meta: { conversationId: "chat" } }); + return (await service.completeTurn("turn", { sessionId: session.sessionId, namespace, ...turnInput })).l1MemoryId!; + }))); + repos.memories.softDelete(memoryIds[0]!); + expect(repos.memories.nativeDuplicates(repos.memories.get(memoryIds[1]!)!)).toEqual({ ids: [memoryIds[1]], deleted: false }); + }); + + it("keeps legacy records with absent or contradictory capture evidence separate", async () => { + const { db, service } = fixture.createTestService(); + const repos = new Repositories(db.db); + const ids = (await Promise.all(["a", "b"].map(async key => { + const session = service.openSession({ namespace: { source: "codex", profileId: "default", sessionKey: key }, meta: { conversationId: "chat" } }); + return (await service.completeTurn("turn", { sessionId: session.sessionId, ...turnInput })).l1MemoryId!; + }))); + db.db.exec("UPDATE memories SET info_json=json_remove(info_json,'$.native_origin_v1'),properties_json=json_remove(properties_json,'$.info.native_origin_v1'); UPDATE raw_turns SET message_payload_json=json_remove(message_payload_json,'$.native_origin_v1')"); + repos.memories.softDelete(ids[0]!); + expect(repos.memories.nativeDuplicates(repos.memories.get(ids[1]!)!)).toEqual({ ids: [ids[1]], deleted: false }); + }); + + it.each([true, false])("keeps origins across restart and bundle restore (raw=%s)", async includeRawText => { + const { db, service, root } = fixture.createTestService(); + const session = service.openSession({ namespace: { source: "codex", profileId: "default", sessionKey: "a" }, meta: { conversationId: "chat" } }); + const turn = (await service.completeTurn("turn", { sessionId: session.sessionId, ...turnInput })); + service.openSession({ sessionId: session.sessionId, namespace: { source: "codex", profileId: "default", projectId: "later" } }); + const bundle = service.exportBundle({ includeRawText }); + await service.stop(); db.close(); + const reopened = new MemoryDb({ path: join(root, "memory.sqlite") }); + try { + expect(new Repositories(reopened.db).memories.get(turn.l1MemoryId!)!.info.native_origin_v1).toMatchObject({ projectId: null }); + const restored = fixture.createTestService(); + restored.service.importBundle({ bundle }); + expect(new Repositories(restored.db.db).memories.get(turn.l1MemoryId!)!.info.native_origin_v1).toMatchObject({ projectId: null }); + } finally { reopened.close(); } + }); + + it.each(["skip", "replace", "error"] as const)("applies the requested conflict strategy: %s", conflictStrategy => { + const { tr, bundle } = deletedBackup(); + if (conflictStrategy === "error") expect(() => tr.runtime.importBundleTables(bundle, { conflictStrategy })).toThrow("import conflict"); + else tr.runtime.importBundleTables(bundle, { conflictStrategy }); + expect(tr.l3WorldModels.fields(null, "project-p").projectContract).toBe(conflictStrategy === "replace" ? null : "Keep target rule"); + }); + + it.each(["partial", "different-id", "same-batch-alias", "alias-chain", "changed-input-project"])("protects target scope for %s deletion backups", mode => { + const { tr, bundle, memory } = deletedBackup(); + if (mode === "partial") delete bundle.memories; + if (mode === "different-id") for (const row of bundle.memories!) row.id = "different-deleted-id"; + if (mode === "same-batch-alias" || mode === "alias-chain") { + bundle.memory_project_aliases = [{ legacy_id: "old-p", project_id: "project-p", workspace_uri: "", created_at: at }]; + if (mode === "alias-chain") { + bundle.memory_project_aliases[0]!.project_id = "middle-p"; + bundle.memory_project_aliases.push({ legacy_id: "middle-p", project_id: "project-p", workspace_uri: "", created_at: at }); + } + for (const row of bundle.memory_l3_overrides!) row.project_key = "old-p"; + } + if (mode === "changed-input-project") { + const row = bundle.memories!.find(row => row.id === memory.id)!; + row.info_json = JSON.stringify({ project_id: "misleading-input-project" }); + } + tr.runtime.importBundleTables(bundle, { conflictStrategy: "skip" }); + expect(tr.l3WorldModels.fields(null, "project-p").projectContract).toBe("Keep target rule"); + expect(tr.db.prepare("SELECT * FROM memory_l3_overrides").all()).toEqual([]); + }); + + it("rolls back new aliases and data together when a late insert fails", () => { + const { target, tr } = deletedBackup(); + const bundle = { memory_project_aliases: [{ legacy_id: "new-alias", project_id: "new-project", workspace_uri: "", created_at: at }], + audit_logs: [{ id: "bad-audit" }] }; + expect(() => tr.runtime.importBundleTables(bundle)).toThrow(); + expect(target.db.db.prepare("SELECT * FROM memory_project_aliases WHERE legacy_id='new-alias'").all()).toEqual([]); + expect(canonicalProjectId(target.db.db, "new-alias")).toBe("new-alias"); + expect(tr.l3WorldModels.fields(null, "project-p").projectContract).toBe("Keep target rule"); + }); + + it("rejects contradictory duplicate rows before writing any part of a bundle", () => { + const { tr, bundle, memory } = deletedBackup(); + bundle.memories!.push({ ...bundle.memories![0], id: memory.id, memory_value: "contradictory" }); + expect(() => tr.runtime.importBundleTables(bundle)).toThrow("contradictory duplicate identity"); + expect(tr.l3WorldModels.fields(null, "project-p").projectContract).toBe("Keep target rule"); + }); + + it("rolls back the service-level import when vector persistence fails after table writes", () => { + const { target, source, sr, tr } = deletedBackup(); + sr.l3WorldModels.upsertField({ userId: null, projectId: "new-project", targetField: "project_contract", value: "New rule", at }); + const bundle = source.service.exportBundle({ includeRawText: true }); + bundle.tables.memory_project_aliases!.push({ legacy_id: "new-alias", project_id: "new-project", workspace_uri: "", created_at: at }); + vi.spyOn(SqliteVecStore.prototype, "importRows").mockImplementationOnce(() => { throw new Error("vector write failed"); }); + expect(() => target.service.importBundle({ bundle })).toThrow("vector write failed"); + expect(tr.l3WorldModels.fields(null, "project-p").projectContract).toBe("Keep target rule"); + expect(tr.l3WorldModels.fields(null, "new-project").projectContract).toBeNull(); + expect(canonicalProjectId(target.db.db, "new-alias")).toBe("new-alias"); + }); + + it("keeps skipped L3 vectors unchanged after imported retry work is drained", async () => { + const texts: string[] = []; + const embedder = { + config: { ...createCapturingEmbedder([]).config, model: "review-fixture" }, + embed: async (input: string[]) => { texts.push(...input); return input.map(() => [0, 1, 0]); }, + embedOne: async (input: string) => { texts.push(input); return [0, 1, 0]; }, + isConfigured: () => true, + isRemote: () => false, + status: () => ({ provider: "local", model: "review-fixture", configured: true, remote: false }) + }; + const target = fixture.createTestService({ embedder }), source = fixture.createTestService(); + const tr = new Repositories(target.db.db), sr = new Repositories(source.db.db); + const memory = tr.l3WorldModels.upsertField({ userId: null, projectId: "project-p", targetField: "project_contract", value: "Keep target rule", at })!; + tr.vectors.upsert(memory.id, { vectorField: "vec", vector: [1, 0, 0], embeddingModel: "review-fixture", embeddingProvider: "local" }, at); + const before = tr.vectors.exportRows(); + sr.runtime.importBundleTables(tr.runtime.exportBundleTables(true)); + const retry = sr.runtime.enqueueEmbeddingRetry({ targetKind: "world_model", targetId: memory.id, vectorField: "vec", + sourceText: embeddingTextForMemory(sr.memories.get(memory.id)!), embedRole: "query", now: 1 }); + const result = target.service.importBundle({ bundle: source.service.exportBundle({ includeRawText: true }), conflictStrategy: "skip" }); + await target.service.runWorkerOnce(10); + expect(tr.runtime.getEmbeddingRetry(retry.id)).toBeUndefined(); + expect(result.skipped.embedding_retry_queue).toBe(1); + expect(tr.vectors.exportRows()).toEqual(before); + expect(texts).toEqual([]); + }); + + it("preserves skip results across reopen and a second export/import while allowing empty-store deletion restore", async () => { + const { target, tr, bundle } = deletedBackup(); + tr.runtime.importBundleTables(bundle); + await target.service.stop(); target.db.close(); + const reopened = new MemoryDb({ path: join(target.root, "memory.sqlite") }); + try { + const rr = new Repositories(reopened.db); + expect(rr.l3WorldModels.fields(null, "project-p").projectContract).toBe("Keep target rule"); + const restored = fixture.createTestService(), restoredRepos = new Repositories(restored.db.db); + restoredRepos.runtime.importBundleTables(rr.runtime.exportBundleTables(true)); + expect(restoredRepos.l3WorldModels.fields(null, "project-p").projectContract).toBe("Keep target rule"); + const empty = fixture.createTestService(), emptyRepos = new Repositories(empty.db.db); + emptyRepos.runtime.importBundleTables(bundle); + expect(emptyRepos.l3WorldModels.fields(null, "project-p").projectContract).toBeNull(); + expect(empty.db.db.prepare("SELECT COUNT(*) AS n FROM memory_l3_overrides WHERE deleted=1").get()).toEqual({ n: 4 }); + } finally { reopened.close(); } + }); +}); diff --git a/Memory/tests/service/embedding/embedding-processing.test.ts b/Memory/tests/service/embedding/embedding-processing.test.ts index d38e8e6b7..fc87bbf8a 100644 --- a/Memory/tests/service/embedding/embedding-processing.test.ts +++ b/Memory/tests/service/embedding/embedding-processing.test.ts @@ -58,9 +58,9 @@ describe("MemoryService / embedding / processing", () => { const repos = new Repositories(db.db); if (queue === "fresh") { const session = service.openSession({ namespace: { source: "codex", profileId: "shared-error", userId: "test" } }); - const ids = await Promise.all(["GOOD_A", "GOOD_B", "GOOD_C"].map(async (answer, i) => (await service.completeTurn(`turn-${i}`, { + const ids = await Promise.all((await Promise.all(["GOOD_A", "GOOD_B", "GOOD_C"].map(async (answer, i) => (await service.completeTurn(`turn-${i}`, { sessionId: session.sessionId, query: "Remember this.", answer - })).l1MemoryId)); + })).l1MemoryId)))); await service.runWorkerOnce(20, { priorityCohortOnly: true }); await service.runWorkerOnce(20, { priorityCohortOnly: true }); for (const id of ids) expect(repos.processing.get(id)).toMatchObject({ @@ -105,9 +105,9 @@ describe("MemoryService / embedding / processing", () => { const texts = ["GOOD_A", "GOOD_B", "GOOD_C", "BAD_EMBEDDING_ITEM"]; if (queue === "fresh") { const session = service.openSession({ namespace: { source: "codex", profileId: "isolation", userId: "test" } }); - const ids = await Promise.all(texts.map(async (answer, i) => (await service.completeTurn(`turn-${i}`, { + const ids = await Promise.all((await Promise.all(texts.map(async (answer, i) => (await service.completeTurn(`turn-${i}`, { sessionId: session.sessionId, query: "Remember this.", answer - })).l1MemoryId)); + })).l1MemoryId)))); await service.runWorkerOnce(20, { priorityCohortOnly: true }); await service.runWorkerOnce(20, { priorityCohortOnly: true }); expect(ids.map(id => repos.processing.get(id)?.state)).toEqual([ diff --git a/Memory/tests/service/evidence/history-skill-automatic.test.ts b/Memory/tests/service/evidence/history-skill-automatic.test.ts index b4299f4e2..ded5ada0b 100644 --- a/Memory/tests/service/evidence/history-skill-automatic.test.ts +++ b/Memory/tests/service/evidence/history-skill-automatic.test.ts @@ -242,9 +242,8 @@ describe("automatic History Skill lifecycle", () => { const result = await service.suggestExternalEvidenceSkill(suggestion); const original = repos.memories.get(result.id!)!; const body = '---\nname: review_release_revised\ndescription: >\n Human revised release review\n with a verification step.\n---\n\n# Revised release review\n\n### When to use\nWhen the current release is ready for review\n\n### Procedure\n1. Review the draft.\n2. Verify the current result.'; - expect(() => service.updateExternalEvidenceSkill(original.id, { body, expectedVersion: original.version }, - { namespace: { ...namespace, userId: "other-user" } })).toThrow(/not found/u); - const saved = service.updateExternalEvidenceSkill(original.id, { body, expectedVersion: original.version }, { namespace }); + const saved = service.updateExternalEvidenceSkill(original.id, { body, expectedVersion: original.version }, + { namespace: { ...namespace, userId: "other-user" } }); expect(saved).toMatchObject({ status: "activated", version: original.version + 1 }); expect(() => service.updateExternalEvidenceSkill(original.id, { body: "Stale write", expectedVersion: original.version }, { namespace })) .toThrow(/changed since editing/u); diff --git a/Memory/tests/service/evolution/l3-world-model.test.ts b/Memory/tests/service/evolution/l3-world-model.test.ts index f69af61d4..30abf192b 100644 --- a/Memory/tests/service/evolution/l3-world-model.test.ts +++ b/Memory/tests/service/evolution/l3-world-model.test.ts @@ -321,6 +321,77 @@ describe("L3 World Model trace field pipeline", () => { db.close(); }); + it("keeps an explicitly archived project L3 suppressed when a queued trace result arrives", async () => { + const complete = vi.fn().mockResolvedValue(JSON.stringify({ + reason: "A queued result must not reopen a user-archived project model.", + op: "update", + project_contract: "Late worker result" + })); + const llm = strictCompletionLlm(complete); + const { db, service } = createTestService({ skillLlm: llm }); + const opened = openProject(service, "explicit-archive-user", "explicit-archive-session"); + const repos = new Repositories(db.db); + const created = repos.l3WorldModels.upsertField({ + userId: "explicit-archive-user", + projectId: opened.projectId, + targetField: "project_contract", + value: "Original project contract" + })!; + (await service.completeTurn("explicit-archive-queued-turn", { + sessionId: opened.sessionId, + query: "Keep the current project boundary.", + answer: "The current boundary is still in effect.", + status: "succeeded" + })); + service.closeSession(opened.sessionId); + const job = repos.runtime.listJobs("queued", 100).find( + candidate => candidate.jobType === "l3_world_model_update" && + candidate.payload.targetField === "project_contract" + ); + expect(job).toBeDefined(); + + service.archiveMemory(created.id, { + namespace: { + source: "codex", + profileId: "default", + projectId: opened.projectId ?? undefined, + userId: "explicit-archive-user" + } + }); + const archivedRow = db.db.prepare("SELECT properties_json FROM memories WHERE id = ?") + .get(created.id) as { properties_json: string }; + expect(JSON.parse(archivedRow.properties_json).internal_info.l3_world_model_explicitly_archived).toBe(true); + const legacyProperties = JSON.parse(archivedRow.properties_json) as Record; + const legacyInternal = legacyProperties.internal_info as Record; + delete legacyInternal.l3_world_model_explicitly_archived; + db.db.prepare("UPDATE memories SET properties_json = ? WHERE id = ?") + .run(JSON.stringify(legacyProperties), created.id); + expect(repos.l3WorldModels.getMemory(null, opened.projectId)?.status).toBe("archived"); + + await new L3WorldModelTraceFieldPipeline({ repos, skillLlm: llm }).updateField(job!); + repos.runtime.completeJob(job!.id); + + const archived = repos.l3WorldModels.getMemory(null, opened.projectId)!; + expect(archived.status).toBe("archived"); + expect(archived.memoryValue).toContain("Original project contract"); + expect(archived.memoryValue).not.toContain("Late worker result"); + expect(archived.properties.internal_info.l3_world_model_explicitly_archived).toBe(true); + expect(repos.l3WorldModels.getTarget( + String(job!.payload.batchId), + "project_contract" + )).toMatchObject({ status: "applied", noChange: true }); + expect(repos.l3WorldModels.upsertField({ + userId: "explicit-archive-user", + projectId: opened.projectId, + targetField: "project_contract", + value: "Later direct field write" + })?.status).toBe("archived"); + expect(repos.l3WorldModels.getMemory(null, opened.projectId)?.status).toBe("archived"); + expect(complete).toHaveBeenCalledTimes(1); + + db.close(); + }); + it("keeps source IDs in evidence order and caps them at the latest 256 across opposite field completion orders", async () => { const complete = vi.fn(async (messages) => { const system = messages[0]?.content ?? ""; @@ -525,8 +596,8 @@ describe("L3 World Model trace field pipeline", () => { status: "succeeded" })); service.closeSession(opened.sessionId); - db.db.prepare(`UPDATE raw_turns SET user_id = 'another-user' WHERE id = ?`) - .run(completed.rawTurnId); + const other = service.openSession({namespace:{source:"codex",profileId:"default",sessionKey:"other-session"}}); + db.db.prepare(`UPDATE raw_turns SET session_id = ? WHERE id = ?`).run(other.sessionId,completed.rawTurnId); const repos = new Repositories(db.db); const job = repos.runtime.listJobs("queued", 100).find( (candidate) => candidate.jobType === "l3_world_model_update" diff --git a/Memory/tests/service/evolution/negative-experience.test.ts b/Memory/tests/service/evolution/negative-experience.test.ts index 8a88de01a..a786e4c11 100644 --- a/Memory/tests/service/evolution/negative-experience.test.ts +++ b/Memory/tests/service/evolution/negative-experience.test.ts @@ -177,8 +177,8 @@ describe("MemoryService / evolution / negative experience", () => { `SELECT COUNT(*) AS count FROM evolution_jobs WHERE job_type = 'negative_experience' - AND user_id = ?` - ).get(namespace.userId) as { count: number }; + AND user_id IS ?` + ).get(null) as { count: number }; expect(negativeJobCount.count).toBe(1); await service.runWorkerOnce(50); diff --git a/Memory/tests/service/evolution/orchestration.test.ts b/Memory/tests/service/evolution/orchestration.test.ts index 2071d6eae..c6181f21a 100644 --- a/Memory/tests/service/evolution/orchestration.test.ts +++ b/Memory/tests/service/evolution/orchestration.test.ts @@ -109,7 +109,7 @@ describe("MemoryService / evolution / orchestration", () => { expect(promotedCandidates.count).toBeGreaterThanOrEqual(1); const l2Row = db.db.prepare( `SELECT properties_json FROM memories - WHERE user_id = 'user-2' AND memory_layer = 'L2' + WHERE user_id IS NULL AND memory_layer = 'L2' LIMIT 1` ).get() as { properties_json: string }; const l2Internal = (JSON.parse(l2Row.properties_json) as { @@ -151,7 +151,7 @@ describe("MemoryService / evolution / orchestration", () => { ).all() as Array<{ namespace_id: string | null; kind: string | null; op: string | null; entity_id: string | null }>; expect(workerMemoryChanges.length).toBeGreaterThan(0); for (const change of workerMemoryChanges) { - expect(change.namespace_id).toContain("user-2"); + expect(change.namespace_id).not.toContain("user-2"); expect(change.kind).toMatch(/^(trace|policy|world_model|skill)$/); expect(change.op).toMatch(/^(created|updated)$/); expect(change.entity_id).toBeTruthy(); @@ -473,7 +473,7 @@ describe("MemoryService / evolution / orchestration", () => { `SELECT id FROM memories WHERE memory_layer = 'L2' - AND user_id = 'user-2' + AND user_id IS NULL LIMIT 1` ).get() as { id: string } | undefined; expect(policyForRepeat).toBeTruthy(); @@ -699,7 +699,7 @@ describe("MemoryService / evolution / orchestration", () => { const worlds = db.db.prepare( `SELECT info_json, properties_json FROM memories - WHERE user_id = 'shared-downstream-user' + WHERE user_id IS NULL AND memory_layer = 'L3'` ).all() as Array<{ info_json: string; properties_json: string }>; expect(worlds).toHaveLength(0); @@ -707,7 +707,7 @@ describe("MemoryService / evolution / orchestration", () => { const skills = db.db.prepare( `SELECT info_json, properties_json FROM memories - WHERE user_id = 'shared-downstream-user' + WHERE user_id IS NULL AND memory_layer = 'Skill'` ).all() as Array<{ info_json: string; properties_json: string }>; expect(skills).toHaveLength(2); @@ -842,7 +842,7 @@ describe("MemoryService / evolution / orchestration", () => { const l2Count = db.db.prepare( `SELECT COUNT(*) AS count FROM memories - WHERE user_id = 'user-l3-skill-prompt' AND memory_layer = 'L2'` + WHERE user_id IS NULL AND memory_layer = 'L2'` ).get() as { count: number }; if (l2Count.count > 0) { policyCreated = true; @@ -854,7 +854,7 @@ describe("MemoryService / evolution / orchestration", () => { const policyRow = db.db.prepare( `SELECT id, properties_json FROM memories - WHERE user_id = 'user-l3-skill-prompt' AND memory_layer = 'L2' + WHERE user_id IS NULL AND memory_layer = 'L2' LIMIT 1` ).get() as { id: string; properties_json: string }; const policyProperties = JSON.parse(policyRow.properties_json) as { @@ -897,7 +897,7 @@ describe("MemoryService / evolution / orchestration", () => { const l1Rows = db.db.prepare( `SELECT * FROM memories - WHERE user_id = 'user-l3-skill-prompt' AND memory_layer = 'L1'` + WHERE user_id IS NULL AND memory_layer = 'L1'` ).all() as Array & { id: string; properties_json: string }>; const sourceL1Row = l1Rows.find((row) => { const properties = JSON.parse(row.properties_json) as { @@ -990,7 +990,7 @@ describe("MemoryService / evolution / orchestration", () => { const worldRow = db.db.prepare( `SELECT properties_json FROM memories - WHERE user_id = 'user-l3-skill-prompt' AND memory_layer = 'L3' + WHERE user_id IS NULL AND memory_layer = 'L3' LIMIT 1` ).get() as { properties_json: string } | undefined; if (worldRow) { @@ -1042,7 +1042,7 @@ describe("MemoryService / evolution / orchestration", () => { const skillRow = db.db.prepare( `SELECT memory_key, properties_json FROM memories - WHERE user_id = 'user-l3-skill-prompt' AND memory_layer = 'Skill' + WHERE user_id IS NULL AND memory_layer = 'Skill' LIMIT 1` ).get() as { memory_key: string; properties_json: string }; const skillProperties = JSON.parse(skillRow.properties_json) as { diff --git a/Memory/tests/service/evolution/policy-induction.test.ts b/Memory/tests/service/evolution/policy-induction.test.ts index 006a025c2..e6e81f1c4 100644 --- a/Memory/tests/service/evolution/policy-induction.test.ts +++ b/Memory/tests/service/evolution/policy-induction.test.ts @@ -169,19 +169,19 @@ describe("MemoryService / evolution / policy induction", () => { expect(rows).toEqual([ { source_memory_id: "mem_candidate-user-a", - user_id: "candidate-user-a", + user_id: null, status: "promoted", policy_id: "policy-user-a" }, { source_memory_id: "mem_candidate-user-a_other-profile", - user_id: "candidate-user-a", + user_id: null, status: "pending", policy_id: null }, { source_memory_id: "mem_candidate-user-b", - user_id: "candidate-user-b", + user_id: null, status: "pending", policy_id: null } @@ -578,12 +578,12 @@ describe("MemoryService / evolution / policy induction", () => { workspaceId: "workspace-key" }); const signature = "python|pytest|_|_"; - const turnsA = (await Promise.all(["a1", "a2"].map(async (suffix) => (await service.completeTurn(`turn-shared-key-${suffix}`, { + const turnsA = (await Promise.all((await Promise.all(["a1", "a2"].map(async (suffix) => (await service.completeTurn(`turn-shared-key-${suffix}`, { sessionId: profileA.sessionId, episodeId: `episode-shared-key-${suffix}`, query: "python pytest retry workflow should stay scoped to profile a", answer: "Run pytest, inspect the failure, and retry after fixing the issue." - }))))); + }))))))); for (const turn of turnsA) { await addPositiveFeedbackForTurn(service, profileA.sessionId, turn); } @@ -601,12 +601,12 @@ describe("MemoryService / evolution / policy induction", () => { db.db.prepare(`UPDATE evolution_jobs SET status = 'succeeded' WHERE job_type <> 'l2_induction'`).run(); await service.runWorkerOnce(20); - const turnsB = (await Promise.all(["b1", "b2"].map(async (suffix) => (await service.completeTurn(`turn-shared-key-${suffix}`, { + const turnsB = (await Promise.all((await Promise.all(["b1", "b2"].map(async (suffix) => (await service.completeTurn(`turn-shared-key-${suffix}`, { sessionId: profileB.sessionId, episodeId: `episode-shared-key-${suffix}`, query: "python pytest retry workflow should stay scoped to profile b", answer: "Run pytest, inspect the failure, and retry after fixing the issue." - }))))); + }))))))); for (const turn of turnsB) { await addPositiveFeedbackForTurn(service, profileB.sessionId, turn); } @@ -627,7 +627,7 @@ describe("MemoryService / evolution / policy induction", () => { const policies = db.db.prepare( `SELECT memory_key, info_json FROM memories - WHERE user_id = 'shared-key-user' + WHERE user_id IS NULL AND memory_layer = 'L2' ORDER BY memory_key` ).all() as Array<{ memory_key: string; info_json: string }>; @@ -951,7 +951,7 @@ describe("MemoryService / evolution / policy induction", () => { const row = db.db.prepare( `SELECT properties_json FROM memories - WHERE user_id = 'user-l2-support-subset' + WHERE user_id IS NULL AND memory_layer = 'L2' LIMIT 1` ).get() as { properties_json: string } | undefined; @@ -1053,18 +1053,18 @@ describe("MemoryService / evolution / policy induction", () => { const l2Count = db.db.prepare( `SELECT COUNT(*) AS count FROM memories - WHERE user_id = ? AND memory_layer = 'L2'` - ).get(userId) as { count: number }; + WHERE user_id IS ? AND memory_layer = 'L2'` + ).get(null) as { count: number }; expect(l2Count.count).toBe(0); const skippedRows = db.db.prepare( `SELECT after_json FROM memory_change_log - WHERE user_id = ? + WHERE user_id IS ? AND kind = 'policy' AND op = 'skipped' AND change_type = 'l2_induction_skipped' ORDER BY seq DESC` - ).all(userId) as Array<{ after_json: string }>; + ).all(null) as Array<{ after_json: string }>; const skippedReasons = skippedRows.map((row) => { const after = JSON.parse(row.after_json) as { reason?: string }; return after.reason; @@ -1074,10 +1074,10 @@ describe("MemoryService / evolution / policy induction", () => { const job = db.db.prepare( `SELECT status, attempts, last_error FROM evolution_jobs - WHERE user_id = ? AND job_type = 'l2_induction' + WHERE user_id IS ? AND job_type = 'l2_induction' ORDER BY created_at DESC LIMIT 1` - ).get(userId) as { status: string; attempts: number; last_error: string | null }; + ).get(null) as { status: string; attempts: number; last_error: string | null }; expect(job).toMatchObject({ status: "succeeded", attempts: 1, @@ -1086,7 +1086,7 @@ describe("MemoryService / evolution / policy induction", () => { const downstreamCount = db.db.prepare( `SELECT COUNT(*) AS count FROM evolution_jobs - WHERE user_id = ? + WHERE user_id IS ? AND ( job_type IN ('l3_abstraction', 'skill_crystallization') OR ( @@ -1094,7 +1094,7 @@ describe("MemoryService / evolution / policy induction", () => { AND json_extract(payload_json, '$.reason') = 'l2.upserted' ) )` - ).get(userId) as { count: number }; + ).get(null) as { count: number }; expect(downstreamCount.count).toBe(0); db.close(); @@ -1185,23 +1185,23 @@ describe("MemoryService / evolution / policy induction", () => { const l2Rows = db.db.prepare( `SELECT id FROM memories - WHERE user_id = ? AND memory_layer = 'L2'` - ).all(userId) as Array<{ id: string }>; + WHERE user_id IS ? AND memory_layer = 'L2'` + ).all(null) as Array<{ id: string }>; expect(l2Rows).toHaveLength(1); const skippedCount = db.db.prepare( `SELECT COUNT(*) AS count FROM memory_change_log - WHERE user_id = ? + WHERE user_id IS ? AND change_type = 'l2_induction_skipped'` - ).get(userId) as { count: number }; + ).get(null) as { count: number }; expect(skippedCount.count).toBe(0); const job = db.db.prepare( `SELECT status, attempts, last_error FROM evolution_jobs - WHERE user_id = ? AND job_type = 'l2_induction' + WHERE user_id IS ? AND job_type = 'l2_induction' ORDER BY created_at DESC LIMIT 1` - ).get(userId) as { status: string; attempts: number; last_error: string | null }; + ).get(null) as { status: string; attempts: number; last_error: string | null }; expect(job).toMatchObject({ status: "succeeded", attempts: 1, @@ -1210,7 +1210,7 @@ describe("MemoryService / evolution / policy induction", () => { const downstreamJobs = db.db.prepare( `SELECT job_type FROM evolution_jobs - WHERE user_id = ? + WHERE user_id IS ? AND ( job_type = 'l3_abstraction' OR ( @@ -1220,7 +1220,7 @@ describe("MemoryService / evolution / policy induction", () => { ) ) ORDER BY job_type` - ).all(userId, l2Rows[0]!.id) as Array<{ job_type: string }>; + ).all(null, l2Rows[0]!.id) as Array<{ job_type: string }>; expect(downstreamJobs.map((item) => item.job_type)).toEqual(["embedding"]); db.close(); diff --git a/Memory/tests/service/evolution/skill-lifecycle.test.ts b/Memory/tests/service/evolution/skill-lifecycle.test.ts index 14fd4cef0..00258f09c 100644 --- a/Memory/tests/service/evolution/skill-lifecycle.test.ts +++ b/Memory/tests/service/evolution/skill-lifecycle.test.ts @@ -195,7 +195,7 @@ describe("MemoryService / evolution / skill lifecycle", () => { const skillCount = db.db.prepare( `SELECT COUNT(*) AS count FROM memories - WHERE user_id = 'user-skill-guard' + WHERE 1=1 AND memory_layer = 'Skill'` ).get() as { count: number }; expect(skillCount.count).toBe(0); @@ -249,7 +249,7 @@ describe("MemoryService / evolution / skill lifecycle", () => { const createdSkill = db.db.prepare( `SELECT id, updated_at FROM memories - WHERE user_id = 'user-fresh-skill-skip' + WHERE 1=1 AND memory_layer = 'Skill' LIMIT 1` ).get() as { id: string; updated_at: string } | undefined; @@ -393,9 +393,9 @@ describe("MemoryService / evolution / skill lifecycle", () => { const rows = db.db.prepare( `SELECT id, status, properties_json FROM memories - WHERE user_id = ? AND memory_layer = 'Skill' + WHERE 1=1 AND memory_layer = 'Skill' ORDER BY id` - ).all(userId) as Array<{ id: string; status: string; properties_json: string }>; + ).all() as Array<{ id: string; status: string; properties_json: string }>; expect(rows).toHaveLength(2); const active = rows.find((row) => row.status !== "archived"); const archived = rows.find((row) => row.status === "archived"); @@ -492,9 +492,9 @@ describe("MemoryService / evolution / skill lifecycle", () => { const rows = db.db.prepare( `SELECT id, status FROM memories - WHERE user_id = ? AND memory_layer = 'Skill' + WHERE 1=1 AND memory_layer = 'Skill' ORDER BY id` - ).all(userId) as Array<{ id: string; status: string }>; + ).all() as Array<{ id: string; status: string }>; expect(rows).toHaveLength(2); expect(rows.map((row) => row.id)).toEqual([ "skill-shared-policy-pytest", @@ -503,8 +503,8 @@ describe("MemoryService / evolution / skill lifecycle", () => { const merges = db.db.prepare( `SELECT entity_id, after_json FROM memory_change_log - WHERE user_id = ? AND change_type = 'skill_merged_into_canonical'` - ).all(userId) as Array<{ entity_id: string; after_json: string }>; + WHERE 1=1 AND change_type = 'skill_merged_into_canonical'` + ).all() as Array<{ entity_id: string; after_json: string }>; expect(merges).toEqual([]); expect(rows.map((row) => row.status)).toEqual(["resolving", "resolving"]); @@ -586,7 +586,7 @@ describe("MemoryService / evolution / skill lifecycle", () => { const createdSkill = db.db.prepare( `SELECT id, updated_at FROM memories - WHERE user_id = 'user-skill-cooldown' + WHERE 1=1 AND memory_layer = 'Skill' LIMIT 1` ).get() as { id: string; updated_at: string } | undefined; @@ -619,7 +619,7 @@ describe("MemoryService / evolution / skill lifecycle", () => { const skills = db.db.prepare( `SELECT id, updated_at FROM memories - WHERE user_id = 'user-skill-cooldown' + WHERE 1=1 AND memory_layer = 'Skill'` ).all() as Array<{ id: string; updated_at: string }>; expect(skills).toEqual([createdSkill]); @@ -762,7 +762,7 @@ describe("MemoryService / evolution / skill lifecycle", () => { const skillRows = db.db.prepare( `SELECT id, memory_key FROM memories - WHERE user_id = 'user-source-policy-skill-skip' + WHERE 1=1 AND memory_layer = 'Skill'` ).all() as Array<{ id: string; memory_key: string }>; expect(skillRows).toEqual([{ @@ -900,7 +900,7 @@ describe("MemoryService / evolution / skill lifecycle", () => { const skillRows = db.db.prepare( `SELECT id, status, updated_at, properties_json FROM memories - WHERE user_id = 'user-archived-skill-recreate' + WHERE 1=1 AND memory_layer = 'Skill' ORDER BY created_at ASC` ).all() as Array<{ id: string; status: string; updated_at: string; properties_json: string }>; @@ -919,7 +919,7 @@ describe("MemoryService / evolution / skill lifecycle", () => { const created = db.db.prepare( `SELECT memory_id, before_json, after_json FROM memory_change_log - WHERE user_id = 'user-archived-skill-recreate' + WHERE 1=1 AND kind = 'skill' AND change_type = 'create' ORDER BY seq DESC @@ -979,7 +979,7 @@ describe("MemoryService / evolution / skill lifecycle", () => { const skillRow = db.db.prepare( `SELECT id FROM memories - WHERE user_id = 'user-skill-reward-drift' + WHERE 1=1 AND memory_layer = 'Skill' LIMIT 1` ).get() as { id: string } | undefined; @@ -1190,9 +1190,9 @@ describe("MemoryService / evolution / skill lifecycle", () => { const skillRow = db.db.prepare( `SELECT memory_key, properties_json FROM memories - WHERE user_id = ? AND memory_layer = 'Skill' + WHERE 1=1 AND memory_layer = 'Skill' LIMIT 1` - ).get(userId) as { memory_key: string; properties_json: string }; + ).get() as { memory_key: string; properties_json: string }; const skillSourceProperties = JSON.parse(skillRow.properties_json) as { internal_info?: { source_policy_ids?: string[] }; }; @@ -1201,9 +1201,9 @@ describe("MemoryService / evolution / skill lifecycle", () => { const policyRow = db.db.prepare( `SELECT id, properties_json FROM memories - WHERE id = ? AND user_id = ? AND memory_layer = 'L2' + WHERE id = ? AND 1=1 AND memory_layer = 'L2' LIMIT 1` - ).get(sourcePolicyId!, userId) as { id: string; properties_json: string }; + ).get(sourcePolicyId!) as { id: string; properties_json: string }; const policyProperties = JSON.parse(policyRow.properties_json) as { internal_info?: { title?: string; @@ -1384,18 +1384,18 @@ describe("MemoryService / evolution / skill lifecycle", () => { const skillCount = db.db.prepare( `SELECT COUNT(*) AS count FROM memories - WHERE user_id = ? AND memory_layer = 'Skill'` - ).get(userId) as { count: number }; + WHERE 1=1 AND memory_layer = 'Skill'` + ).get() as { count: number }; expect(skillCount.count).toBe(0); const skippedRows = db.db.prepare( `SELECT after_json FROM memory_change_log - WHERE user_id = ? + WHERE 1=1 AND kind = 'skill' AND op = 'skipped' AND change_type = 'skill_crystallization_skipped' ORDER BY seq DESC` - ).all(userId) as Array<{ after_json: string }>; + ).all() as Array<{ after_json: string }>; const skippedReasons = skippedRows.map((row) => { const after = JSON.parse(row.after_json) as { reason?: string }; return after.reason; @@ -1528,18 +1528,18 @@ describe("MemoryService / evolution / skill lifecycle", () => { const skillCount = db.db.prepare( `SELECT COUNT(*) AS count FROM memories - WHERE user_id = ? AND memory_layer = 'Skill'` - ).get(userId) as { count: number }; + WHERE 1=1 AND memory_layer = 'Skill'` + ).get() as { count: number }; expect(skillCount.count).toBe(0); const skippedRows = db.db.prepare( `SELECT after_json FROM memory_change_log - WHERE user_id = ? + WHERE 1=1 AND kind = 'skill' AND op = 'skipped' AND change_type = 'skill_crystallization_skipped' ORDER BY seq DESC` - ).all(userId) as Array<{ after_json: string }>; + ).all() as Array<{ after_json: string }>; const skippedReasons = skippedRows.map((row) => { const after = JSON.parse(row.after_json) as { reason?: string }; return after.reason; diff --git a/Memory/tests/service/evolution/span-big-turn.test.ts b/Memory/tests/service/evolution/span-big-turn.test.ts index 6300e103a..17cb241a6 100644 --- a/Memory/tests/service/evolution/span-big-turn.test.ts +++ b/Memory/tests/service/evolution/span-big-turn.test.ts @@ -335,10 +335,10 @@ describe("MemoryService / evolution / span big turn", () => { }); it.each([ - { name: "after the parent trace was deleted", deleteParent: true, splitAgain: false }, - { name: "after a second successful split with different ranges", deleteParent: false, splitAgain: true }, - { name: "after both re-splitting and deleting the parent", deleteParent: true, splitAgain: true } - ])("deletes all task spans $name", async ({ deleteParent, splitAgain }) => { + { name: "after the parent trace was deleted", deleteParent: true, splitAgain: false, episodeUserId: null }, + { name: "after a second successful split with different ranges", deleteParent: false, splitAgain: true, episodeUserId: null }, + { name: "after both re-splitting and deleting a historical parent", deleteParent: true, splitAgain: true, episodeUserId: "legacy-task-owner" } + ])("deletes all task spans $name", async ({ deleteParent, splitAgain, episodeUserId }) => { const calls: Array<{ messages: LlmMessage[]; options: LlmCompletionOptions }> = []; const firstSplit = { spans: [ @@ -416,6 +416,13 @@ describe("MemoryService / evolution / span big turn", () => { } const targetIds = spanIds(assignedEpisodeId(service, task)); expect(targetIds).toHaveLength(splitAgain ? 5 : 3); + // An upgraded device can contain both retained account-owned spans and new + // spans without a UID. Task ownership comes from episode/source links. + db.db.prepare("UPDATE episodes SET user_id = ? WHERE id = ?").run(episodeUserId, assignedEpisodeId(service, task)); + targetIds.forEach((id, index) => { + db.db.prepare("UPDATE memories SET user_id = ? WHERE id = ?") + .run(index % 2 === 0 ? "legacy-span-owner" : null, id); + }); const control = await createTask("cascade-control"); await reward(control); const controlIds = spanIds(assignedEpisodeId(service, control)); diff --git a/Memory/tests/service/feedback/experience.test.ts b/Memory/tests/service/feedback/experience.test.ts index 44195c37a..dbd27439a 100644 --- a/Memory/tests/service/feedback/experience.test.ts +++ b/Memory/tests/service/feedback/experience.test.ts @@ -110,7 +110,7 @@ describe("MemoryService / feedback / experience", () => { const created = db.db.prepare( `SELECT id, memory_value, properties_json FROM memories - WHERE user_id = 'user-feedback-experience' + WHERE user_id IS NULL AND memory_layer = 'L2'` ).all() as Array<{ id: string; memory_value: string; properties_json: string }>; expect(created).toHaveLength(1); @@ -148,7 +148,7 @@ describe("MemoryService / feedback / experience", () => { const beforeWorker = db.db.prepare( `SELECT id, properties_json FROM memories - WHERE user_id = 'user-feedback-experience' + WHERE user_id IS NULL AND memory_layer = 'L2'` ).all() as Array<{ id: string; properties_json: string }>; expect(beforeWorker).toHaveLength(1); @@ -166,7 +166,7 @@ describe("MemoryService / feedback / experience", () => { const policies = db.db.prepare( `SELECT id, properties_json FROM memories - WHERE user_id = 'user-feedback-experience' + WHERE user_id IS NULL AND memory_layer = 'L2'` ).all() as Array<{ id: string; properties_json: string }>; expect(policies).toHaveLength(2); @@ -281,7 +281,7 @@ describe("MemoryService / feedback / experience", () => { const row = db.db.prepare( `SELECT properties_json FROM memories - WHERE user_id = 'user-feedback-refiner' + WHERE user_id IS NULL AND memory_layer = 'L2' LIMIT 1` ).get() as { properties_json: string } | undefined; @@ -308,7 +308,7 @@ describe("MemoryService / feedback / experience", () => { const skillRow = db.db.prepare( `SELECT id, properties_json FROM memories - WHERE user_id = 'user-feedback-refiner' + WHERE user_id IS NULL AND memory_layer = 'Skill' LIMIT 1` ).get() as { id: string; properties_json: string } | undefined; diff --git a/Memory/tests/service/import/memory-capture-dedup.test.ts b/Memory/tests/service/import/memory-capture-dedup.test.ts index b87c1fe3f..d8037a47c 100644 --- a/Memory/tests/service/import/memory-capture-dedup.test.ts +++ b/Memory/tests/service/import/memory-capture-dedup.test.ts @@ -2,19 +2,148 @@ import { assignedEpisodeId } from "../../fixtures/memory-service-fixture.js"; import { afterEach, describe, expect, it } from "vitest"; import type { MemoryService } from "../../../src/index.js"; import { createMemoryServiceFixture } from "../../fixtures/memory-service-fixture.js"; +import { memoryCaptureQaHash } from "../../../src/utils/memory-capture-claim.js"; +import type { RuntimeNamespace } from "../../../src/types.js"; +import { MemoryDb } from "../../../src/storage/db.js"; +import { join } from "node:path"; -const { cleanup, createTestService } = createMemoryServiceFixture(); +const { cleanup, createTestService, createTestMemoryService } = createMemoryServiceFixture(); afterEach(cleanup); const namespace = { source: "codex", profileId: "jiang", + sessionKey: "native-capture-conversation", userId: "qa-dedup-user" }; describe("MemoryService / import / QA capture dedup", () => { - it("keeps the hook capture when an agent-source scan later sees the same QA", async () => { + it.each([false, true])("uses the native conversation ID of a prefixed Hook session (legacy claim=%s)", async (legacy) => { + const { db, service } = createTestService(); + const ns = { ...namespace, sessionKey: "codex-memory-native-conversation" }; + const session = service.openSession({ namespace: ns, meta: { conversationId: "native-conversation" } }); + const query = "Read package.json and report the test command."; + const answer = "The test command is npm test."; + const first = (await service.completeTurn("native-turn", { + adapterId: "memmy-codex-hook", requestId: "prefixed-hook-turn", + sessionId: session.sessionId, namespace: ns, query, answer, + toolCalls: [{ id: "read", name: "read_file", input: { path: "package.json" } }], + toolResults: [{ id: "read", output: "npm test" }] + })); + if (legacy) { + db.db.prepare("UPDATE memories SET user_id='historical-account' WHERE id=?").run(first.l1MemoryId); + db.db.prepare("UPDATE memory_capture_claims SET user_id='historical-account',qa_hash=? WHERE primary_memory_id=?") + .run(memoryCaptureQaHash(query, answer), first.l1MemoryId); + } + const before = db.db.prepare("SELECT * FROM memories WHERE id=?").get(first.l1MemoryId); + const scanned = addScannedTurn(service, { namespace: { ...ns, sessionKey: "native-conversation", userId: "account-b" }, requestId: "scan-native", turnId: "native-turn", query, answer }); + expect(scanned).toMatchObject({ id: first.l1MemoryId, duplicate: true }); + expect(l1Count(db.db)).toBe(1); + expect(db.db.prepare("SELECT * FROM memories WHERE id=?").get(first.l1MemoryId)).toEqual(before); + }); + + it("retains a proven historical scan ID across QA revisions, account switches and restart", async () => { + const { db, service, root } = createTestService(); + const ns = { ...namespace, projectId: "legacy-project" }; + const sessionId = service.openSession({ namespace: ns }).sessionId; + const input = { namespace: ns, sessionId, requestId: "legacy-first", turnId: "legacy-turn", query: "Test command?", answer: "npm test" }; + const first = addScannedTurn(service, input); + const key = "memory.add:agent-source:codex:turn:legacy-turn"; + db.db.prepare("UPDATE memories SET memory_key=?,user_id='historical-account' WHERE id=?").run(key, first.id); + db.db.prepare("UPDATE memory_capture_claims SET qa_hash=?,user_id='historical-account' WHERE primary_memory_id=?") + .run(memoryCaptureQaHash(input.query, input.answer), first.id); + const oldClaim = db.db.prepare("SELECT * FROM memory_capture_claims WHERE user_id='historical-account'").get(); + const revised = { ...input, namespace: { ...ns, userId: "another-account" }, requestId: "legacy-revised", answer: "npm run test" }; + expect(addScannedTurn(service, revised).id).toBe(first.id); + expect(addScannedTurn(service, { ...revised, requestId: "legacy-replay" })).toMatchObject({ id: first.id, duplicate: true }); + expect(db.db.prepare("SELECT memory_key,user_id,memory_value FROM memories WHERE id=?").get(first.id)) + .toMatchObject({ memory_key: key, user_id: "historical-account", memory_value: expect.stringContaining(revised.answer) }); + expect(db.db.prepare("SELECT * FROM memory_capture_claims WHERE user_id='historical-account'").get()).toEqual(oldClaim); + expect(l1Count(db.db)).toBe(1); + expect(db.db.prepare("SELECT DISTINCT primary_memory_id FROM memory_capture_claims").all()).toEqual([{ primary_memory_id: first.id }]); + await service.stop(); db.close(); + const reopenedDb = new MemoryDb({ path: join(root, "memory.sqlite") }); + const restarted = createTestMemoryService({ db: reopenedDb, mode: "dev" }); + expect(addScannedTurn(restarted, { ...revised, requestId: "after-restart", answer: "npm run test:unit" }).id).toBe(first.id); + expect(l1Count(reopenedDb.db)).toBe(1); + expect(reopenedDb.db.prepare("SELECT DISTINCT primary_memory_id FROM memory_capture_claims").all()).toEqual([{ primary_memory_id: first.id }]); + }); + + it.each(["projectId", "workspaceId", "tenantId", "profileId"] as const)("retains %s boundaries when conversation information is missing", (field) => { + const { db, service } = createTestService(); + const ns = { ...namespace, sessionKey: undefined, projectId: "project-a", workspaceId: "workspace-a", tenantId: "tenant-a" }; + const input = { namespace: ns, requestId: "missing-first", turnId: "same-turn", query: "Test command?", answer: "first answer" }; + const first = addScannedTurn(service, input); + const before = db.db.prepare("SELECT * FROM memories WHERE id=?").get(first.id); + const other = addScannedTurn(service, { ...input, namespace: { ...ns, [field]: "different", userId: "other-account" }, requestId: "missing-other", answer: "other answer" }); + expect(other.id).not.toBe(first.id); + expect(db.db.prepare("SELECT * FROM memories WHERE id=?").get(first.id)).toEqual(before); + const revised = addScannedTurn(service, { ...input, namespace: { ...ns, userId: "other-account" }, requestId: "missing-revised", answer: "updated answer" }); + expect(revised.id).toBe(first.id); + expect(l1Count(db.db)).toBe(2); + expect(db.db.prepare("SELECT COUNT(*) n FROM memories WHERE user_id IS NOT NULL").get()).toEqual({ n: 0 }); + expect(db.db.prepare("SELECT COUNT(*) n FROM memory_capture_claims").get()).toEqual({ n: 0 }); + }); + + it.each(["project", "conversation", "missing-session"] as const)("does not adopt a historical key with %s mismatch or missing evidence", (difference) => { + const { db, service } = createTestService(); + const ns = { ...namespace, projectId: "legacy-project" }; + const sessionId = difference === "missing-session" ? undefined : service.openSession({ namespace: ns }).sessionId; + const input = { namespace: ns, sessionId, requestId: "legacy-first", turnId: "legacy-turn", query: "Test command?", answer: "old answer" }; + const first = addScannedTurn(service, input); + if (difference === "missing-session") { + // Model the old writer, which had neither a Session nor a persisted origin. + db.db.prepare("UPDATE memories SET info_json=json_remove(info_json,'$.native_origin_v1'),properties_json=json_remove(properties_json,'$.info.native_origin_v1') WHERE id=?").run(first.id); + } + db.db.prepare("UPDATE memories SET memory_key=? WHERE id=?").run("memory.add:agent-source:codex:turn:legacy-turn", first.id); + db.db.prepare("UPDATE memory_capture_claims SET qa_hash=? WHERE primary_memory_id=?").run(memoryCaptureQaHash(input.query, input.answer), first.id); + const before = db.db.prepare("SELECT * FROM memories WHERE id=?").get(first.id); + const otherNamespace = { ...ns, ...(difference === "project" ? { projectId: "different" } : {}), ...(difference === "conversation" ? { sessionKey: "different" } : {}) }; + const otherSession = service.openSession({ namespace: otherNamespace }).sessionId; + const other = addScannedTurn(service, { ...input, namespace: otherNamespace, sessionId: otherSession, requestId: "legacy-other", answer: "new answer" }); + expect(other.id).not.toBe(first.id); + expect(db.db.prepare("SELECT * FROM memories WHERE id=?").get(first.id)).toEqual(before); + }); + + it.each([false, true])("updates only the verified historical ID when another project shares its old key (changed QA=%s)", (changedQa) => { + const { db, service } = createTestService(); + const ns = { ...namespace, projectId: "project-a" }; + const sessionId = service.openSession({ namespace: ns }).sessionId; + const input = { namespace: ns, sessionId, requestId: "original", turnId: "shared-old-key", query: "Test command?", answer: "old answer" }; + const original = addScannedTurn(service, input); + const otherNs = { ...namespace, projectId: "project-b", sessionKey: "other-conversation" }; + const otherSession = service.openSession({ namespace: otherNs }).sessionId; + const other = addScannedTurn(service, { ...input, namespace: otherNs, sessionId: otherSession, requestId: "other", answer: "other answer" }); + const key = "memory.add:agent-source:codex:turn:shared-old-key"; + db.db.prepare("UPDATE memories SET memory_key=?,user_id=?,updated_at=? WHERE id=?").run(key, "old-a", "2026-01-01T00:00:00.000Z", original.id); + db.db.prepare("UPDATE memories SET memory_key=?,user_id=?,updated_at=? WHERE id=?").run(key, "old-b", "2099-01-01T00:00:00.000Z", other.id); + for (const [id, answer] of [[original.id, input.answer], [other.id, "other answer"]]) { + db.db.prepare("UPDATE memory_capture_claims SET qa_hash=? WHERE primary_memory_id=?").run(memoryCaptureQaHash(input.query, answer!), id); + } + const otherBefore = db.db.prepare("SELECT * FROM memories WHERE id=?").get(other.id); + const revised = addScannedTurn(service, { ...input, namespace: { ...ns, userId: "new-account" }, requestId: "revised", answer: changedQa ? "revised answer" : input.answer }); + expect(revised.id).toBe(original.id); + expect(db.db.prepare("SELECT * FROM memories WHERE id=?").get(other.id)).toEqual(otherBefore); + expect(db.db.prepare("SELECT user_id,memory_key FROM memories WHERE id=?").get(original.id)).toEqual({ user_id: "old-a", memory_key: key }); + expect(l1Count(db.db)).toBe(2); + }); + + it.each(["legacy", "scoped"])("preserves a deleted %s scan when a later QA revision arrives", (keyKind) => { + const { db, service } = createTestService(); + const sessionId = service.openSession({ namespace }).sessionId; + const input = { sessionId, requestId: "deleted-first", turnId: "deleted-turn", query: "Test command?", answer: "old answer" }; + const first = addScannedTurn(service, input); + if (keyKind === "legacy") db.db.prepare("UPDATE memories SET memory_key=? WHERE id=?").run("memory.add:agent-source:codex:turn:deleted-turn", first.id); + db.db.prepare("UPDATE memories SET status='deleted',deleted_at=? WHERE id=?").run("2026-10-01T00:00:00.000Z", first.id); + const before = db.db.prepare("SELECT * FROM memories WHERE id=?").get(first.id); + expect(addScannedTurn(service, { ...input, requestId: "deleted-revision", answer: "new answer" })) + .toMatchObject({ id: first.id, status: "deleted", duplicate: true }); + expect(db.db.prepare("SELECT * FROM memories WHERE id=?").get(first.id)).toEqual(before); + expect(l1Count(db.db)).toBe(0); + }); + + it.each([false, true])("keeps a proven same-turn hook capture across accounts (legacy claim=%s)", async (legacy) => { const { db, service } = createTestService(); const session = service.openSession({ adapterId: "memmy-codex-hook", @@ -33,10 +162,13 @@ describe("MemoryService / import / QA capture dedup", () => { toolResults: [{ id: "read-1", name: "read_file", output: "packageManager: pnpm" }] })); const beforeCount = l1Count(db.db); + if (legacy) db.db.prepare("UPDATE memory_capture_claims SET qa_hash = ?, user_id = 'old-account'") + .run(memoryCaptureQaHash(query, answer)); const scanned = addScannedTurn(service, { requestId: "scan-after-hook", - turnId: "codex:scan-after-hook", + turnId: "hook-first-turn", + namespace: { ...namespace, userId: "another-account" }, query, answer, intermediateAssistant: "我先读取 package.json。" @@ -47,8 +179,8 @@ describe("MemoryService / import / QA capture dedup", () => { expect(l1Count(db.db)).toBe(beforeCount); expect(db.db.prepare( `SELECT captured_by FROM memory_capture_claims - WHERE user_id = ? AND source = 'codex'` - ).get(namespace.userId)).toEqual({ captured_by: "turn_complete" }); + WHERE source = 'codex'` + ).get()).toEqual({ captured_by: "turn_complete" }); }); it("keeps the scan capture and still records the hook raw turn and episode link", async () => { @@ -57,7 +189,7 @@ describe("MemoryService / import / QA capture dedup", () => { const answer = "运行 npm test 即可执行完整测试。"; const scanned = addScannedTurn(service, { requestId: "scan-first", - turnId: "codex:scan-first", + turnId: "scan-first-hook-turn", query, answer, intermediateAssistant: "我先检查 package.json scripts。" @@ -91,8 +223,8 @@ describe("MemoryService / import / QA capture dedup", () => { expect(JSON.parse(episode.raw_turn_ids_json)).toContain(completed.rawTurnId); expect(db.db.prepare( `SELECT captured_by FROM memory_capture_claims - WHERE user_id = ? AND source = 'codex'` - ).get(namespace.userId)).toEqual({ captured_by: "agent_source_scan" }); + WHERE user_id IS ? AND source = 'codex'` + ).get(null)).toEqual({ captured_by: "agent_source_scan" }); }); it("stores a second memory when the final assistant answer differs", () => { @@ -117,6 +249,31 @@ describe("MemoryService / import / QA capture dedup", () => { expect(l1Count(db.db)).toBe(2); }); + it.each(["project", "conversation", "turn", "missing-origin"] as const)("does not merge equal QA with a different %s", async (difference) => { + const { db, service } = createTestService(); + const firstNamespace = { ...namespace, projectId: "project-a" }; + const otherNamespace = { ...firstNamespace, userId: "account-b", + ...(difference === "project" ? { projectId: "project-b" } : {}), + ...(difference === "conversation" ? { sessionKey: "other-conversation" } : {}), + ...(difference === "missing-origin" ? { sessionKey: undefined } : {}) }; + const scanned = addScannedTurn(service, { namespace: firstNamespace, requestId: "first", turnId: "native-turn", query: "Run tests", answer: "Tests passed" }); + const session = service.openSession({ namespace: otherNamespace }); + const completed = (await service.completeTurn(difference === "turn" ? "another-turn" : "native-turn", { + sessionId: session.sessionId, query: "Run tests", answer: "Tests passed" + })); + expect(completed.l1MemoryIds).not.toContain(scanned.id); + expect(l1Count(db.db)).toBe(2); + }); + + it("does not merge scans with the same turn ID in different projects", () => { + const { db, service } = createTestService(); + const input = { requestId: "scan-a", turnId: "turn-1", query: "Run tests", answer: "Tests passed" }; + const a = addScannedTurn(service, { ...input, namespace: { ...namespace, projectId: "a" } }); + const b = addScannedTurn(service, { ...input, requestId: "scan-b", namespace: { ...namespace, projectId: "b", userId: "b" } }); + expect(b.id).not.toBe(a.id); + expect(l1Count(db.db)).toBe(2); + }); + it("keeps revised scanner turns on the existing memory id without leaving dangling claims", () => { const { db, service } = createTestService(); const first = addScannedTurn(service, { @@ -159,10 +316,13 @@ function addScannedTurn( query: string; answer: string; intermediateAssistant?: string; + namespace?: RuntimeNamespace; + sessionId?: string; } ): ReturnType { return service.addMemory({ - namespace, + namespace: input.namespace ?? namespace, + sessionId: input.sessionId, adapterId: "agent-source:codex", requestId: input.requestId, layer: "L1", diff --git a/Memory/tests/service/memory-token-budget-review.test.ts b/Memory/tests/service/memory-token-budget-review.test.ts index 1b7ccbf3f..7b650218d 100644 --- a/Memory/tests/service/memory-token-budget-review.test.ts +++ b/Memory/tests/service/memory-token-budget-review.test.ts @@ -140,7 +140,7 @@ describe("memory token budget review regressions", () => { payload: { feedbackId: feedback.feedbackId, contextHash: "ctx-budget-revision", - namespace + namespace: { ...namespace, userId: null } } }); await service.runWorkerOnce(10); @@ -153,7 +153,7 @@ describe("memory token budget review regressions", () => { expect(repairs()).toHaveLength(1); expect(repairs()[0]).toMatchObject({ feedbackId: feedback.feedbackId, - userId: namespace.userId, + userId: null, projectId: namespace.projectId, preference: expect.stringContaining("Inspect migration output"), source: { synthesis: "llm" } @@ -161,7 +161,7 @@ describe("memory token budget review regressions", () => { expect(repos.runtime.listChanges(undefined, 50) .filter((change) => change.changeType === "decision_repair_created") .map((change) => change.namespaceId)).toEqual([ - "user-budget-revision:project-budget-revision:codex:budget-revision-profile" + "project-budget-revision:codex:budget-revision-profile" ]); await service.runWorkerOnce(10); expect(calls.filter((operation) => operation === "decision.repair.v1")).toHaveLength(1); @@ -535,7 +535,7 @@ describe("memory token budget review regressions", () => { const row = db.db.prepare( `SELECT id FROM memories - WHERE user_id = 'user-feedback-rule' + WHERE user_id IS NULL AND memory_layer = 'L2' LIMIT 1` ).get() as { id: string } | undefined; diff --git a/Memory/tests/service/read-model/l3-world-model-context.test.ts b/Memory/tests/service/read-model/l3-world-model-context.test.ts index e0f29f7e5..a257993cb 100644 --- a/Memory/tests/service/read-model/l3-world-model-context.test.ts +++ b/Memory/tests/service/read-model/l3-world-model-context.test.ts @@ -12,6 +12,32 @@ afterEach(() => { }); describe("Session L3 World Model context read model", () => { + it.each(["project_contract", "domain_knowledge"] as const)("refreshes context after an equal-length tail edit to long %s", (targetField) => { + const { db, service } = createTestService(); + const namespace = { source: "codex", profileId: "default", sessionKey: "long-context", userId: "account-a" }; + const session = service.openSession({ namespace, workspaceUri: "file:///tmp/long-context", workspaceHostId: "b".repeat(64), l3WorldModelProtocolVersion: 2, l3WorldModelTransition: "resume_only" }); + const scopedNamespace = { ...namespace, projectId: session.projectId! }; + const repos = new Repositories(db.db); + const prefix = "规则x".repeat(10_000); + const write = (suffix: string, sources = ["historical-l1"]) => repos.l3WorldModels.upsertField({ + userId: namespace.userId, projectId: session.projectId!, targetField, value: prefix + suffix, eligibleL1MemoryIds: sources + }); + write("A"); + const before = service.l3WorldModelContext(session.sessionId, envelope(scopedNamespace)); + write("B"); + const after = service.l3WorldModelContext(session.sessionId, envelope({ ...scopedNamespace, userId: "account-b" })); + expect(after.memoryId).toBe(before.memoryId); + expect(after.memoryVersion).not.toBe(before.memoryVersion); + expect(after.renderedContext).toContain(prefix + "B"); + expect(after.renderedContext).not.toContain(prefix + "A"); + expect(service.l3WorldModelContext(session.sessionId, envelope(scopedNamespace)).memoryVersion).toBe(after.memoryVersion); + write("B", ["new-l1"]); + const withNewSource = service.l3WorldModelContext(session.sessionId, envelope(scopedNamespace)); + expect(withNewSource.renderedContext).toBe(after.renderedContext); + expect(withNewSource.memoryVersion).not.toBe(after.memoryVersion); + expect(withNewSource.sourceMemoryIds).toEqual(expect.arrayContaining(["historical-l1", "new-l1"])); + }); + it("loads one exact no-project record without truncating it", () => { const { db, service } = createTestService(); const namespace = { @@ -38,7 +64,7 @@ describe("Session L3 World Model context read model", () => { schemaVersion: 2, projectId: null, memoryId: memory?.id, - memoryVersion: memory?.version, + memoryVersion: expect.any(Number), renderedContext: `## 通用规则与安全约束\n${content}`, generalRulesAndSafetyConstraints: content, projectEnvironmentProfile: null, @@ -49,6 +75,38 @@ describe("Session L3 World Model context read model", () => { db.close(); }); + it("keeps an explicitly archived project L3 archived in the panel and out of session context", () => { + const { db, service } = createTestService(); + const namespace = { source: "codex", profileId: "default", sessionKey: "archive-context", userId: "archive-user" }; + const opened = service.openSession({ + l3WorldModelProtocolVersion: 2, + l3WorldModelTransition: "resume_only", + workspaceUri: "file:///tmp/archive-context", + workspaceHostId: "a".repeat(64), + namespace + }); + const repos = new Repositories(db.db); + const memory = repos.l3WorldModels.upsertField({ + userId: namespace.userId, + projectId: opened.projectId, + targetField: "project_contract", + value: "Archived content must stay out of the session context." + })!; + expect(service.l3WorldModelContext(opened.sessionId, envelope({ ...namespace, projectId: opened.projectId! })).renderedContext) + .toContain("Archived content"); + + service.archiveMemory(memory.id, { namespace }); + + expect(repos.l3WorldModels.getMemory(null, opened.projectId)?.status).toBe("archived"); + expect(service.l3WorldModelContext(opened.sessionId, envelope({ ...namespace, projectId: opened.projectId! }))) + .toMatchObject({ renderedContext: "", projectContract: null, memoryId: null }); + expect(service.panelItems({ layer: "L3", status: "activated" }).items).toEqual([]); + expect(service.panelItems({ layer: "L3", status: "archived" }).items).toContainEqual( + expect.objectContaining({ id: memory.id, status: "archived" }) + ); + db.close(); + }); + it("projects out a stale environment profile while preserving contract and knowledge", () => { const { db, service } = createTestService(); const namespace = { @@ -89,8 +147,8 @@ describe("Session L3 World Model context read model", () => { db.db.prepare( `UPDATE l3_world_model_project_environment_state SET project_kind = 'code', status = 'clean', applied_scan_id = 'scan-1', updated_at = ? - WHERE user_id = ? AND project_id = ?` - ).run("2026-01-01T00:00:00.000Z", namespace.userId, projectId); + WHERE user_id IS ? AND project_id = ?` + ).run("2026-01-01T00:00:00.000Z", null, projectId); const context = service.l3WorldModelContext(opened.sessionId, envelope(scopedNamespace)); expect(context).toMatchObject({ @@ -102,9 +160,11 @@ describe("Session L3 World Model context read model", () => { const before = repos.memories.get(memory.id)!; db.db.prepare( `UPDATE l3_world_model_project_environment_state - SET applied_scan_id = 'scan-2' WHERE user_id = ? AND project_id = ?` - ).run(namespace.userId, projectId); + SET applied_scan_id = 'scan-2' WHERE user_id IS ? AND project_id = ?` + ).run(null, projectId); const projected = service.l3WorldModelContext(opened.sessionId, envelope(scopedNamespace)); + expect(projected.memoryVersion).not.toBe(context.memoryVersion); + expect(service.l3WorldModelContext(opened.sessionId, envelope(scopedNamespace)).memoryVersion).toBe(projected.memoryVersion); expect(projected.projectEnvironmentProfile).toBeNull(); expect(projected.projectContract).toBe("提交前运行测试。"); expect(projected.domainKnowledge).toBe("Node 22 -> 可使用原生 TypeScript strip types。"); diff --git a/Memory/tests/service/read-model/panel-read.test.ts b/Memory/tests/service/read-model/panel-read.test.ts index e5e350ff6..e198ce544 100644 --- a/Memory/tests/service/read-model/panel-read.test.ts +++ b/Memory/tests/service/read-model/panel-read.test.ts @@ -419,14 +419,14 @@ describe("MemoryService / read model / panel", () => { userId: "user-panel-pagination" }; const session = service.openSession({ namespace }); - const completed = (await Promise.all(["alpha", "beta", "gamma"].map(async (suffix) => + const completed = (await Promise.all((await Promise.all(["alpha", "beta", "gamma"].map(async (suffix) => (await service.completeTurn(`turn-panel-pagination-${suffix}`, { sessionId: session.sessionId, query: `panel pagination needle ${suffix}`, answer: `stored panel pagination needle ${suffix}`, tags: ["panel-pagination"] })) - ))); + ))))); const firstPage = service.panelItems({ namespace, @@ -622,6 +622,45 @@ describe("MemoryService / read model / panel", () => { db.close(); }); + it("filters the merged L3 status after collapsing archived legacy account copies", () => { + const { db, service } = createTestService(); + const repos = (service as unknown as { repos: Repositories }).repos; + const active = repos.l3WorldModels.upsertField({ + userId: "panel-active-account", + projectId: "panel-archived-legacy-project", + targetField: "project_contract", + value: "The shared project rule remains active.", + at: "2026-06-01T00:00:00.000Z" + })!; + const source = db.db.prepare("SELECT info_json, properties_json FROM memories WHERE id = ?") + .get(active.id) as { info_json: string; properties_json: string }; + const properties = JSON.parse(source.properties_json) as { + status?: string; + internal_info: { world_model: Record }; + }; + properties.status = "archived"; + properties.internal_info.world_model.project_contract = null; + db.db.prepare(`INSERT INTO memories ( + id, timeline, user_id, memory_key, memory_value, info_json, properties_json, + memory_layer, status, created_at, updated_at + ) VALUES (?, ?, ?, ?, '', ?, ?, 'L3', 'archived', ?, ?)`) + .run("empty-archived-legacy-copy", "2026-06-02T00:00:00.000Z", "panel-old-account", + "world_model:project:panel-old-account:panel-archived-legacy-project", source.info_json, + JSON.stringify(properties), "2026-06-02T00:00:00.000Z", "2026-06-02T00:00:00.000Z"); + db.db.prepare(`INSERT INTO l3_world_model_scopes ( + scope_key, user_id, project_id, memory_id, updated_at + ) VALUES (?, ?, ?, ?, ?)`) + .run("empty-archived-legacy-scope", "panel-old-account", "panel-archived-legacy-project", + "empty-archived-legacy-copy", "2026-06-02T00:00:00.000Z"); + + expect(service.panelItems({ layer: "L3", status: "archived" }).items).toEqual([]); + expect(service.panelItems({ layer: "L3", status: "activated" }).items).toEqual([ + expect.objectContaining({ id: active.id, status: "activated" }) + ]); + + db.close(); + }); + it("adds typed general and project scope to L3 panel items with one batched lookup", () => { const { db, service } = createTestService(); const repos = (service as unknown as { repos: Repositories }).repos; @@ -679,7 +718,7 @@ describe("MemoryService / read model / panel", () => { targetField: "project_environment_profile", value: "Project profile." })!; - db.db.prepare(`UPDATE memories SET user_id = 'different-owner' WHERE id = ?`).run(project.id); + db.db.prepare(`UPDATE memories SET info_json = json_set(info_json, '$.project_id', 'different-project') WHERE id = ?`).run(project.id); expect(service.panelItems({ layer: "L3" }).items.find((item) => item.id === project.id)) .not.toHaveProperty("worldModelScope"); diff --git a/Memory/tests/service/retrieval/same-source-l1-dedup.test.ts b/Memory/tests/service/retrieval/same-source-l1-dedup.test.ts new file mode 100644 index 000000000..8ecef30f6 --- /dev/null +++ b/Memory/tests/service/retrieval/same-source-l1-dedup.test.ts @@ -0,0 +1,308 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; +import { DEFAULT_MEMMY_CONFIG, type LlmClient, type MemoryRow } from "../../../src/index.js"; +import { Repositories } from "../../../src/storage/repositories.js"; +import { refreshProjectAliases } from "../../../src/storage/device-scope.js"; +import { sha256Hex } from "../../../src/contracts/index.js"; +import { stableHash } from "../../../src/utils/id.js"; +import { attachMemoryVector, memoryVectorEntries } from "../../../src/storage/memory-vector-state.js"; +import { compileRetrievalQuery, retrievePluginMemories } from "../../../src/algorithm/plugin-algorithms.js"; +import { deduplicateSameSourceL1Candidates } from "../../../src/service/retrieval/same-source-l1-dedup.js"; +import { IndexedCandidatePool } from "../../../src/service/retrieval/indexed-candidate-pool.js"; +import { createMemoryServiceFixture } from "../../fixtures/memory-service-fixture.js"; + +const fixture = createMemoryServiceFixture(); +afterEach(() => { vi.restoreAllMocks(); fixture.cleanup(); }); +const at = "2026-10-01T00:00:00.000Z"; +const uri = "file:///workspace/shared", host = "a".repeat(64); +function setup(time = false, rewrite = false) { + const config = structuredClone(DEFAULT_MEMMY_CONFIG); + config.algorithm.enableQueryRewrite = rewrite; + Object.assign(config.algorithm.retrieval, { minRecallScore: 0, relativeThresholdFloor: 0, smartSeed: false, llmFilterEnabled: false }); + const llm: LlmClient = { + config: config.summary, isConfigured: () => time || rewrite, + complete: async () => "{}", + completeJson: async >(_messages: unknown, options: {operation?: string}) => + (options.operation?.includes("rewrite") ? { queries: ["shared evidence alpha", "shared evidence beta"] } : time + ? {queryVecText: "", keywords: [], timeFilter: {startAt: at, endAt: "2026-10-02T00:00:00.000Z"}} + : {queryVecText: "shared evidence", keywords: ["shared"]}) as unknown as T, + status: () => ({provider: "host", model: "fixture", configured: time || rewrite, remote: false}) + }; + const state = fixture.createTestService({ config, llm }); + const repos = new Repositories(state.db.db); + function seed(id: string, owner: string | null, options: { + turn?: string; step?: number | null; source?: string; profile?: string; project?: string; + conversation?: string; tenant?: string; missingSource?: boolean; layer?: "L1" | "L2"; work?: boolean; + } = {}) { + const opened = state.service.openSession({ namespace: {source: options.source ?? "codex", profileId: options.profile ?? "default", sessionKey: id, userId: owner ?? undefined} }); + const workspaceId = owner ? sha256Hex(`${owner}\0local\0${host}\0${uri}`) : "device-workspace"; + const projectId = options.project ?? (owner ? `ws_${workspaceId}` : "device-project"); + const meta = {workspace_uri: options.project ? undefined : uri, workspace_host_id: host, + tenant_id: options.tenant ?? null, + source_namespace_key: stableHash({tenantId: options.tenant ?? null, projectId, workspaceId})}; + state.db.db.prepare("UPDATE sessions SET user_id=?,project_id=?,workspace_id=?,conversation_id=?,meta_json=? WHERE id=?") + .run(owner, projectId, workspaceId, options.missingSource ? null : options.conversation ?? "native-chat", JSON.stringify(meta), opened.sessionId); + refreshProjectAliases(state.db.db); + const turn = options.turn ?? "native-turn"; + const row: MemoryRow = { + id, userId: null, sessionId: opened.sessionId, timeline: at, agentId: options.source ?? "codex", + memoryLayer: options.layer ?? "L1", memoryType: "LongTermMemory", status: "activated", visibility: "private", + memoryKey: `trace:${id}`, memoryValue: "shared evidence unchanged", tags: ["trace"], appId: workspaceId, info: {turn_id: turn, project_id: projectId}, + properties: {internal_info: {memory_layer: options.layer ?? "L1", memory_kind: options.work ? "work_memory" : "trace", trace: { + key: `trace:${id}`, ts: Date.parse(at), turn_id: turn, raw_turn_id: `raw-${id}`, + ...(options.step === null ? {} : {step_index: options.step ?? 0}), sub_step_total: 1, + userText: "shared evidence", agentText: "shared evidence", summary: "shared evidence", + tool_calls: [], reflection: null, alpha: 0, tags: ["trace"], value: 1, priority: 1, + error_signatures: [], vec_summary: [1,0,0], vec_action: [1,0,0], embedding_model: "capturing-test-embedding" + }}}, contentHash: "same-content", version: 1, createdAt: at, updatedAt: at + }; + repos.memories.insert(row); + state.db.db.prepare("UPDATE memories SET user_id=? WHERE id=?").run(owner, id); + return repos.memories.get(id)!; + } + return {...state, repos, seed}; +} + +describe("same-source L1 deduplication without UID", () => { + it.each(["profile", "source", "conversation"])("preserves the recorded %s when the same session is reopened", async boundary => { + const {db, repos, service} = setup(); + const rows: MemoryRow[] = []; + for (const profile of ["a", "b"]) { + const namespace = {source: boundary === "source" && profile === "b" ? "claude" : "codex", + profileId: boundary === "profile" ? `profile-${profile}` : "default", userId: `user-${profile}`, + sessionKey: `host-${profile}`, projectId: "project-p", workspaceId: "workspace-w"}; + const opened = service.openSession({namespace, meta: {conversationId: boundary === "conversation" ? `conversation-${profile}` : "native-conversation"}}); + const turn = (await service.completeTurn("native-turn", {sessionId: opened.sessionId, namespace, + query: "Please inspect the project and report its test command.", answer: "The test command is npm test.", + toolCalls: [{id: "read", name: "read_file", input: {path: "package.json"}}], toolResults: [{id: "read", output: "npm test"}]})); + const row = repos.memories.get(turn.l1MemoryId!)!; + expect(row.info.profile_id).toBe(namespace.profileId); rows.push(row); + repos.runtime.upsertCandidatePoolTrace({id: `candidate-${profile}`, userId: null, sessionId: opened.sessionId, + sourceMemoryId: row.id, candidateKey: "policy", candidateValue: "same evidence", score: 1, + evidence: {}, createdAt: at, updatedAt: at}); + } + const a = rows[0]!, b = rows[1]!; + service.openSession({sessionId: a.sessionId, namespace: {source: b.agentId!, profileId: b.info.profile_id as string, + projectId: "project-p", workspaceId: "workspace-w"}, meta: {conversationId: b.conversationId}}); + repos.runtime.markCandidatePoolPromoted({candidateKey: "policy", sourceMemoryIds: [b.id], policyId: "policy-b", at}); + expect(db.db.prepare("SELECT id,status FROM l2_candidate_pool ORDER BY id").all()).toEqual([ + {id: "candidate-a", status: "pending"}, {id: "candidate-b", status: "promoted"} + ]); + repos.memories.softDelete(a.id); + const history = db.db.prepare("SELECT * FROM memories ORDER BY id").all(); + expect(repos.memories.nativeDuplicates(b)).toEqual({ids: [b.id], deleted: false}); + expect(deduplicateSameSourceL1Candidates({repos, memories: [b]}).memories.map(row => row.id)).toEqual([b.id]); + expect(db.db.prepare("SELECT * FROM memories ORDER BY id").all()).toEqual(history); + }); + + it.each([ + ["project", false], ["project", true], ["workspace", false], ["workspace", true] + ] as const)("does not propagate deletion or L2 promotion after a session changes %s (deleted before move=%s)", async (boundary, deletedBeforeMove) => { + const {db, repos, service, seed} = setup(); + const a = seed("a", "account-a", {project: boundary === "project" ? "project-a" : "shared-project"}); + const b = seed("b", "account-b", {project: boundary === "project" ? "project-b" : "shared-project"}); + // Historical hook sessions need not have a frozen source_namespace_key. + db.db.prepare("UPDATE sessions SET meta_json='{}' WHERE id IN (?,?)").run(a.sessionId, b.sessionId); + for (const memory of [a, b]) repos.runtime.upsertCandidatePoolTrace({ + id: `candidate-${memory.id}`, userId: null, sessionId: memory.sessionId, sourceMemoryId: memory.id, + candidateKey: "shared-policy", candidateValue: "shared evidence", score: 1, evidence: {}, createdAt: at, updatedAt: at + }); + if (deletedBeforeMove) repos.memories.softDelete("a"); + service.openSession({sessionId: a.sessionId, namespace: {source: "codex", profileId: "default", + userId: "account-a", projectId: b.info.project_id as string, workspaceId: b.appId}}); + expect(repos.memories.nativeDuplicates(a)).toEqual({ids: ["a"], deleted: deletedBeforeMove}); + expect(repos.memories.nativeDuplicates(b)).toEqual({ids: ["b"], deleted: false}); + if (!deletedBeforeMove) { + expect(deduplicateSameSourceL1Candidates({repos, memories: [a, b]}).memories).toHaveLength(2); + repos.memories.softDelete("a"); + } + const history = db.db.prepare("SELECT * FROM memories ORDER BY id").all(); + for (const userId of ["account-a", "account-b", "new-account"]) { + const result = await service.search({namespace: {source: "codex", profileId: "default", userId}, + query: "shared evidence", layers: ["L1"], limit: 10}); + expect(result.hits.map(hit => hit.id)).toEqual(["b"]); + expect(repos.runtime.listPendingCandidatePool({userId}).map(row => row.sourceMemoryId)).toEqual(["b"]); + } + expect(repos.memories.nativeDuplicates(repos.memories.getIncludingDeleted("a")!)).toEqual({ids: ["a"], deleted: true}); + repos.runtime.markCandidatePoolPromoted({candidateKey: "shared-policy", sourceMemoryIds: ["b"], policyId: "policy-b", at}); + expect(db.db.prepare("SELECT source_memory_id,status FROM l2_candidate_pool ORDER BY source_memory_id").all()) + .toEqual([{source_memory_id: "a", status: "pending"}, {source_memory_id: "b", status: "promoted"}]); + expect(db.db.prepare("SELECT * FROM memories ORDER BY id").all()).toEqual(history); + }); + + it.each([["account-a", "account-b"], ["account-a", "account-a"], [null, null], ["account-a", null]])( + "folds same-source copies (%s / %s), preserving rows, content and channel scores", (a, b) => { + const {db, repos, seed} = setup(); + seed("copy-a", a); seed("copy-b", b); + const before = db.db.prepare("SELECT * FROM memories ORDER BY id").all(); + const memories = [repos.memories.get("copy-b")!, repos.memories.get("copy-a")!]; + const result = deduplicateSameSourceL1Candidates({memories, repos, + channelScoresByMemory: new Map([["copy-a", {fts: 0.2}], ["copy-b", {fts: 0.9, vec_summary: 0.8}]])}); + expect(result.memories.map(m => m.id)).toEqual(["copy-a"]); + expect(result.channelScoresByMemory.get("copy-a")).toMatchObject({fts: 0.9, vec_summary: 0.8}); + expect(db.db.prepare("SELECT * FROM memories ORDER BY id").all()).toEqual(before); + } + ); + + it("uses the latest same-source revision when only an older copy is a candidate", () => { + const makeMemory = (input: {id: string; value: string; summary: string; updatedAt: string; version: number}): MemoryRow => ({ + id: input.id, timeline: at, userId: null, agentId: "codex", memoryType: "LongTermMemory", + status: "activated", visibility: "private", memoryKey: `trace:${input.id}`, memoryValue: input.value, + tags: ["trace"], info: {}, memoryLayer: "L1", contentHash: stableHash(input.value), version: input.version, + createdAt: at, updatedAt: input.updatedAt, + properties: {internal_info: {memory_layer: "L1", memory_kind: "trace", trace: { + summary: input.summary, userText: "shared evidence", agentText: input.value, tool_calls: [], reflection: null + }}} + }); + const older = attachMemoryVector(makeMemory({id: "copy-a", value: "original evidence", summary: "original summary", updatedAt: at, version: 1}), { + vectorField: "vec_summary", vector: [1, 0, 0], embeddingModel: "old-model" + }); + const newer = attachMemoryVector(makeMemory({id: "copy-b", value: "revised evidence", summary: "revised summary", + updatedAt: "2026-10-02T00:00:00.000Z", version: 2}), { + vectorField: "vec_summary", vector: [0, 1, 0], embeddingModel: "new-model" + }); + const byId = new Map([[older.id, older], [newer.id, newer]]); + const repos = {memories: { + nativeDuplicates: () => ({ids: ["copy-a", "copy-b"], deleted: false}), + getMany: (ids: string[]) => ids.flatMap(id => byId.get(id) ? [byId.get(id)!] : []) + }} as unknown as Repositories; + + const result = deduplicateSameSourceL1Candidates({ + memories: [older], repos, + channelScoresByMemory: new Map([ + ["copy-a", {fts: 0.99, vec_summary: 0.95}], + ["copy-b", {fts: 0.1}] + ]) + }); + + expect(result.memories.map(memory => memory.id)).toEqual(["copy-b"]); + expect(result.memories[0]?.memoryValue).toBe("revised evidence"); + expect(memoryVectorEntries(result.memories[0]!)).toMatchObject([ + {vectorField: "vec_summary", vector: [0, 1, 0], embeddingModel: "new-model"} + ]); + expect(result.channelScoresByMemory.get("copy-b")).toBeUndefined(); + expect(result.memoryIdByMemoryId).toEqual(new Map([["copy-a", "copy-b"], ["copy-b", "copy-b"]])); + expect(memoryVectorEntries(older)).toHaveLength(1); + + const config = structuredClone(DEFAULT_MEMMY_CONFIG); + Object.assign(config.algorithm.retrieval, { + minRecallScore: 0, relativeThresholdFloor: 0, smartSeed: false, llmFilterEnabled: false, minTraceSim: 0.1 + }); + const retrieval = retrievePluginMemories({ + query: "revised evidence", queryVector: [0, 1, 0], memories: result.memories, layers: ["L1"], limit: 1, + channelScoresByMemory: result.channelScoresByMemory, config: config.algorithm.retrieval + }); + expect(retrieval.hits.map(hit => hit.id)).toEqual(["copy-b"]); + + const fallback = deduplicateSameSourceL1Candidates({ + memories: [older], repos, isMemoryEligible: memory => memory.id === "copy-a" + }); + expect(fallback.memories.map(memory => memory.id)).toEqual(["copy-a"]); + + const outOfScopeLatest = deduplicateSameSourceL1Candidates({ + memories: [older], repos, + isMemoryEligible: () => true, + isMemoryInScope: memory => memory.id === "copy-a" + }); + expect(outOfScopeLatest.memories).toEqual([]); + }); + + it("expands an L1 route when same-source copies fill its candidate slots", async () => { + const {repos, seed} = setup(); + seed("copy-a", "account-a"); + seed("copy-b", "account-b"); + seed("distinct-c", "account-c", {turn: "distinct-turn"}); + const routeFilters: Array<{excludedIds?: string[]}> = []; + vi.spyOn(repos.memories, "searchFtsIds").mockImplementation((_match, filter = {}, limit = 20) => { + routeFilters.push(filter); + return ["copy-a", "copy-b", "distinct-c"] + .filter(id => !filter.excludedIds?.includes(id)) + .slice(0, limit) + .map((id, index) => ({id, score: 1 / (index + 1), channel: "fts"})); + }); + const candidatePool = new IndexedCandidatePool({repos, memoryHasImportPipeline: () => false}); + const result = await candidatePool.indexedRetrievalCandidatePool({ + userId: null, + compiledQuery: { + ...compileRetrievalQuery("shared evidence"), + patternTerms: [], + structuralFragments: [] + }, + layers: ["L1"], + config: { + tier1TopK: 1, + tier2TopK: 2, + tier3TopK: 1, + candidatePoolFactor: 1, + keywordTopK: 2, + tagFilter: "off" + } + }); + const collapsed = deduplicateSameSourceL1Candidates({memories: result.memories, repos}); + + expect(routeFilters).toHaveLength(2); + expect(routeFilters[0]?.excludedIds).toEqual([]); + expect(routeFilters[1]?.excludedIds).toEqual(expect.arrayContaining(["copy-a", "copy-b"])); + expect(collapsed.memories.map(memory => memory.id)).toEqual(["copy-a", "distinct-c"]); + }); + + it.each([ + {turn: "other-turn"}, {step: 1}, {project: "other-project"}, {conversation: "other-chat"}, + {source: "cursor"}, {profile: "other-profile"}, {tenant: "other-tenant"}, + {missingSource: true}, {layer: "L2" as const}, {work: true} + ])("does not collapse equal content with a different or incomplete source: %j", options => { + const {repos, seed} = setup(); + seed("a", "account-a"); seed("b", "account-b", options); + expect(deduplicateSameSourceL1Candidates({repos, memories: repos.memories.list({}, 100)}).memories).toHaveLength(2); + }); + + it("does not let an unreturned deleted copy or recent raw turn reappear through a sibling", () => { + const {repos, seed} = setup(); + seed("a", "account-a"); const b = seed("b", "account-b"); + expect(deduplicateSameSourceL1Candidates({repos, memories: [b], excludeTraceRawTurnIds: new Set(["raw-a"])}).memories).toEqual([]); + repos.memories.softDelete("a"); + expect(deduplicateSameSourceL1Candidates({repos, memories: [b]}).memories).toEqual([]); + }); + + it.each([false, true])("search integration returns one historical copy for every account (time filter=%s)", async time => { + const {db, repos, service, seed} = setup(time); + seed("a", "account-a"); seed("b", "account-b"); seed("c", null, {turn: "different-turn"}); + const before = db.db.prepare("SELECT * FROM memories ORDER BY id").all(); + for (const userId of ["account-a", "account-b", "new-account"]) { + const result = await service.search({namespace: {source: "codex", profileId: "default", userId}, query: "shared evidence", layers: ["L1"], limit: 10}); + expect(result.hits.map(hit => hit.id).sort()).toEqual(["a", "c"]); + expect(result.sourceMemoryIds.sort()).toEqual(["a", "c"]); + } + expect(db.db.prepare("SELECT * FROM memories ORDER BY id").all()).toEqual(before); + repos.memories.softDelete("b"); + const result = await service.search({namespace: {source: "codex", profileId: "default", userId: "new-account"}, query: "shared evidence", layers: ["L1"], limit: 10}); + expect(result.hits.map(hit => hit.id)).toEqual(["c"]); + }); + + it("uses the same historical source for repair suggestions across accounts", async () => { + const {db, repos, service, seed} = setup(); + seed("a", "account-a"); seed("b", "account-b"); + const before = db.db.prepare("SELECT * FROM memories ORDER BY id").all(); + for (const userId of ["account-a", "account-b", "new-account"]) { + const session = service.openSession({namespace: {source: "codex", profileId: "default", userId}}); + const result = await service.repairSuggestion({sessionId: session.sessionId, issue: "shared evidence"}); + expect(result.sourceMemoryIds).toEqual(["a"]); + } + expect(db.db.prepare("SELECT * FROM memories ORDER BY id").all()).toEqual(before); + repos.memories.softDelete("a"); + const session = service.openSession({namespace: {source: "codex", profileId: "default", userId: "account-b"}}); + const result = await service.repairSuggestion({sessionId: session.sessionId, issue: "shared evidence"}); + expect(result.sourceMemoryIds).toEqual([]); + }); + + it("folds copies returned by separate query rewrites before final fusion", async () => { + const {repos, service, seed} = setup(false, true); + seed("a", "account-a"); seed("b", "account-b"); + const pool = vi.spyOn(IndexedCandidatePool.prototype, "indexedRetrievalCandidatePool"); + pool.mockResolvedValueOnce({memories: [repos.memories.get("a")!], channelScoresByMemory: new Map([["a", {vec_summary: 1}], ["b", {vec_summary: 1}]])}); + pool.mockResolvedValueOnce({memories: [repos.memories.get("b")!], channelScoresByMemory: new Map([["a", {vec_summary: 1}], ["b", {vec_summary: 1}]])}); + const result = await service.search({namespace: {source: "codex", profileId: "default", userId: "account-b"}, query: "shared evidence", layers: ["L1"], limit: 10}); + expect(pool).toHaveBeenCalledTimes(2); + expect(result.hits.map(hit => hit.id)).toEqual(["a"]); + expect(result.sourceMemoryIds).toEqual(["a"]); + }); +}); diff --git a/Memory/tests/service/session/episode-relation.test.ts b/Memory/tests/service/session/episode-relation.test.ts index 227d72333..72e1b8a00 100644 --- a/Memory/tests/service/session/episode-relation.test.ts +++ b/Memory/tests/service/session/episode-relation.test.ts @@ -422,8 +422,8 @@ describe("MemoryService / session / episode relation", () => { l1_memory_ids_json: JSON.stringify([first.l1MemoryId, completed.l1MemoryId]) }); expect(db.db.prepare( - `SELECT content, source_turn_id FROM user_memories WHERE user_id = ? AND status = 'active'` - ).all("user-end-topic-feedback")).toEqual([{ + `SELECT content, source_turn_id FROM user_memories WHERE user_id IS ? AND status = 'active'` + ).all(null)).toEqual([{ content: closingQuery, source_turn_id: completed.rawTurnId }]); @@ -864,7 +864,7 @@ describe("MemoryService / session / episode relation", () => { const feedbackCount = db.db.prepare( `SELECT COUNT(*) AS count FROM feedback - WHERE user_id = 'user-relation-follow-up'` + WHERE user_id IS NULL` ).get() as { count: number }; expect(feedbackCount.count).toBe(0); @@ -1279,7 +1279,7 @@ describe("MemoryService / session / episode relation", () => { }); expect(prepared).not.toHaveProperty("episodeId"); expect(db.db.prepare( - "SELECT COUNT(*) AS count FROM feedback WHERE user_id = 'user-relation-revision'" + "SELECT COUNT(*) AS count FROM feedback WHERE user_id IS NULL" ).get()).toEqual({ count: 0 }); const correction = (await service.completeTurn("turn-relation-revision-fix", { sessionId: session.sessionId, @@ -1289,13 +1289,13 @@ describe("MemoryService / session / episode relation", () => { await settleCaptureAssignments(service); expect(assignedEpisodeId(service, correction)).toBe(assignedEpisodeId(service, first)); expect(db.db.prepare( - "SELECT COUNT(*) AS count FROM feedback WHERE user_id = 'user-relation-revision'" + "SELECT COUNT(*) AS count FROM feedback WHERE user_id IS NULL" ).get()).toEqual({ count: 1 }); const feedback = db.db.prepare( `SELECT id, l1_memory_id, raw_turn_id, polarity, raw_payload_json FROM feedback - WHERE user_id = 'user-relation-revision'` + WHERE user_id IS NULL` ).get() as { id: string; l1_memory_id: string | null; @@ -1466,7 +1466,7 @@ describe("MemoryService / session / episode relation", () => { expect(prepared).not.toHaveProperty("episodeId"); expect(prepared).not.toHaveProperty("closedEpisodeIds"); expect(db.db.prepare( - "SELECT COUNT(*) AS count FROM feedback WHERE user_id = 'user-implicit-turn-feedback'" + "SELECT COUNT(*) AS count FROM feedback WHERE user_id IS NULL" ).get()).toEqual({ count: 0 }); expect(service.getMemory(assignedEpisodeId(service, first))).toMatchObject({ kind: "episode", @@ -1483,7 +1483,7 @@ describe("MemoryService / session / episode relation", () => { const feedback = db.db.prepare( `SELECT id, channel, polarity, magnitude, l1_memory_id, raw_turn_id, raw_payload_json FROM feedback - WHERE user_id = 'user-implicit-turn-feedback'` + WHERE user_id IS NULL` ).get() as { id: string; channel: string; diff --git a/Memory/tests/service/session/session-lifecycle.test.ts b/Memory/tests/service/session/session-lifecycle.test.ts index 026f0ecd4..43d625ad7 100644 --- a/Memory/tests/service/session/session-lifecycle.test.ts +++ b/Memory/tests/service/session/session-lifecycle.test.ts @@ -1,10 +1,12 @@ import { assignedEpisodeId } from "../../fixtures/memory-service-fixture.js"; import { afterEach, describe, expect, it } from "vitest"; import { deriveWorkspaceHostId } from "../../../src/contracts/index.js"; +import { DEFAULT_MEMMY_CONFIG } from "../../../src/config/index.js"; import { createMemoryServiceFixture } from "../../fixtures/memory-service-fixture.js"; const { cleanup, + createTestMemoryService, createTestService } = createMemoryServiceFixture(); @@ -253,7 +255,7 @@ describe("MemoryService / session / lifecycle", () => { injected_memory_ids_json: string; outcome: string; } | undefined; - expect(recallRow?.namespace_id).toContain("user-1"); + expect(recallRow?.namespace_id).not.toContain("user-1"); expect(recallRow?.query_hash).toBeTruthy(); expect(JSON.parse(recallRow!.candidate_memory_ids_json)).toEqual(recall.candidateMemoryIds); expect(JSON.parse(recallRow!.injected_memory_ids_json)).toEqual(recall.sourceMemoryIds); @@ -315,6 +317,199 @@ describe("MemoryService / session / lifecycle", () => { db.close(); }); + it("keeps host-key session recovery isolated by tenant while sharing within a tenant across accounts", async () => { + const { db, service } = createTestService(); + const open = (tenantId: string | undefined, userId: string) => service.openSession({ + namespace: { source: "codex", profileId: "default", sessionKey: "shared-host-key", tenantId, userId } + }); + + const tenantA = open("tenant-a", "account-a"); + const tenantB = open("tenant-b", "account-b"); + expect(tenantB.sessionId).not.toBe(tenantA.sessionId); + expect(tenantB.resumed).toBe(false); + expect(open("tenant-a", "account-c")).toMatchObject({ sessionId: tenantA.sessionId, resumed: true }); + expect(open(undefined, "account-d").sessionId).not.toBe(tenantA.sessionId); + + const row = db.db.prepare("SELECT meta_json FROM sessions WHERE id = ?").get(tenantB.sessionId) as { meta_json: string }; + expect(JSON.parse(row.meta_json).tenant_id).toBe("tenant-b"); + const camelOnlyMeta = JSON.parse(row.meta_json) as Record; + delete camelOnlyMeta.tenant_id; + camelOnlyMeta.tenantId = "tenant-b"; + db.db.prepare("UPDATE sessions SET meta_json = ? WHERE id = ?").run(JSON.stringify(camelOnlyMeta), tenantB.sessionId); + expect(open("tenant-b", "account-e")).toMatchObject({ sessionId: tenantB.sessionId, resumed: true }); + + const captured = (await service.completeTurn("tenant-a-native-turn", { + sessionId: tenantA.sessionId, + query: "Tenant source metadata must follow the session.", + answer: "The session tenant is retained in capture context.", + status: "succeeded" + })); + const rawTurn = db.db.prepare("SELECT message_payload_json FROM raw_turns WHERE id = ?") + .get(captured.rawTurnId) as { message_payload_json: string }; + expect(JSON.parse(rawTurn.message_payload_json)).toMatchObject({ + native_origin_v1: { tenantId: "tenant-a" } + }); + expect(db.db.prepare( + "SELECT namespace_id FROM memory_change_log WHERE entity_id = ? AND change_type = 'raw_turn_created'" + ).get(captured.rawTurnId)).toEqual({ namespace_id: "tenant-a:codex:default" }); + + const beforeExplicitOpen = db.db.prepare( + "SELECT status, last_seen_at, updated_at FROM sessions WHERE id = ?" + ).get(tenantA.sessionId); + expect(() => service.openSession({ + sessionId: tenantA.sessionId, + namespace: { source: "codex", profileId: "default", sessionKey: "shared-host-key", tenantId: "tenant-b" } + })).toThrow(/different tenant/u); + expect(db.db.prepare( + "SELECT status, last_seen_at, updated_at FROM sessions WHERE id = ?" + ).get(tenantA.sessionId)).toEqual(beforeExplicitOpen); + expect(() => service.closeSession(tenantA.sessionId, { + namespace: { source: "codex", profileId: "default", sessionKey: "shared-host-key", tenantId: "tenant-b" } + })).toThrow(/different tenant/u); + expect(db.db.prepare("SELECT status FROM sessions WHERE id = ?").get(tenantA.sessionId)).toEqual({ status: "open" }); + + const untrustedMetaSession = service.openSession({ + namespace: { source: "codex", profileId: "default", sessionKey: "meta-tenant-spoof", userId: "account-e" }, + meta: { tenant_id: "spoofed-tenant", tenantId: "also-spoofed" } + }); + const untrustedMeta = db.db.prepare("SELECT meta_json FROM sessions WHERE id = ?").get(untrustedMetaSession.sessionId) as { meta_json: string }; + expect(JSON.parse(untrustedMeta.meta_json)).not.toHaveProperty("tenant_id"); + expect(JSON.parse(untrustedMeta.meta_json)).not.toHaveProperty("tenantId"); + + const conflicting = open("tenant-c", "account-f"); + const conflictingMetaRow = db.db.prepare("SELECT meta_json FROM sessions WHERE id = ?") + .get(conflicting.sessionId) as { meta_json: string }; + const conflictingMeta = JSON.parse(conflictingMetaRow.meta_json) as Record; + conflictingMeta.tenantId = "tenant-other"; + db.db.prepare("UPDATE sessions SET meta_json = ? WHERE id = ?") + .run(JSON.stringify(conflictingMeta), conflicting.sessionId); + expect(() => service.closeSession(conflicting.sessionId)).toThrow(/conflicting tenant metadata/u); + expect(db.db.prepare("SELECT status FROM sessions WHERE id = ?").get(conflicting.sessionId)).toEqual({ status: "open" }); + db.close(); + }); + + it("keeps protocol-v2 host-key recovery and explicit resume isolated by tenant", () => { + const { db, service } = createTestService(); + const openV2 = (tenantId: string, userId: string) => service.openSession({ + l3WorldModelProtocolVersion: 2, + l3WorldModelTransition: "resume_only", + workspaceUri: "file:///tmp/v2-tenant-isolation", + workspaceHostId: "c".repeat(64), + namespace: { + source: "codex", + profileId: "default", + sessionKey: "v2-shared-host-key", + tenantId, + userId + } + }); + + const tenantA = openV2("tenant-a", "account-a"); + const tenantB = openV2("tenant-b", "account-b"); + expect(tenantB.sessionId).not.toBe(tenantA.sessionId); + expect(tenantB.resumed).toBe(false); + expect(openV2("tenant-a", "account-c")).toMatchObject({ sessionId: tenantA.sessionId, resumed: true }); + const before = db.db.prepare( + "SELECT last_seen_at, updated_at FROM sessions WHERE id = ?" + ).get(tenantA.sessionId); + expect(() => service.openSession({ + sessionId: tenantA.sessionId, + l3WorldModelProtocolVersion: 2, + l3WorldModelTransition: "resume_only", + workspaceUri: "file:///tmp/v2-tenant-isolation", + workspaceHostId: "c".repeat(64), + namespace: { + source: "codex", + profileId: "default", + sessionKey: "v2-shared-host-key", + tenantId: "tenant-b", + userId: "account-b" + } + })).toThrow(/scope_conflict/u); + expect(db.db.prepare( + "SELECT last_seen_at, updated_at FROM sessions WHERE id = ?" + ).get(tenantA.sessionId)).toEqual(before); + db.close(); + }); + + it("keeps read-only session lookup tenant-scoped without partitioning by account UID", () => { + const { db, service: writer } = createTestService(); + const existing = writer.openSession({ + namespace: { + source: "codex", + profileId: "default", + sessionKey: "readonly-shared-host-key", + tenantId: "tenant-a", + userId: "account-a" + } + }); + const readOnlyConfig = { + ...DEFAULT_MEMMY_CONFIG, + algorithm: { ...DEFAULT_MEMMY_CONFIG.algorithm, enableMemoryAdd: false } + }; + const reader = createTestMemoryService({ db, mode: "dev", config: readOnlyConfig }); + const tenantB = reader.openSession({ + namespace: { + source: "codex", + profileId: "default", + sessionKey: "readonly-shared-host-key", + tenantId: "tenant-b", + userId: "account-b" + } + }); + const sameTenantDifferentAccount = reader.openSession({ + namespace: { + source: "codex", + profileId: "default", + sessionKey: "readonly-shared-host-key", + tenantId: "tenant-a", + userId: "account-c" + } + }); + expect(tenantB).toMatchObject({ resumed: false }); + expect(tenantB.sessionId).not.toBe(existing.sessionId); + expect(sameTenantDifferentAccount).toMatchObject({ sessionId: existing.sessionId, resumed: true }); + + const fallbackA = reader.openSession({ + namespace: { source: "codex", profileId: "default", tenantId: "tenant-a", userId: "account-a" } + }); + const fallbackAWithOtherAccount = reader.openSession({ + namespace: { source: "codex", profileId: "default", tenantId: "tenant-a", userId: "account-c" } + }); + const fallbackB = reader.openSession({ + namespace: { source: "codex", profileId: "default", tenantId: "tenant-b", userId: "account-b" } + }); + expect(fallbackAWithOtherAccount.sessionId).toBe(fallbackA.sessionId); + expect(fallbackB.sessionId).not.toBe(fallbackA.sessionId); + + const projectA = reader.openSession({ + namespace: { source: "codex", profileId: "default", tenantId: "tenant-a", userId: "account-a" }, + projectId: "project-a" + }); + const projectAWithOtherAccount = reader.openSession({ + namespace: { source: "codex", profileId: "default", tenantId: "tenant-a", userId: "account-c" }, + projectId: "project-a" + }); + const projectB = reader.openSession({ + namespace: { source: "codex", profileId: "default", tenantId: "tenant-a", userId: "account-a" }, + projectId: "project-b" + }); + expect(projectAWithOtherAccount.sessionId).toBe(projectA.sessionId); + expect(projectB.sessionId).not.toBe(projectA.sessionId); + + const conversationA = reader.openSession({ + namespace: { source: "codex", profileId: "default", tenantId: "tenant-a" }, + meta: { conversationId: "conversation-a" } + }); + const conversationB = reader.openSession({ + namespace: { source: "codex", profileId: "default", tenantId: "tenant-a" }, + meta: { conversationId: "conversation-b" } + }); + expect(conversationB.sessionId).not.toBe(conversationA.sessionId); + expect(db.db.prepare("SELECT COUNT(*) AS count FROM sessions").get()).toEqual({ count: 1 }); + db.close(); + }); + it("refreshes explicit source when reopening an existing session", async () => { const { db, service } = createTestService(); const sessionId = "openclaw-memory-default"; @@ -444,7 +639,7 @@ describe("MemoryService / session / lifecycle", () => { `SELECT user_id, project_id, workspace_uri FROM l3_world_model_scopes WHERE project_id = ?` ).get(opened.projectId)).toEqual({ - user_id: "v2-user", + user_id: null, project_id: opened.projectId, workspace_uri: base.workspaceUri }); @@ -514,8 +709,8 @@ describe("MemoryService / session / lifecycle", () => { db.db.prepare( `UPDATE l3_world_model_scopes SET workspace_uri = 'file:///workspace/conflict' - WHERE user_id = ? AND project_id = ?` - ).run(request.namespace.userId, opened.projectId); + WHERE user_id IS ? AND project_id = ?` + ).run(null, opened.projectId); expect(() => service.openSession({ ...request, sessionId: opened.sessionId, @@ -527,8 +722,8 @@ describe("MemoryService / session / lifecycle", () => { ).get(opened.sessionId)).toEqual(before); db.db.prepare( - `UPDATE l3_world_model_scopes SET workspace_uri = ? WHERE user_id = ? AND project_id = ?` - ).run(request.workspaceUri, request.namespace.userId, opened.projectId); + `UPDATE l3_world_model_scopes SET workspace_uri = ? WHERE user_id IS ? AND project_id = ?` + ).run(request.workspaceUri, null, opened.projectId); db.db.prepare( `UPDATE sessions SET meta_json = json_remove(meta_json, '$.workspace_uri') WHERE id = ?` ).run(opened.sessionId); @@ -625,7 +820,7 @@ describe("MemoryService / session / lifecycle", () => { session_id: session.sessionId, episode_id: assignedEpisodeId(service, complete), raw_turn_id: complete.rawTurnId, - user_id: namespace.userId, + user_id: null, kind: "file", uri: "file:///tmp/memmy-report.txt" }); @@ -775,7 +970,7 @@ describe("L3 World Model trace boundaries", () => { requestId: "e4880aaa-e635-4c60-a58f-eec7c21677af", adapterId: "codex-memory", source: "codex", - namespace: { ...namespace, userId: "other-user" } + namespace: { ...namespace, profileId: "other-profile" } })).toThrow("l3_world_model_session_scope_conflict"); expect(() => service.l3WorldModelBoundary(opened.sessionId, { requestId: "9c7e85f5-5fb3-44b5-a91e-6c92d3ae5239", diff --git a/Memory/tests/service/session/source-turn-capture.test.ts b/Memory/tests/service/session/source-turn-capture.test.ts index 3665bdda5..e1c80a389 100644 --- a/Memory/tests/service/session/source-turn-capture.test.ts +++ b/Memory/tests/service/session/source-turn-capture.test.ts @@ -165,7 +165,9 @@ describe("native source turn submission", () => { const { db, service } = createTestService(); const input = request(); const opened = service.openSession({ namespace: { ...input.namespace!, sessionKey: input.sourceTurn.conversationId }, meta: { conversationId: input.sourceTurn.conversationId } }); - new Repositories(db.db).captureClaims.claim({ userId: "source-user", source: "codex", qaHash: memoryCaptureQaHash(input.query, input.answer), primaryMemoryId: "deleted-old-memory", capturedBy: "agent_source_scan", createdAt: "2026-01-01" }); + db.db.prepare(`INSERT INTO memory_capture_claims(user_id,source,qa_hash,primary_memory_id,captured_by,created_at) + VALUES ('source-user','codex',?,'deleted-old-memory','agent_source_scan','2026-01-01')`) + .run(memoryCaptureQaHash(input.query, input.answer)); const stored = (await service.completeSourceTurn(request({ channel: "agent_source_scan" }))); expect(stored.result?.sessionId).toBe(opened.sessionId); expect(counts(db).sessions).toBe(1); @@ -191,12 +193,12 @@ describe("native source turn submission", () => { expect(second).toMatchObject({ status: "existing", result: { sessionId: opened.sessionId, l1MemoryId: first.result!.l1MemoryId } }); expect(counts(db).sessions).toBe(1); expect(runtime.getSession(opened.sessionId)).toMatchObject({ - userId: "source-user", hostSessionKey: original.hostSessionKey, conversationId: input.sourceTurn.conversationId, + userId: null, hostSessionKey: original.hostSessionKey, conversationId: input.sourceTurn.conversationId, projectId: original.projectId, workspaceId: original.workspaceId, meta: original.meta }); - expect(runtime.getRawTurn(first.result!.rawTurnId)).toMatchObject({ userId: "source-user", conversationId: input.sourceTurn.conversationId }); + expect(runtime.getRawTurn(first.result!.rawTurnId)).toMatchObject({ userId: null, conversationId: input.sourceTurn.conversationId }); const memory = new Repositories(db.db).memories.get(first.result!.l1MemoryId)!; - expect(memory).toMatchObject({ userId: "source-user", conversationId: input.sourceTurn.conversationId, appId: original.workspaceId }); + expect(memory).toMatchObject({ userId: null, conversationId: input.sourceTurn.conversationId, appId: original.workspaceId }); expect(memory.info.project_id).toBe(original.projectId); service.closeSession(opened.sessionId, { namespace: input.namespace }); expect(runtime.getEpisode(first.result!.episodeId)?.status).toBe("closed"); @@ -281,7 +283,7 @@ describe("native source turn submission", () => { expect(new Repositories(db.db).memories.get(completed.result!.l1MemoryId)?.conversationId).toBe(input.sourceTurn.conversationId); }); - it.each(["user", "conversation"] as const)("refuses an observed RawTurn with a conflicting %s", async (conflict) => { + it.each(["user", "conversation"] as const)("handles observed RawTurn legacy %s identity", async (conflict) => { const { db, service } = createTestService(); const input = request(); const opened = service.openSession({ namespace: { ...input.namespace!, sessionKey: `codex-memory-${input.sourceTurn.conversationId}` } }); @@ -292,7 +294,12 @@ describe("native source turn submission", () => { const runtime = new Repositories(db.db).runtime; const original = runtime.getRawTurn(observed.rawTurnId!)!; const before = counts(db); - await expect(async () => (await service.completeSourceTurn(request({ sessionId: opened.sessionId })))).rejects.toThrow("source_raw_turn_scope_conflict"); + if (conflict === "user") { + expect((await service.completeSourceTurn(request({ sessionId: opened.sessionId }))).status).toBe("stored"); + expect(runtime.getRawTurn(observed.rawTurnId!)?.userId).toBeNull(); + return; + } + await expect(service.completeSourceTurn(request({ sessionId: opened.sessionId }))).rejects.toThrow("source_raw_turn_scope_conflict"); expect(counts(db)).toEqual(before); expect(runtime.getRawTurn(observed.rawTurnId!)).toEqual(original); expect(runtime.getSession(opened.sessionId)?.conversationId).toBeUndefined(); @@ -347,7 +354,7 @@ describe("native source turn submission", () => { sessionId: opened.sessionId, episodeId: first.result!.episodeId } }); expect(second.result!.l1MemoryId).not.toBe(first.result!.l1MemoryId); - expect(db.db.prepare("SELECT DISTINCT user_id FROM source_turn_captures").all()).toEqual([{ user_id: "installed-owner" }]); + expect(db.db.prepare("SELECT DISTINCT user_id FROM source_turn_captures").all()).toEqual([{ user_id: null }]); expect(counts(db)).toMatchObject({ sessions: 1, episodes: 1, raw_turns: 2, source_turn_captures: 2 }); await runWorkerRounds(service, 4); const recall = await service.search({ query: input.query, layers: ["L1"], limit: 10 }); @@ -370,7 +377,7 @@ describe("native source turn submission", () => { expect(scanned).toMatchObject({ status: "stored", result: { sessionId: opened.sessionId } }); expect((await service.completeSourceTurn(input)).status).toBe("existing"); expect(counts(db)).toMatchObject({ sessions: 1, episodes: 1, raw_turns: 1, source_turn_captures: 1 }); - expect(new Repositories(db.db).memories.get(scanned.result!.l1MemoryId)?.userId).toBe(input.namespace!.userId); + expect(new Repositories(db.db).memories.get(scanned.result!.l1MemoryId)?.userId).toBe(null); }); it("retains cross-account conflicts and tombstones after the Episode closes", async () => { @@ -711,7 +718,7 @@ describe("native source turn submission", () => { expect(activeMemories()).toBe(deletedBefore); }); - it("reuses an old agent-source import and keeps it deleted", async () => { + it.each(["legacy", "scoped", "session"])("reuses an agent-source import (%s key) and keeps it deleted", async (keyKind) => { const { db, service } = createTestService(); const conversationId = request().sourceTurn.conversationId; const firstUserMessageId = "legacy-user-message"; @@ -721,7 +728,7 @@ describe("native source turn submission", () => { answer: "Implemented and verified the capture identity transaction." }; const imported = service.addMemory({ - namespace: request().namespace, + namespace: { ...request().namespace!, ...(keyKind === "session" ? { sessionKey: conversationId } : {}) }, adapterId: "agent-source:codex", turnId: importedTurnId, source: "codex", @@ -730,6 +737,9 @@ describe("native source turn submission", () => { title: content.query, tags: ["agent-source", "codex"] }); + if (keyKind === "legacy") { + db.db.prepare("UPDATE memories SET memory_key=? WHERE id=?").run(`memory.add:agent-source:codex:turn:${importedTurnId}`, imported.id); + } const historical = async () => (await service.completeSourceTurn(request({ channel: "agent_source_scan", captureLegacyHistory: true, @@ -862,7 +872,7 @@ describe("native source turn submission", () => { ); it("still treats an unkeyed project session as completed for a default-scope scan", async () => { - const { service } = createTestService(); + const { db, service } = createTestService(); const conversationId = "unkeyed-project-to-default"; const content = { query: "Implement capture identity transactions.", @@ -873,6 +883,8 @@ describe("native source turn submission", () => { meta: { conversationId } }); (await service.completeTurn("same-native-turn", { sessionId: opened.sessionId, ...content })); + // This compatibility case models an old writer without immutable provenance. + db.db.prepare("UPDATE raw_turns SET message_payload_json=json_remove(message_payload_json,'$.native_origin_v1') WHERE session_id=?").run(opened.sessionId); service.closeSession(opened.sessionId); expect((await service.completeSourceTurn(request({ channel: "agent_source_scan", @@ -1020,6 +1032,30 @@ describe("native source turn submission", () => { startedAt: "2099-01-01T10:02:00.000Z", completedAt: "2099-01-01T10:03:00.000Z" } }))).status).toBe("stored"); }); + + it.each([false, true])("continues the same native conversation immediately after full clear (restart=%s)", async (restart) => { + const {root,db,service} = createTestService(); + const first = (await service.completeSourceTurn(request())); + service.deleteMemory(first.result!.l1MemoryId); + expect((await service.completeSourceTurn(request()))).toMatchObject({status:"rejected",reason:"capture_deleted"}); + const activation = new Repositories(db.db).runtime.getKv("source_turn_capture_activated_at"); + service.clearAllData(); + expect(counts(db).source_turn_captures).toBe(0); + let activeDb = db, activeService = service; + if (restart) { + db.close(); + activeDb = new MemoryDb({path:join(root,"memory.sqlite")}); + activeService = createTestMemoryService({db:activeDb}); + } + const next = request({namespace:{...request().namespace!,userId:"another-account"},sourceTurn:{...request().sourceTurn, + turnId:"next-after-clear",sequence:2,startedAt:"2099-01-01T10:02:00.000Z",completedAt:"2099-01-01T10:03:00.000Z"}}); + const stored = (await activeService.completeSourceTurn(next)); + expect(stored.status).toBe("stored"); + expect(stored.result!.sessionId).not.toBe(first.result!.sessionId); + expect((await activeService.completeSourceTurn(next))).toMatchObject({status:"existing",result:{l1MemoryId:stored.result!.l1MemoryId}}); + expect(new Repositories(activeDb.db).runtime.getKv("source_turn_capture_activated_at")).toEqual(activation); + }); + it("reimports pre-activation scan history after full clear while preserving the hook boundary", async () => { const { db, service } = createTestService(); const historical = request({ channel: "agent_source_scan", captureLegacyHistory: true, diff --git a/Memory/tests/service/user-memory/user-memory.test.ts b/Memory/tests/service/user-memory/user-memory.test.ts index e2501806b..63d773a23 100644 --- a/Memory/tests/service/user-memory/user-memory.test.ts +++ b/Memory/tests/service/user-memory/user-memory.test.ts @@ -119,11 +119,11 @@ describe("User Memory", () => { }) }); const session = open(service, "one-off-task-user"); - const turns = (await Promise.all(requests.map(async (query, index) => (await service.completeTurn(`one-off-task-${index}`, { + const turns = (await Promise.all((await Promise.all(requests.map(async (query, index) => (await service.completeTurn(`one-off-task-${index}`, { sessionId: session.sessionId, query, answer: "好的。" - }))))); + }))))))); await service.runWorkerOnce(50, { priorityCohortOnly: true }); @@ -937,7 +937,7 @@ describe("User Memory", () => { db.close(); }); - it("lists User Memory in its own panel layer with user isolation and search", async () => { + it("lists User Memory in its own panel layer across accounts with search", async () => { const { db, service } = createTestService(); const session = open(service, "panel-user"); (await service.completeTurn("turn-panel-user-memory", { @@ -962,13 +962,13 @@ describe("User Memory", () => { expect(service.panelItems({ namespace: { source: "codex", profileId: "default", userId: "another-user" }, layer: "UserMemory" - }).items).toEqual([]); + }).items).toEqual(panel.items); expect(service.panelOverviewSummary({ namespace: { source: "codex", profileId: "default", userId: "panel-user" } }).counts.userMemories).toBe(1); expect(service.panelOverviewSummary({ namespace: { source: "codex", profileId: "default", userId: "another-user" } - }).counts.userMemories).toBe(0); + }).counts.userMemories).toBe(1); db.close(); }); diff --git a/Memory/tests/service/work-memory/work-memory-pipeline.test.ts b/Memory/tests/service/work-memory/work-memory-pipeline.test.ts index 0ad8eb4f4..4309a8dc6 100644 --- a/Memory/tests/service/work-memory/work-memory-pipeline.test.ts +++ b/Memory/tests/service/work-memory/work-memory-pipeline.test.ts @@ -7,6 +7,7 @@ import { canonicalWorkMemoryText, WorkMemoryPipeline } from "../../../src/service/work-memory/work-memory-pipeline.js"; +import { refreshProjectAliases } from "../../../src/storage/device-scope.js"; import { stableHash } from "../../../src/utils/id.js"; import { createCapturingEmbedder, @@ -73,7 +74,7 @@ describe("Work Memory pipeline", () => { })); }); - it("第一阶段只抽取 requirement/reason,第二阶段 create 时生成 topic 并写入 L1 Work Memory", async () => { + it.each(["new", "new-on-legacy-session", "legacy-job"])("第一阶段抽取并写入 L1 Work Memory (%s)", async (mode) => { const { db, service } = createTestService(); const namespace = { source: "codex", @@ -104,7 +105,17 @@ describe("Work Memory pipeline", () => { throughL1MemoryId: completed.l1MemoryId }); const repos = new Repositories(db.db); - const job = findWorkMemoryJob(repos); + let job = findWorkMemoryJob(repos); + if (mode !== "new") { + db.db.prepare("UPDATE sessions SET user_id='legacy-owner', project_id='legacy-project' WHERE id=?").run(opened.sessionId); + refreshProjectAliases(db.db); + if (mode === "legacy-job") { + db.db.prepare("UPDATE evolution_jobs SET user_id='legacy-owner', scope_key=?, dedupe_key=? WHERE id=?").run( + stableHash(["work_memory", "legacy-owner", "legacy-project"]), + stableHash(["work_memory_extract", "legacy-owner", "codex", "legacy-project", job.payload.trajectoryHash]), job.id); + } + job = repos.runtime.getJob(job.id)!; + } const completeJson = vi.fn() .mockResolvedValueOnce({ work_memories: [{ requirement: "固定数据清洗流水线", reason: "保证训练结果可复现" }] }) .mockResolvedValueOnce({ decisions: [{ @@ -478,7 +489,7 @@ function workMemoryNamespace( ): { source: string; profileId: string; - userId: string; + userId: string | null; sessionKey: string; projectId?: string; } { @@ -551,7 +562,7 @@ function testLlm(completeJson: LlmClient["completeJson"]): LlmClient { }; } -function workMemoryRow(userId: string, projectId: string | null): MemoryRow { +function workMemoryRow(userId: string | null, projectId: string | null): MemoryRow { const at = "2026-01-01T00:00:00.000Z"; const topic = "SFT training pipeline"; const requirement = "固定 SFT 数据清洗流程"; diff --git a/Memory/tests/service/worker/embedding-maintenance-freshness.test.ts b/Memory/tests/service/worker/embedding-maintenance-freshness.test.ts index 85c05d063..f03a28c56 100644 --- a/Memory/tests/service/worker/embedding-maintenance-freshness.test.ts +++ b/Memory/tests/service/worker/embedding-maintenance-freshness.test.ts @@ -9,9 +9,10 @@ afterEach(() => { vi.restoreAllMocks(); cleanup(); }); function seed(repos: Repositories, userMemory: boolean, id = "edited-memory") { const at = new Date().toISOString(); if (userMemory) { + const content = `old content ${id}`; repos.userMemories.insert({ - id, sourceTurnId: id, userId: "local-user", memoryTypes: [], content: "old content", - normalizedUserTextHash: stableHash("old content"), sourceTurnRefs: [], status: "active", createdAt: at, updatedAt: at + id, sourceTurnId: id, userId: "local-user", memoryTypes: [], content, + normalizedUserTextHash: stableHash(content), sourceTurnRefs: [], status: "active", createdAt: at, updatedAt: at }); } else { repos.memories.insert({ diff --git a/Memory/tests/service/worker/embedding-retry-reproduction.test.ts b/Memory/tests/service/worker/embedding-retry-reproduction.test.ts index cacbdbd73..cd1cf53ec 100644 --- a/Memory/tests/service/worker/embedding-retry-reproduction.test.ts +++ b/Memory/tests/service/worker/embedding-retry-reproduction.test.ts @@ -144,7 +144,7 @@ describe("v1.2.6 embedding retry reproduction", () => { expect(probe).toHaveBeenCalledTimes(2); }); - it("combines both embedding queues and excludes unrelated or other users' jobs", () => { + it("combines both embedding queues across device-scoped user memories and excludes unrelated jobs", () => { const { db } = createTestService(); const runtime = new Repositories(db.db).runtime; const at = new Date(1_000).toISOString(); @@ -158,7 +158,7 @@ describe("v1.2.6 embedding retry reproduction", () => { createdAt: at, updatedAt: at }); } runtime.enqueueEmbeddingRetry({ targetKind: "trace", targetId: "combined", vectorField: "vec_summary", sourceText: "test", now: 1_000 }); - expect(runtime.embeddingMaintenanceQueueStats("local-user", 2_000)).toEqual({ pending: 3, active: 0, stale: 0, failed: 0 }); + expect(runtime.embeddingMaintenanceQueueStats(null, 2_000)).toEqual({ pending: 4, active: 0, stale: 0, failed: 0 }); }); it("queues only the three missing vectors from the Viewer repair request", async () => { diff --git a/Memory/tests/storage/token-usage-outbox.test.ts b/Memory/tests/storage/token-usage-outbox.test.ts index fd5e05618..c7192aa3b 100644 --- a/Memory/tests/storage/token-usage-outbox.test.ts +++ b/Memory/tests/storage/token-usage-outbox.test.ts @@ -3,6 +3,7 @@ import { tmpdir } from "node:os"; import { join } from "node:path"; import { afterEach, describe, expect, it } from "vitest"; import { MemoryDb } from "../../src/index.js"; +import { SCHEMA_VERSION, SCHEMA_MIGRATION_ID } from "../../src/storage/schema.js"; import { Repositories } from "../../src/storage/repositories.js"; import { TokenUsageOutbox } from "../../src/storage/token-usage-outbox.js"; @@ -64,8 +65,8 @@ describe("token usage outbox", () => { const upgraded = new MemoryDb({ path: dbPath }); expect(upgraded.schemaVersion()).toMatchObject({ - version: 10, - lastMigrationId: "010_pending_episode_assignment" + version: SCHEMA_VERSION, + lastMigrationId: SCHEMA_MIGRATION_ID }); expect(upgraded.db.prepare(`SELECT id FROM memories`).get()).toEqual({ id: "memory-keep" }); expect(upgraded.db.prepare( @@ -75,7 +76,7 @@ describe("token usage outbox", () => { upgraded.close(); const reopened = new MemoryDb({ path: dbPath }); - expect(reopened.schemaVersion().version).toBe(10); + expect(reopened.schemaVersion().version).toBe(SCHEMA_VERSION); expect(reopened.db.prepare(`SELECT COUNT(*) AS count FROM memories`).get()).toEqual({ count: 1 }); reopened.close(); });